openstack/swift
Code Issues Proposed changes

Merge from trunk (i18n)

Browse Source
This commit is contained in:
gholt 2010-12-22 12:33:06 -08:00
commit a7edf44b16
54 changed files with 411 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/st
View File

@ -44,6 +44,8 @@ except:
try: try:
from swift.common.bufferedhttp \ from swift.common.bufferedhttp \
import BufferedHTTPConnection as HTTPConnection import BufferedHTTPConnection as HTTPConnection
import gettext
gettext.install('swift', unicode=1)
except: except:
from httplib import HTTPConnection from httplib import HTTPConnection

2
bin/swauth-add-account
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import gettext
from optparse import OptionParser from optparse import OptionParser
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
@ -23,6 +24,7 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
parser = OptionParser(usage='Usage: %prog [options] <account>') parser = OptionParser(usage='Usage: %prog [options] <account>')
parser.add_option('-s', '--suffix', dest='suffix', parser.add_option('-s', '--suffix', dest='suffix',
default='', help='The suffix to use with the reseller prefix as the ' default='', help='The suffix to use with the reseller prefix as the '

2
bin/swauth-add-user
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import gettext
from optparse import OptionParser from optparse import OptionParser
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
@ -23,6 +24,7 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
parser = OptionParser( parser = OptionParser(
usage='Usage: %prog [options] <account> <user> <password>') usage='Usage: %prog [options] <account> <user> <password>')
parser.add_option('-a', '--admin', dest='admin', action='store_true', parser.add_option('-a', '--admin', dest='admin', action='store_true',

2
bin/swauth-cleanup-tokens
View File

@ -18,6 +18,7 @@ try:
import simplejson as json import simplejson as json
except ImportError: except ImportError:
import json import json
import gettext
import re import re
from datetime import datetime, timedelta from datetime import datetime, timedelta
from optparse import OptionParser from optparse import OptionParser
@ -28,6 +29,7 @@ from swift.common.client import Connection
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
parser = OptionParser(usage='Usage: %prog [options]') parser = OptionParser(usage='Usage: %prog [options]')
parser.add_option('-t', '--token-life', dest='token_life', parser.add_option('-t', '--token-life', dest='token_life',
default='86400', help='The expected life of tokens; token objects ' default='86400', help='The expected life of tokens; token objects '

2
bin/swauth-delete-account
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import gettext
from optparse import OptionParser from optparse import OptionParser
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
@ -23,6 +24,7 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
parser = OptionParser(usage='Usage: %prog [options] <account>') parser = OptionParser(usage='Usage: %prog [options] <account>')
parser.add_option('-A', '--admin-url', dest='admin_url', parser.add_option('-A', '--admin-url', dest='admin_url',
default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' default='http://127.0.0.1:8080/auth/', help='The URL to the auth '

2
bin/swauth-delete-user
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import gettext
from optparse import OptionParser from optparse import OptionParser
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
@ -23,6 +24,7 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
parser = OptionParser(usage='Usage: %prog [options] <account> <user>') parser = OptionParser(usage='Usage: %prog [options] <account> <user>')
parser.add_option('-A', '--admin-url', dest='admin_url', parser.add_option('-A', '--admin-url', dest='admin_url',
default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' default='http://127.0.0.1:8080/auth/', help='The URL to the auth '

2
bin/swauth-list
View File

@ -18,6 +18,7 @@ try:
import simplejson as json import simplejson as json
except ImportError: except ImportError:
import json import json
import gettext
from optparse import OptionParser from optparse import OptionParser
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
@ -27,6 +28,7 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
parser = OptionParser(usage=''' parser = OptionParser(usage='''
Usage: %prog [options] [account] [user] Usage: %prog [options] [account] [user]

2
bin/swauth-prep
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import gettext
from optparse import OptionParser from optparse import OptionParser
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
@ -23,6 +24,7 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
parser = OptionParser(usage='Usage: %prog [options]') parser = OptionParser(usage='Usage: %prog [options]')
parser.add_option('-A', '--admin-url', dest='admin_url', parser.add_option('-A', '--admin-url', dest='admin_url',
default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' default='http://127.0.0.1:8080/auth/', help='The URL to the auth '

2
bin/swauth-set-account-service
View File

@ -18,6 +18,7 @@ try:
import simplejson as json import simplejson as json
except ImportError: except ImportError:
import json import json
import gettext
from optparse import OptionParser from optparse import OptionParser
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
@ -27,6 +28,7 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
parser = OptionParser(usage=''' parser = OptionParser(usage='''
Usage: %prog [options] <account> <service> <name> <value> Usage: %prog [options] <account> <service> <name> <value>

2
bin/swift-account-audit
View File

@ -20,6 +20,7 @@ from urllib import quote
from hashlib import md5 from hashlib import md5
import getopt import getopt
from itertools import chain from itertools import chain
import gettext
import simplejson import simplejson
from eventlet.greenpool import GreenPool from eventlet.greenpool import GreenPool
@ -324,6 +325,7 @@ class Auditor(object):
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
try: try:
optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d') optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d')
except getopt.GetoptError, err: except getopt.GetoptError, err:

2
bin/swift-auth-add-user
View File

@ -18,11 +18,13 @@ from ConfigParser import ConfigParser
from optparse import OptionParser from optparse import OptionParser
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
import gettext
from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
default_conf = '/etc/swift/auth-server.conf' default_conf = '/etc/swift/auth-server.conf'
parser = OptionParser( parser = OptionParser(
usage='Usage: %prog [options] <account> <user> <password>') usage='Usage: %prog [options] <account> <user> <password>')

2
bin/swift-auth-recreate-accounts
View File

@ -17,10 +17,12 @@
from ConfigParser import ConfigParser from ConfigParser import ConfigParser
from optparse import OptionParser from optparse import OptionParser
from sys import argv, exit from sys import argv, exit
import gettext
from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
default_conf = '/etc/swift/auth-server.conf' default_conf = '/etc/swift/auth-server.conf'
parser = OptionParser(usage='Usage: %prog [options]') parser = OptionParser(usage='Usage: %prog [options]')
parser.add_option('-c', '--conf', dest='conf', default=default_conf, parser.add_option('-c', '--conf', dest='conf', default=default_conf,

2
bin/swift-auth-to-swauth
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import gettext
from subprocess import call from subprocess import call
from sys import argv, exit from sys import argv, exit
@ -21,6 +22,7 @@ import sqlite3
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
if len(argv) != 4 or argv[1] != '-K': if len(argv) != 4 or argv[1] != '-K':
exit('Syntax: %s -K <super_admin_key> <path to auth.db>' % argv[0]) exit('Syntax: %s -K <super_admin_key> <path to auth.db>' % argv[0])
_, _, super_admin_key, auth_db = argv _, _, super_admin_key, auth_db = argv

2
bin/swift-auth-update-reseller-prefixes
View File

@ -16,11 +16,13 @@
from os.path import basename from os.path import basename
from sys import argv, exit from sys import argv, exit
import gettext
from swift.common.db import get_db_connection from swift.common.db import get_db_connection
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
app = basename(argv[0]) app = basename(argv[0])
if len(argv) != 3: if len(argv) != 3:
exit(''' exit('''

2
bin/swift-bench
View File

@ -20,6 +20,7 @@ import sys
import signal import signal
import uuid import uuid
from optparse import OptionParser from optparse import OptionParser
import gettext
from swift.common.bench import BenchController from swift.common.bench import BenchController
from swift.common.utils import readconf, NamedLogger from swift.common.utils import readconf, NamedLogger
@ -55,6 +56,7 @@ SAIO_DEFAULTS = {
} }
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
usage = "usage: %prog [OPTIONS] [CONF_FILE]" usage = "usage: %prog [OPTIONS] [CONF_FILE]"
usage += """\n\nConf file with SAIO defaults: usage += """\n\nConf file with SAIO defaults:

2
bin/swift-drive-audit
View File

@ -20,6 +20,7 @@ import re
import subprocess import subprocess
import sys import sys
from ConfigParser import ConfigParser from ConfigParser import ConfigParser
import gettext
from swift.common.utils import get_logger from swift.common.utils import get_logger
@ -86,6 +87,7 @@ def comment_fstab(mount_point):
os.rename('/etc/fstab.new', '/etc/fstab') os.rename('/etc/fstab.new', '/etc/fstab')
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
c = ConfigParser() c = ConfigParser()
try: try:
conf_path = sys.argv[1] conf_path = sys.argv[1]

3
bin/swift-get-nodes
View File

@ -16,11 +16,14 @@
import sys import sys
import urllib import urllib
import gettext
from swift.common.ring import Ring from swift.common.ring import Ring
from swift.common.utils import hash_path from swift.common.utils import hash_path
gettext.install('swift', unicode=1)
if len(sys.argv) < 3 or len(sys.argv) > 5: if len(sys.argv) < 3 or len(sys.argv) > 5:
print 'Usage: %s <ring.gz> <account> [<container>] [<object>]' \ print 'Usage: %s <ring.gz> <account> [<container>] [<object>]' \
% sys.argv[0] % sys.argv[0]

2
bin/swift-log-uploader
View File

@ -15,12 +15,14 @@
# limitations under the License. # limitations under the License.
import sys import sys
import gettext
from swift.stats.log_uploader import LogUploader from swift.stats.log_uploader import LogUploader
from swift.common.utils import parse_options from swift.common.utils import parse_options
from swift.common import utils from swift.common import utils
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN") conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN")
try: try:
plugin = options['extra_args'][0] plugin = options['extra_args'][0]

2
bin/swift-object-info
View File

@ -18,12 +18,14 @@ import sys
import cPickle as pickle import cPickle as pickle
from datetime import datetime from datetime import datetime
from hashlib import md5 from hashlib import md5
import gettext
from swift.common.ring import Ring from swift.common.ring import Ring
from swift.obj.server import read_metadata from swift.obj.server import read_metadata
from swift.common.utils import hash_path from swift.common.utils import hash_path
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
if len(sys.argv) <= 1: if len(sys.argv) <= 1:
print "Usage: %s OBJECT_FILE" % sys.argv[0] print "Usage: %s OBJECT_FILE" % sys.argv[0]
sys.exit(1) sys.exit(1)

2
bin/swift-ring-builder
View File

@ -21,6 +21,7 @@ from os import mkdir
from os.path import basename, dirname, exists, join as pathjoin from os.path import basename, dirname, exists, join as pathjoin
from sys import argv, exit from sys import argv, exit
from time import time from time import time
import gettext
from swift.common.ring import RingBuilder from swift.common.ring import RingBuilder
@ -174,6 +175,7 @@ swift-ring-builder <builder_file> set_min_part_hours <hours>
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
if len(argv) < 2: if len(argv) < 2:
print ''' print '''
swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s

2
bin/swift-stats-populate
View File

@ -21,6 +21,7 @@ from optparse import OptionParser
from sys import exit, argv from sys import exit, argv
from time import time from time import time
from uuid import uuid4 from uuid import uuid4
import gettext
from eventlet import GreenPool, patcher, sleep from eventlet import GreenPool, patcher, sleep
from eventlet.pools import Pool from eventlet.pools import Pool
@ -75,6 +76,7 @@ def report(success):
if __name__ == '__main__': if __name__ == '__main__':
global begun, created, item_type, next_report, need_to_create, retries_done global begun, created, item_type, next_report, need_to_create, retries_done
gettext.install('swift', unicode=1)
patcher.monkey_patch() patcher.monkey_patch()
parser = OptionParser() parser = OptionParser()

2
bin/swift-stats-report
View File

@ -23,6 +23,7 @@ from optparse import OptionParser
from sys import argv, exit, stderr from sys import argv, exit, stderr
from time import time from time import time
from uuid import uuid4 from uuid import uuid4
import gettext
from eventlet import GreenPool, hubs, patcher, sleep, Timeout from eventlet import GreenPool, hubs, patcher, sleep, Timeout
from eventlet.pools import Pool from eventlet.pools import Pool
@ -746,6 +747,7 @@ def object_delete_report(coropool, connpool, options):
if __name__ == '__main__': if __name__ == '__main__':
gettext.install('swift', unicode=1)
patcher.monkey_patch() patcher.monkey_patch()
hubs.get_hub().debug_exceptions = False hubs.get_hub().debug_exceptions = False

26
swift/account/auditor.py
View File

@ -49,11 +49,11 @@ class AccountAuditor(Daemon):
for path, device, partition in all_locs: for path, device, partition in all_locs:
self.account_audit(path) self.account_audit(path)
if time.time() - reported >= 3600: # once an hour if time.time() - reported >= 3600: # once an hour
self.logger.info( self.logger.info(_('Since %(time)s: Account audits: '
'Since %s: Account audits: %s passed audit, ' '%(passed)s passed audit, %(failed)s failed audit'),
'%s failed audit' % (time.ctime(reported), {'time': time.ctime(reported),
self.account_passes, 'passed': self.account_passes,
self.account_failures)) 'failed': self.account_failures})
reported = time.time() reported = time.time()
self.account_passes = 0 self.account_passes = 0
self.account_failures = 0 self.account_failures = 0
@ -72,17 +72,17 @@ class AccountAuditor(Daemon):
for path, device, partition in all_locs: for path, device, partition in all_locs:
self.account_audit(path) self.account_audit(path)
if time.time() - reported >= 3600: # once an hour if time.time() - reported >= 3600: # once an hour
self.logger.info( self.logger.info(_('Since %(time)s: Account audits: '
'Since %s: Account audits: %s passed audit, ' '%(passed)s passed audit, %(failed)s failed audit'),
'%s failed audit' % (time.ctime(reported), {'time': time.ctime(reported),
self.account_passes, 'passed': self.account_passes,
self.account_failures)) 'failed': self.account_failures})
reported = time.time() reported = time.time()
self.account_passes = 0 self.account_passes = 0
self.account_failures = 0 self.account_failures = 0
elapsed = time.time() - begin elapsed = time.time() - begin
self.logger.info( self.logger.info(
'Account audit "once" mode completed: %.02fs' % elapsed) 'Account audit "once" mode completed: %.02fs', elapsed)
def account_audit(self, path): def account_audit(self, path):
""" """
@ -97,8 +97,8 @@ class AccountAuditor(Daemon):
if not broker.is_deleted(): if not broker.is_deleted():
info = broker.get_info() info = broker.get_info()
self.account_passes += 1 self.account_passes += 1
self.logger.debug('Audit passed for %s' % broker.db_file) self.logger.debug(_('Audit passed for %s') % broker.db_file)
except Exception: except Exception:
self.account_failures += 1 self.account_failures += 1
self.logger.exception('ERROR Could not get account info %s' % self.logger.exception(_('ERROR Could not get account info %s'),
(broker.db_file)) (broker.db_file))

31
swift/account/reaper.py
View File

@ -77,7 +77,7 @@ class AccountReaper(Daemon):
""" The account :class:`swift.common.ring.Ring` for the cluster. """ """ The account :class:`swift.common.ring.Ring` for the cluster. """
if not self.account_ring: if not self.account_ring:
self.logger.debug( self.logger.debug(
'Loading account ring from %s' % self.account_ring_path) _('Loading account ring from %s'), self.account_ring_path)
self.account_ring = Ring(self.account_ring_path) self.account_ring = Ring(self.account_ring_path)
return self.account_ring return self.account_ring
@ -85,7 +85,7 @@ class AccountReaper(Daemon):
""" The container :class:`swift.common.ring.Ring` for the cluster. """ """ The container :class:`swift.common.ring.Ring` for the cluster. """
if not self.container_ring: if not self.container_ring:
self.logger.debug( self.logger.debug(
'Loading container ring from %s' % self.container_ring_path) _('Loading container ring from %s'), self.container_ring_path)
self.container_ring = Ring(self.container_ring_path) self.container_ring = Ring(self.container_ring_path)
return self.container_ring return self.container_ring
@ -93,7 +93,7 @@ class AccountReaper(Daemon):
""" The object :class:`swift.common.ring.Ring` for the cluster. """ """ The object :class:`swift.common.ring.Ring` for the cluster. """
if not self.object_ring: if not self.object_ring:
self.logger.debug( self.logger.debug(
'Loading object ring from %s' % self.object_ring_path) _('Loading object ring from %s'), self.object_ring_path)
self.object_ring = Ring(self.object_ring_path) self.object_ring = Ring(self.object_ring_path)
return self.object_ring return self.object_ring
@ -103,7 +103,7 @@ class AccountReaper(Daemon):
This repeatedly calls :func:`reap_once` no quicker than the This repeatedly calls :func:`reap_once` no quicker than the
configuration interval. configuration interval.
""" """
self.logger.debug('Daemon started.') self.logger.debug(_('Daemon started.'))
sleep(random.random() * self.interval) sleep(random.random() * self.interval)
while True: while True:
begin = time() begin = time()
@ -119,17 +119,17 @@ class AccountReaper(Daemon):
repeatedly by :func:`run_forever`. This will call :func:`reap_device` repeatedly by :func:`run_forever`. This will call :func:`reap_device`
once for each device on the server. once for each device on the server.
""" """
self.logger.debug('Begin devices pass: %s' % self.devices) self.logger.debug(_('Begin devices pass: %s'), self.devices)
begin = time() begin = time()
for device in os.listdir(self.devices): for device in os.listdir(self.devices):
if self.mount_check and \ if self.mount_check and \
not os.path.ismount(os.path.join(self.devices, device)): not os.path.ismount(os.path.join(self.devices, device)):
self.logger.debug( self.logger.debug(
'Skipping %s as it is not mounted' % device) _('Skipping %s as it is not mounted'), device)
continue continue
self.reap_device(device) self.reap_device(device)
elapsed = time() - begin elapsed = time() - begin
self.logger.info('Devices pass completed: %.02fs' % elapsed) self.logger.info(_('Devices pass completed: %.02fs'), elapsed)
def reap_device(self, device): def reap_device(self, device):
""" """
@ -212,7 +212,7 @@ class AccountReaper(Daemon):
""" """
begin = time() begin = time()
account = broker.get_info()['account'] account = broker.get_info()['account']
self.logger.info('Beginning pass on account %s' % account) self.logger.info(_('Beginning pass on account %s'), account)
self.stats_return_codes = {} self.stats_return_codes = {}
self.stats_containers_deleted = 0 self.stats_containers_deleted = 0
self.stats_objects_deleted = 0 self.stats_objects_deleted = 0
@ -235,12 +235,12 @@ class AccountReaper(Daemon):
self.container_pool.waitall() self.container_pool.waitall()
except Exception: except Exception:
self.logger.exception( self.logger.exception(
'Exception with containers for account %s' % account) _('Exception with containers for account %s'), account)
marker = containers[-1][0] marker = containers[-1][0]
log = 'Completed pass on account %s' % account log = 'Completed pass on account %s' % account
except Exception: except Exception:
self.logger.exception( self.logger.exception(
'Exception with account %s' % account) _('Exception with account %s'), account)
log = 'Incomplete pass on account %s' % account log = 'Incomplete pass on account %s' % account
if self.stats_containers_deleted: if self.stats_containers_deleted:
log += ', %s containers deleted' % self.stats_containers_deleted log += ', %s containers deleted' % self.stats_containers_deleted
@ -317,7 +317,7 @@ class AccountReaper(Daemon):
except ClientException, err: except ClientException, err:
if self.logger.getEffectiveLevel() <= DEBUG: if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception( self.logger.exception(
'Exception with %(ip)s:%(port)s/%(device)s' % node) _('Exception with %(ip)s:%(port)s/%(device)s'), node)
self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes[err.http_status / 100] = \
self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.stats_return_codes.get(err.http_status / 100, 0) + 1
if not objects: if not objects:
@ -330,8 +330,9 @@ class AccountReaper(Daemon):
nodes, obj['name']) nodes, obj['name'])
pool.waitall() pool.waitall()
except Exception: except Exception:
self.logger.exception('Exception with objects for container ' self.logger.exception(_('Exception with objects for container '
'%s for account %s' % (container, account)) '%(container)s for account %(account)s'),
{'container': container, 'account': account})
marker = objects[-1]['name'] marker = objects[-1]['name']
successes = 0 successes = 0
failures = 0 failures = 0
@ -351,7 +352,7 @@ class AccountReaper(Daemon):
except ClientException, err: except ClientException, err:
if self.logger.getEffectiveLevel() <= DEBUG: if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception( self.logger.exception(
'Exception with %(ip)s:%(port)s/%(device)s' % node) _('Exception with %(ip)s:%(port)s/%(device)s'), node)
failures += 1 failures += 1
self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes[err.http_status / 100] = \
self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.stats_return_codes.get(err.http_status / 100, 0) + 1
@ -402,7 +403,7 @@ class AccountReaper(Daemon):
except ClientException, err: except ClientException, err:
if self.logger.getEffectiveLevel() <= DEBUG: if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception( self.logger.exception(
'Exception with %(ip)s:%(port)s/%(device)s' % node) _('Exception with %(ip)s:%(port)s/%(device)s'), node)
failures += 1 failures += 1
self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes[err.http_status / 100] = \
self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.stats_return_codes.get(err.http_status / 100, 0) + 1

10
swift/account/server.py
View File

@ -18,15 +18,14 @@ from __future__ import with_statement
import os import os
import time import time
import traceback import traceback
from urllib import unquote from urllib import unquote
from xml.sax import saxutils
from webob import Request, Response from webob import Request, Response
from webob.exc import HTTPAccepted, HTTPBadRequest, \ from webob.exc import HTTPAccepted, HTTPBadRequest, \
HTTPCreated, HTTPForbidden, HTTPInternalServerError, \ HTTPCreated, HTTPForbidden, HTTPInternalServerError, \
HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, HTTPPreconditionFailed HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, HTTPPreconditionFailed
import simplejson import simplejson
from xml.sax import saxutils
from swift.common.db import AccountBroker from swift.common.db import AccountBroker
from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.utils import get_logger, get_param, hash_path, \
@ -297,6 +296,7 @@ class AccountController(object):
def __call__(self, env, start_response): def __call__(self, env, start_response):
start_time = time.time() start_time = time.time()
req = Request(env) req = Request(env)
self.logger.txn_id = req.headers.get('x-cf-trans-id', None)
if not check_utf8(req.path_info): if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8') res = HTTPPreconditionFailed(body='Invalid UTF8')
else: else:
@ -306,10 +306,8 @@ class AccountController(object):
else: else:
res = HTTPMethodNotAllowed() res = HTTPMethodNotAllowed()
except: except:
self.logger.exception('ERROR __call__ error with %s %s ' self.logger.exception(_('ERROR __call__ error with %(method)s'
'transaction %s' % (env.get('REQUEST_METHOD', '-'), ' %(path)s '), {'method': req.method, 'path': req.path})
env.get('PATH_INFO', '-'), env.get('HTTP_X_CF_TRANS_ID',
'-')))
res = HTTPInternalServerError(body=traceback.format_exc()) res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = '%.4f' % (time.time() - start_time) trans_time = '%.4f' % (time.time() - start_time)
additional_info = '' additional_info = ''

10
swift/common/bench.py
View File

@ -82,10 +82,10 @@ class Bench(object):
def _log_status(self, title): def _log_status(self, title):
total = time.time() - self.beginbeat total = time.time() - self.beginbeat
self.logger.info('%s %s [%s failures], %.01f/s' % ( self.logger.info(_('%(complete)s %(title)s [%(fail)s failures], '
self.complete, title, self.failures, '%(rate).01f/s'),
(float(self.complete) / total), {'title': title, 'complete': self.complete, 'fail': self.failures,
)) 'rate': (float(self.complete) / total)})
@contextmanager @contextmanager
def connection(self): def connection(self):
@ -94,7 +94,7 @@ class Bench(object):
try: try:
yield hc yield hc
except CannotSendRequest: except CannotSendRequest:
self.logger.info("CannotSendRequest. Skipping...") self.logger.info(_("CannotSendRequest. Skipping..."))
try: try:
hc.close() hc.close()
except: except:

13
swift/common/bufferedhttp.py
View File

@ -82,15 +82,9 @@ class BufferedHTTPConnection(HTTPConnection):
def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0): def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
self._method = method self._method = method
self._path = url self._path = url
self._txn_id = '-'
return HTTPConnection.putrequest(self, method, url, skip_host, return HTTPConnection.putrequest(self, method, url, skip_host,
skip_accept_encoding) skip_accept_encoding)
def putheader(self, header, value):
if header.lower() == 'x-cf-trans-id':
self._txn_id = value
return HTTPConnection.putheader(self, header, value)
def getexpect(self): def getexpect(self):
response = BufferedHTTPResponse(self.sock, strict=self.strict, response = BufferedHTTPResponse(self.sock, strict=self.strict,
method=self._method) method=self._method)
@ -99,9 +93,10 @@ class BufferedHTTPConnection(HTTPConnection):
def getresponse(self): def getresponse(self):
response = HTTPConnection.getresponse(self) response = HTTPConnection.getresponse(self)
logging.debug("HTTP PERF: %.5f seconds to %s %s:%s %s (%s)" % logging.debug(_("HTTP PERF: %(time).5f seconds to %(method)s "
(time.time() - self._connected_time, self._method, self.host, "%(host)s:%(port)s %(path)s)"),
self.port, self._path, self._txn_id)) {'time': time.time() - self._connected_time, 'method': self._method,
'host': self.host, 'port': self.port, 'path': self._path})
return response return response

3
swift/common/daemon.py
View File

@ -16,7 +16,9 @@
import os import os
import sys import sys
import signal import signal
import gettext
from re import sub from re import sub
from swift.common import utils from swift.common import utils
@ -40,6 +42,7 @@ class Daemon(object):
utils.validate_configuration() utils.validate_configuration()
utils.capture_stdio(self.logger, **kwargs) utils.capture_stdio(self.logger, **kwargs)
utils.drop_privileges(self.conf.get('user', 'swift')) utils.drop_privileges(self.conf.get('user', 'swift'))
gettext.install('swift', unicode=1)
def kill_children(*args): def kill_children(*args):
signal.signal(signal.SIGTERM, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN)

10
swift/common/db.py
View File

@ -295,7 +295,7 @@ class DatabaseBroker(object):
self.conn = conn self.conn = conn
except: # pragma: no cover except: # pragma: no cover
logging.exception( logging.exception(
'Broker error trying to rollback locked connection') _('Broker error trying to rollback locked connection'))
conn.close() conn.close()
def newid(self, remote_id): def newid(self, remote_id):
@ -750,8 +750,8 @@ class ContainerBroker(DatabaseBroker):
'deleted': deleted}) 'deleted': deleted})
except: except:
self.logger.exception( self.logger.exception(
'Invalid pending entry %s: %s' _('Invalid pending entry %(file)s: %(entry)s'),
% (self.pending_file, entry)) {'file': self.pending_file, 'entry': entry})
if item_list: if item_list:
self.merge_items(item_list) self.merge_items(item_list)
try: try:
@ -1217,8 +1217,8 @@ class AccountBroker(DatabaseBroker):
'deleted': deleted}) 'deleted': deleted})
except: except:
self.logger.exception( self.logger.exception(
'Invalid pending entry %s: %s' _('Invalid pending entry %(file)s: %(entry)s'),
% (self.pending_file, entry)) {'file': self.pending_file, 'entry': entry})
if item_list: if item_list:
self.merge_items(item_list) self.merge_items(item_list)
try: try:

53
swift/common/db_replicator.py
View File

@ -81,7 +81,7 @@ class ReplConnection(BufferedHTTPConnection):
return response return response
except: except:
self.logger.exception( self.logger.exception(
'ERROR reading HTTP response from %s' % self.node) _('ERROR reading HTTP response from %s'), self.node)
return None return None
@ -120,12 +120,14 @@ class Replicator(Daemon):
def _report_stats(self): def _report_stats(self):
"""Report the current stats to the logs.""" """Report the current stats to the logs."""
self.logger.info( self.logger.info(
'Attempted to replicate %d dbs in %.5f seconds (%.5f/s)' _('Attempted to replicate %(count)d dbs in %(time).5f seconds '
% (self.stats['attempted'], time.time() - self.stats['start'], '(%(rate).5f/s)'),
self.stats['attempted'] / {'count': self.stats['attempted'],
(time.time() - self.stats['start'] + 0.0000001))) 'time': time.time() - self.stats['start'],
self.logger.info('Removed %(remove)d dbs' % self.stats) 'rate': self.stats['attempted'] /
self.logger.info('%(success)s successes, %(failure)s failures' (time.time() - self.stats['start'] + 0.0000001)})
self.logger.info(_('Removed %(remove)d dbs') % self.stats)
self.logger.info(_('%(success)s successes, %(failure)s failures')
% self.stats) % self.stats)
self.logger.info(' '.join(['%s:%s' % item for item in self.logger.info(' '.join(['%s:%s' % item for item in
self.stats.items() if item[0] in self.stats.items() if item[0] in
@ -150,8 +152,8 @@ class Replicator(Daemon):
proc = subprocess.Popen(popen_args) proc = subprocess.Popen(popen_args)
proc.communicate() proc.communicate()
if proc.returncode != 0: if proc.returncode != 0:
self.logger.error('ERROR rsync failed with %s: %s' % self.logger.error(_('ERROR rsync failed with %(code)s: %(args)s'),
(proc.returncode, popen_args)) {'code': proc.returncode, 'args': popen_args})
return proc.returncode == 0 return proc.returncode == 0
def _rsync_db(self, broker, device, http, local_id, def _rsync_db(self, broker, device, http, local_id,
@ -200,7 +202,7 @@ class Replicator(Daemon):
:returns: boolean indicating completion and success :returns: boolean indicating completion and success
""" """
self.stats['diff'] += 1 self.stats['diff'] += 1
self.logger.debug('Syncing chunks with %s', http.host) self.logger.debug(_('Syncing chunks with %s'), http.host)
sync_table = broker.get_syncs() sync_table = broker.get_syncs()
objects = broker.get_items_since(point, self.per_diff) objects = broker.get_items_since(point, self.per_diff)
while len(objects): while len(objects):
@ -208,8 +210,9 @@ class Replicator(Daemon):
response = http.replicate('merge_items', objects, local_id) response = http.replicate('merge_items', objects, local_id)
if not response or response.status >= 300 or response.status < 200: if not response or response.status >= 300 or response.status < 200:
if response: if response:
self.logger.error('ERROR Bad response %s from %s' % self.logger.error(_('ERROR Bad response %(status)s from '
(response.status, http.host)) '%(host)s'),
{'status': response.status, 'host': http.host})
return False return False
point = objects[-1]['ROWID'] point = objects[-1]['ROWID']
objects = broker.get_items_since(point, self.per_diff) objects = broker.get_items_since(point, self.per_diff)
@ -272,7 +275,7 @@ class Replicator(Daemon):
http = self._http_connect(node, partition, broker.db_file) http = self._http_connect(node, partition, broker.db_file)
if not http: if not http:
self.logger.error( self.logger.error(
'ERROR Unable to connect to remote server: %s' % node) _('ERROR Unable to connect to remote server: %s'), node)
return False return False
with Timeout(self.node_timeout): with Timeout(self.node_timeout):
response = http.replicate('sync', info['max_row'], info['hash'], response = http.replicate('sync', info['max_row'], info['hash'],
@ -310,7 +313,7 @@ class Replicator(Daemon):
:param object_file: DB file name to be replicated :param object_file: DB file name to be replicated
:param node_id: node id of the node to be replicated to :param node_id: node id of the node to be replicated to
""" """
self.logger.debug('Replicating db %s' % object_file) self.logger.debug(_('Replicating db %s'), object_file)
self.stats['attempted'] += 1 self.stats['attempted'] += 1
try: try:
broker = self.brokerclass(object_file, pending_timeout=30) broker = self.brokerclass(object_file, pending_timeout=30)
@ -319,10 +322,10 @@ class Replicator(Daemon):
info = broker.get_replication_info() info = broker.get_replication_info()
except Exception, e: except Exception, e:
if 'no such table' in str(e): if 'no such table' in str(e):
self.logger.error('Quarantining DB %s' % object_file) self.logger.error(_('Quarantining DB %s'), object_file)
quarantine_db(broker.db_file, broker.db_type) quarantine_db(broker.db_file, broker.db_type)
else: else:
self.logger.exception('ERROR reading db %s' % object_file) self.logger.exception(_('ERROR reading db %s'), object_file)
self.stats['failure'] += 1 self.stats['failure'] += 1
return return
# The db is considered deleted if the delete_timestamp value is greater # The db is considered deleted if the delete_timestamp value is greater
@ -355,10 +358,10 @@ class Replicator(Daemon):
success = self._repl_to_node(node, broker, partition, info) success = self._repl_to_node(node, broker, partition, info)
except DriveNotMounted: except DriveNotMounted:
repl_nodes.append(more_nodes.next()) repl_nodes.append(more_nodes.next())
self.logger.error('ERROR Remote drive not mounted %s' % node) self.logger.error(_('ERROR Remote drive not mounted %s'), node)
except: except:
self.logger.exception('ERROR syncing %s with node %s' % self.logger.exception(_('ERROR syncing %(file)s with node'
(object_file, node)) ' %(node)s'), {'file': object_file, 'node': node})
self.stats['success' if success else 'failure'] += 1 self.stats['success' if success else 'failure'] += 1
responses.append(success) responses.append(success)
if not shouldbehere and all(responses): if not shouldbehere and all(responses):
@ -399,14 +402,14 @@ class Replicator(Daemon):
dirs = [] dirs = []
ips = whataremyips() ips = whataremyips()
if not ips: if not ips:
self.logger.error('ERROR Failed to get my own IPs?') self.logger.error(_('ERROR Failed to get my own IPs?'))
return return
for node in self.ring.devs: for node in self.ring.devs:
if node and node['ip'] in ips and node['port'] == self.port: if node and node['ip'] in ips and node['port'] == self.port:
if self.mount_check and not os.path.ismount( if self.mount_check and not os.path.ismount(
os.path.join(self.root, node['device'])): os.path.join(self.root, node['device'])):
self.logger.warn( self.logger.warn(
'Skipping %(device)s as it is not mounted' % node) _('Skipping %(device)s as it is not mounted') % node)
continue continue
unlink_older_than( unlink_older_than(
os.path.join(self.root, node['device'], 'tmp'), os.path.join(self.root, node['device'], 'tmp'),
@ -414,12 +417,12 @@ class Replicator(Daemon):
datadir = os.path.join(self.root, node['device'], self.datadir) datadir = os.path.join(self.root, node['device'], self.datadir)
if os.path.isdir(datadir): if os.path.isdir(datadir):
dirs.append((datadir, node['id'])) dirs.append((datadir, node['id']))
self.logger.info('Beginning replication run') self.logger.info(_('Beginning replication run'))
for part, object_file, node_id in self.roundrobin_datadirs(dirs): for part, object_file, node_id in self.roundrobin_datadirs(dirs):
self.cpool.spawn_n( self.cpool.spawn_n(
self._replicate_object, part, object_file, node_id) self._replicate_object, part, object_file, node_id)
self.cpool.waitall() self.cpool.waitall()
self.logger.info('Replication run OVER') self.logger.info(_('Replication run OVER'))
self._report_stats() self._report_stats()
def run_forever(self): def run_forever(self):
@ -430,7 +433,7 @@ class Replicator(Daemon):
try: try:
self.run_once() self.run_once()
except: except:
self.logger.exception('ERROR trying to replicate') self.logger.exception(_('ERROR trying to replicate'))
sleep(self.run_pause) sleep(self.run_pause)
@ -473,7 +476,7 @@ class ReplicatorRpc(object):
except Exception, e: except Exception, e:
if 'no such table' in str(e): if 'no such table' in str(e):
# TODO(unknown): find a real logger # TODO(unknown): find a real logger
print "Quarantining DB %s" % broker.db_file print _("Quarantining DB %s") % broker.db_file
quarantine_db(broker.db_file, broker.db_type) quarantine_db(broker.db_file, broker.db_type)
return HTTPNotFound() return HTTPNotFound()
raise raise

9
swift/common/memcached.py
View File

@ -27,7 +27,6 @@ import time
from bisect import bisect from bisect import bisect
from hashlib import md5 from hashlib import md5
CONN_TIMEOUT = 0.3 CONN_TIMEOUT = 0.3
IO_TIMEOUT = 2.0 IO_TIMEOUT = 2.0
PICKLE_FLAG = 1 PICKLE_FLAG = 1
@ -67,9 +66,11 @@ class MemcacheRing(object):
def _exception_occurred(self, server, e, action='talking'): def _exception_occurred(self, server, e, action='talking'):
if isinstance(e, socket.timeout): if isinstance(e, socket.timeout):
logging.error("Timeout %s to memcached: %s" % (action, server)) logging.error(_("Timeout %(action)s to memcached: %(server)s"),
{'action': action, 'server': server})
else: else:
logging.exception("Error %s to memcached: %s" % (action, server)) logging.exception(_("Error %(action)s to memcached: %(server)s"),
{'action': action, 'server': server})
now = time.time() now = time.time()
self._errors[server].append(time.time()) self._errors[server].append(time.time())
if len(self._errors[server]) > ERROR_LIMIT_COUNT: if len(self._errors[server]) > ERROR_LIMIT_COUNT:
@ -77,7 +78,7 @@ class MemcacheRing(object):
if err > now - ERROR_LIMIT_TIME] if err > now - ERROR_LIMIT_TIME]
if len(self._errors[server]) > ERROR_LIMIT_COUNT: if len(self._errors[server]) > ERROR_LIMIT_COUNT:
self._error_limited[server] = now + ERROR_LIMIT_DURATION self._error_limited[server] = now + ERROR_LIMIT_DURATION
logging.error('Error limiting server %s' % server) logging.error(_('Error limiting server %s'), server)
def _get_conns(self, key): def _get_conns(self, key):
""" """

2
swift/common/middleware/catch_errors.py
View File

@ -32,7 +32,7 @@ class CatchErrorMiddleware(object):
try: try:
return self.app(env, start_response) return self.app(env, start_response)
except Exception, err: except Exception, err:
self.logger.exception('Error: %s' % err) self.logger.exception(_('Error: %s'), err)
resp = HTTPServerError(request=Request(env), resp = HTTPServerError(request=Request(env),
body='An error occurred', body='An error occurred',
content_type='text/plain') content_type='text/plain')

15
swift/common/middleware/ratelimit.py
View File

@ -167,7 +167,7 @@ class RateLimitMiddleware(object):
:param obj_name: object name from path :param obj_name: object name from path
''' '''
if account_name in self.ratelimit_blacklist: if account_name in self.ratelimit_blacklist:
self.logger.error('Returning 497 because of blacklisting') self.logger.error(_('Returning 497 because of blacklisting'))
return Response(status='497 Blacklisted', return Response(status='497 Blacklisted',
body='Your account has been blacklisted', request=req) body='Your account has been blacklisted', request=req)
if account_name in self.ratelimit_whitelist: if account_name in self.ratelimit_whitelist:
@ -181,14 +181,15 @@ class RateLimitMiddleware(object):
need_to_sleep = self._get_sleep_time(key, max_rate) need_to_sleep = self._get_sleep_time(key, max_rate)
if self.log_sleep_time_seconds and \ if self.log_sleep_time_seconds and \
need_to_sleep > self.log_sleep_time_seconds: need_to_sleep > self.log_sleep_time_seconds:
self.logger.info("Ratelimit sleep log: %s for %s/%s/%s" % ( self.logger.info(_("Ratelimit sleep log: %(sleep)s for "
need_to_sleep, account_name, "%(account)s/%(container)s/%(object)s"),
container_name, obj_name)) {'sleep': need_to_sleep, 'account': account_name,
'container': container_name, 'object': obj_name})
if need_to_sleep > 0: if need_to_sleep > 0:
eventlet.sleep(need_to_sleep) eventlet.sleep(need_to_sleep)
except MaxSleepTimeHit, e: except MaxSleepTimeHit, e:
self.logger.error('Returning 498 because of ops ' + \ self.logger.error(_('Returning 498 because of ops rate '
'rate limiting (Max Sleep) %s' % e) 'limiting (Max Sleep) %s') % str(e))
error_resp = Response(status='498 Rate Limited', error_resp = Response(status='498 Rate Limited',
body='Slow down', request=req) body='Slow down', request=req)
return error_resp return error_resp
@ -207,7 +208,7 @@ class RateLimitMiddleware(object):
self.memcache_client = cache_from_env(env) self.memcache_client = cache_from_env(env)
if not self.memcache_client: if not self.memcache_client:
self.logger.warning( self.logger.warning(
'Warning: Cannot ratelimit without a memcached client') _('Warning: Cannot ratelimit without a memcached client'))
return self.app(env, start_response) return self.app(env, start_response)
try: try:
version, account, container, obj = split_path(req.path, 1, 4, True) version, account, container, obj = split_path(req.path, 1, 4, True)

72
swift/common/utils.py
View File

@ -35,7 +35,6 @@ from optparse import OptionParser
from tempfile import mkstemp from tempfile import mkstemp
import cPickle as pickle import cPickle as pickle
import eventlet import eventlet
from eventlet import greenio, GreenPool, sleep, Timeout, listen from eventlet import greenio, GreenPool, sleep, Timeout, listen
from eventlet.green import socket, subprocess, ssl, thread, threading from eventlet.green import socket, subprocess, ssl, thread, threading
@ -85,8 +84,8 @@ def load_libc_function(func_name):
libc = ctypes.CDLL(ctypes.util.find_library('c')) libc = ctypes.CDLL(ctypes.util.find_library('c'))
return getattr(libc, func_name) return getattr(libc, func_name)
except AttributeError: except AttributeError:
logging.warn("Unable to locate %s in libc. Leaving as a no-op." logging.warn(_("Unable to locate %s in libc. Leaving as a no-op."),
% func_name) func_name)
def noop_libc_function(*args): def noop_libc_function(*args):
return 0 return 0
@ -252,12 +251,12 @@ class LoggerFileObject(object):
value = value.strip() value = value.strip()
if value: if value:
if 'Connection reset by peer' in value: if 'Connection reset by peer' in value:
self.logger.error('STDOUT: Connection reset by peer') self.logger.error(_('STDOUT: Connection reset by peer'))
else: else:
self.logger.error('STDOUT: %s' % value) self.logger.error(_('STDOUT: %s'), value)
def writelines(self, values): def writelines(self, values):
self.logger.error('STDOUT: %s' % '#012'.join(values)) self.logger.error(_('STDOUT: %s'), '#012'.join(values))
def close(self): def close(self):
pass pass
@ -284,23 +283,24 @@ class LoggerFileObject(object):
return self return self
class NamedLogger(object): class LogAdapter(object):
"""Cheesy version of the LoggerAdapter available in Python 3""" """Cheesy version of the LoggerAdapter available in Python 3"""
def __init__(self, logger, server): def __init__(self, logger):
self.logger = logger self.logger = logger
self.server = server self._txn_id = threading.local()
for proxied_method in ('debug', 'info', 'log', 'warn', 'warning', for proxied_method in ('debug', 'log', 'warn', 'warning', 'error',
'error', 'critical'): 'critical', 'info'):
setattr(self, proxied_method, setattr(self, proxied_method, getattr(logger, proxied_method))
self._proxy(getattr(logger, proxied_method)))
def _proxy(self, logger_meth): @property
def txn_id(self):
if hasattr(self._txn_id, 'value'):
return self._txn_id.value
def _inner_proxy(msg, *args, **kwargs): @txn_id.setter
msg = '%s %s' % (self.server, msg) def txn_id(self, value):
logger_meth(msg, *args, **kwargs) self._txn_id.value = value
return _inner_proxy
def getEffectiveLevel(self): def getEffectiveLevel(self):
return self.logger.getEffectiveLevel() return self.logger.getEffectiveLevel()
@ -330,7 +330,22 @@ class NamedLogger(object):
emsg += ' %s' % exc.msg emsg += ' %s' % exc.msg
else: else:
call = self.logger.exception call = self.logger.exception
call('%s %s: %s' % (self.server, msg, emsg), *args) call('%s: %s' % (msg, emsg), *args)
class NamedFormatter(logging.Formatter):
def __init__(self, server, logger):
logging.Formatter.__init__(self)
self.server = server
self.logger = logger
def format(self, record):
msg = logging.Formatter.format(self, record)
if self.logger.txn_id and (record.levelno != logging.INFO or
self.logger.txn_id not in msg):
return '%s %s (txn: %s)' % (self.server, msg, self.logger.txn_id)
else:
return '%s %s' % (self.server, msg)
def get_logger(conf, name=None, log_to_console=False): def get_logger(conf, name=None, log_to_console=False):
@ -359,7 +374,8 @@ def get_logger(conf, name=None, log_to_console=False):
root_logger.addHandler(get_logger.console) root_logger.addHandler(get_logger.console)
if conf is None: if conf is None:
root_logger.setLevel(logging.INFO) root_logger.setLevel(logging.INFO)
return NamedLogger(root_logger, name) adapted_logger = LogAdapter(root_logger)
return adapted_logger
if name is None: if name is None:
name = conf.get('log_name', 'swift') name = conf.get('log_name', 'swift')
get_logger.handler = SysLogHandler(address='/dev/log', get_logger.handler = SysLogHandler(address='/dev/log',
@ -369,7 +385,9 @@ def get_logger(conf, name=None, log_to_console=False):
root_logger.addHandler(get_logger.handler) root_logger.addHandler(get_logger.handler)
root_logger.setLevel( root_logger.setLevel(
getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO)) getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO))
return NamedLogger(root_logger, name) adapted_logger = LogAdapter(root_logger)
get_logger.handler.setFormatter(NamedFormatter(name, adapted_logger))
return adapted_logger
def drop_privileges(user): def drop_privileges(user):
@ -444,12 +462,12 @@ def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None):
if not args: if not args:
parser.print_usage() parser.print_usage()
print "Error: missing config file argument" print _("Error: missing config file argument")
sys.exit(1) sys.exit(1)
config = os.path.abspath(args.pop(0)) config = os.path.abspath(args.pop(0))
if not os.path.exists(config): if not os.path.exists(config):
parser.print_usage() parser.print_usage()
print "Error: unable to locate %s" % config print _("Error: unable to locate %s") % config
sys.exit(1) sys.exit(1)
extra_args = [] extra_args = []
@ -672,14 +690,14 @@ def readconf(conf, section_name=None, log_name=None, defaults=None):
defaults = {} defaults = {}
c = ConfigParser(defaults) c = ConfigParser(defaults)
if not c.read(conf): if not c.read(conf):
print "Unable to read config file %s" % conf print _("Unable to read config file %s") % conf
sys.exit(1) sys.exit(1)
if section_name: if section_name:
if c.has_section(section_name): if c.has_section(section_name):
conf = dict(c.items(section_name)) conf = dict(c.items(section_name))
else: else:
print "Unable to find %s config section in %s" % (section_name, print _("Unable to find %s config section in %s") % \
conf) (section_name, conf)
sys.exit(1) sys.exit(1)
if "log_name" not in conf: if "log_name" not in conf:
if log_name is not None: if log_name is not None:
@ -731,7 +749,7 @@ def audit_location_generator(devices, datadir, mount_check=True, logger=None):
os.path.ismount(os.path.join(devices, device)): os.path.ismount(os.path.join(devices, device)):
if logger: if logger:
logger.debug( logger.debug(
'Skipping %s as it is not mounted' % device) _('Skipping %s as it is not mounted'), device)
continue continue
datadir = os.path.join(devices, device, datadir) datadir = os.path.join(devices, device, datadir)
if not os.path.exists(datadir): if not os.path.exists(datadir):

2
swift/common/wsgi.py
View File

@ -21,6 +21,7 @@ import signal
import sys import sys
import time import time
import mimetools import mimetools
import gettext
import eventlet import eventlet
from eventlet import greenio, GreenPool, sleep, wsgi, listen from eventlet import greenio, GreenPool, sleep, wsgi, listen
@ -120,6 +121,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
sock = get_socket(conf, default_port=kwargs.get('default_port', 8080)) sock = get_socket(conf, default_port=kwargs.get('default_port', 8080))
# remaining tasks should not require elevated privileges # remaining tasks should not require elevated privileges
drop_privileges(conf.get('user', 'swift')) drop_privileges(conf.get('user', 'swift'))
gettext.install('swift', unicode=1)
# finally after binding to ports and privilege drop, run app __init__ code # finally after binding to ports and privilege drop, run app __init__ code
app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name}) app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})

26
swift/container/auditor.py
View File

@ -51,10 +51,11 @@ class ContainerAuditor(Daemon):
self.container_audit(path) self.container_audit(path)
if time.time() - reported >= 3600: # once an hour if time.time() - reported >= 3600: # once an hour
self.logger.info( self.logger.info(
'Since %s: Container audits: %s passed audit, ' _('Since %(time)s: Container audits: %(pass)s passed '
'%s failed audit' % (time.ctime(reported), 'audit, %(fail)s failed audit'),
self.container_passes, {'time': time.ctime(reported),
self.container_failures)) 'pass': self.container_passes,
'fail': self.container_failures})
reported = time.time() reported = time.time()
self.container_passes = 0 self.container_passes = 0
self.container_failures = 0 self.container_failures = 0
@ -64,7 +65,7 @@ class ContainerAuditor(Daemon):
def run_once(self): def run_once(self):
"""Run the container audit once.""" """Run the container audit once."""
self.logger.info('Begin container audit "once" mode') self.logger.info(_('Begin container audit "once" mode'))
begin = reported = time.time() begin = reported = time.time()
all_locs = audit_location_generator(self.devices, all_locs = audit_location_generator(self.devices,
container_server.DATADIR, container_server.DATADIR,
@ -74,16 +75,17 @@ class ContainerAuditor(Daemon):
self.container_audit(path) self.container_audit(path)
if time.time() - reported >= 3600: # once an hour if time.time() - reported >= 3600: # once an hour
self.logger.info( self.logger.info(
'Since %s: Container audits: %s passed audit, ' _('Since %(time)s: Container audits: %(pass)s passed '
'%s failed audit' % (time.ctime(reported), 'audit, %(fail)s failed audit'),
self.container_passes, {'time': time.ctime(reported),
self.container_failures)) 'pass': self.container_passes,
'fail': self.container_failures})
reported = time.time() reported = time.time()
self.container_passes = 0 self.container_passes = 0
self.container_failures = 0 self.container_failures = 0
elapsed = time.time() - begin elapsed = time.time() - begin
self.logger.info( self.logger.info(
'Container audit "once" mode completed: %.02fs' % elapsed) _('Container audit "once" mode completed: %.02fs'), elapsed)
def container_audit(self, path): def container_audit(self, path):
""" """
@ -98,8 +100,8 @@ class ContainerAuditor(Daemon):
if not broker.is_deleted(): if not broker.is_deleted():
info = broker.get_info() info = broker.get_info()
self.container_passes += 1 self.container_passes += 1
self.logger.debug('Audit passed for %s' % broker.db_file) self.logger.debug(_('Audit passed for %s'), broker.db_file)
except Exception: except Exception:
self.container_failures += 1 self.container_failures += 1
self.logger.exception('ERROR Could not get container info %s' % self.logger.exception(_('ERROR Could not get container info %s'),
(broker.db_file)) (broker.db_file))

29
swift/container/server.py
View File

@ -111,18 +111,18 @@ class ContainerController(object):
return HTTPNotFound(request=req) return HTTPNotFound(request=req)
elif account_response.status < 200 or \ elif account_response.status < 200 or \
account_response.status > 299: account_response.status > 299:
self.logger.error('ERROR Account update failed ' self.logger.error(_('ERROR Account update failed '
'with %s:%s/%s transaction %s (will retry ' 'with %(ip)s:%(port)s/%(device)s (will retry '
'later): Response %s %s' % (account_ip, 'later): Response %(status)s %(reason)s'),
account_port, account_device, {'ip': account_ip, 'port': account_port,
req.headers.get('x-cf-trans-id'), 'device': account_device,
account_response.status, 'status': account_response.status,
account_response.reason)) 'reason': account_response.reason})
except: except:
self.logger.exception('ERROR account update failed with ' self.logger.exception(_('ERROR account update failed with '
'%s:%s/%s transaction %s (will retry later)' % '%(ip)s:%(port)s/%(device)s (will retry later)'),
(account_ip, account_port, account_device, {'ip': account_ip, 'port': account_port,
req.headers.get('x-cf-trans-id', '-'))) 'device': account_device})
return None return None
def DELETE(self, req): def DELETE(self, req):
@ -384,6 +384,7 @@ class ContainerController(object):
def __call__(self, env, start_response): def __call__(self, env, start_response):
start_time = time.time() start_time = time.time()
req = Request(env) req = Request(env)
self.logger.txn_id = req.headers.get('x-cf-trans-id', None)
if not check_utf8(req.path_info): if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8') res = HTTPPreconditionFailed(body='Invalid UTF8')
else: else:
@ -393,10 +394,8 @@ class ContainerController(object):
else: else:
res = HTTPMethodNotAllowed() res = HTTPMethodNotAllowed()
except: except:
self.logger.exception('ERROR __call__ error with %s %s ' self.logger.exception(_('ERROR __call__ error with %(method)s'
'transaction %s' % (env.get('REQUEST_METHOD', '-'), ' %(path)s '), {'method': req.method, 'path': req.path})
env.get('PATH_INFO', '-'), env.get('HTTP_X_CF_TRANS_ID',
'-')))
res = HTTPInternalServerError(body=traceback.format_exc()) res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = '%.4f' % (time.time() - start_time) trans_time = '%.4f' % (time.time() - start_time)
log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % ( log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % (

38
swift/container/updater.py
View File

@ -56,7 +56,7 @@ class ContainerUpdater(Daemon):
"""Get the account ring. Load it if it hasn't been yet.""" """Get the account ring. Load it if it hasn't been yet."""
if not self.account_ring: if not self.account_ring:
self.logger.debug( self.logger.debug(
'Loading account ring from %s' % self.account_ring_path) _('Loading account ring from %s'), self.account_ring_path)
self.account_ring = Ring(self.account_ring_path) self.account_ring = Ring(self.account_ring_path)
return self.account_ring return self.account_ring
@ -70,7 +70,7 @@ class ContainerUpdater(Daemon):
for device in os.listdir(self.devices): for device in os.listdir(self.devices):
dev_path = os.path.join(self.devices, device) dev_path = os.path.join(self.devices, device)
if self.mount_check and not os.path.ismount(dev_path): if self.mount_check and not os.path.ismount(dev_path):
self.logger.warn('%s is not mounted' % device) self.logger.warn(_('%s is not mounted'), device)
continue continue
con_path = os.path.join(dev_path, DATADIR) con_path = os.path.join(dev_path, DATADIR)
if not os.path.exists(con_path): if not os.path.exists(con_path):
@ -86,7 +86,7 @@ class ContainerUpdater(Daemon):
""" """
time.sleep(random() * self.interval) time.sleep(random() * self.interval)
while True: while True:
self.logger.info('Begin container update sweep') self.logger.info(_('Begin container update sweep'))
begin = time.time() begin = time.time()
pids = [] pids = []
# read from account ring to ensure it's fresh # read from account ring to ensure it's fresh
@ -107,15 +107,17 @@ class ContainerUpdater(Daemon):
self.container_sweep(path) self.container_sweep(path)
elapsed = time.time() - forkbegin elapsed = time.time() - forkbegin
self.logger.debug( self.logger.debug(
'Container update sweep of %s completed: ' _('Container update sweep of %(path)s completed: '
'%.02fs, %s successes, %s failures, %s with no changes' '%(elapsed).02fs, %(success)s successes, %(fail)s '
% (path, elapsed, self.successes, self.failures, 'failures, %(no_change)s with no changes'),
self.no_changes)) {'path': path, 'elapsed': elapsed,
'success': self.successes, 'fail': self.failures,
'no_change': self.no_changes})
sys.exit() sys.exit()
while pids: while pids:
pids.remove(os.wait()[0]) pids.remove(os.wait()[0])
elapsed = time.time() - begin elapsed = time.time() - begin
self.logger.info('Container update sweep completed: %.02fs' % self.logger.info(_('Container update sweep completed: %.02fs'),
elapsed) elapsed)
if elapsed < self.interval: if elapsed < self.interval:
time.sleep(self.interval - elapsed) time.sleep(self.interval - elapsed)
@ -133,9 +135,11 @@ class ContainerUpdater(Daemon):
for path in self.get_paths(): for path in self.get_paths():
self.container_sweep(path) self.container_sweep(path)
elapsed = time.time() - begin elapsed = time.time() - begin
self.logger.info('Container update single threaded sweep completed: ' self.logger.info(_('Container update single threaded sweep completed: '
'%.02fs, %s successes, %s failures, %s with no changes' % '%(elapsed).02fs, %(success)s successes, %(fail)s failures, '
(elapsed, self.successes, self.failures, self.no_changes)) '%(no_change)s with no changes'),
{'elapsed': elapsed, 'success': self.successes,
'fail': self.failures, 'no_change': self.no_changes})
def container_sweep(self, path): def container_sweep(self, path):
""" """
@ -181,14 +185,16 @@ class ContainerUpdater(Daemon):
if successes > failures: if successes > failures:
self.successes += 1 self.successes += 1
self.logger.debug( self.logger.debug(
'Update report sent for %s %s' % (container, dbfile)) _('Update report sent for %(container)s %(dbfile)s'),
{'container': container, 'dbfile': dbfile})
broker.reported(info['put_timestamp'], broker.reported(info['put_timestamp'],
info['delete_timestamp'], info['object_count'], info['delete_timestamp'], info['object_count'],
info['bytes_used']) info['bytes_used'])
else: else:
self.failures += 1 self.failures += 1
self.logger.debug( self.logger.debug(
'Update report failed for %s %s' % (container, dbfile)) _('Update report failed for %(container)s %(dbfile)s'),
{'container': container, 'dbfile': dbfile})
else: else:
self.no_changes += 1 self.no_changes += 1
@ -216,8 +222,8 @@ class ContainerUpdater(Daemon):
'X-Bytes-Used': bytes, 'X-Bytes-Used': bytes,
'X-Account-Override-Deleted': 'yes'}) 'X-Account-Override-Deleted': 'yes'})
except: except:
self.logger.exception('ERROR account update failed with ' self.logger.exception(_('ERROR account update failed with '
'%(ip)s:%(port)s/%(device)s (will retry later): ' % node) '%(ip)s:%(port)s/%(device)s (will retry later): '), node)
return 500 return 500
with Timeout(self.node_timeout): with Timeout(self.node_timeout):
try: try:
@ -227,5 +233,5 @@ class ContainerUpdater(Daemon):
except: except:
if self.logger.getEffectiveLevel() <= logging.DEBUG: if self.logger.getEffectiveLevel() <= logging.DEBUG:
self.logger.exception( self.logger.exception(
'Exception with %(ip)s:%(port)s/%(device)s' % node) _('Exception with %(ip)s:%(port)s/%(device)s'), node)
return 500 return 500

26
swift/obj/auditor.py
View File

@ -52,10 +52,10 @@ class ObjectAuditor(Daemon):
for path, device, partition in all_locs: for path, device, partition in all_locs:
self.object_audit(path, device, partition) self.object_audit(path, device, partition)
if time.time() - reported >= 3600: # once an hour if time.time() - reported >= 3600: # once an hour
self.logger.info( self.logger.info(_('Since %(time)s: Locally: %(pass)d '
'Since %s: Locally: %d passed audit, %d quarantined, ' 'passed audit, %(quar)d quarantined, %(error)d errors'),
'%d errors' % (time.ctime(reported), self.passes, {'time': time.ctime(reported), 'pass': self.passes,
self.quarantines, self.errors)) 'quar': self.quarantines, 'error': self.errors})
reported = time.time() reported = time.time()
self.passes = 0 self.passes = 0
self.quarantines = 0 self.quarantines = 0
@ -66,7 +66,7 @@ class ObjectAuditor(Daemon):
def run_once(self): def run_once(self):
"""Run the object audit once.""" """Run the object audit once."""
self.logger.info('Begin object audit "once" mode') self.logger.info(_('Begin object audit "once" mode'))
begin = reported = time.time() begin = reported = time.time()
all_locs = audit_location_generator(self.devices, all_locs = audit_location_generator(self.devices,
object_server.DATADIR, object_server.DATADIR,
@ -75,17 +75,17 @@ class ObjectAuditor(Daemon):
for path, device, partition in all_locs: for path, device, partition in all_locs:
self.object_audit(path, device, partition) self.object_audit(path, device, partition)
if time.time() - reported >= 3600: # once an hour if time.time() - reported >= 3600: # once an hour
self.logger.info( self.logger.info(_('Since %(time)s: Locally: %(pass)d '
'Since %s: Locally: %d passed audit, %d quarantined, ' 'passed audit, %(quar)d quarantined, %(error)d errors'),
'%d errors' % (time.ctime(reported), self.passes, {'time': time.ctime(reported), 'pass': self.passes,
self.quarantines, self.errors)) 'quar': self.quarantines, 'error': self.errors})
reported = time.time() reported = time.time()
self.passes = 0 self.passes = 0
self.quarantines = 0 self.quarantines = 0
self.errors = 0 self.errors = 0
elapsed = time.time() - begin elapsed = time.time() - begin
self.logger.info( self.logger.info(
'Object audit "once" mode completed: %.02fs' % elapsed) _('Object audit "once" mode completed: %.02fs'), elapsed)
def object_audit(self, path, device, partition): def object_audit(self, path, device, partition):
""" """
@ -124,8 +124,8 @@ class ObjectAuditor(Daemon):
"%s" % (df.metadata['ETag'], etag)) "%s" % (df.metadata['ETag'], etag))
except AuditException, err: except AuditException, err:
self.quarantines += 1 self.quarantines += 1
self.logger.error('ERROR Object %s failed audit and will be ' self.logger.error(_('ERROR Object %(obj)s failed audit and will be '
'quarantined: %s' % (path, err)) 'quarantined: %(err)s'), {'obj': path, 'err': err})
invalidate_hash(os.path.dirname(path)) invalidate_hash(os.path.dirname(path))
renamer_path = os.path.dirname(path) renamer_path = os.path.dirname(path)
renamer(renamer_path, os.path.join(self.devices, device, renamer(renamer_path, os.path.join(self.devices, device,
@ -133,6 +133,6 @@ class ObjectAuditor(Daemon):
return return
except Exception: except Exception:
self.errors += 1 self.errors += 1
self.logger.exception('ERROR Trying to audit %s' % path) self.logger.exception(_('ERROR Trying to audit %s'), path)
return return
self.passes += 1 self.passes += 1

91
swift/obj/replicator.py
View File

@ -243,26 +243,27 @@ class ObjectReplicator(Daemon):
results = proc.stdout.read() results = proc.stdout.read()
ret_val = proc.wait() ret_val = proc.wait()
except Timeout: except Timeout:
self.logger.error("Killing long-running rsync: %s" % str(args)) self.logger.error(_("Killing long-running rsync: %s"), str(args))
proc.kill() proc.kill()
return 1 # failure response code return 1 # failure response code
total_time = time.time() - start_time total_time = time.time() - start_time
if results:
for result in results.split('\n'): for result in results.split('\n'):
if result == '': if result == '':
continue continue
if result.startswith('cd+'): if result.startswith('cd+'):
continue continue
self.logger.info(result) self.logger.info(result)
if ret_val:
self.logger.error(_('Bad rsync return code: %s -> %d'),
(str(args), ret_val))
elif results:
self.logger.info( self.logger.info(
"Sync of %s at %s complete (%.03f) [%d]" % ( _("Successful rsync of %(src)s at %(dst)s (%(time).03f)"),
args[-2], args[-1], total_time, ret_val)) {'src': args[-2], 'dst': args[-1], 'time': total_time})
else: else:
self.logger.debug( self.logger.debug(
"Sync of %s at %s complete (%.03f) [%d]" % ( _("Successful rsync of %(src)s at %(dst)s (%(time).03f)"),
args[-2], args[-1], total_time, ret_val)) {'src': args[-2], 'dst': args[-1], 'time': total_time})
if ret_val:
self.logger.error('Bad rsync return code: %d' % ret_val)
return ret_val return ret_val
def rsync(self, node, job, suffixes): def rsync(self, node, job, suffixes):
@ -346,10 +347,10 @@ class ObjectReplicator(Daemon):
responses.append(success) responses.append(success)
if not suffixes or (len(responses) == \ if not suffixes or (len(responses) == \
self.object_ring.replica_count and all(responses)): self.object_ring.replica_count and all(responses)):
self.logger.info("Removing partition: %s" % job['path']) self.logger.info(_("Removing partition: %s"), job['path'])
tpool.execute(shutil.rmtree, job['path'], ignore_errors=True) tpool.execute(shutil.rmtree, job['path'], ignore_errors=True)
except (Exception, Timeout): except (Exception, Timeout):
self.logger.exception("Error syncing handoff partition") self.logger.exception(_("Error syncing handoff partition"))
finally: finally:
self.partition_times.append(time.time() - begin) self.partition_times.append(time.time() - begin)
@ -379,13 +380,14 @@ class ObjectReplicator(Daemon):
node['device'], job['partition'], 'REPLICATE', node['device'], job['partition'], 'REPLICATE',
'', headers={'Content-Length': '0'}).getresponse() '', headers={'Content-Length': '0'}).getresponse()
if resp.status == 507: if resp.status == 507:
self.logger.error('%s/%s responded as unmounted' % self.logger.error(_('%(ip)s/%(device)s responded'
(node['ip'], node['device'])) ' as unmounted'), node)
attempts_left += 1 attempts_left += 1
continue continue
if resp.status != 200: if resp.status != 200:
self.logger.error("Invalid response %s from %s" % self.logger.error(_("Invalid response %(resp)s "
(resp.status, node['ip'])) "from %(ip)s"),
{'resp': resp.status, 'ip': node['ip']})
continue continue
remote_hash = pickle.loads(resp.read()) remote_hash = pickle.loads(resp.read())
del resp del resp
@ -408,7 +410,7 @@ class ObjectReplicator(Daemon):
logging.exception("Error syncing with node: %s" % node) logging.exception("Error syncing with node: %s" % node)
self.suffix_count += len(local_hash) self.suffix_count += len(local_hash)
except (Exception, Timeout): except (Exception, Timeout):
self.logger.exception("Error syncing partition") self.logger.exception(_("Error syncing partition"))
finally: finally:
self.partition_times.append(time.time() - begin) self.partition_times.append(time.time() - begin)
@ -418,27 +420,30 @@ class ObjectReplicator(Daemon):
""" """
if self.replication_count: if self.replication_count:
rate = self.replication_count / (time.time() - self.start) rate = self.replication_count / (time.time() - self.start)
self.logger.info("%d/%d (%.2f%%) partitions replicated in %.2f " self.logger.info(_("%(replicated)d/%(total)d (%(percentage).2f%%)"
"seconds (%.2f/sec, %s remaining)" " partitions replicated in %(time).2fs (%(rate).2f/sec, "
% (self.replication_count, self.job_count, "%(remaining)s remaining)"),
self.replication_count * 100.0 / self.job_count, {'replicated': self.replication_count, 'total': self.job_count,
time.time() - self.start, rate, 'percentage': self.replication_count * 100.0 / self.job_count,
'%d%s' % compute_eta(self.start, 'time': time.time() - self.start, 'rate': rate,
self.replication_count, self.job_count))) 'remaining': '%d%s' % compute_eta(self.start,
self.replication_count, self.job_count)})
if self.suffix_count: if self.suffix_count:
self.logger.info("%d suffixes checked - %.2f%% hashed, " self.logger.info(_("%(checked)d suffixes checked - "
"%.2f%% synced" % "%(hashed).2f%% hashed, %(synced).2f%% synced"),
(self.suffix_count, {'checked': self.suffix_count,
(self.suffix_hash * 100.0) / self.suffix_count, 'hashed': (self.suffix_hash * 100.0) / self.suffix_count,
(self.suffix_sync * 100.0) / self.suffix_count)) 'synced': (self.suffix_sync * 100.0) / self.suffix_count})
self.partition_times.sort() self.partition_times.sort()
self.logger.info("Partition times: max %.4fs, min %.4fs, " self.logger.info(_("Partition times: max %(max).4fs, "
"med %.4fs" "min %(min).4fs, med %(med).4fs"),
% (self.partition_times[-1], self.partition_times[0], {'max': self.partition_times[-1],
self.partition_times[len(self.partition_times) // 2])) 'min': self.partition_times[0],
'med': self.partition_times[
len(self.partition_times) // 2]})
else: else:
self.logger.info("Nothing replicated for %s seconds." self.logger.info(_("Nothing replicated for %s seconds."),
% (time.time() - self.start)) (time.time() - self.start))
def kill_coros(self): def kill_coros(self):
"""Utility function that kills all coroutines currently running.""" """Utility function that kills all coroutines currently running."""
@ -466,7 +471,7 @@ class ObjectReplicator(Daemon):
while True: while True:
eventlet.sleep(self.lockup_timeout) eventlet.sleep(self.lockup_timeout)
if self.replication_count == self.last_replication_count: if self.replication_count == self.last_replication_count:
self.logger.error("Lockup detected.. killing live coros.") self.logger.error(_("Lockup detected.. killing live coros."))
self.kill_coros() self.kill_coros()
self.last_replication_count = self.replication_count self.last_replication_count = self.replication_count
@ -483,7 +488,7 @@ class ObjectReplicator(Daemon):
obj_path = join(dev_path, 'objects') obj_path = join(dev_path, 'objects')
tmp_path = join(dev_path, 'tmp') tmp_path = join(dev_path, 'tmp')
if self.mount_check and not os.path.ismount(dev_path): if self.mount_check and not os.path.ismount(dev_path):
self.logger.warn('%s is not mounted' % local_dev['device']) self.logger.warn(_('%s is not mounted'), local_dev['device'])
continue continue
unlink_older_than(tmp_path, time.time() - self.reclaim_age) unlink_older_than(tmp_path, time.time() - self.reclaim_age)
if not os.path.exists(obj_path): if not os.path.exists(obj_path):
@ -521,8 +526,8 @@ class ObjectReplicator(Daemon):
jobs = self.collect_jobs() jobs = self.collect_jobs()
for job in jobs: for job in jobs:
if not self.check_ring(): if not self.check_ring():
self.logger.info( self.logger.info(_("Ring change detected. Aborting "
"Ring change detected. Aborting current replication pass.") "current replication pass."))
return return
if job['delete']: if job['delete']:
self.run_pool.spawn(self.update_deleted, job) self.run_pool.spawn(self.update_deleted, job)
@ -531,7 +536,7 @@ class ObjectReplicator(Daemon):
with Timeout(self.lockup_timeout): with Timeout(self.lockup_timeout):
self.run_pool.waitall() self.run_pool.waitall()
except (Exception, Timeout): except (Exception, Timeout):
self.logger.exception("Exception in top-level replication loop") self.logger.exception(_("Exception in top-level replication loop"))
self.kill_coros() self.kill_coros()
finally: finally:
stats.kill() stats.kill()
@ -540,23 +545,23 @@ class ObjectReplicator(Daemon):
def run_once(self): def run_once(self):
start = time.time() start = time.time()
self.logger.info("Running object replicator in script mode.") self.logger.info(_("Running object replicator in script mode."))
self.replicate() self.replicate()
total = (time.time() - start) / 60 total = (time.time() - start) / 60
self.logger.info( self.logger.info(
"Object replication complete. (%.02f minutes)" % total) _("Object replication complete. (%.02f minutes)"), total)
def run_forever(self): def run_forever(self):
self.logger.info("Starting object replicator in daemon mode.") self.logger.info("Starting object replicator in daemon mode.")
# Run the replicator continually # Run the replicator continually
while True: while True:
start = time.time() start = time.time()
self.logger.info("Starting object replication pass.") self.logger.info(_("Starting object replication pass."))
# Run the replicator # Run the replicator
self.replicate() self.replicate()
total = (time.time() - start) / 60 total = (time.time() - start) / 60
self.logger.info( self.logger.info(
"Object replication complete. (%.02f minutes)" % total) _("Object replication complete. (%.02f minutes)"), total)
self.logger.debug('Replication sleeping for %s seconds.' % self.logger.debug(_('Replication sleeping for %s seconds.'),
self.run_pause) self.run_pause)
sleep(self.run_pause) sleep(self.run_pause)

21
swift/obj/server.py
View File

@ -292,13 +292,15 @@ class ObjectController(object):
if 200 <= response.status < 300: if 200 <= response.status < 300:
return return
else: else:
self.logger.error('ERROR Container update failed (saving ' self.logger.error(_('ERROR Container update failed '
'for async update later): %d response from %s:%s/%s' % '(saving for async update later): %(status)d '
(response.status, ip, port, contdevice)) 'response from %(ip)s:%(port)s/%(dev)s'),
{'status': response.status, 'ip': ip, 'port': port,
'dev': contdevice})
except: except:
self.logger.exception('ERROR container update failed with ' self.logger.exception(_('ERROR container update failed with '
'%s:%s/%s transaction %s (saving for async update later)' % '%(ip)s:%(port)s/%(dev)s (saving for async update later)'),
(ip, port, contdevice, headers_in.get('x-cf-trans-id', '-'))) {'ip': ip, 'port': port, 'dev': contdevice})
async_dir = os.path.join(self.devices, objdevice, ASYNCDIR) async_dir = os.path.join(self.devices, objdevice, ASYNCDIR)
ohash = hash_path(account, container, obj) ohash = hash_path(account, container, obj)
write_pickle( write_pickle(
@ -560,6 +562,7 @@ class ObjectController(object):
"""WSGI Application entry point for the Swift Object Server.""" """WSGI Application entry point for the Swift Object Server."""
start_time = time.time() start_time = time.time()
req = Request(env) req = Request(env)
self.logger.txn_id = req.headers.get('x-cf-trans-id', None)
if not check_utf8(req.path_info): if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8') res = HTTPPreconditionFailed(body='Invalid UTF8')
else: else:
@ -569,10 +572,8 @@ class ObjectController(object):
else: else:
res = HTTPMethodNotAllowed() res = HTTPMethodNotAllowed()
except: except:
self.logger.exception('ERROR __call__ error with %s %s ' self.logger.exception(_('ERROR __call__ error with %(method)s'
'transaction %s' % (env.get('REQUEST_METHOD', '-'), ' %(path)s '), {'method': req.method, 'path': req.path})
env.get('PATH_INFO', '-'), env.get('HTTP_X_CF_TRANS_ID',
'-')))
res = HTTPInternalServerError(body=traceback.format_exc()) res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = time.time() - start_time trans_time = time.time() - start_time
if self.log_requests: if self.log_requests:

40
swift/obj/updater.py
View File

@ -54,7 +54,7 @@ class ObjectUpdater(Daemon):
"""Get the container ring. Load it, if it hasn't been yet.""" """Get the container ring. Load it, if it hasn't been yet."""
if not self.container_ring: if not self.container_ring:
self.logger.debug( self.logger.debug(
'Loading container ring from %s' % self.container_ring_path) _('Loading container ring from %s'), self.container_ring_path)
self.container_ring = Ring(self.container_ring_path) self.container_ring = Ring(self.container_ring_path)
return self.container_ring return self.container_ring
@ -62,7 +62,7 @@ class ObjectUpdater(Daemon):
"""Run the updater continuously.""" """Run the updater continuously."""
time.sleep(random() * self.interval) time.sleep(random() * self.interval)
while True: while True:
self.logger.info('Begin object update sweep') self.logger.info(_('Begin object update sweep'))
begin = time.time() begin = time.time()
pids = [] pids = []
# read from container ring to ensure it's fresh # read from container ring to ensure it's fresh
@ -71,7 +71,7 @@ class ObjectUpdater(Daemon):
if self.mount_check and not \ if self.mount_check and not \
os.path.ismount(os.path.join(self.devices, device)): os.path.ismount(os.path.join(self.devices, device)):
self.logger.warn( self.logger.warn(
'Skipping %s as it is not mounted' % device) _('Skipping %s as it is not mounted'), device)
continue continue
while len(pids) >= self.concurrency: while len(pids) >= self.concurrency:
pids.remove(os.wait()[0]) pids.remove(os.wait()[0])
@ -86,20 +86,23 @@ class ObjectUpdater(Daemon):
forkbegin = time.time() forkbegin = time.time()
self.object_sweep(os.path.join(self.devices, device)) self.object_sweep(os.path.join(self.devices, device))
elapsed = time.time() - forkbegin elapsed = time.time() - forkbegin
self.logger.info('Object update sweep of %s completed: ' self.logger.info(_('Object update sweep of %(device)s'
'%.02fs, %s successes, %s failures' % ' completed: %(elapsed).02fs, %(success)s successes'
(device, elapsed, self.successes, self.failures)) ', %(fail)s failures'),
{'device': device, 'elapsed': elapsed,
'success': self.successes, 'fail': self.failures})
sys.exit() sys.exit()
while pids: while pids:
pids.remove(os.wait()[0]) pids.remove(os.wait()[0])
elapsed = time.time() - begin elapsed = time.time() - begin
self.logger.info('Object update sweep completed: %.02fs' % elapsed) self.logger.info(_('Object update sweep completed: %.02fs'),
elapsed)
if elapsed < self.interval: if elapsed < self.interval:
time.sleep(self.interval - elapsed) time.sleep(self.interval - elapsed)
def run_once(self): def run_once(self):
"""Run the updater once""" """Run the updater once"""
self.logger.info('Begin object update single threaded sweep') self.logger.info(_('Begin object update single threaded sweep'))
begin = time.time() begin = time.time()
self.successes = 0 self.successes = 0
self.failures = 0 self.failures = 0
@ -107,13 +110,14 @@ class ObjectUpdater(Daemon):
if self.mount_check and \ if self.mount_check and \
not os.path.ismount(os.path.join(self.devices, device)): not os.path.ismount(os.path.join(self.devices, device)):
self.logger.warn( self.logger.warn(
'Skipping %s as it is not mounted' % device) _('Skipping %s as it is not mounted'), device)
continue continue
self.object_sweep(os.path.join(self.devices, device)) self.object_sweep(os.path.join(self.devices, device))
elapsed = time.time() - begin elapsed = time.time() - begin
self.logger.info('Object update single threaded sweep completed: ' self.logger.info(_('Object update single threaded sweep completed: '
'%.02fs, %s successes, %s failures' % '%(elapsed).02fs, %(success)s successes, %(fail)s failures'),
(elapsed, self.successes, self.failures)) {'elapsed': elapsed, 'success': self.successes,
'fail': self.failures})
def object_sweep(self, device): def object_sweep(self, device):
""" """
@ -150,7 +154,7 @@ class ObjectUpdater(Daemon):
update = pickle.load(open(update_path, 'rb')) update = pickle.load(open(update_path, 'rb'))
except Exception: except Exception:
self.logger.exception( self.logger.exception(
'ERROR Pickle problem, quarantining %s' % update_path) _('ERROR Pickle problem, quarantining %s'), update_path)
renamer(update_path, os.path.join(device, renamer(update_path, os.path.join(device,
'quarantined', 'objects', os.path.basename(update_path))) 'quarantined', 'objects', os.path.basename(update_path)))
return return
@ -170,11 +174,13 @@ class ObjectUpdater(Daemon):
successes.append(node['id']) successes.append(node['id'])
if success: if success:
self.successes += 1 self.successes += 1
self.logger.debug('Update sent for %s %s' % (obj, update_path)) self.logger.debug(_('Update sent for %(obj)s %(path)s'),
{'obj': obj, 'path': update_path})
os.unlink(update_path) os.unlink(update_path)
else: else:
self.failures += 1 self.failures += 1
self.logger.debug('Update failed for %s %s' % (obj, update_path)) self.logger.debug(_('Update failed for %(obj)s %(path)s'),
{'obj': obj, 'path': update_path})
update['successes'] = successes update['successes'] = successes
write_pickle(update, update_path, os.path.join(device, 'tmp')) write_pickle(update, update_path, os.path.join(device, 'tmp'))
@ -197,6 +203,6 @@ class ObjectUpdater(Daemon):
resp.read() resp.read()
return resp.status return resp.status
except: except:
self.logger.exception('ERROR with remote server ' self.logger.exception(_('ERROR with remote server '
'%(ip)s:%(port)s/%(device)s' % node) '%(ip)s:%(port)s/%(device)s'), node)
return 500 return 500

89
swift/proxy/server.py
View File

@ -171,10 +171,11 @@ class SegmentedIterable(object):
raise raise
except Exception, err: except Exception, err:
if not getattr(err, 'swift_logged', False): if not getattr(err, 'swift_logged', False):
self.controller.app.logger.exception('ERROR: While processing ' self.controller.app.logger.exception(_('ERROR: While '
'manifest /%s/%s/%s %s' % (self.controller.account_name, 'processing manifest /%(acc)s/%(cont)s/%(obj)s'),
self.controller.container_name, {'acc': self.controller.account_name,
self.controller.object_name, self.controller.trans_id)) 'cont': self.controller.container_name,
'obj': self.controller.object_name})
err.swift_logged = True err.swift_logged = True
self.response.status_int = 503 self.response.status_int = 503
raise raise
@ -203,10 +204,11 @@ class SegmentedIterable(object):
raise raise
except Exception, err: except Exception, err:
if not getattr(err, 'swift_logged', False): if not getattr(err, 'swift_logged', False):
self.controller.app.logger.exception('ERROR: While processing ' self.controller.app.logger.exception(_('ERROR: While '
'manifest /%s/%s/%s %s' % (self.controller.account_name, 'processing manifest /%(acc)s/%(cont)s/%(obj)s'),
self.controller.container_name, {'acc': self.controller.account_name,
self.controller.object_name, self.controller.trans_id)) 'cont': self.controller.container_name,
'obj': self.controller.object_name})
err.swift_logged = True err.swift_logged = True
self.response.status_int = 503 self.response.status_int = 503
raise raise
@ -249,10 +251,11 @@ class SegmentedIterable(object):
raise raise
except Exception, err: except Exception, err:
if not getattr(err, 'swift_logged', False): if not getattr(err, 'swift_logged', False):
self.controller.app.logger.exception('ERROR: While processing ' self.controller.app.logger.exception(_('ERROR: While '
'manifest /%s/%s/%s %s' % (self.controller.account_name, 'processing manifest /%(acc)s/%(cont)s/%(obj)s'),
self.controller.container_name, {'acc': self.controller.account_name,
self.controller.object_name, self.controller.trans_id)) 'cont': self.controller.container_name,
'obj': self.controller.object_name})
err.swift_logged = True err.swift_logged = True
self.response.status_int = 503 self.response.status_int = 503
raise raise
@ -283,8 +286,8 @@ class Controller(object):
:param msg: error message :param msg: error message
""" """
self.error_increment(node) self.error_increment(node)
self.app.logger.error( self.app.logger.error(_('%(msg)s %(ip)s:%(port)s'),
'%s %s:%s' % (msg, node['ip'], node['port'])) {'msg': msg, 'ip': node['ip'], 'port': node['port']})
def exception_occurred(self, node, typ, additional_info): def exception_occurred(self, node, typ, additional_info):
""" """
@ -295,9 +298,9 @@ class Controller(object):
:param additional_info: additional information to log :param additional_info: additional information to log
""" """
self.app.logger.exception( self.app.logger.exception(
'ERROR with %s server %s:%s/%s transaction %s re: %s' % (typ, _('ERROR with %(type)s server %(ip)s:%(port)s/%(device)s re: %(info)s'),
node['ip'], node['port'], node['device'], self.trans_id, {'type': typ, 'ip': node['ip'], 'port': node['port'],
additional_info)) 'device': node['device'], 'info': additional_info})
def error_limited(self, node): def error_limited(self, node):
""" """
@ -318,8 +321,7 @@ class Controller(object):
limited = node['errors'] > self.app.error_suppression_limit limited = node['errors'] > self.app.error_suppression_limit
if limited: if limited:
self.app.logger.debug( self.app.logger.debug(
'Node error limited %s:%s (%s)' % ( _('Node error limited %(ip)s:%(port)s (%(device)s)'), node)
node['ip'], node['port'], node['device']))
return limited return limited
def error_limit(self, node): def error_limit(self, node):
@ -543,8 +545,8 @@ class Controller(object):
if etag: if etag:
resp.headers['etag'] = etag.strip('"') resp.headers['etag'] = etag.strip('"')
return resp return resp
self.app.logger.error('%s returning 503 for %s, transaction %s' % self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'),
(server_type, statuses, self.trans_id)) {'type': server_type, 'statuses': statuses})
resp.status = '503 Internal Server Error' resp.status = '503 Internal Server Error'
return resp return resp
@ -617,9 +619,7 @@ class Controller(object):
res.bytes_transferred += len(chunk) res.bytes_transferred += len(chunk)
except GeneratorExit: except GeneratorExit:
res.client_disconnect = True res.client_disconnect = True
self.app.logger.info( self.app.logger.info(_('Client disconnected on read'))
'Client disconnected on read transaction %s' %
self.trans_id)
except: except:
self.exception_occurred(node, 'Object', self.exception_occurred(node, 'Object',
'Trying to read during GET of %s' % req.path) 'Trying to read during GET of %s' % req.path)
@ -852,7 +852,7 @@ class ObjectController(Controller):
error_response = check_metadata(req, 'object') error_response = check_metadata(req, 'object')
if error_response: if error_response:
return error_response return error_response
container_partition, containers, _, req.acl = \ container_partition, containers, _junk, req.acl = \
self.container_info(self.account_name, self.container_name) self.container_info(self.account_name, self.container_name)
if 'swift.authorize' in req.environ: if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req) aresp = req.environ['swift.authorize'](req)
@ -894,7 +894,7 @@ class ObjectController(Controller):
@delay_denial @delay_denial
def PUT(self, req): def PUT(self, req):
"""HTTP PUT request handler.""" """HTTP PUT request handler."""
container_partition, containers, _, req.acl = \ container_partition, containers, _junk, req.acl = \
self.container_info(self.account_name, self.container_name) self.container_info(self.account_name, self.container_name)
if 'swift.authorize' in req.environ: if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req) aresp = req.environ['swift.authorize'](req)
@ -909,7 +909,7 @@ class ObjectController(Controller):
req.headers['X-Timestamp'] = normalize_timestamp(time.time()) req.headers['X-Timestamp'] = normalize_timestamp(time.time())
# Sometimes the 'content-type' header exists, but is set to None. # Sometimes the 'content-type' header exists, but is set to None.
if not req.headers.get('content-type'): if not req.headers.get('content-type'):
guessed_type, _ = mimetypes.guess_type(req.path_info) guessed_type, _junk = mimetypes.guess_type(req.path_info)
if not guessed_type: if not guessed_type:
req.headers['Content-Type'] = 'application/octet-stream' req.headers['Content-Type'] = 'application/octet-stream'
else: else:
@ -995,9 +995,9 @@ class ObjectController(Controller):
containers.insert(0, container) containers.insert(0, container)
if len(conns) <= len(nodes) / 2: if len(conns) <= len(nodes) / 2:
self.app.logger.error( self.app.logger.error(
'Object PUT returning 503, %s/%s required connections, ' _('Object PUT returning 503, %(conns)s/%(nodes)s '
'transaction %s' % 'required connections'),
(len(conns), len(nodes) / 2 + 1, self.trans_id)) {'conns': len(conns), 'nodes': len(nodes) // 2 + 1})
return HTTPServiceUnavailable(request=req) return HTTPServiceUnavailable(request=req)
try: try:
req.bytes_transferred = 0 req.bytes_transferred = 0
@ -1027,27 +1027,26 @@ class ObjectController(Controller):
conns.remove(conn) conns.remove(conn)
if len(conns) <= len(nodes) / 2: if len(conns) <= len(nodes) / 2:
self.app.logger.error( self.app.logger.error(
'Object PUT exceptions during send, %s/%s ' _('Object PUT exceptions during send, '
'required connections, transaction %s' % '%(conns)s/%(nodes)s required connections'),
(len(conns), len(nodes) // 2 + 1, {'conns': len(conns),
self.trans_id)) 'nodes': len(nodes) // 2 + 1})
return HTTPServiceUnavailable(request=req) return HTTPServiceUnavailable(request=req)
if req.headers.get('transfer-encoding') and chunk == '': if req.headers.get('transfer-encoding') and chunk == '':
break break
except ChunkReadTimeout, err: except ChunkReadTimeout, err:
self.app.logger.info( self.app.logger.info(
'ERROR Client read timeout (%ss)' % err.seconds) _('ERROR Client read timeout (%ss)'), err.seconds)
return HTTPRequestTimeout(request=req) return HTTPRequestTimeout(request=req)
except: except:
req.client_disconnect = True req.client_disconnect = True
self.app.logger.exception( self.app.logger.exception(
'ERROR Exception causing client disconnect') _('ERROR Exception causing client disconnect'))
return Response(status='499 Client Disconnect') return Response(status='499 Client Disconnect')
if req.content_length and req.bytes_transferred < req.content_length: if req.content_length and req.bytes_transferred < req.content_length:
req.client_disconnect = True req.client_disconnect = True
self.app.logger.info( self.app.logger.info(
'Client disconnected without sending enough data %s' % _('Client disconnected without sending enough data'))
self.trans_id)
return Response(status='499 Client Disconnect') return Response(status='499 Client Disconnect')
statuses = [] statuses = []
reasons = [] reasons = []
@ -1071,7 +1070,7 @@ class ObjectController(Controller):
'Trying to get final status of PUT to %s' % req.path) 'Trying to get final status of PUT to %s' % req.path)
if len(etags) > 1: if len(etags) > 1:
self.app.logger.error( self.app.logger.error(
'Object servers returned %s mismatched etags' % len(etags)) _('Object servers returned %s mismatched etags'), len(etags))
return HTTPServerError(request=req) return HTTPServerError(request=req)
etag = len(etags) and etags.pop() or None etag = len(etags) and etags.pop() or None
while len(statuses) < len(nodes): while len(statuses) < len(nodes):
@ -1095,7 +1094,7 @@ class ObjectController(Controller):
@delay_denial @delay_denial
def DELETE(self, req): def DELETE(self, req):
"""HTTP DELETE request handler.""" """HTTP DELETE request handler."""
container_partition, containers, _, req.acl = \ container_partition, containers, _junk, req.acl = \
self.container_info(self.account_name, self.container_name) self.container_info(self.account_name, self.container_name)
if 'swift.authorize' in req.environ: if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req) aresp = req.environ['swift.authorize'](req)
@ -1145,7 +1144,7 @@ class ObjectController(Controller):
if not dest.startswith('/'): if not dest.startswith('/'):
dest = '/' + dest dest = '/' + dest
try: try:
_, dest_container, dest_object = dest.split('/', 2) _junk, dest_container, dest_object = dest.split('/', 2)
except ValueError: except ValueError:
return HTTPPreconditionFailed(request=req, return HTTPPreconditionFailed(request=req,
body='Destination header must be of the form ' body='Destination header must be of the form '
@ -1413,9 +1412,8 @@ class ContainerController(Controller):
# If even one node doesn't do the delete, we can't be sure # If even one node doesn't do the delete, we can't be sure
# what the outcome will be once everything is in sync; so # what the outcome will be once everything is in sync; so
# we 503. # we 503.
self.app.logger.error('Returning 503 because not all ' self.app.logger.error(_('Returning 503 because not all '
'container nodes confirmed DELETE, transaction %s' % 'container nodes confirmed DELETE'))
self.trans_id)
return HTTPServiceUnavailable(request=req) return HTTPServiceUnavailable(request=req)
if resp.status_int == 202: # Indicates no server had the container if resp.status_int == 202: # Indicates no server had the container
return HTTPNotFound(request=req) return HTTPNotFound(request=req)
@ -1710,6 +1708,7 @@ class BaseApplication(object):
controller = controller(self, **path_parts) controller = controller(self, **path_parts)
controller.trans_id = req.headers.get('x-cf-trans-id', '-') controller.trans_id = req.headers.get('x-cf-trans-id', '-')
self.logger.txn_id = req.headers.get('x-cf-trans-id', None)
try: try:
handler = getattr(controller, req.method) handler = getattr(controller, req.method)
if not getattr(handler, 'publicly_accessible'): if not getattr(handler, 'publicly_accessible'):
@ -1737,7 +1736,7 @@ class BaseApplication(object):
return resp return resp
return handler(req) return handler(req)
except Exception: except Exception:
self.logger.exception('ERROR Unhandled exception in request') self.logger.exception(_('ERROR Unhandled exception in request'))
return HTTPServerError(request=req) return HTTPServerError(request=req)

7
test/__init__.py
View File

@ -0,0 +1,7 @@
# See http://code.google.com/p/python-nose/issues/detail?id=373
# The code below enables nosetests to work with i18n _() blocks
import __builtin__
setattr(__builtin__, '_', lambda x: x)

0
test/probe/__init__.py Normal file
View File

2
test/probe/test_account_failures.py
View File

@ -21,7 +21,7 @@ from subprocess import Popen
from time import sleep from time import sleep
from swift.common import client from swift.common import client
from common import get_to_final_state, kill_pids, reset_environment from test.probe.common import get_to_final_state, kill_pids, reset_environment
class TestAccountFailures(unittest.TestCase): class TestAccountFailures(unittest.TestCase):

2
test/probe/test_container_failures.py
View File

@ -23,7 +23,7 @@ from uuid import uuid4
from swift.common import client from swift.common import client
from common import get_to_final_state, kill_pids, reset_environment from test.probe.common import get_to_final_state, kill_pids, reset_environment
class TestContainerFailures(unittest.TestCase): class TestContainerFailures(unittest.TestCase):

2
test/probe/test_object_async_update.py
View File

@ -23,7 +23,7 @@ from uuid import uuid4
from swift.common import client, direct_client from swift.common import client, direct_client
from common import kill_pids, reset_environment from test.probe.common import kill_pids, reset_environment
class TestObjectAsyncUpdate(unittest.TestCase): class TestObjectAsyncUpdate(unittest.TestCase):

2
test/probe/test_object_handoff.py
View File

@ -23,7 +23,7 @@ from uuid import uuid4
from swift.common import client, direct_client from swift.common import client, direct_client
from common import kill_pids, reset_environment from test.probe.common import kill_pids, reset_environment
class TestObjectHandoff(unittest.TestCase): class TestObjectHandoff(unittest.TestCase):

2
test/probe/test_running_with_each_type_down.py
View File

@ -22,7 +22,7 @@ from time import sleep
from swift.common import client from swift.common import client
from common import get_to_final_state, kill_pids, reset_environment from test.probe.common import get_to_final_state, kill_pids, reset_environment
class TestRunningWithEachTypeDown(unittest.TestCase): class TestRunningWithEachTypeDown(unittest.TestCase):

8
test/unit/auth/test_server.py
View File

@ -462,7 +462,7 @@ class TestAuthServer(unittest.TestCase):
auth_server.http_connect = fake_http_connect(201) auth_server.http_connect = fake_http_connect(201)
url = self.controller.create_user('test', 'tester', 'testing') url = self.controller.create_user('test', 'tester', 'testing')
self.assertEquals(log.getvalue().rsplit(' ', 1)[0], self.assertEquals(log.getvalue().rsplit(' ', 1)[0],
"auth SUCCESS create_user('test', 'tester', _, False, False) " "SUCCESS create_user('test', 'tester', _, False, False) "
"= %s" % repr(url)) "= %s" % repr(url))
log.truncate(0) log.truncate(0)
def start_response(*args): def start_response(*args):
@ -491,7 +491,7 @@ class TestAuthServer(unittest.TestCase):
logsegs[1] = '[01/Jan/2001:01:02:03 +0000]' logsegs[1] = '[01/Jan/2001:01:02:03 +0000]'
logsegs[2:] = logsegs[2].split(' ') logsegs[2:] = logsegs[2].split(' ')
logsegs[-1] = '0.1234' logsegs[-1] = '0.1234'
self.assertEquals(' '.join(logsegs), 'auth testhost - - ' self.assertEquals(' '.join(logsegs), 'testhost - - '
'[01/Jan/2001:01:02:03 +0000] "GET /v1/test/auth?test=True ' '[01/Jan/2001:01:02:03 +0000] "GET /v1/test/auth?test=True '
'HTTP/1.0" 204 - "-" "-" - - - - - - - - - "-" "None" "-" ' 'HTTP/1.0" 204 - "-" "-" - - - - - - - - - "-" "None" "-" '
'0.1234') '0.1234')
@ -519,7 +519,7 @@ class TestAuthServer(unittest.TestCase):
logsegs[1] = '[01/Jan/2001:01:02:03 +0000]' logsegs[1] = '[01/Jan/2001:01:02:03 +0000]'
logsegs[2:] = logsegs[2].split(' ') logsegs[2:] = logsegs[2].split(' ')
logsegs[-1] = '0.1234' logsegs[-1] = '0.1234'
self.assertEquals(' '.join(logsegs), 'auth None - - [01/Jan/2001:' self.assertEquals(' '.join(logsegs), 'None - - [01/Jan/2001:'
'01:02:03 +0000] "GET /v1/test/auth HTTP/1.0" 204 - "-" "-" - ' '01:02:03 +0000] "GET /v1/test/auth HTTP/1.0" 204 - "-" "-" - '
'- - - - - - - - "-" "None" "Content-Length: 0\n' '- - - - - - - - "-" "None" "Content-Length: 0\n'
'X-Storage-User: tester\nX-Storage-Pass: testing" 0.1234') 'X-Storage-User: tester\nX-Storage-Pass: testing" 0.1234')
@ -556,7 +556,7 @@ class TestAuthServer(unittest.TestCase):
'HTTP_X_STORAGE_PASS': 'testing'}, 'HTTP_X_STORAGE_PASS': 'testing'},
start_response) start_response)
self.assert_(log.getvalue().startswith( self.assert_(log.getvalue().startswith(
'auth ERROR Unhandled exception in ReST request'), 'ERROR Unhandled exception in ReST request'),
log.getvalue()) log.getvalue())
log.truncate(0) log.truncate(0)
finally: finally:

2
test/unit/common/test_daemon.py
View File

@ -50,7 +50,7 @@ class TestDaemon(unittest.TestCase):
def test_create(self): def test_create(self):
d = daemon.Daemon({}) d = daemon.Daemon({})
self.assertEquals(d.conf, {}) self.assertEquals(d.conf, {})
self.assert_(isinstance(d.logger, utils.NamedLogger)) self.assert_(isinstance(d.logger, utils.LogAdapter))
def test_stubs(self): def test_stubs(self):
d = daemon.Daemon({}) d = daemon.Daemon({})

18
test/unit/common/test_utils.py
View File

@ -283,35 +283,27 @@ Error: unable to locate %s
utils.sys.stdout = orig_stdout utils.sys.stdout = orig_stdout
utils.sys.stderr = orig_stderr utils.sys.stderr = orig_stderr
def test_NamedLogger(self):
sio = StringIO()
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(sio))
nl = utils.NamedLogger(logger, 'server')
nl.warn('test')
self.assertEquals(sio.getvalue(), 'server test\n')
def test_get_logger(self): def test_get_logger(self):
sio = StringIO() sio = StringIO()
logger = logging.getLogger() logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(sio)) logger.addHandler(logging.StreamHandler(sio))
logger = utils.get_logger(None, 'server') logger = utils.get_logger(None, 'server')
logger.warn('test1') logger.warn('test1')
self.assertEquals(sio.getvalue(), 'server test1\n') self.assertEquals(sio.getvalue(), 'test1\n')
logger.debug('test2') logger.debug('test2')
self.assertEquals(sio.getvalue(), 'server test1\n') self.assertEquals(sio.getvalue(), 'test1\n')
logger = utils.get_logger({'log_level': 'DEBUG'}, 'server') logger = utils.get_logger({'log_level': 'DEBUG'}, 'server')
logger.debug('test3') logger.debug('test3')
self.assertEquals(sio.getvalue(), 'server test1\nserver test3\n') self.assertEquals(sio.getvalue(), 'test1\ntest3\n')
# Doesn't really test that the log facility is truly being used all the # Doesn't really test that the log facility is truly being used all the
# way to syslog; but exercises the code. # way to syslog; but exercises the code.
logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server') logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server')
logger.warn('test4') logger.warn('test4')
self.assertEquals(sio.getvalue(), self.assertEquals(sio.getvalue(),
'server test1\nserver test3\nserver test4\n') 'test1\ntest3\ntest4\n')
logger.debug('test5') logger.debug('test5')
self.assertEquals(sio.getvalue(), self.assertEquals(sio.getvalue(),
'server test1\nserver test3\nserver test4\n') 'test1\ntest3\ntest4\n')
def test_storage_directory(self): def test_storage_directory(self):
self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'),

20
test/unit/proxy/test_server.py
View File

@ -3019,8 +3019,8 @@ class TestSegmentedIterable(unittest.TestCase):
self.assertRaises(Exception, self.assertRaises(Exception,
proxy_server.SegmentedIterable(self.controller, None, proxy_server.SegmentedIterable(self.controller, None,
[None])._load_next_segment) [None])._load_next_segment)
self.assertEquals(self.controller.exception_args[0], self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest /a/c/o tx1') 'ERROR: While processing manifest'))
def test_load_next_segment_with_no_segments(self): def test_load_next_segment_with_no_segments(self):
self.assertRaises(StopIteration, self.assertRaises(StopIteration,
@ -3079,8 +3079,8 @@ class TestSegmentedIterable(unittest.TestCase):
self.assertRaises(Exception, self.assertRaises(Exception,
proxy_server.SegmentedIterable(self.controller, 'lc', [{'name': proxy_server.SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}])._load_next_segment) 'o1'}])._load_next_segment)
self.assertEquals(self.controller.exception_args[0], self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest /a/c/o tx1') 'ERROR: While processing manifest'))
self.assertEquals(str(self.controller.exception_info[1]), self.assertEquals(str(self.controller.exception_info[1]),
'Could not load object segment /a/lc/o1: 404') 'Could not load object segment /a/lc/o1: 404')
@ -3088,8 +3088,8 @@ class TestSegmentedIterable(unittest.TestCase):
# Iterator value isn't a dict # Iterator value isn't a dict
self.assertRaises(Exception, ''.join, self.assertRaises(Exception, ''.join,
proxy_server.SegmentedIterable(self.controller, None, [None])) proxy_server.SegmentedIterable(self.controller, None, [None]))
self.assertEquals(self.controller.exception_args[0], self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest /a/c/o tx1') 'ERROR: While processing manifest'))
def test_iter_with_no_segments(self): def test_iter_with_no_segments(self):
segit = proxy_server.SegmentedIterable(self.controller, 'lc', []) segit = proxy_server.SegmentedIterable(self.controller, 'lc', [])
@ -3118,8 +3118,8 @@ class TestSegmentedIterable(unittest.TestCase):
self.assertRaises(Exception, ''.join, self.assertRaises(Exception, ''.join,
proxy_server.SegmentedIterable(self.controller, 'lc', [{'name': proxy_server.SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}])) 'o1'}]))
self.assertEquals(self.controller.exception_args[0], self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest /a/c/o tx1') 'ERROR: While processing manifest'))
self.assertEquals(str(self.controller.exception_info[1]), self.assertEquals(str(self.controller.exception_info[1]),
'Could not load object segment /a/lc/o1: 404') 'Could not load object segment /a/lc/o1: 404')
@ -3128,8 +3128,8 @@ class TestSegmentedIterable(unittest.TestCase):
self.assertRaises(Exception, self.assertRaises(Exception,
proxy_server.SegmentedIterable(self.controller, None, proxy_server.SegmentedIterable(self.controller, None,
[None]).app_iter_range(None, None).next) [None]).app_iter_range(None, None).next)
self.assertEquals(self.controller.exception_args[0], self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest /a/c/o tx1') 'ERROR: While processing manifest'))
def test_app_iter_range_with_no_segments(self): def test_app_iter_range_with_no_segments(self):
self.assertEquals(''.join(proxy_server.SegmentedIterable( self.assertEquals(''.join(proxy_server.SegmentedIterable(