openstack/swift
Code Issues Proposed changes

Patch for Swift Solaris (Illumos) compability.

Browse Source 
* Add new configuration option log_address.

Change-Id: I636bd4116687629c997b70a0d804b7ed4bc46032
This commit is contained in:
Victor Rodionov 2012-05-17 15:46:38 -07:00
parent 95786e577c
commit 13e4de1899
12 changed files with 122 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/manpages/account-server.conf.5
View File

@ -112,6 +112,8 @@ Syslog log facility. The default is LOG_LOCAL0.
Logging level. The default is INFO.
.IP "\fB set log_requests\fR
Enables request logging. The default is True.
.IP "\fB set log_address\fR
Logging address. The default is /dev/log.
.RE
.PD
@ -148,7 +150,9 @@ Label used when logging. The default is account-replicator.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBvm_test_mode\fR
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBvm_test_mode\fR
Indicates that you are using a VM environment. The default is no.
.IP \fBper_diff\fR
The default is 1000.
@ -187,6 +191,8 @@ Label used when logging. The default is account-auditor.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBinterval\fR
Will audit, at most, 1 account per device per interval. The default is 1800 seconds.
.RE
@ -203,6 +209,8 @@ Label used when logging. The default is account-reaper.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBconcurrency\fR
Number of reaper workers to spawn. The default is 25.
.IP \fBinterval\fR
@ -216,7 +224,7 @@ Connection timeout to external services. The default is 0.5 seconds.
.SH DOCUMENTATION
.LP
More in depth documentation about the swift-account-server and

12
doc/manpages/container-server.conf.5
View File

@ -76,6 +76,8 @@ Label used when logging. The default is swift.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.RE
.PD
@ -112,6 +114,8 @@ Syslog log facility. The default is LOG_LOCAL0.
Logging level. The default is INFO.
.IP "\fB set log_requests\fR
Enables request logging. The default is True.
.IP "\fB set log_address\fR
Logging address. The default is /dev/log.
.IP \fBnode_timeout\fR
Request timeout to external services. The default is 3 seconds.
.IP \fBconn_timeout\fR
@ -152,6 +156,8 @@ Label used when logging. The default is container-replicator.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBvm_test_mode\fR
Indicates that you are using a VM environment. The default is no.
.IP \fBer_diff\fR
@ -184,6 +190,8 @@ Label used when logging. The default is container-updater.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBinterval\fR
Minimum time for a pass to take. The default is 300 seconds.
.IP \fBconcurrency\fR
@ -210,6 +218,8 @@ Label used when logging. The default is container-auditor.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBinterval\fR
Will audit, at most, 1 container per device per interval. The default is 1800 seconds.
.RE
@ -226,6 +236,8 @@ Label used when logging. The default is container-sync.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBsync_proxy\fR
If you need to use an HTTP Proxy, set it here; defaults to no proxy.
.IP \fBinterval\fR

2
doc/manpages/object-expirer.conf.5
View File

@ -63,6 +63,8 @@ Label used when logging. The default is swift.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.RE
.PD

14
doc/manpages/object-server.conf.5
View File

@ -76,6 +76,8 @@ Label used when logging. The default is swift.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.RE
.PD
@ -112,7 +114,9 @@ Syslog log facility. The default is LOG_LOCAL0.
Logging level. The default is INFO.
.IP "\fB set log_requests\fR
Enables request logging. The default is True.
.IP \fBnode_timeout\fR
.IP "\fB set log_address\fR
Logging address. The default is /dev/log.
.IP \fBnode_timeout\fR
Request timeout to external services. The default is 3 seconds.
.IP \fBconn_timeout\fR
Connection timeout to external services. The default is 0.5 seconds.
@ -165,6 +169,8 @@ Label used when logging. The default is object-replicator.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBvm_test_mode\fR
Indicates that you are using a VM environment. The default is no.
.IP \fBdaemonize\fR
@ -206,6 +212,8 @@ Label used when logging. The default is object-updater.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBinterval\fR
Minimum time for a pass to take. The default is 300 seconds.
.IP \fBconcurrency\fR
@ -230,6 +238,8 @@ Label used when logging. The default is object-auditor.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBfiles_per_second\fR
Maximum files audited per second. Should be tuned according to individual
@ -245,7 +255,7 @@ The default is 50.
.SH DOCUMENTATION
.LP
More in depth documentation about the swift-object-server and

20
doc/manpages/proxy-server.conf.5
View File

@ -76,6 +76,8 @@ Label used when logging. The default is swift.
Syslog log facility. The default is LOG_LOCAL0.
.IP \fBlog_level\fR
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.RE
.PD
@ -124,6 +126,8 @@ Label used when logging. The default is tempauth.
Syslog log facility. The default is LOG_LOCAL0.
.IP "\fBset log_level\fR "
Logging level. The default is INFO.
.IP "\fBset log_address\fR"
Logging address. The default is /dev/log.
.IP "\fBset log_headers\fR "
Enables the ability to log request headers. The default is False.
.IP \fBreseller_prefix\fR
@ -179,6 +183,8 @@ Label used when logging. The default is healthcheck.
Syslog log facility. The default is LOG_LOCAL0.
.IP "\fBset log_level\fR "
Logging level. The default is INFO.
.IP "\fBset log_address\fR"
Logging address. The default is /dev/log.
.IP "\fBset log_headers\fR "
Enables the ability to log request headers. The default is False.
.RE
@ -201,6 +207,8 @@ Label used when logging. The default is memcache.
Syslog log facility. The default is LOG_LOCAL0.
.IP "\fBset log_level\fR "
Logging level. The default is INFO.
.IP "\fBset log_address\fR"
Logging address. The default is /dev/log.
.IP "\fBset log_headers\fR "
Enables the ability to log request headers. The default is False.
.IP \fBmemcache_servers\fR
@ -225,6 +233,8 @@ Label used when logging. The default is ratelimit.
Syslog log facility. The default is LOG_LOCAL0.
.IP "\fBset log_level\fR "
Logging level. The default is INFO.
.IP "\fBset log_address\fR"
Logging address. The default is /dev/log.
.IP "\fBset log_headers\fR "
Enables the ability to log request headers. The default is False.
.IP \fBclock_accuracy\fR
@ -267,6 +277,8 @@ Entry point for paste.deploy for the domain_remap middleware. This is the refere
The default is \fBegg:swift#domain_remap\fR.
.IP "\fBset log_name\fR"
Label used when logging. The default is domain_remap.
.IP "\fBset log_address\fR"
Logging address. The default is /dev/log.
.IP "\fBset log_headers\fR"
Enables the ability to log request headers. The default is False.
.IP \fBstorage_domain\fR
@ -298,6 +310,8 @@ Label used when logging. The default is catch_errors.
Syslog log facility. The default is LOG_LOCAL0.
.IP "\fBset log_level\fR "
Logging level. The default is INFO.
.IP "\fBset log_address\fR "
Logging address. The default is /dev/log.
.IP "\fBset log_headers\fR"
Enables the ability to log request headers. The default is False.
.RE
@ -320,6 +334,8 @@ Label used when logging. The default is cname_lookup.
Syslog log facility. The default is LOG_LOCAL0.
.IP "\fBset log_level\fR "
Logging level. The default is INFO.
.IP "\fBset log_address\fR"
Logging address. The default is /dev/log.
.IP "\fBset log_headers\fR"
Enables the ability to log request headers. The default is False.
.IP \fBstorage_domain\fR
@ -349,6 +365,8 @@ Label used when logging. The default is staticweb.
Syslog log facility. The default is LOG_LOCAL0.
.IP "\fBset log_level\fR "
Logging level. The default is INFO.
.IP "\fBset log_address\fR "
Logging address. The default is /dev/log.
.IP "\fBset log_headers\fR"
Enables the ability to log request headers. The default is False.
.IP "\fBset access_log_name\fR"
@ -431,6 +449,8 @@ Label used when logging. The default is proxy-server.
Syslog log facility. The default is LOG_LOCAL0.
.IP "\fB set log_level\fR
Logging level. The default is INFO.
.IP "\fB set log_address\fR
Logging address. The default is /dev/log.
.IP "\fBset access_log_name\fR"
Label used when logging. The default is proxy-server.
.IP "\fBset access_log_facility\fR"

5
etc/account-server.conf-sample
View File

@ -11,6 +11,7 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# You can enable default statsD logging here and/or override it in sections
# below:
# log_statsd_host = localhost
@ -31,6 +32,7 @@ use = egg:swift#account
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_requests = True
# set log_address = /dev/log
# auto_create_account_prefix = .
[filter:recon]
@ -42,6 +44,7 @@ use = egg:swift#recon
# log_name = account-replicator
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# vm_test_mode = no
# per_diff = 1000
# max_diffs = 100
@ -65,6 +68,7 @@ use = egg:swift#recon
# log_name = account-auditor
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# Will audit, at most, 1 account per device per interval
# interval = 1800
# log_facility = LOG_LOCAL0
@ -76,6 +80,7 @@ use = egg:swift#recon
# log_name = account-reaper
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# concurrency = 25
# interval = 3600
# node_timeout = 10

6
etc/container-server.conf-sample
View File

@ -14,6 +14,7 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# You can enable default statsD logging here and/or override it in sections
# below:
# log_statsd_host = localhost
@ -34,6 +35,7 @@ use = egg:swift#container
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_requests = True
# set log_address = /dev/log
# node_timeout = 3
# conn_timeout = 0.5
# allow_versions = False
@ -48,6 +50,7 @@ use = egg:swift#recon
# log_name = container-replicator
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# vm_test_mode = no
# per_diff = 1000
# max_diffs = 100
@ -66,6 +69,7 @@ use = egg:swift#recon
# log_name = container-updater
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# interval = 300
# concurrency = 4
# node_timeout = 3
@ -81,6 +85,7 @@ use = egg:swift#recon
# log_name = container-auditor
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# Will audit, at most, 1 container per device per interval
# interval = 1800
# recon_cache_path = /var/cache/swift
@ -90,6 +95,7 @@ use = egg:swift#recon
# log_name = container-sync
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# If you need to use an HTTP Proxy, set it here; defaults to no proxy.
# sync_proxy = http://127.0.0.1:8888
# Will sync, at most, each container once per interval

1
etc/drive-audit.conf-sample
View File

@ -2,5 +2,6 @@
# device_dir = /srv/node
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# minutes = 60
# error_limit = 1

1
etc/object-expirer.conf-sample
View File

@ -5,6 +5,7 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# You can enable default statsD logging here if you want:
# log_statsd_host = localhost
# log_statsd_port = 8125

5
etc/object-server.conf-sample
View File

@ -12,6 +12,7 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# You can enable default statsD logging here and/or override it in sections
# below:
# log_statsd_host = localhost
@ -29,6 +30,7 @@ use = egg:swift#object
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_requests = True
# set log_address = /dev/log
# node_timeout = 3
# conn_timeout = 0.5
# network_chunk_size = 65536
@ -58,6 +60,7 @@ use = egg:swift#recon
# log_name = object-replicator
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# vm_test_mode = no
# daemonize = on
# run_pause = 30
@ -81,6 +84,7 @@ use = egg:swift#recon
# log_name = object-updater
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# interval = 300
# concurrency = 1
# node_timeout = 10
@ -94,6 +98,7 @@ use = egg:swift#recon
# log_name = object-auditor
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# files_per_second = 20
# bytes_per_second = 10000000
# log_time = 3600

10
etc/proxy-server.conf-sample
View File

@ -13,6 +13,7 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# You can enable default statsD logging here and/or override it in sections
# below:
# log_statsd_host = localhost
@ -29,6 +30,7 @@ use = egg:swift#proxy
# set log_name = proxy-server
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_address = /dev/log
# set access_log_name = proxy-server
# set access_log_facility = LOG_LOCAL0
# set access_log_level = INFO
@ -78,6 +80,7 @@ use = egg:swift#tempauth
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = False
# set log_address = /dev/log
# The reseller prefix will verify a token begins with this prefix before even
# attempting to validate it. Also, with authorization, only Swift storage
# accounts with this prefix will be authorized by this middleware. Useful if
@ -122,6 +125,7 @@ use = egg:swift#healthcheck
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = False
# set log_address = /dev/log
[filter:cache]
use = egg:swift#memcache
@ -130,6 +134,7 @@ use = egg:swift#memcache
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = False
# set log_address = /dev/log
# Default for memcache_servers is to try to read the property from
# memcache.conf (see memcache.conf-sample) or lacking that file, it will
# default to the value below. You can specify multiple servers separated with
@ -143,6 +148,7 @@ use = egg:swift#ratelimit
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = False
# set log_address = /dev/log
# clock_accuracy should represent how accurate the proxy servers' system clocks
# are with each other. 1000 means that all the proxies' clock are accurate to
# each other within 1 millisecond. No ratelimit should be higher than the
@ -175,6 +181,7 @@ use = egg:swift#domain_remap
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = False
# set log_address = /dev/log
# storage_domain = example.com
# path_root = v1
# reseller_prefixes = AUTH
@ -186,6 +193,7 @@ use = egg:swift#catch_errors
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = False
# set log_address = /dev/log
[filter:cname_lookup]
# Note: this middleware requires python-dnspython
@ -195,6 +203,7 @@ use = egg:swift#cname_lookup
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = False
# set log_address = /dev/log
# storage_domain = example.com
# lookup_depth = 1
@ -207,6 +216,7 @@ use = egg:swift#staticweb
# set log_name = staticweb
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_address = /dev/log
# set access_log_name = staticweb
# set access_log_facility = LOG_LOCAL0
# set access_log_level = INFO

46
swift/common/utils.py
View File

@ -39,6 +39,7 @@ except ImportError:
import cPickle as pickle
import glob
from urlparse import urlparse as stdlib_urlparse, ParseResult
import socket
import eventlet
from eventlet import GreenPool, sleep, Timeout
@ -78,6 +79,10 @@ if hash_conf.read('/etc/swift/swift.conf'):
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
def noop_libc_function(*args):
return 0
def validate_configuration():
if HASH_PATH_SUFFIX == '':
sys.exit("Error: [swift-hash]: swift_hash_path_suffix missing "
@ -96,9 +101,6 @@ def load_libc_function(func_name):
except AttributeError:
logging.warn(_("Unable to locate %s in libc. Leaving as a no-op."),
func_name)
def noop_libc_function(*args):
return 0
return noop_libc_function
@ -118,6 +120,23 @@ def get_param(req, name, default=None):
return value
class FallocateWrapper(object):
def __init__(self):
for func in ('fallocate', 'posix_fallocate'):
self.func_name = func
self.fallocate = load_libc_function(func)
if self.fallocate is not noop_libc_function:
break
def __call__(self, fd, mode, offset, len):
args = {
'fallocate': (fd, mode, offset, len),
'posix_fallocate': (fd, offset, len)
}
return self.fallocate(*args[self.func_name])
def fallocate(fd, size):
"""
Pre-allocate disk space for a file file.
@ -127,7 +146,7 @@ def fallocate(fd, size):
"""
global _sys_fallocate
if _sys_fallocate is None:
_sys_fallocate = load_libc_function('fallocate')
_sys_fallocate = FallocateWrapper()
if size > 0:
# 1 means "FALLOC_FL_KEEP_SIZE", which means it pre-allocates invisibly
ret = _sys_fallocate(fd, 1, 0, ctypes.c_uint64(size))
@ -193,7 +212,7 @@ def renamer(old, new):
try:
mkdirs(os.path.dirname(new))
os.rename(old, new)
except OSError:
except OSError, err:
mkdirs(os.path.dirname(new))
os.rename(old, new)
@ -523,7 +542,13 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None,
# facility for this logger will be set by last call wins
facility = getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'),
SysLogHandler.LOG_LOCAL0)
handler = SysLogHandler(address='/dev/log', facility=facility)
log_address = conf.get('log_address', '/dev/log')
try:
handler = SysLogHandler(address=log_address, facility=facility)
except socket.error, e:
if e.errno != errno.ENOTSOCK: # Socket operation on non-socket
raise e
handler = SysLogHandler(facility=facility)
handler.setFormatter(formatter)
logger.addHandler(handler)
get_logger.handler4logger[logger] = handler
@ -723,13 +748,18 @@ def lock_path(directory, timeout=10):
the lock can be acquired, or the timeout time has expired (whichever occurs
first).
For locking exclusively, file or directory has to be opened in Write mode.
Python doesn't allow directories to be opened in Write Mode. So we
workaround by locking a hidden file in the directory.
:param directory: directory to be locked
:param timeout: timeout (in seconds)
"""
mkdirs(directory)
fd = os.open(directory, os.O_RDONLY)
lockpath = '%s/.lock' % directory
fd = os.open(lockpath, os.O_WRONLY | os.O_CREAT)
try:
with LockTimeout(timeout, directory):
with LockTimeout(timeout, lockpath):
while True:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)