diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5 index b9db0d68e6..3b90246cc3 100644 --- a/doc/manpages/account-server.conf.5 +++ b/doc/manpages/account-server.conf.5 @@ -112,6 +112,8 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP "\fB set log_requests\fR Enables request logging. The default is True. +.IP "\fB set log_address\fR +Logging address. The default is /dev/log. .RE .PD @@ -148,7 +150,9 @@ Label used when logging. The default is account-replicator. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. -.IP \fBvm_test_mode\fR +.IP \fBlog_address\fR +Logging address. The default is /dev/log. +.IP \fBvm_test_mode\fR Indicates that you are using a VM environment. The default is no. .IP \fBper_diff\fR The default is 1000. @@ -187,6 +191,8 @@ Label used when logging. The default is account-auditor. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBinterval\fR Will audit, at most, 1 account per device per interval. The default is 1800 seconds. .RE @@ -203,6 +209,8 @@ Label used when logging. The default is account-reaper. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBconcurrency\fR Number of reaper workers to spawn. The default is 25. .IP \fBinterval\fR @@ -216,7 +224,7 @@ Connection timeout to external services. The default is 0.5 seconds. - + .SH DOCUMENTATION .LP More in depth documentation about the swift-account-server and diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index a1990b9ca4..5aef25b7d1 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -76,6 +76,8 @@ Label used when logging. The default is swift. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .RE .PD @@ -112,6 +114,8 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP "\fB set log_requests\fR Enables request logging. The default is True. +.IP "\fB set log_address\fR +Logging address. The default is /dev/log. .IP \fBnode_timeout\fR Request timeout to external services. The default is 3 seconds. .IP \fBconn_timeout\fR @@ -152,6 +156,8 @@ Label used when logging. The default is container-replicator. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBvm_test_mode\fR Indicates that you are using a VM environment. The default is no. .IP \fBer_diff\fR @@ -184,6 +190,8 @@ Label used when logging. The default is container-updater. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBinterval\fR Minimum time for a pass to take. The default is 300 seconds. .IP \fBconcurrency\fR @@ -210,6 +218,8 @@ Label used when logging. The default is container-auditor. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBinterval\fR Will audit, at most, 1 container per device per interval. The default is 1800 seconds. .RE @@ -226,6 +236,8 @@ Label used when logging. The default is container-sync. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBsync_proxy\fR If you need to use an HTTP Proxy, set it here; defaults to no proxy. .IP \fBinterval\fR diff --git a/doc/manpages/object-expirer.conf.5 b/doc/manpages/object-expirer.conf.5 index 3dd74eb0c0..8de307ee22 100644 --- a/doc/manpages/object-expirer.conf.5 +++ b/doc/manpages/object-expirer.conf.5 @@ -63,6 +63,8 @@ Label used when logging. The default is swift. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .RE .PD diff --git a/doc/manpages/object-server.conf.5 b/doc/manpages/object-server.conf.5 index 82d9c4f466..42b621b491 100644 --- a/doc/manpages/object-server.conf.5 +++ b/doc/manpages/object-server.conf.5 @@ -76,6 +76,8 @@ Label used when logging. The default is swift. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .RE .PD @@ -112,7 +114,9 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP "\fB set log_requests\fR Enables request logging. The default is True. -.IP \fBnode_timeout\fR +.IP "\fB set log_address\fR +Logging address. The default is /dev/log. +.IP \fBnode_timeout\fR Request timeout to external services. The default is 3 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. @@ -165,6 +169,8 @@ Label used when logging. The default is object-replicator. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBvm_test_mode\fR Indicates that you are using a VM environment. The default is no. .IP \fBdaemonize\fR @@ -206,6 +212,8 @@ Label used when logging. The default is object-updater. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBinterval\fR Minimum time for a pass to take. The default is 300 seconds. .IP \fBconcurrency\fR @@ -230,6 +238,8 @@ Label used when logging. The default is object-auditor. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .IP \fBfiles_per_second\fR Maximum files audited per second. Should be tuned according to individual @@ -245,7 +255,7 @@ The default is 50. - + .SH DOCUMENTATION .LP More in depth documentation about the swift-object-server and diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index af2f648ea1..3af49c223b 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -76,6 +76,8 @@ Label used when logging. The default is swift. Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. +.IP \fBlog_address\fR +Logging address. The default is /dev/log. .RE .PD @@ -124,6 +126,8 @@ Label used when logging. The default is tempauth. Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. +.IP "\fBset log_address\fR" +Logging address. The default is /dev/log. .IP "\fBset log_headers\fR " Enables the ability to log request headers. The default is False. .IP \fBreseller_prefix\fR @@ -179,6 +183,8 @@ Label used when logging. The default is healthcheck. Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. +.IP "\fBset log_address\fR" +Logging address. The default is /dev/log. .IP "\fBset log_headers\fR " Enables the ability to log request headers. The default is False. .RE @@ -201,6 +207,8 @@ Label used when logging. The default is memcache. Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. +.IP "\fBset log_address\fR" +Logging address. The default is /dev/log. .IP "\fBset log_headers\fR " Enables the ability to log request headers. The default is False. .IP \fBmemcache_servers\fR @@ -225,6 +233,8 @@ Label used when logging. The default is ratelimit. Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. +.IP "\fBset log_address\fR" +Logging address. The default is /dev/log. .IP "\fBset log_headers\fR " Enables the ability to log request headers. The default is False. .IP \fBclock_accuracy\fR @@ -267,6 +277,8 @@ Entry point for paste.deploy for the domain_remap middleware. This is the refere The default is \fBegg:swift#domain_remap\fR. .IP "\fBset log_name\fR" Label used when logging. The default is domain_remap. +.IP "\fBset log_address\fR" +Logging address. The default is /dev/log. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. .IP \fBstorage_domain\fR @@ -298,6 +310,8 @@ Label used when logging. The default is catch_errors. Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. +.IP "\fBset log_address\fR " +Logging address. The default is /dev/log. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. .RE @@ -320,6 +334,8 @@ Label used when logging. The default is cname_lookup. Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. +.IP "\fBset log_address\fR" +Logging address. The default is /dev/log. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. .IP \fBstorage_domain\fR @@ -349,6 +365,8 @@ Label used when logging. The default is staticweb. Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. +.IP "\fBset log_address\fR " +Logging address. The default is /dev/log. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. .IP "\fBset access_log_name\fR" @@ -431,6 +449,8 @@ Label used when logging. The default is proxy-server. Syslog log facility. The default is LOG_LOCAL0. .IP "\fB set log_level\fR Logging level. The default is INFO. +.IP "\fB set log_address\fR +Logging address. The default is /dev/log. .IP "\fBset access_log_name\fR" Label used when logging. The default is proxy-server. .IP "\fBset access_log_facility\fR" diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index a0abb765ea..e06f1b2aaa 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -11,6 +11,7 @@ # log_name = swift # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # You can enable default statsD logging here and/or override it in sections # below: # log_statsd_host = localhost @@ -31,6 +32,7 @@ use = egg:swift#account # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_requests = True +# set log_address = /dev/log # auto_create_account_prefix = . [filter:recon] @@ -42,6 +44,7 @@ use = egg:swift#recon # log_name = account-replicator # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # vm_test_mode = no # per_diff = 1000 # max_diffs = 100 @@ -65,6 +68,7 @@ use = egg:swift#recon # log_name = account-auditor # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # Will audit, at most, 1 account per device per interval # interval = 1800 # log_facility = LOG_LOCAL0 @@ -76,6 +80,7 @@ use = egg:swift#recon # log_name = account-reaper # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # concurrency = 25 # interval = 3600 # node_timeout = 10 diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 6ee91bd455..287085544a 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -14,6 +14,7 @@ # log_name = swift # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # You can enable default statsD logging here and/or override it in sections # below: # log_statsd_host = localhost @@ -34,6 +35,7 @@ use = egg:swift#container # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_requests = True +# set log_address = /dev/log # node_timeout = 3 # conn_timeout = 0.5 # allow_versions = False @@ -48,6 +50,7 @@ use = egg:swift#recon # log_name = container-replicator # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # vm_test_mode = no # per_diff = 1000 # max_diffs = 100 @@ -66,6 +69,7 @@ use = egg:swift#recon # log_name = container-updater # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # interval = 300 # concurrency = 4 # node_timeout = 3 @@ -81,6 +85,7 @@ use = egg:swift#recon # log_name = container-auditor # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # Will audit, at most, 1 container per device per interval # interval = 1800 # recon_cache_path = /var/cache/swift @@ -90,6 +95,7 @@ use = egg:swift#recon # log_name = container-sync # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # If you need to use an HTTP Proxy, set it here; defaults to no proxy. # sync_proxy = http://127.0.0.1:8888 # Will sync, at most, each container once per interval diff --git a/etc/drive-audit.conf-sample b/etc/drive-audit.conf-sample index 9ffa94760c..98ef4eb1be 100644 --- a/etc/drive-audit.conf-sample +++ b/etc/drive-audit.conf-sample @@ -2,5 +2,6 @@ # device_dir = /srv/node # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # minutes = 60 # error_limit = 1 diff --git a/etc/object-expirer.conf-sample b/etc/object-expirer.conf-sample index 5fe9793121..00ef3a63e9 100644 --- a/etc/object-expirer.conf-sample +++ b/etc/object-expirer.conf-sample @@ -5,6 +5,7 @@ # log_name = swift # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # You can enable default statsD logging here if you want: # log_statsd_host = localhost # log_statsd_port = 8125 diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index f7cc09617f..96ca259275 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -12,6 +12,7 @@ # log_name = swift # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # You can enable default statsD logging here and/or override it in sections # below: # log_statsd_host = localhost @@ -29,6 +30,7 @@ use = egg:swift#object # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_requests = True +# set log_address = /dev/log # node_timeout = 3 # conn_timeout = 0.5 # network_chunk_size = 65536 @@ -58,6 +60,7 @@ use = egg:swift#recon # log_name = object-replicator # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # vm_test_mode = no # daemonize = on # run_pause = 30 @@ -81,6 +84,7 @@ use = egg:swift#recon # log_name = object-updater # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # interval = 300 # concurrency = 1 # node_timeout = 10 @@ -94,6 +98,7 @@ use = egg:swift#recon # log_name = object-auditor # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # files_per_second = 20 # bytes_per_second = 10000000 # log_time = 3600 diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 13ee342f33..a8cf050a10 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -13,6 +13,7 @@ # log_name = swift # log_facility = LOG_LOCAL0 # log_level = INFO +# log_address = /dev/log # You can enable default statsD logging here and/or override it in sections # below: # log_statsd_host = localhost @@ -29,6 +30,7 @@ use = egg:swift#proxy # set log_name = proxy-server # set log_facility = LOG_LOCAL0 # set log_level = INFO +# set log_address = /dev/log # set access_log_name = proxy-server # set access_log_facility = LOG_LOCAL0 # set access_log_level = INFO @@ -78,6 +80,7 @@ use = egg:swift#tempauth # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False +# set log_address = /dev/log # The reseller prefix will verify a token begins with this prefix before even # attempting to validate it. Also, with authorization, only Swift storage # accounts with this prefix will be authorized by this middleware. Useful if @@ -122,6 +125,7 @@ use = egg:swift#healthcheck # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False +# set log_address = /dev/log [filter:cache] use = egg:swift#memcache @@ -130,6 +134,7 @@ use = egg:swift#memcache # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False +# set log_address = /dev/log # Default for memcache_servers is to try to read the property from # memcache.conf (see memcache.conf-sample) or lacking that file, it will # default to the value below. You can specify multiple servers separated with @@ -143,6 +148,7 @@ use = egg:swift#ratelimit # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False +# set log_address = /dev/log # clock_accuracy should represent how accurate the proxy servers' system clocks # are with each other. 1000 means that all the proxies' clock are accurate to # each other within 1 millisecond. No ratelimit should be higher than the @@ -175,6 +181,7 @@ use = egg:swift#domain_remap # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False +# set log_address = /dev/log # storage_domain = example.com # path_root = v1 # reseller_prefixes = AUTH @@ -186,6 +193,7 @@ use = egg:swift#catch_errors # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False +# set log_address = /dev/log [filter:cname_lookup] # Note: this middleware requires python-dnspython @@ -195,6 +203,7 @@ use = egg:swift#cname_lookup # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False +# set log_address = /dev/log # storage_domain = example.com # lookup_depth = 1 @@ -207,6 +216,7 @@ use = egg:swift#staticweb # set log_name = staticweb # set log_facility = LOG_LOCAL0 # set log_level = INFO +# set log_address = /dev/log # set access_log_name = staticweb # set access_log_facility = LOG_LOCAL0 # set access_log_level = INFO diff --git a/swift/common/utils.py b/swift/common/utils.py index 5c2382e492..a353a5c122 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -39,6 +39,7 @@ except ImportError: import cPickle as pickle import glob from urlparse import urlparse as stdlib_urlparse, ParseResult +import socket import eventlet from eventlet import GreenPool, sleep, Timeout @@ -78,6 +79,10 @@ if hash_conf.read('/etc/swift/swift.conf'): TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) +def noop_libc_function(*args): + return 0 + + def validate_configuration(): if HASH_PATH_SUFFIX == '': sys.exit("Error: [swift-hash]: swift_hash_path_suffix missing " @@ -96,9 +101,6 @@ def load_libc_function(func_name): except AttributeError: logging.warn(_("Unable to locate %s in libc. Leaving as a no-op."), func_name) - - def noop_libc_function(*args): - return 0 return noop_libc_function @@ -118,6 +120,23 @@ def get_param(req, name, default=None): return value +class FallocateWrapper(object): + + def __init__(self): + for func in ('fallocate', 'posix_fallocate'): + self.func_name = func + self.fallocate = load_libc_function(func) + if self.fallocate is not noop_libc_function: + break + + def __call__(self, fd, mode, offset, len): + args = { + 'fallocate': (fd, mode, offset, len), + 'posix_fallocate': (fd, offset, len) + } + return self.fallocate(*args[self.func_name]) + + def fallocate(fd, size): """ Pre-allocate disk space for a file file. @@ -127,7 +146,7 @@ def fallocate(fd, size): """ global _sys_fallocate if _sys_fallocate is None: - _sys_fallocate = load_libc_function('fallocate') + _sys_fallocate = FallocateWrapper() if size > 0: # 1 means "FALLOC_FL_KEEP_SIZE", which means it pre-allocates invisibly ret = _sys_fallocate(fd, 1, 0, ctypes.c_uint64(size)) @@ -193,7 +212,7 @@ def renamer(old, new): try: mkdirs(os.path.dirname(new)) os.rename(old, new) - except OSError: + except OSError, err: mkdirs(os.path.dirname(new)) os.rename(old, new) @@ -523,7 +542,13 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None, # facility for this logger will be set by last call wins facility = getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), SysLogHandler.LOG_LOCAL0) - handler = SysLogHandler(address='/dev/log', facility=facility) + log_address = conf.get('log_address', '/dev/log') + try: + handler = SysLogHandler(address=log_address, facility=facility) + except socket.error, e: + if e.errno != errno.ENOTSOCK: # Socket operation on non-socket + raise e + handler = SysLogHandler(facility=facility) handler.setFormatter(formatter) logger.addHandler(handler) get_logger.handler4logger[logger] = handler @@ -723,13 +748,18 @@ def lock_path(directory, timeout=10): the lock can be acquired, or the timeout time has expired (whichever occurs first). + For locking exclusively, file or directory has to be opened in Write mode. + Python doesn't allow directories to be opened in Write Mode. So we + workaround by locking a hidden file in the directory. + :param directory: directory to be locked :param timeout: timeout (in seconds) """ mkdirs(directory) - fd = os.open(directory, os.O_RDONLY) + lockpath = '%s/.lock' % directory + fd = os.open(lockpath, os.O_WRONLY | os.O_CREAT) try: - with LockTimeout(timeout, directory): + with LockTimeout(timeout, lockpath): while True: try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)