openstack/swift
Code Issues Proposed changes

updated daemonize process, added option for servers/daemons to log to console

Browse Source
This commit is contained in:
Clay Gerrard 2010-11-19 22:52:31 +00:00 committed by Tarmac
commit 111ebb5a09
26 changed files with 651 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/swift-account-auditor
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.account.auditor import AccountAuditor
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-account-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-auditor')
auditor = AccountAuditor(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(AccountAuditor, conf_file, **options)

13
bin/swift-account-reaper
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.account.reaper import AccountReaper
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: account-reaper CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-reaper')
reaper = AccountReaper(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(AccountReaper, conf_file, **options)

13
bin/swift-account-replicator
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common import utils
from swift.account.replicator import AccountReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-account-replicator CONFIG_FILE [once]"
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-replicator')
AccountReplicator(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(AccountReplicator, conf_file, **options)

9
bin/swift-account-server
View File

@ -14,12 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'account-server', default_port=6002)
conf_file, options = parse_options()
run_wsgi(conf_file, 'account-server', default_port=6002, **options)

15
bin/swift-account-stats-logger
View File

@ -14,14 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.stats.account_stats import AccountStat
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-account-stats-logger CONFIG_FILE"
sys.exit()
stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats')
stats = AccountStat(stats_conf).run(once=True)
conf_file, options = parse_options()
# currently AccountStat only supports run_once
options['once'] = True
run_daemon(AccountStat, conf_file, section_name='log-processor-stats',
**options)

8
bin/swift-auth-server
View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'auth-server', default_port=11000)
conf_file, options = parse_options()
run_wsgi(conf_file, 'auth-server', default_port=11000, **options)

13
bin/swift-container-auditor
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.container.auditor import ContainerAuditor
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-container-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-auditor')
ContainerAuditor(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ContainerAuditor, conf_file, **options)

14
bin/swift-container-replicator
View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common import db, utils
from swift.container.replicator import ContainerReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-container-replicator CONFIG_FILE [once]"
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-replicator')
ContainerReplicator(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ContainerReplicator, conf_file, **options)

8
bin/swift-container-server
View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'container-server', default_port=6001)
conf_file, options = parse_options()
run_wsgi(conf_file, 'container-server', default_port=6001, **options)

13
bin/swift-container-updater
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.container.updater import ContainerUpdater
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-container-updater CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-updater')
ContainerUpdater(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ContainerUpdater, conf_file, **options)

15
bin/swift-log-stats-collector
View File

@ -14,14 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.stats.log_processor import LogProcessorDaemon
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-log-stats-collector CONFIG_FILE"
sys.exit()
conf = utils.readconf(sys.argv[1], log_name='log-stats-collector')
stats = LogProcessorDaemon(conf).run(once=True)
conf_file, options = parse_options()
# currently the LogProcessorDaemon only supports run_once
options['once'] = True
run_daemon(LogProcessorDaemon, conf_file, section_name=None,
log_name='log-stats-collector', **options)

24
bin/swift-log-uploader
View File

@ -17,15 +17,25 @@
import sys
from swift.stats.log_uploader import LogUploader
from swift.common.utils import parse_options
from swift.common import utils
if __name__ == '__main__':
if len(sys.argv) < 3:
print "Usage: swift-log-uploader CONFIG_FILE plugin"
sys.exit()
uploader_conf = utils.readconf(sys.argv[1], 'log-processor')
plugin = sys.argv[2]
conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN")
try:
plugin = options['extra_args'][0]
except IndexError:
print "Error: missing plugin name"
sys.exit(1)
uploader_conf = utils.readconf(conf_file, 'log-processor')
section_name = 'log-processor-%s' % plugin
plugin_conf = utils.readconf(sys.argv[1], section_name)
plugin_conf = utils.readconf(conf_file, section_name)
uploader_conf.update(plugin_conf)
uploader = LogUploader(uploader_conf, plugin).run(once=True)
# pre-configure logger
logger = utils.get_logger(uploader_conf, plugin,
log_to_console=options.get('verbose', False))
# currently LogUploader only supports run_once
options['once'] = True
uploader = LogUploader(uploader_conf, plugin).run(**options)

14
bin/swift-object-auditor
View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.obj.auditor import ObjectAuditor
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-object-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'object-auditor')
ObjectAuditor(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ObjectAuditor, conf_file, **options)

14
bin/swift-object-replicator
View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.obj.replicator import ObjectReplicator
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-object-replicator CONFIG_FILE [once]"
sys.exit()
conf = utils.readconf(sys.argv[1], "object-replicator")
once = (len(sys.argv) > 2 and sys.argv[2] == 'once') or \
conf.get('daemonize', 'true') not in utils.TRUE_VALUES
ObjectReplicator(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ObjectReplicator, conf_file, **options)

8
bin/swift-object-server
View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'object-server', default_port=6000)
conf_file, options = parse_options()
run_wsgi(conf_file, 'object-server', default_port=6000, **options)

13
bin/swift-object-updater
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.obj.updater import ObjectUpdater
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-object-updater CONFIG_FILE [once]"
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'object-updater')
ObjectUpdater(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ObjectUpdater, conf_file, **options)

8
bin/swift-proxy-server
View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'proxy-server', default_port=8080)
conf_file, options = parse_options()
run_wsgi(conf_file, 'proxy-server', default_port=8080, **options)

56
swift/common/daemon.py
View File

@ -16,6 +16,7 @@
import os
import sys
import signal
from re import sub
from swift.common import utils
@ -34,23 +35,11 @@ class Daemon(object):
"""Override this to run forever"""
raise NotImplementedError('run_forever not implemented')
def run(self, once=False, capture_stdout=True, capture_stderr=True):
def run(self, once=False, **kwargs):
"""Run the daemon"""
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
if capture_stdout:
sys.stdout = utils.LoggerFileObject(self.logger)
if capture_stderr:
sys.stderr = utils.LoggerFileObject(self.logger)
utils.drop_privileges(self.conf.get('user', 'swift'))
utils.validate_configuration()
try:
os.setsid()
except OSError:
pass
utils.capture_stdio(self.logger, **kwargs)
utils.drop_privileges(self.conf.get('user', 'swift'))
def kill_children(*args):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
@ -63,3 +52,40 @@ class Daemon(object):
self.run_once()
else:
self.run_forever()
def run_daemon(klass, conf_file, section_name='',
once=False, **kwargs):
"""
Loads settings from conf, then instantiates daemon "klass" and runs the
daemon with the specified once kwarg. The section_name will be derived
from the daemon "klass" if not provided (e.g. ObjectReplicator =>
object-replicator).
:param klass: Class to instantiate, subclass of common.daemon.Daemon
:param conf_file: Path to configuration file
:param section_name: Section name from conf file to load config from
:param once: Passed to daemon run method
"""
# very often the config section_name is based on the class name
# the None singleton will be passed through to readconf as is
if section_name is '':
section_name = sub(r'([a-z])([A-Z])', r'\1-\2',
klass.__name__).lower()
conf = utils.readconf(conf_file, section_name,
log_name=kwargs.get('log_name'))
# once on command line (i.e. daemonize=false) will over-ride config
once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES
# pre-configure logger
if 'logger' in kwargs:
logger = kwargs.pop('logger')
else:
logger = utils.get_logger(conf, conf.get('log_name', section_name),
log_to_console=kwargs.pop('verbose', False))
try:
klass(conf).run(once=once, **kwargs)
except KeyboardInterrupt:
logger.info('User quit')
logger.info('Exited')

4
swift/common/db_replicator.py
View File

@ -93,10 +93,6 @@ class Replicator(Daemon):
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = LoggerFileObject(self.logger)
self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')

114
swift/common/utils.py
View File

@ -31,6 +31,7 @@ import ctypes
import ctypes.util
import struct
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from optparse import OptionParser
from tempfile import mkstemp
import cPickle as pickle
@ -283,17 +284,6 @@ class LoggerFileObject(object):
return self
def drop_privileges(user):
"""
Sets the userid of the current process
:param user: User id to change privileges to
"""
user = pwd.getpwnam(user)
os.setgid(user[3])
os.setuid(user[2])
class NamedLogger(object):
"""Cheesy version of the LoggerAdapter available in Python 3"""
@ -343,7 +333,7 @@ class NamedLogger(object):
call('%s %s: %s' % (self.server, msg, emsg), *args)
def get_logger(conf, name=None):
def get_logger(conf, name=None, log_to_console=False):
"""
Get the current system logger using config settings.
@ -355,11 +345,18 @@ def get_logger(conf, name=None):
:param conf: Configuration dict to read settings from
:param name: Name of the logger
:param log_to_console: Add handler which writes to console on stderr
"""
root_logger = logging.getLogger()
if hasattr(get_logger, 'handler') and get_logger.handler:
root_logger.removeHandler(get_logger.handler)
get_logger.handler = None
if log_to_console:
# check if a previous call to get_logger already added a console logger
if hasattr(get_logger, 'console') and get_logger.console:
root_logger.removeHandler(get_logger.console)
get_logger.console = logging.StreamHandler(sys.__stderr__)
root_logger.addHandler(get_logger.console)
if conf is None:
root_logger.setLevel(logging.INFO)
return NamedLogger(root_logger, name)
@ -375,6 +372,99 @@ def get_logger(conf, name=None):
return NamedLogger(root_logger, name)
def drop_privileges(user):
"""
Sets the userid/groupid of the current process, get session leader, etc.
:param user: User name to change privileges to
"""
user = pwd.getpwnam(user)
os.setgid(user[3])
os.setuid(user[2])
try:
os.setsid()
except OSError:
pass
os.chdir('/') # in case you need to rmdir on where you started the daemon
os.umask(0) # ensure files are created with the correct privileges
def capture_stdio(logger, **kwargs):
"""
Log unhandled exceptions, close stdio, capture stdout and stderr.
param logger: Logger object to use
"""
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
# collect stdio file desc not in use for logging
stdio_fds = [0, 1, 2]
if hasattr(get_logger, 'console'):
stdio_fds.remove(get_logger.console.stream.fileno())
with open(os.devnull, 'r+b') as nullfile:
# close stdio (excludes fds open for logging)
for desc in stdio_fds:
try:
os.dup2(nullfile.fileno(), desc)
except OSError:
pass
# redirect stdio
if kwargs.pop('capture_stdout', True):
sys.stdout = LoggerFileObject(logger)
if kwargs.pop('capture_stderr', True):
sys.stderr = LoggerFileObject(logger)
def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None):
"""
Parse standard swift server/daemon options with optparse.OptionParser.
:param usage: String describing usage
:param once: Boolean indicating the "once" option is available
:param test_args: Override sys.argv; used in testing
:returns : Tuple of (config, options); config is an absolute path to the
config file, options is the parser options as a dictionary.
:raises SystemExit: First arg (CONFIG) is required, file must exist
"""
parser = OptionParser(usage)
parser.add_option("-v", "--verbose", default=False, action="store_true",
help="log to console")
if once:
parser.add_option("-o", "--once", default=False, action="store_true",
help="only run one pass of daemon")
# if test_args is None, optparse will use sys.argv[:1]
options, args = parser.parse_args(args=test_args)
if not args:
parser.print_usage()
print "Error: missing config file argument"
sys.exit(1)
config = os.path.abspath(args.pop(0))
if not os.path.exists(config):
parser.print_usage()
print "Error: unable to locate %s" % config
sys.exit(1)
extra_args = []
# if any named options appear in remaining args, set the option to True
for arg in args:
if arg in options.__dict__:
setattr(options, arg, True)
else:
extra_args.append(arg)
options = vars(options)
options['extra_args'] = extra_args
return config, options
def whataremyips():
"""
Get the machine's ip addresses using ifconfig

84
swift/common/wsgi.py
View File

@ -34,7 +34,7 @@ wsgi.ACCEPT_ERRNO.add(ECONNRESET)
from eventlet.green import socket, ssl
from swift.common.utils import get_logger, drop_privileges, \
validate_configuration, LoggerFileObject, NullLogger
validate_configuration, capture_stdio, NullLogger
def monkey_patch_mimetools():
@ -56,41 +56,17 @@ def monkey_patch_mimetools():
mimetools.Message.parsetype = parsetype
def get_socket(conf, default_port=8080):
"""Bind socket to bind ip:port in conf
# We might be able to pull pieces of this out to test, but right now it seems
# like more work than it's worth.
def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
:param conf: Configuration dict to read settings from
:param default_port: port to use if not specified in conf
:returns : a socket object as returned from socket.listen or ssl.wrap_socket
if conf specifies cert_file
"""
Loads common settings from conf, then instantiates app and runs
the server using the specified number of workers.
:param conf_file: Path to paste.deploy style configuration file
:param app_section: App name from conf file to load config from
"""
try:
conf = appconfig('config:%s' % conf_file, name=app_section)
log_name = conf.get('log_name', app_section)
app = loadapp('config:%s' % conf_file,
global_conf={'log_name': log_name})
except Exception, e:
print "Error trying to load config %s: %s" % (conf_file, e)
return
if 'logger' in kwargs:
logger = kwargs['logger']
else:
logger = get_logger(conf, log_name)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = LoggerFileObject(logger)
try:
os.setsid()
except OSError:
no_cover = True # pass
bind_addr = (conf.get('bind_ip', '0.0.0.0'),
int(conf.get('bind_port', kwargs.get('default_port', 8080))))
int(conf.get('bind_port', default_port)))
sock = None
retry_until = time.time() + 30
while not sock and time.time() < retry_until:
@ -110,10 +86,44 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
# in my experience, sockets can hang around forever without keepalive
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600)
worker_count = int(conf.get('workers', '1'))
drop_privileges(conf.get('user', 'swift'))
return sock
# TODO: pull pieces of this out to test
def run_wsgi(conf_file, app_section, *args, **kwargs):
"""
Loads common settings from conf, then instantiates app and runs
the server using the specified number of workers.
:param conf_file: Path to paste.deploy style configuration file
:param app_section: App name from conf file to load config from
"""
try:
conf = appconfig('config:%s' % conf_file, name=app_section)
except Exception, e:
print "Error trying to load config %s: %s" % (conf_file, e)
return
validate_configuration()
# pre-configure logger
log_name = conf.get('log_name', app_section)
if 'logger' in kwargs:
logger = kwargs.pop('logger')
else:
logger = get_logger(conf, log_name,
log_to_console=kwargs.pop('verbose', False))
# redirect errors to logger and close stdio
capture_stdio(logger)
# bind to address and port
sock = get_socket(conf, default_port=kwargs.get('default_port', 8080))
# remaining tasks should not require elevated privileges
drop_privileges(conf.get('user', 'swift'))
# finally after binding to ports and privilege drop, run app __init__ code
app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})
def run_server():
wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
eventlet.hubs.use_hub('poll')
@ -127,6 +137,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
raise
pool.waitall()
worker_count = int(conf.get('workers', '1'))
# Useful for profiling [no forks].
if worker_count == 0:
run_server()
@ -169,6 +180,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
except OSError, err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
except KeyboardInterrupt:
logger.info('User quit')
break
greenio.shutdown_safe(sock)
sock.close()
logger.info('Exited')

15
test/unit/__init__.py
View File

@ -1,5 +1,8 @@
""" Swift tests """
import os
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from eventlet.green import socket
@ -23,6 +26,18 @@ def connect_tcp(hostport):
rv.connect(hostport)
return rv
@contextmanager
def tmpfile(content):
with NamedTemporaryFile('w', delete=False) as f:
file_name = f.name
f.write(str(content))
try:
yield file_name
finally:
os.unlink(file_name)
class MockTrue(object):
"""
Instances of MockTrue evaluate like True

87
test/unit/common/test_daemon.py
View File

@ -13,16 +13,95 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO: Tests
# TODO: Test kill_children signal handlers
import unittest
from swift.common import daemon
from getpass import getuser
import logging
from StringIO import StringIO
from test.unit import tmpfile
from swift.common import daemon, utils
class MyDaemon(daemon.Daemon):
def __init__(self, conf):
self.conf = conf
self.logger = utils.get_logger(None)
MyDaemon.forever_called = False
MyDaemon.once_called = False
def run_forever(self):
MyDaemon.forever_called = True
def run_once(self):
MyDaemon.once_called = True
def run_raise(self):
raise OSError
def run_quit(self):
raise KeyboardInterrupt
class TestDaemon(unittest.TestCase):
def test_placeholder(self):
pass
def test_create(self):
d = daemon.Daemon({})
self.assertEquals(d.conf, {})
self.assert_(isinstance(d.logger, utils.NamedLogger))
def test_stubs(self):
d = daemon.Daemon({})
self.assertRaises(NotImplementedError, d.run_once)
self.assertRaises(NotImplementedError, d.run_forever)
class TestRunDaemon(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.drop_privileges = lambda *args: None
utils.capture_stdio = lambda *args: None
def tearDown(self):
reload(utils)
def test_run(self):
d = MyDaemon({})
self.assertFalse(MyDaemon.forever_called)
self.assertFalse(MyDaemon.once_called)
# test default
d.run()
self.assertEquals(d.forever_called, True)
# test once
d.run(once=True)
self.assertEquals(d.once_called, True)
def test_run_daemon(self):
sample_conf = """[my-daemon]
user = %s
""" % getuser()
with tmpfile(sample_conf) as conf_file:
daemon.run_daemon(MyDaemon, conf_file)
self.assertEquals(MyDaemon.forever_called, True)
daemon.run_daemon(MyDaemon, conf_file, once=True)
self.assertEquals(MyDaemon.once_called, True)
# test raise in daemon code
MyDaemon.run_once = MyDaemon.run_raise
self.assertRaises(OSError, daemon.run_daemon, MyDaemon,
conf_file, once=True)
# test user quit
MyDaemon.run_forever = MyDaemon.run_quit
sio = StringIO()
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(sio))
logger = utils.get_logger(None, 'server')
daemon.run_daemon(MyDaemon, conf_file, logger=logger)
self.assert_('user quit' in sio.getvalue().lower())
if __name__ == '__main__':

179
test/unit/common/test_utils.py
View File

@ -25,12 +25,55 @@ import unittest
from getpass import getuser
from shutil import rmtree
from StringIO import StringIO
from functools import partial
from tempfile import NamedTemporaryFile
from eventlet import sleep
from swift.common import utils
class MockOs():
def __init__(self, pass_funcs=[], called_funcs=[], raise_funcs=[]):
self.closed_fds = []
for func in pass_funcs:
setattr(self, func, self.pass_func)
self.called_funcs = {}
for func in called_funcs:
c_func = partial(self.called_func, func)
setattr(self, func, c_func)
for func in raise_funcs:
r_func = partial(self.raise_func, func)
setattr(self, func, r_func)
def pass_func(self, *args, **kwargs):
pass
chdir = setsid = setgid = setuid = umask = pass_func
def called_func(self, name, *args, **kwargs):
self.called_funcs[name] = True
def raise_func(self, name, *args, **kwargs):
self.called_funcs[name] = True
raise OSError()
def dup2(self, source, target):
self.closed_fds.append(target)
def __getattr__(self, name):
# I only over-ride portions of the os module
try:
return object.__getattr__(self, name)
except AttributeError:
return getattr(os, name)
class MockSys():
__stderr__ = sys.__stderr__
class TestUtils(unittest.TestCase):
""" Tests for swift.common.utils """
@ -182,10 +225,63 @@ class TestUtils(unittest.TestCase):
self.assertRaises(IOError, lfo.readline, 1024)
lfo.tell()
def test_drop_privileges(self):
# Note that this doesn't really drop privileges as it just sets them to
# what they already are; but it exercises the code at least.
utils.drop_privileges(getuser())
def test_parse_options(self):
# use mkstemp to get a file that is definately on disk
with NamedTemporaryFile() as f:
conf_file = f.name
conf, options = utils.parse_options(test_args=[conf_file])
self.assertEquals(conf, conf_file)
# assert defaults
self.assertEquals(options['verbose'], False)
self.assert_('once' not in options)
# assert verbose as option
conf, options = utils.parse_options(test_args=[conf_file, '-v'])
self.assertEquals(options['verbose'], True)
# check once option
conf, options = utils.parse_options(test_args=[conf_file],
once=True)
self.assertEquals(options['once'], False)
test_args = [conf_file, '--once']
conf, options = utils.parse_options(test_args=test_args, once=True)
self.assertEquals(options['once'], True)
# check options as arg parsing
test_args = [conf_file, 'once', 'plugin_name', 'verbose']
conf, options = utils.parse_options(test_args=test_args, once=True)
self.assertEquals(options['verbose'], True)
self.assertEquals(options['once'], True)
self.assertEquals(options['extra_args'], ['plugin_name'])
def test_parse_options_errors(self):
orig_stdout = sys.stdout
orig_stderr = sys.stderr
stdo = StringIO()
stde = StringIO()
utils.sys.stdout = stdo
utils.sys.stderr = stde
err_msg = """Usage: test usage
Error: missing config file argument
"""
test_args = []
self.assertRaises(SystemExit, utils.parse_options, 'test usage', True,
test_args)
self.assertEquals(stdo.getvalue(), err_msg)
# verify conf file must exist, context manager will delete temp file
with NamedTemporaryFile() as f:
conf_file = f.name
err_msg += """Usage: test usage
Error: unable to locate %s
""" % conf_file
test_args = [conf_file]
self.assertRaises(SystemExit, utils.parse_options, 'test usage', True,
test_args)
self.assertEquals(stdo.getvalue(), err_msg)
# reset stdio
utils.sys.stdout = orig_stdout
utils.sys.stderr = orig_stderr
def test_NamedLogger(self):
sio = StringIO()
@ -275,5 +371,80 @@ log_name = yarr'''
self.assertEquals(result, expected)
os.unlink('/tmp/test')
def test_drop_privileges(self):
user = getuser()
# over-ride os with mock
required_func_calls = ('setgid', 'setuid', 'setsid', 'chdir', 'umask')
utils.os = MockOs(called_funcs=required_func_calls)
# exercise the code
utils.drop_privileges(user)
for func in required_func_calls:
self.assert_(utils.os.called_funcs[func])
# reset; test same args, OSError trying to get session leader
utils.os = MockOs(called_funcs=required_func_calls,
raise_funcs=('setsid',))
for func in required_func_calls:
self.assertFalse(utils.os.called_funcs.get(func, False))
utils.drop_privileges(user)
for func in required_func_calls:
self.assert_(utils.os.called_funcs[func])
def test_capture_stdio(self):
# stubs
logger = utils.get_logger(None, 'dummy')
# mock utils system modules
utils.sys = MockSys()
utils.os = MockOs()
# basic test
utils.capture_stdio(logger)
self.assert_(utils.sys.excepthook is not None)
self.assertEquals(utils.os.closed_fds, [0, 1, 2])
self.assert_(utils.sys.stdout is not None)
self.assert_(utils.sys.stderr is not None)
# reset; test same args, but exc when trying to close stdio
utils.os = MockOs(raise_funcs=('dup2',))
utils.sys = MockSys()
# test unable to close stdio
utils.capture_stdio(logger)
self.assert_(utils.sys.excepthook is not None)
self.assertEquals(utils.os.closed_fds, [])
self.assert_(utils.sys.stdout is not None)
self.assert_(utils.sys.stderr is not None)
# reset; test some other args
logger = utils.get_logger(None, log_to_console=True)
utils.os = MockOs()
utils.sys = MockSys()
# test console log
utils.capture_stdio(logger, capture_stdout=False,
capture_stderr=False)
self.assert_(utils.sys.excepthook is not None)
# when logging to console, stderr remains open
self.assertEquals(utils.os.closed_fds, [0, 1])
logger.logger.removeHandler(utils.get_logger.console)
# stdio not captured
self.assertFalse(hasattr(utils.sys, 'stdout'))
self.assertFalse(hasattr(utils.sys, 'stderr'))
def test_get_logger_console(self):
reload(utils) # reset get_logger attrs
logger = utils.get_logger(None)
self.assertFalse(hasattr(utils.get_logger, 'console'))
logger = utils.get_logger(None, log_to_console=True)
self.assert_(hasattr(utils.get_logger, 'console'))
self.assert_(isinstance(utils.get_logger.console,
logging.StreamHandler))
# make sure you can't have two console handlers
old_handler = utils.get_logger.console
logger = utils.get_logger(None, log_to_console=True)
self.assertNotEquals(utils.get_logger.console, old_handler)
logger.logger.removeHandler(utils.get_logger.console)
if __name__ == '__main__':
unittest.main()

104
test/unit/common/test_wsgi.py
View File

@ -25,12 +25,12 @@ import unittest
from getpass import getuser
from shutil import rmtree
from StringIO import StringIO
from collections import defaultdict
from eventlet import sleep
from swift.common import wsgi
class TestWSGI(unittest.TestCase):
""" Tests for swift.common.wsgi """
@ -72,5 +72,107 @@ class TestWSGI(unittest.TestCase):
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEquals(mimetools.Message(sio).subtype, 'html')
def test_get_socket(self):
# stubs
conf = {}
ssl_conf = {
'cert_file': '',
'key_file': '',
}
# mocks
class MockSocket():
def __init__(self):
self.opts = defaultdict(dict)
def setsockopt(self, level, optname, value):
self.opts[level][optname] = value
def mock_listen(*args, **kwargs):
return MockSocket()
class MockSsl():
def __init__(self):
self.wrap_socket_called = []
def wrap_socket(self, sock, **kwargs):
self.wrap_socket_called.append(kwargs)
return sock
# patch
old_listen = wsgi.listen
old_ssl = wsgi.ssl
try:
wsgi.listen = mock_listen
wsgi.ssl = MockSsl()
# test
sock = wsgi.get_socket(conf)
# assert
self.assert_(isinstance(sock, MockSocket))
expected_socket_opts = {
socket.SOL_SOCKET: {
socket.SO_REUSEADDR: 1,
socket.SO_KEEPALIVE: 1,
},
socket.IPPROTO_TCP: {
socket.TCP_KEEPIDLE: 600,
},
}
self.assertEquals(sock.opts, expected_socket_opts)
# test ssl
sock = wsgi.get_socket(ssl_conf)
expected_kwargs = {
'certfile': '',
'keyfile': '',
}
self.assertEquals(wsgi.ssl.wrap_socket_called, [expected_kwargs])
finally:
wsgi.listen = old_listen
wsgi.ssl = old_ssl
def test_address_in_use(self):
# stubs
conf = {}
# mocks
def mock_listen(*args, **kwargs):
raise socket.error(errno.EADDRINUSE)
def value_error_listen(*args, **kwargs):
raise ValueError('fake')
def mock_sleep(*args):
pass
class MockTime():
"""Fast clock advances 10 seconds after every call to time
"""
def __init__(self):
self.current_time = old_time.time()
def time(self, *args, **kwargs):
rv = self.current_time
# advance for next call
self.current_time += 10
return rv
old_listen = wsgi.listen
old_sleep = wsgi.sleep
old_time = wsgi.time
try:
wsgi.listen = mock_listen
wsgi.sleep = mock_sleep
wsgi.time = MockTime()
# test error
self.assertRaises(Exception, wsgi.get_socket, conf)
# different error
wsgi.listen = value_error_listen
self.assertRaises(ValueError, wsgi.get_socket, conf)
finally:
wsgi.listen = old_listen
wsgi.sleep = old_sleep
wsgi.time = old_time
if __name__ == '__main__':
unittest.main()

15
test/unit/stats/test_log_processor.py
View File

@ -14,25 +14,12 @@
# limitations under the License.
import unittest
import os
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from test.unit import tmpfile
from swift.common import internal_proxy
from swift.stats import log_processor
@contextmanager
def tmpfile(content):
with NamedTemporaryFile('w', delete=False) as f:
file_name = f.name
f.write(str(content))
try:
yield file_name
finally:
os.unlink(file_name)
class FakeUploadApp(object):
def __init__(self, *args, **kwargs):
pass