From 57a35f0d7c85bae8bf78fbc915e5cf73bdc8e4ff Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 11 Nov 2010 16:41:07 -0600 Subject: [PATCH] added helper/util to parse command line args; removed some duplicated code in server/daemon bin scripts; more standized python/linux daemonization procedures; fixed lp:666957 "devauth server creates auth.db with the wrong privileges"; new run_daemon helper based on run_wsgi simplifies daemon launching/testing; new - all servers/daemons support verbose option when started interactivlty which will log to the console; fixed lp:667839 "can't start servers with relative paths to configs"; added tests --- bin/swift-account-auditor | 13 +- bin/swift-account-reaper | 13 +- bin/swift-account-replicator | 13 +- bin/swift-account-server | 9 +- bin/swift-account-stats-logger | 15 +-- bin/swift-auth-server | 8 +- bin/swift-container-auditor | 13 +- bin/swift-container-replicator | 14 +- bin/swift-container-server | 8 +- bin/swift-container-updater | 13 +- bin/swift-log-stats-collector | 15 +-- bin/swift-log-uploader | 22 ++- bin/swift-object-auditor | 14 +- bin/swift-object-replicator | 14 +- bin/swift-object-server | 8 +- bin/swift-object-updater | 13 +- bin/swift-proxy-server | 8 +- swift/common/daemon.py | 55 +++++--- swift/common/utils.py | 126 ++++++++++++++++-- swift/common/wsgi.py | 41 +++--- test/unit/__init__.py | 15 +++ test/unit/common/test_daemon.py | 86 +++++++++++- test/unit/common/test_utils.py | 185 +++++++++++++++++++++++++- test/unit/stats/test_log_processor.py | 15 +-- 24 files changed, 535 insertions(+), 201 deletions(-) diff --git a/bin/swift-account-auditor b/bin/swift-account-auditor index a715979599..5707dfd515 100755 --- a/bin/swift-account-auditor +++ b/bin/swift-account-auditor @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.account.auditor import AccountAuditor -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-account-auditor CONFIG_FILE [once]" - sys.exit() - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'account-auditor') - auditor = AccountAuditor(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(AccountAuditor, conf_file, **options) diff --git a/bin/swift-account-reaper b/bin/swift-account-reaper index 90496c64e8..688b19b14d 100755 --- a/bin/swift-account-reaper +++ b/bin/swift-account-reaper @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.account.reaper import AccountReaper -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: account-reaper CONFIG_FILE [once]" - sys.exit() - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'account-reaper') - reaper = AccountReaper(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(AccountReaper, conf_file, **options) diff --git a/bin/swift-account-replicator b/bin/swift-account-replicator index c71c326b8b..8edc7cf406 100755 --- a/bin/swift-account-replicator +++ b/bin/swift-account-replicator @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -from swift.common import utils from swift.account.replicator import AccountReplicator +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-account-replicator CONFIG_FILE [once]" - sys.exit(1) - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'account-replicator') - AccountReplicator(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(AccountReplicator, conf_file, **options) diff --git a/bin/swift-account-server b/bin/swift-account-server index 55e2a10857..8c627afa59 100755 --- a/bin/swift-account-server +++ b/bin/swift-account-server @@ -14,12 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'account-server', default_port=6002) - + conf_file, options = parse_options() + run_wsgi(conf_file, 'account-server', default_port=6002, **options) diff --git a/bin/swift-account-stats-logger b/bin/swift-account-stats-logger index c42554de82..7e9d26ba50 100755 --- a/bin/swift-account-stats-logger +++ b/bin/swift-account-stats-logger @@ -14,14 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.stats.account_stats import AccountStat -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-account-stats-logger CONFIG_FILE" - sys.exit() - stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats') - stats = AccountStat(stats_conf).run(once=True) + conf_file, options = parse_options() + # currently AccountStat only supports run_once + options['once'] = True + run_daemon(AccountStat, conf_file, section_name='log-processor-stats', + **options) diff --git a/bin/swift-auth-server b/bin/swift-auth-server index a8f03c6e92..80c652f5b4 100755 --- a/bin/swift-auth-server +++ b/bin/swift-auth-server @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'auth-server', default_port=11000) + conf_file, options = parse_options() + run_wsgi(conf_file, 'auth-server', default_port=11000, **options) diff --git a/bin/swift-container-auditor b/bin/swift-container-auditor index b3472c54af..62ff797535 100755 --- a/bin/swift-container-auditor +++ b/bin/swift-container-auditor @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.container.auditor import ContainerAuditor -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-container-auditor CONFIG_FILE [once]" - sys.exit() - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'container-auditor') - ContainerAuditor(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ContainerAuditor, conf_file, **options) diff --git a/bin/swift-container-replicator b/bin/swift-container-replicator index a0594142df..34e1dae8c7 100755 --- a/bin/swift-container-replicator +++ b/bin/swift-container-replicator @@ -14,16 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -from swift.common import db, utils from swift.container.replicator import ContainerReplicator +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-container-replicator CONFIG_FILE [once]" - sys.exit(1) - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'container-replicator') - ContainerReplicator(conf).run(once) - + conf_file, options = parse_options(once=True) + run_daemon(ContainerReplicator, conf_file, **options) diff --git a/bin/swift-container-server b/bin/swift-container-server index a15a35cf04..c6d4cf154b 100755 --- a/bin/swift-container-server +++ b/bin/swift-container-server @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'container-server', default_port=6001) + conf_file, options = parse_options() + run_wsgi(conf_file, 'container-server', default_port=6001, **options) diff --git a/bin/swift-container-updater b/bin/swift-container-updater index ed22d29901..28bdaf1eae 100755 --- a/bin/swift-container-updater +++ b/bin/swift-container-updater @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.container.updater import ContainerUpdater -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-container-updater CONFIG_FILE [once]" - sys.exit() - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'container-updater') - ContainerUpdater(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ContainerUpdater, conf_file, **options) diff --git a/bin/swift-log-stats-collector b/bin/swift-log-stats-collector index d21135b35c..950a0f5fe0 100755 --- a/bin/swift-log-stats-collector +++ b/bin/swift-log-stats-collector @@ -14,14 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.stats.log_processor import LogProcessorDaemon -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-log-stats-collector CONFIG_FILE" - sys.exit() - conf = utils.readconf(sys.argv[1], log_name='log-stats-collector') - stats = LogProcessorDaemon(conf).run(once=True) + conf_file, options = parse_options() + # currently the LogProcessorDaemon only supports run_once + options['once'] = True + run_daemon(LogProcessorDaemon, conf_file, section_name=None, + log_name='log-stats-collector', **options) diff --git a/bin/swift-log-uploader b/bin/swift-log-uploader index b557e4c167..617380ca9f 100755 --- a/bin/swift-log-uploader +++ b/bin/swift-log-uploader @@ -17,15 +17,25 @@ import sys from swift.stats.log_uploader import LogUploader +from swift.common.utils import parse_options from swift.common import utils if __name__ == '__main__': - if len(sys.argv) < 3: - print "Usage: swift-log-uploader CONFIG_FILE plugin" - sys.exit() - uploader_conf = utils.readconf(sys.argv[1], 'log-processor') - plugin = sys.argv[2] + conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN") + try: + plugin = options['extra_args'][0] + except IndexError: + print "Error: missing plugin name" + sys.exit(1) + + uploader_conf = utils.readconf(conf_file, 'log-processor') section_name = 'log-processor-%s' % plugin - plugin_conf = utils.readconf(sys.argv[1], section_name) + plugin_conf = utils.readconf(conf_file, section_name) uploader_conf.update(plugin_conf) + + # pre-configure logger + logger = utils.get_logger(uploader_conf, plugin, + log_to_console=options.get('verbose', False)) + # currently LogUploader only supports run_once + options['once'] = True uploader = LogUploader(uploader_conf, plugin).run(once=True) diff --git a/bin/swift-object-auditor b/bin/swift-object-auditor index d60bcb6148..033249ecca 100755 --- a/bin/swift-object-auditor +++ b/bin/swift-object-auditor @@ -14,16 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.obj.auditor import ObjectAuditor -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-object-auditor CONFIG_FILE [once]" - sys.exit() - - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'object-auditor') - ObjectAuditor(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ObjectAuditor, conf_file, **options) diff --git a/bin/swift-object-replicator b/bin/swift-object-replicator index cc8c32fdde..53a48e3942 100755 --- a/bin/swift-object-replicator +++ b/bin/swift-object-replicator @@ -14,16 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.obj.replicator import ObjectReplicator -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-object-replicator CONFIG_FILE [once]" - sys.exit() - conf = utils.readconf(sys.argv[1], "object-replicator") - once = (len(sys.argv) > 2 and sys.argv[2] == 'once') or \ - conf.get('daemonize', 'true') not in utils.TRUE_VALUES - ObjectReplicator(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ObjectReplicator, conf_file, **options) diff --git a/bin/swift-object-server b/bin/swift-object-server index 39c9246d6b..5984fe6c69 100755 --- a/bin/swift-object-server +++ b/bin/swift-object-server @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'object-server', default_port=6000) + conf_file, options = parse_options() + run_wsgi(conf_file, 'object-server', default_port=6000, **options) diff --git a/bin/swift-object-updater b/bin/swift-object-updater index d24cdbcfe9..6d68c57c6e 100755 --- a/bin/swift-object-updater +++ b/bin/swift-object-updater @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.obj.updater import ObjectUpdater -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-object-updater CONFIG_FILE [once]" - sys.exit(1) - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'object-updater') - ObjectUpdater(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ObjectUpdater, conf_file, **options) diff --git a/bin/swift-proxy-server b/bin/swift-proxy-server index 7599f87280..baccf568e6 100755 --- a/bin/swift-proxy-server +++ b/bin/swift-proxy-server @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'proxy-server', default_port=8080) + conf_file, options = parse_options() + run_wsgi(conf_file, 'proxy-server', default_port=8080, **options) diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 26892824b8..faa4112a1d 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -16,6 +16,7 @@ import os import sys import signal +from re import sub from swift.common import utils @@ -34,23 +35,10 @@ class Daemon(object): """Override this to run forever""" raise NotImplementedError('run_forever not implemented') - def run(self, once=False, capture_stdout=True, capture_stderr=True): + def run(self, once=False, **kwargs): """Run the daemon""" - # log uncaught exceptions - sys.excepthook = lambda *exc_info: \ - self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) - if capture_stdout: - sys.stdout = utils.LoggerFileObject(self.logger) - if capture_stderr: - sys.stderr = utils.LoggerFileObject(self.logger) - - utils.drop_privileges(self.conf.get('user', 'swift')) utils.validate_configuration() - - try: - os.setsid() - except OSError: - pass + utils.daemonize(self.conf, self.logger, **kwargs) def kill_children(*args): signal.signal(signal.SIGTERM, signal.SIG_IGN) @@ -63,3 +51,40 @@ class Daemon(object): self.run_once() else: self.run_forever() + + +def run_daemon(klass, conf_file, section_name='', + once=False, **kwargs): + """ + Loads settings from conf, then instantiates daemon "klass" and runs the + daemon with the specified once kwarg. The section_name will be derived + from the daemon "klass" if not provided (e.g. ObjectReplicator => + object-replicator). + + :param klass: Class to instantiate, subclass of common.daemon.Daemon + :param conf_file: Path to configuration file + :param section_name: Section name from conf file to load config from + :param once: Passed to daemon run method + """ + # very often the config section_name is based on the class name + # the None singleton will be passed through to readconf as is + if section_name is '': + section_name = sub(r'([a-z])([A-Z])', r'\1-\2', + klass.__name__).lower() + conf = utils.readconf(conf_file, section_name, + log_name=kwargs.get('log_name')) + + # once on command line (i.e. daemonize=false) will over-ride config + once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES + + # pre-configure logger + if 'logger' in kwargs: + logger = kwargs.pop('logger') + else: + logger = utils.get_logger(conf, conf.get('log_name', section_name), + log_to_console=kwargs.pop('verbose', False)) + try: + klass(conf).run(once=once, **kwargs) + except KeyboardInterrupt: + logger.info('User quit') + logger.info('Exited') diff --git a/swift/common/utils.py b/swift/common/utils.py index bb635725c8..7a70acd64d 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -32,6 +32,7 @@ import ctypes.util import fcntl import struct from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from optparse import OptionParser from tempfile import mkstemp import cPickle as pickle @@ -284,17 +285,6 @@ class LoggerFileObject(object): return self -def drop_privileges(user): - """ - Sets the userid of the current process - - :param user: User id to change privileges to - """ - user = pwd.getpwnam(user) - os.setgid(user[3]) - os.setuid(user[2]) - - class NamedLogger(object): """Cheesy version of the LoggerAdapter available in Python 3""" @@ -344,7 +334,7 @@ class NamedLogger(object): call('%s %s: %s' % (self.server, msg, emsg), *args) -def get_logger(conf, name=None): +def get_logger(conf, name=None, log_to_console=False): """ Get the current system logger using config settings. @@ -356,11 +346,18 @@ def get_logger(conf, name=None): :param conf: Configuration dict to read settings from :param name: Name of the logger + :param log_to_console: Add handler which writes to console on stderr """ root_logger = logging.getLogger() if hasattr(get_logger, 'handler') and get_logger.handler: root_logger.removeHandler(get_logger.handler) get_logger.handler = None + if log_to_console: + # check if a previous call to get_logger already added a console logger + if hasattr(get_logger, 'console') and get_logger.console: + root_logger.removeHandler(get_logger.console) + get_logger.console = logging.StreamHandler(sys.__stderr__) + root_logger.addHandler(get_logger.console) if conf is None: root_logger.setLevel(logging.INFO) return NamedLogger(root_logger, name) @@ -376,6 +373,111 @@ def get_logger(conf, name=None): return NamedLogger(root_logger, name) +def drop_privileges(user): + """ + Sets the userid/groupid of the current process, get session leader, etc. + + :param user: User name to change privileges to + """ + user = pwd.getpwnam(user) + os.setgid(user[3]) + os.setuid(user[2]) + try: + os.setsid() + except OSError: + pass + os.chdir('/') # in case you need to rmdir on where you started the daemon + os.umask(0) # ensure files are created with the correct privledges + + +def capture_stdio(logger, **kwargs): + """ + Log unhaneled exceptions, close stdio, capture stdout and stderr. + + param logger: Logger object to use + """ + # log uncaught exceptions + sys.excepthook = lambda * exc_info: \ + logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + + # collect stdio file desc not in use for logging + stdio_fds = [0, 1, 2] + if hasattr(get_logger, 'console'): + stdio_fds.remove(get_logger.console.stream.fileno()) + + with open(os.devnull, 'r+b') as nullfile: + # close stdio (excludes fds open for logging) + for desc in stdio_fds: + try: + os.dup2(nullfile.fileno(), desc) + except OSError: + pass + + # redirect stdio + if kwargs.pop('capture_stdout', True): + sys.stdout = LoggerFileObject(logger) + if kwargs.pop('capture_stderr', True): + sys.stderr = LoggerFileObject(logger) + + +def daemonize(conf, logger, **kwargs): + """ + Perform standard python/linux daemonization operations. + + :param user: Configuration dict to read settings from (i.e. user) + :param logger: Logger object to handle stdio redirect and uncaught exc + """ + + drop_privileges(conf.get('user', 'swift')) + capture_stdio(logger, **kwargs) + + +def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None): + """ + Parse standard swift server/daemon options with optparse.OptionParser. + + :param usage: String describing usage + :param once: Boolean indicating the "once" option is avaiable + :param test_args: Override sys.argv; used in testing + + :returns : Tuple of (config, options); config is an absolute path to the + config file, options is the parser options as a dictionary. + + :raises SystemExit: First arg (CONFIG) is required, file must exist + """ + parser = OptionParser(usage) + parser.add_option("-v", "--verbose", default=False, action="store_true", + help="log to console") + if once: + parser.add_option("-o", "--once", default=False, action="store_true", + help="only run one pass of daemon") + + # if test_args is None, optparse will use sys.argv[:1] + options, args = parser.parse_args(args=test_args) + + if not args: + parser.print_usage() + print "Error: missing config file argument" + sys.exit(1) + config = os.path.abspath(args.pop(0)) + if not os.path.exists(config): + parser.print_usage() + print "Error: unable to locate %s" % config + sys.exit(1) + + extra_args = [] + # if any named options appear in remaining args, set the option to True + for arg in args: + if arg in options.__dict__: + setattr(options, arg, True) + else: + extra_args.append(arg) + + options = vars(options) + options['extra_args'] = extra_args + return config, options + + def whataremyips(): """ Get the machine's ip addresses using ifconfig diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 513ae17220..790de69c37 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -34,7 +34,7 @@ wsgi.ACCEPT_ERRNO.add(ECONNRESET) from eventlet.green import socket, ssl from swift.common.utils import get_logger, drop_privileges, \ - validate_configuration, LoggerFileObject, NullLogger + validate_configuration, capture_stdio, NullLogger def monkey_patch_mimetools(): @@ -57,9 +57,8 @@ def monkey_patch_mimetools(): mimetools.Message.parsetype = parsetype -# We might be able to pull pieces of this out to test, but right now it seems -# like more work than it's worth. -def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover +# TODO: pull pieces of this out to test +def run_wsgi(conf_file, app_section, *args, **kwargs): """ Loads common settings from conf, then instantiates app and runs the server using the specified number of workers. @@ -70,25 +69,22 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover try: conf = appconfig('config:%s' % conf_file, name=app_section) - log_name = conf.get('log_name', app_section) - app = loadapp('config:%s' % conf_file, - global_conf={'log_name': log_name}) except Exception, e: print "Error trying to load config %s: %s" % (conf_file, e) return - if 'logger' in kwargs: - logger = kwargs['logger'] - else: - logger = get_logger(conf, log_name) - # log uncaught exceptions - sys.excepthook = lambda * exc_info: \ - logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) - sys.stdout = sys.stderr = LoggerFileObject(logger) + validate_configuration() + + # pre-configure logger + log_name = conf.get('log_name', app_section) + if 'logger' in kwargs: + logger = kwargs.pop('logger') + else: + logger = get_logger(conf, log_name, + log_to_console=kwargs.pop('verbose', False)) + + # redirect errors to logger and close stdio + capture_stdio(logger) - try: - os.setsid() - except OSError: - no_cover = True # pass bind_addr = (conf.get('bind_ip', '0.0.0.0'), int(conf.get('bind_port', kwargs.get('default_port', 8080)))) sock = None @@ -112,7 +108,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) worker_count = int(conf.get('workers', '1')) drop_privileges(conf.get('user', 'swift')) - validate_configuration() + + # finally after binding to ports and privilege drop, run app __init__ code + app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name}) def run_server(): wsgi.HttpProtocol.default_request_version = "HTTP/1.0" @@ -169,6 +167,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover except OSError, err: if err.errno not in (errno.EINTR, errno.ECHILD): raise + except KeyboardInterrupt: + logger.info('User quit') + break greenio.shutdown_safe(sock) sock.close() logger.info('Exited') diff --git a/test/unit/__init__.py b/test/unit/__init__.py index cfd6cf1ad8..1895098c2e 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -1,5 +1,8 @@ """ Swift tests """ +import os +from contextlib import contextmanager +from tempfile import NamedTemporaryFile from eventlet.green import socket @@ -23,6 +26,18 @@ def connect_tcp(hostport): rv.connect(hostport) return rv + +@contextmanager +def tmpfile(content): + with NamedTemporaryFile('w', delete=False) as f: + file_name = f.name + f.write(str(content)) + try: + yield file_name + finally: + os.unlink(file_name) + + class MockTrue(object): """ Instances of MockTrue evaluate like True diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index e2db43caa6..83bb971907 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -13,16 +13,94 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Tests +# TODO: Test kill_children signal handlers import unittest -from swift.common import daemon +from getpass import getuser +import logging +from StringIO import StringIO +from test.unit import tmpfile + +from swift.common import daemon, utils + + +class MyDaemon(daemon.Daemon): + + def __init__(self, conf): + self.conf = conf + self.logger = utils.get_logger(None) + MyDaemon.forever_called = False + MyDaemon.once_called = False + + def run_forever(self): + MyDaemon.forever_called = True + + def run_once(self): + MyDaemon.once_called = True + + def run_raise(self): + raise OSError + + def run_quit(self): + raise KeyboardInterrupt class TestDaemon(unittest.TestCase): - def test_placeholder(self): - pass + def test_create(self): + d = daemon.Daemon({}) + self.assertEquals(d.conf, {}) + self.assert_(isinstance(d.logger, utils.NamedLogger)) + + def test_stubs(self): + d = daemon.Daemon({}) + self.assertRaises(NotImplementedError, d.run_once) + self.assertRaises(NotImplementedError, d.run_forever) + + +class TestRunDaemon(unittest.TestCase): + + def setUp(self): + utils.HASH_PATH_SUFFIX = 'endcap' + utils.daemonize = lambda *args: None + + def tearDown(self): + reload(utils) + + def test_run(self): + d = MyDaemon({}) + self.assertFalse(MyDaemon.forever_called) + self.assertFalse(MyDaemon.once_called) + # test default + d.run() + self.assertEquals(d.forever_called, True) + # test once + d.run(once=True) + self.assertEquals(d.once_called, True) + + def test_run_daemon(self): + sample_conf = """[my-daemon] +user = %s +""" % getuser() + with tmpfile(sample_conf) as conf_file: + daemon.run_daemon(MyDaemon, conf_file) + self.assertEquals(MyDaemon.forever_called, True) + daemon.run_daemon(MyDaemon, conf_file, once=True) + self.assertEquals(MyDaemon.once_called, True) + + # test raise in daemon code + MyDaemon.run_once = MyDaemon.run_raise + self.assertRaises(OSError, daemon.run_daemon, MyDaemon, + conf_file, once=True) + + # test user quit + MyDaemon.run_forever = MyDaemon.run_quit + sio = StringIO() + logger = logging.getLogger() + logger.addHandler(logging.StreamHandler(sio)) + logger = utils.get_logger(None, 'server') + daemon.run_daemon(MyDaemon, conf_file, logger=logger) + self.assert_('user quit' in sio.getvalue().lower()) if __name__ == '__main__': diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 92be1077c0..81ad0fb06b 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -25,17 +25,66 @@ import unittest from getpass import getuser from shutil import rmtree from StringIO import StringIO +from functools import partial +from tempfile import NamedTemporaryFile from eventlet import sleep from swift.common import utils +class MockOs(): + def __init__(self, pass_funcs=[], called_funcs=[], raise_funcs=[]): + self.closed_fds = [] + for func in pass_funcs: + setattr(self, func, self.pass_func) + self.called_funcs = {} + for func in called_funcs: + c_func = partial(self.called_func, name) + setattr(self, func, c_func) + for func in raise_funcs: + setattr(self, func, self.raise_func) + + def pass_func(self, *args, **kwargs): + pass + + chdir = setsid = setgid = setuid = umask = pass_func + + def called_func(self, name, *args, **kwargs): + self.called_funcs[name] = True + + def raise_func(self, *args, **kwargs): + raise OSError() + + def dup2(self, source, target): + self.closed_fds.append(target) + + def __getattr__(self, name): + # I only over-ride portions of the os module + try: + return object.__getattr__(self, name) + except AttributeError: + return getattr(os, name) + + +class MockSys(): + + __stderr__ = sys.__stderr__ + + class TestUtils(unittest.TestCase): """ Tests for swift.common.utils """ def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' + self.logger = logging.getLogger() + self.starting_handlers = list(self.logger.handlers) + + def tearDown(self): + # don't let extra handlers pile up redirecting stdio and other stuff... + for handler in self.logger.handlers: + if handler not in self.starting_handlers: + self.logger.removeHandler(handler) def test_normalize_timestamp(self): """ Test swift.common.utils.normalize_timestamp """ @@ -127,6 +176,16 @@ class TestUtils(unittest.TestCase): self.assertEquals(sio.getvalue(), '') def test_LoggerFileObject(self): + if isinstance(sys.stdout, utils.LoggerFileObject): + # This may happen if some other not so nice test allowed stdout to + # be caputred by daemonize w/o cleaning up after itself (i.e. + # test_db_replicator.TestDBReplicator.test_run_once). Normally + # nose would clean this up for us (which works well and is + # probably the best solution). But when running with --nocapture, + # this condition would cause the first print to acctually be + # redirected to a log call and the test would fail - so we have to + # go old school + sys.stdout = sys.__stdout__ orig_stdout = sys.stdout orig_stderr = sys.stderr sio = StringIO() @@ -182,10 +241,63 @@ class TestUtils(unittest.TestCase): self.assertRaises(IOError, lfo.readline, 1024) lfo.tell() - def test_drop_privileges(self): - # Note that this doesn't really drop privileges as it just sets them to - # what they already are; but it exercises the code at least. - utils.drop_privileges(getuser()) + def test_parse_options(self): + # use mkstemp to get a file that is definately on disk + with NamedTemporaryFile() as f: + conf_file = f.name + conf, options = utils.parse_options(test_args=[conf_file]) + self.assertEquals(conf, conf_file) + # assert defaults + self.assertEquals(options['verbose'], False) + self.assert_('once' not in options) + # assert verbose as option + conf, options = utils.parse_options(test_args=[conf_file, '-v']) + self.assertEquals(options['verbose'], True) + # check once option + conf, options = utils.parse_options(test_args=[conf_file], + once=True) + self.assertEquals(options['once'], False) + test_args = [conf_file, '--once'] + conf, options = utils.parse_options(test_args=test_args, once=True) + self.assertEquals(options['once'], True) + # check options as arg parsing + test_args = [conf_file, 'once', 'plugin_name', 'verbose'] + conf, options = utils.parse_options(test_args=test_args, once=True) + self.assertEquals(options['verbose'], True) + self.assertEquals(options['once'], True) + self.assertEquals(options['extra_args'], ['plugin_name']) + + def test_parse_options_errors(self): + orig_stdout = sys.stdout + orig_stderr = sys.stderr + stdo = StringIO() + stde = StringIO() + utils.sys.stdout = stdo + utils.sys.stderr = stde + err_msg = """Usage: test usage + +Error: missing config file argument +""" + test_args = [] + self.assertRaises(SystemExit, utils.parse_options, 'test usage', True, + test_args) + self.assertEquals(stdo.getvalue(), err_msg) + + # verify conf file must exist, context manager will delete temp file + with NamedTemporaryFile() as f: + conf_file = f.name + err_msg += """Usage: test usage + +Error: unable to locate %s +""" % conf_file + test_args = [conf_file] + self.assertRaises(SystemExit, utils.parse_options, 'test usage', True, + test_args) + self.assertEquals(stdo.getvalue(), err_msg) + + # reset stdio + utils.sys.stdout = orig_stdout + utils.sys.stderr = orig_stderr def test_NamedLogger(self): sio = StringIO() @@ -275,5 +387,70 @@ log_name = yarr''' self.assertEquals(result, expected) os.unlink('/tmp/test') + def test_daemonize(self): + # default args + conf = {'user': getuser()} + logger = utils.get_logger(None, 'daemon') + + # over-ride utils system modules with mocks + utils.os = MockOs() + utils.sys = MockSys() + + utils.daemonize(conf, logger) + self.assert_(utils.sys.excepthook is not None) + self.assertEquals(utils.os.closed_fds, [0, 1, 2]) + self.assert_(utils.sys.stdout is not None) + self.assert_(utils.sys.stderr is not None) + + # reset; test same args, OSError trying to get session leader + utils.os = MockOs(raise_funcs=('setsid',)) + utils.sys = MockSys() + + utils.daemonize(conf, logger) + self.assert_(utils.sys.excepthook is not None) + self.assertEquals(utils.os.closed_fds, [0, 1, 2]) + self.assert_(utils.sys.stdout is not None) + self.assert_(utils.sys.stderr is not None) + + # reset; test same args, exc when trying to close stdio + utils.os = MockOs(raise_funcs=('dup2',)) + utils.sys = MockSys() + + utils.daemonize(conf, logger) + self.assert_(utils.sys.excepthook is not None) + # unable to close stdio + self.assertEquals(utils.os.closed_fds, []) + self.assert_(utils.sys.stdout is not None) + self.assert_(utils.sys.stderr is not None) + + # reset; test some other args + utils.os = MockOs() + utils.sys = MockSys() + + conf = {'user': getuser()} + logger = utils.get_logger(None, log_to_console=True) + logger = logging.getLogger() + utils.daemonize(conf, logger, capture_stdout=False, + capture_stderr=False) + self.assert_(utils.sys.excepthook is not None) + # when logging to console, stderr remains open + self.assertEquals(utils.os.closed_fds, [0, 1]) + # stdio not captured + self.assertFalse(hasattr(utils.sys, 'stdout')) + self.assertFalse(hasattr(utils.sys, 'stderr')) + + def test_get_logger_console(self): + reload(utils) # reset get_logger attrs + logger = utils.get_logger(None) + self.assertFalse(hasattr(utils.get_logger, 'console')) + logger = utils.get_logger(None, log_to_console=True) + self.assert_(hasattr(utils.get_logger, 'console')) + self.assert_(isinstance(utils.get_logger.console, + logging.StreamHandler)) + # make sure you can't have two console handlers + old_handler = utils.get_logger.console + logger = utils.get_logger(None, log_to_console=True) + self.assertNotEquals(utils.get_logger.console, old_handler) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/stats/test_log_processor.py b/test/unit/stats/test_log_processor.py index 3cb773e7bb..1231c4a33c 100644 --- a/test/unit/stats/test_log_processor.py +++ b/test/unit/stats/test_log_processor.py @@ -14,25 +14,12 @@ # limitations under the License. import unittest -import os -from contextlib import contextmanager -from tempfile import NamedTemporaryFile +from test.unit import tmpfile from swift.common import internal_proxy from swift.stats import log_processor -@contextmanager -def tmpfile(content): - with NamedTemporaryFile('w', delete=False) as f: - file_name = f.name - f.write(str(content)) - try: - yield file_name - finally: - os.unlink(file_name) - - class FakeUploadApp(object): def __init__(self, *args, **kwargs): pass