From 57a35f0d7c85bae8bf78fbc915e5cf73bdc8e4ff Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 11 Nov 2010 16:41:07 -0600 Subject: [PATCH 1/8] added helper/util to parse command line args; removed some duplicated code in server/daemon bin scripts; more standized python/linux daemonization procedures; fixed lp:666957 "devauth server creates auth.db with the wrong privileges"; new run_daemon helper based on run_wsgi simplifies daemon launching/testing; new - all servers/daemons support verbose option when started interactivlty which will log to the console; fixed lp:667839 "can't start servers with relative paths to configs"; added tests --- bin/swift-account-auditor | 13 +- bin/swift-account-reaper | 13 +- bin/swift-account-replicator | 13 +- bin/swift-account-server | 9 +- bin/swift-account-stats-logger | 15 +-- bin/swift-auth-server | 8 +- bin/swift-container-auditor | 13 +- bin/swift-container-replicator | 14 +- bin/swift-container-server | 8 +- bin/swift-container-updater | 13 +- bin/swift-log-stats-collector | 15 +-- bin/swift-log-uploader | 22 ++- bin/swift-object-auditor | 14 +- bin/swift-object-replicator | 14 +- bin/swift-object-server | 8 +- bin/swift-object-updater | 13 +- bin/swift-proxy-server | 8 +- swift/common/daemon.py | 55 +++++--- swift/common/utils.py | 126 ++++++++++++++++-- swift/common/wsgi.py | 41 +++--- test/unit/__init__.py | 15 +++ test/unit/common/test_daemon.py | 86 +++++++++++- test/unit/common/test_utils.py | 185 +++++++++++++++++++++++++- test/unit/stats/test_log_processor.py | 15 +-- 24 files changed, 535 insertions(+), 201 deletions(-) diff --git a/bin/swift-account-auditor b/bin/swift-account-auditor index a715979599..5707dfd515 100755 --- a/bin/swift-account-auditor +++ b/bin/swift-account-auditor @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.account.auditor import AccountAuditor -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-account-auditor CONFIG_FILE [once]" - sys.exit() - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'account-auditor') - auditor = AccountAuditor(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(AccountAuditor, conf_file, **options) diff --git a/bin/swift-account-reaper b/bin/swift-account-reaper index 90496c64e8..688b19b14d 100755 --- a/bin/swift-account-reaper +++ b/bin/swift-account-reaper @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.account.reaper import AccountReaper -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: account-reaper CONFIG_FILE [once]" - sys.exit() - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'account-reaper') - reaper = AccountReaper(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(AccountReaper, conf_file, **options) diff --git a/bin/swift-account-replicator b/bin/swift-account-replicator index c71c326b8b..8edc7cf406 100755 --- a/bin/swift-account-replicator +++ b/bin/swift-account-replicator @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -from swift.common import utils from swift.account.replicator import AccountReplicator +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-account-replicator CONFIG_FILE [once]" - sys.exit(1) - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'account-replicator') - AccountReplicator(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(AccountReplicator, conf_file, **options) diff --git a/bin/swift-account-server b/bin/swift-account-server index 55e2a10857..8c627afa59 100755 --- a/bin/swift-account-server +++ b/bin/swift-account-server @@ -14,12 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'account-server', default_port=6002) - + conf_file, options = parse_options() + run_wsgi(conf_file, 'account-server', default_port=6002, **options) diff --git a/bin/swift-account-stats-logger b/bin/swift-account-stats-logger index c42554de82..7e9d26ba50 100755 --- a/bin/swift-account-stats-logger +++ b/bin/swift-account-stats-logger @@ -14,14 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.stats.account_stats import AccountStat -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-account-stats-logger CONFIG_FILE" - sys.exit() - stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats') - stats = AccountStat(stats_conf).run(once=True) + conf_file, options = parse_options() + # currently AccountStat only supports run_once + options['once'] = True + run_daemon(AccountStat, conf_file, section_name='log-processor-stats', + **options) diff --git a/bin/swift-auth-server b/bin/swift-auth-server index a8f03c6e92..80c652f5b4 100755 --- a/bin/swift-auth-server +++ b/bin/swift-auth-server @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'auth-server', default_port=11000) + conf_file, options = parse_options() + run_wsgi(conf_file, 'auth-server', default_port=11000, **options) diff --git a/bin/swift-container-auditor b/bin/swift-container-auditor index b3472c54af..62ff797535 100755 --- a/bin/swift-container-auditor +++ b/bin/swift-container-auditor @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.container.auditor import ContainerAuditor -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-container-auditor CONFIG_FILE [once]" - sys.exit() - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'container-auditor') - ContainerAuditor(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ContainerAuditor, conf_file, **options) diff --git a/bin/swift-container-replicator b/bin/swift-container-replicator index a0594142df..34e1dae8c7 100755 --- a/bin/swift-container-replicator +++ b/bin/swift-container-replicator @@ -14,16 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -from swift.common import db, utils from swift.container.replicator import ContainerReplicator +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-container-replicator CONFIG_FILE [once]" - sys.exit(1) - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'container-replicator') - ContainerReplicator(conf).run(once) - + conf_file, options = parse_options(once=True) + run_daemon(ContainerReplicator, conf_file, **options) diff --git a/bin/swift-container-server b/bin/swift-container-server index a15a35cf04..c6d4cf154b 100755 --- a/bin/swift-container-server +++ b/bin/swift-container-server @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'container-server', default_port=6001) + conf_file, options = parse_options() + run_wsgi(conf_file, 'container-server', default_port=6001, **options) diff --git a/bin/swift-container-updater b/bin/swift-container-updater index ed22d29901..28bdaf1eae 100755 --- a/bin/swift-container-updater +++ b/bin/swift-container-updater @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.container.updater import ContainerUpdater -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-container-updater CONFIG_FILE [once]" - sys.exit() - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'container-updater') - ContainerUpdater(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ContainerUpdater, conf_file, **options) diff --git a/bin/swift-log-stats-collector b/bin/swift-log-stats-collector index d21135b35c..950a0f5fe0 100755 --- a/bin/swift-log-stats-collector +++ b/bin/swift-log-stats-collector @@ -14,14 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.stats.log_processor import LogProcessorDaemon -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-log-stats-collector CONFIG_FILE" - sys.exit() - conf = utils.readconf(sys.argv[1], log_name='log-stats-collector') - stats = LogProcessorDaemon(conf).run(once=True) + conf_file, options = parse_options() + # currently the LogProcessorDaemon only supports run_once + options['once'] = True + run_daemon(LogProcessorDaemon, conf_file, section_name=None, + log_name='log-stats-collector', **options) diff --git a/bin/swift-log-uploader b/bin/swift-log-uploader index b557e4c167..617380ca9f 100755 --- a/bin/swift-log-uploader +++ b/bin/swift-log-uploader @@ -17,15 +17,25 @@ import sys from swift.stats.log_uploader import LogUploader +from swift.common.utils import parse_options from swift.common import utils if __name__ == '__main__': - if len(sys.argv) < 3: - print "Usage: swift-log-uploader CONFIG_FILE plugin" - sys.exit() - uploader_conf = utils.readconf(sys.argv[1], 'log-processor') - plugin = sys.argv[2] + conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN") + try: + plugin = options['extra_args'][0] + except IndexError: + print "Error: missing plugin name" + sys.exit(1) + + uploader_conf = utils.readconf(conf_file, 'log-processor') section_name = 'log-processor-%s' % plugin - plugin_conf = utils.readconf(sys.argv[1], section_name) + plugin_conf = utils.readconf(conf_file, section_name) uploader_conf.update(plugin_conf) + + # pre-configure logger + logger = utils.get_logger(uploader_conf, plugin, + log_to_console=options.get('verbose', False)) + # currently LogUploader only supports run_once + options['once'] = True uploader = LogUploader(uploader_conf, plugin).run(once=True) diff --git a/bin/swift-object-auditor b/bin/swift-object-auditor index d60bcb6148..033249ecca 100755 --- a/bin/swift-object-auditor +++ b/bin/swift-object-auditor @@ -14,16 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.obj.auditor import ObjectAuditor -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-object-auditor CONFIG_FILE [once]" - sys.exit() - - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'object-auditor') - ObjectAuditor(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ObjectAuditor, conf_file, **options) diff --git a/bin/swift-object-replicator b/bin/swift-object-replicator index cc8c32fdde..53a48e3942 100755 --- a/bin/swift-object-replicator +++ b/bin/swift-object-replicator @@ -14,16 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.obj.replicator import ObjectReplicator -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-object-replicator CONFIG_FILE [once]" - sys.exit() - conf = utils.readconf(sys.argv[1], "object-replicator") - once = (len(sys.argv) > 2 and sys.argv[2] == 'once') or \ - conf.get('daemonize', 'true') not in utils.TRUE_VALUES - ObjectReplicator(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ObjectReplicator, conf_file, **options) diff --git a/bin/swift-object-server b/bin/swift-object-server index 39c9246d6b..5984fe6c69 100755 --- a/bin/swift-object-server +++ b/bin/swift-object-server @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'object-server', default_port=6000) + conf_file, options = parse_options() + run_wsgi(conf_file, 'object-server', default_port=6000, **options) diff --git a/bin/swift-object-updater b/bin/swift-object-updater index d24cdbcfe9..6d68c57c6e 100755 --- a/bin/swift-object-updater +++ b/bin/swift-object-updater @@ -14,15 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from swift.obj.updater import ObjectUpdater -from swift.common import utils +from swift.common.utils import parse_options +from swift.common.daemon import run_daemon if __name__ == '__main__': - if len(sys.argv) < 2: - print "Usage: swift-object-updater CONFIG_FILE [once]" - sys.exit(1) - once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = utils.readconf(sys.argv[1], 'object-updater') - ObjectUpdater(conf).run(once) + conf_file, options = parse_options(once=True) + run_daemon(ObjectUpdater, conf_file, **options) diff --git a/bin/swift-proxy-server b/bin/swift-proxy-server index 7599f87280..baccf568e6 100755 --- a/bin/swift-proxy-server +++ b/bin/swift-proxy-server @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - +from swift.common.utils import parse_options from swift.common.wsgi import run_wsgi if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'proxy-server', default_port=8080) + conf_file, options = parse_options() + run_wsgi(conf_file, 'proxy-server', default_port=8080, **options) diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 26892824b8..faa4112a1d 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -16,6 +16,7 @@ import os import sys import signal +from re import sub from swift.common import utils @@ -34,23 +35,10 @@ class Daemon(object): """Override this to run forever""" raise NotImplementedError('run_forever not implemented') - def run(self, once=False, capture_stdout=True, capture_stderr=True): + def run(self, once=False, **kwargs): """Run the daemon""" - # log uncaught exceptions - sys.excepthook = lambda *exc_info: \ - self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) - if capture_stdout: - sys.stdout = utils.LoggerFileObject(self.logger) - if capture_stderr: - sys.stderr = utils.LoggerFileObject(self.logger) - - utils.drop_privileges(self.conf.get('user', 'swift')) utils.validate_configuration() - - try: - os.setsid() - except OSError: - pass + utils.daemonize(self.conf, self.logger, **kwargs) def kill_children(*args): signal.signal(signal.SIGTERM, signal.SIG_IGN) @@ -63,3 +51,40 @@ class Daemon(object): self.run_once() else: self.run_forever() + + +def run_daemon(klass, conf_file, section_name='', + once=False, **kwargs): + """ + Loads settings from conf, then instantiates daemon "klass" and runs the + daemon with the specified once kwarg. The section_name will be derived + from the daemon "klass" if not provided (e.g. ObjectReplicator => + object-replicator). + + :param klass: Class to instantiate, subclass of common.daemon.Daemon + :param conf_file: Path to configuration file + :param section_name: Section name from conf file to load config from + :param once: Passed to daemon run method + """ + # very often the config section_name is based on the class name + # the None singleton will be passed through to readconf as is + if section_name is '': + section_name = sub(r'([a-z])([A-Z])', r'\1-\2', + klass.__name__).lower() + conf = utils.readconf(conf_file, section_name, + log_name=kwargs.get('log_name')) + + # once on command line (i.e. daemonize=false) will over-ride config + once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES + + # pre-configure logger + if 'logger' in kwargs: + logger = kwargs.pop('logger') + else: + logger = utils.get_logger(conf, conf.get('log_name', section_name), + log_to_console=kwargs.pop('verbose', False)) + try: + klass(conf).run(once=once, **kwargs) + except KeyboardInterrupt: + logger.info('User quit') + logger.info('Exited') diff --git a/swift/common/utils.py b/swift/common/utils.py index bb635725c8..7a70acd64d 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -32,6 +32,7 @@ import ctypes.util import fcntl import struct from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from optparse import OptionParser from tempfile import mkstemp import cPickle as pickle @@ -284,17 +285,6 @@ class LoggerFileObject(object): return self -def drop_privileges(user): - """ - Sets the userid of the current process - - :param user: User id to change privileges to - """ - user = pwd.getpwnam(user) - os.setgid(user[3]) - os.setuid(user[2]) - - class NamedLogger(object): """Cheesy version of the LoggerAdapter available in Python 3""" @@ -344,7 +334,7 @@ class NamedLogger(object): call('%s %s: %s' % (self.server, msg, emsg), *args) -def get_logger(conf, name=None): +def get_logger(conf, name=None, log_to_console=False): """ Get the current system logger using config settings. @@ -356,11 +346,18 @@ def get_logger(conf, name=None): :param conf: Configuration dict to read settings from :param name: Name of the logger + :param log_to_console: Add handler which writes to console on stderr """ root_logger = logging.getLogger() if hasattr(get_logger, 'handler') and get_logger.handler: root_logger.removeHandler(get_logger.handler) get_logger.handler = None + if log_to_console: + # check if a previous call to get_logger already added a console logger + if hasattr(get_logger, 'console') and get_logger.console: + root_logger.removeHandler(get_logger.console) + get_logger.console = logging.StreamHandler(sys.__stderr__) + root_logger.addHandler(get_logger.console) if conf is None: root_logger.setLevel(logging.INFO) return NamedLogger(root_logger, name) @@ -376,6 +373,111 @@ def get_logger(conf, name=None): return NamedLogger(root_logger, name) +def drop_privileges(user): + """ + Sets the userid/groupid of the current process, get session leader, etc. + + :param user: User name to change privileges to + """ + user = pwd.getpwnam(user) + os.setgid(user[3]) + os.setuid(user[2]) + try: + os.setsid() + except OSError: + pass + os.chdir('/') # in case you need to rmdir on where you started the daemon + os.umask(0) # ensure files are created with the correct privledges + + +def capture_stdio(logger, **kwargs): + """ + Log unhaneled exceptions, close stdio, capture stdout and stderr. + + param logger: Logger object to use + """ + # log uncaught exceptions + sys.excepthook = lambda * exc_info: \ + logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + + # collect stdio file desc not in use for logging + stdio_fds = [0, 1, 2] + if hasattr(get_logger, 'console'): + stdio_fds.remove(get_logger.console.stream.fileno()) + + with open(os.devnull, 'r+b') as nullfile: + # close stdio (excludes fds open for logging) + for desc in stdio_fds: + try: + os.dup2(nullfile.fileno(), desc) + except OSError: + pass + + # redirect stdio + if kwargs.pop('capture_stdout', True): + sys.stdout = LoggerFileObject(logger) + if kwargs.pop('capture_stderr', True): + sys.stderr = LoggerFileObject(logger) + + +def daemonize(conf, logger, **kwargs): + """ + Perform standard python/linux daemonization operations. + + :param user: Configuration dict to read settings from (i.e. user) + :param logger: Logger object to handle stdio redirect and uncaught exc + """ + + drop_privileges(conf.get('user', 'swift')) + capture_stdio(logger, **kwargs) + + +def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None): + """ + Parse standard swift server/daemon options with optparse.OptionParser. + + :param usage: String describing usage + :param once: Boolean indicating the "once" option is avaiable + :param test_args: Override sys.argv; used in testing + + :returns : Tuple of (config, options); config is an absolute path to the + config file, options is the parser options as a dictionary. + + :raises SystemExit: First arg (CONFIG) is required, file must exist + """ + parser = OptionParser(usage) + parser.add_option("-v", "--verbose", default=False, action="store_true", + help="log to console") + if once: + parser.add_option("-o", "--once", default=False, action="store_true", + help="only run one pass of daemon") + + # if test_args is None, optparse will use sys.argv[:1] + options, args = parser.parse_args(args=test_args) + + if not args: + parser.print_usage() + print "Error: missing config file argument" + sys.exit(1) + config = os.path.abspath(args.pop(0)) + if not os.path.exists(config): + parser.print_usage() + print "Error: unable to locate %s" % config + sys.exit(1) + + extra_args = [] + # if any named options appear in remaining args, set the option to True + for arg in args: + if arg in options.__dict__: + setattr(options, arg, True) + else: + extra_args.append(arg) + + options = vars(options) + options['extra_args'] = extra_args + return config, options + + def whataremyips(): """ Get the machine's ip addresses using ifconfig diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 513ae17220..790de69c37 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -34,7 +34,7 @@ wsgi.ACCEPT_ERRNO.add(ECONNRESET) from eventlet.green import socket, ssl from swift.common.utils import get_logger, drop_privileges, \ - validate_configuration, LoggerFileObject, NullLogger + validate_configuration, capture_stdio, NullLogger def monkey_patch_mimetools(): @@ -57,9 +57,8 @@ def monkey_patch_mimetools(): mimetools.Message.parsetype = parsetype -# We might be able to pull pieces of this out to test, but right now it seems -# like more work than it's worth. -def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover +# TODO: pull pieces of this out to test +def run_wsgi(conf_file, app_section, *args, **kwargs): """ Loads common settings from conf, then instantiates app and runs the server using the specified number of workers. @@ -70,25 +69,22 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover try: conf = appconfig('config:%s' % conf_file, name=app_section) - log_name = conf.get('log_name', app_section) - app = loadapp('config:%s' % conf_file, - global_conf={'log_name': log_name}) except Exception, e: print "Error trying to load config %s: %s" % (conf_file, e) return - if 'logger' in kwargs: - logger = kwargs['logger'] - else: - logger = get_logger(conf, log_name) - # log uncaught exceptions - sys.excepthook = lambda * exc_info: \ - logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) - sys.stdout = sys.stderr = LoggerFileObject(logger) + validate_configuration() + + # pre-configure logger + log_name = conf.get('log_name', app_section) + if 'logger' in kwargs: + logger = kwargs.pop('logger') + else: + logger = get_logger(conf, log_name, + log_to_console=kwargs.pop('verbose', False)) + + # redirect errors to logger and close stdio + capture_stdio(logger) - try: - os.setsid() - except OSError: - no_cover = True # pass bind_addr = (conf.get('bind_ip', '0.0.0.0'), int(conf.get('bind_port', kwargs.get('default_port', 8080)))) sock = None @@ -112,7 +108,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) worker_count = int(conf.get('workers', '1')) drop_privileges(conf.get('user', 'swift')) - validate_configuration() + + # finally after binding to ports and privilege drop, run app __init__ code + app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name}) def run_server(): wsgi.HttpProtocol.default_request_version = "HTTP/1.0" @@ -169,6 +167,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover except OSError, err: if err.errno not in (errno.EINTR, errno.ECHILD): raise + except KeyboardInterrupt: + logger.info('User quit') + break greenio.shutdown_safe(sock) sock.close() logger.info('Exited') diff --git a/test/unit/__init__.py b/test/unit/__init__.py index cfd6cf1ad8..1895098c2e 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -1,5 +1,8 @@ """ Swift tests """ +import os +from contextlib import contextmanager +from tempfile import NamedTemporaryFile from eventlet.green import socket @@ -23,6 +26,18 @@ def connect_tcp(hostport): rv.connect(hostport) return rv + +@contextmanager +def tmpfile(content): + with NamedTemporaryFile('w', delete=False) as f: + file_name = f.name + f.write(str(content)) + try: + yield file_name + finally: + os.unlink(file_name) + + class MockTrue(object): """ Instances of MockTrue evaluate like True diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index e2db43caa6..83bb971907 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -13,16 +13,94 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Tests +# TODO: Test kill_children signal handlers import unittest -from swift.common import daemon +from getpass import getuser +import logging +from StringIO import StringIO +from test.unit import tmpfile + +from swift.common import daemon, utils + + +class MyDaemon(daemon.Daemon): + + def __init__(self, conf): + self.conf = conf + self.logger = utils.get_logger(None) + MyDaemon.forever_called = False + MyDaemon.once_called = False + + def run_forever(self): + MyDaemon.forever_called = True + + def run_once(self): + MyDaemon.once_called = True + + def run_raise(self): + raise OSError + + def run_quit(self): + raise KeyboardInterrupt class TestDaemon(unittest.TestCase): - def test_placeholder(self): - pass + def test_create(self): + d = daemon.Daemon({}) + self.assertEquals(d.conf, {}) + self.assert_(isinstance(d.logger, utils.NamedLogger)) + + def test_stubs(self): + d = daemon.Daemon({}) + self.assertRaises(NotImplementedError, d.run_once) + self.assertRaises(NotImplementedError, d.run_forever) + + +class TestRunDaemon(unittest.TestCase): + + def setUp(self): + utils.HASH_PATH_SUFFIX = 'endcap' + utils.daemonize = lambda *args: None + + def tearDown(self): + reload(utils) + + def test_run(self): + d = MyDaemon({}) + self.assertFalse(MyDaemon.forever_called) + self.assertFalse(MyDaemon.once_called) + # test default + d.run() + self.assertEquals(d.forever_called, True) + # test once + d.run(once=True) + self.assertEquals(d.once_called, True) + + def test_run_daemon(self): + sample_conf = """[my-daemon] +user = %s +""" % getuser() + with tmpfile(sample_conf) as conf_file: + daemon.run_daemon(MyDaemon, conf_file) + self.assertEquals(MyDaemon.forever_called, True) + daemon.run_daemon(MyDaemon, conf_file, once=True) + self.assertEquals(MyDaemon.once_called, True) + + # test raise in daemon code + MyDaemon.run_once = MyDaemon.run_raise + self.assertRaises(OSError, daemon.run_daemon, MyDaemon, + conf_file, once=True) + + # test user quit + MyDaemon.run_forever = MyDaemon.run_quit + sio = StringIO() + logger = logging.getLogger() + logger.addHandler(logging.StreamHandler(sio)) + logger = utils.get_logger(None, 'server') + daemon.run_daemon(MyDaemon, conf_file, logger=logger) + self.assert_('user quit' in sio.getvalue().lower()) if __name__ == '__main__': diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 92be1077c0..81ad0fb06b 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -25,17 +25,66 @@ import unittest from getpass import getuser from shutil import rmtree from StringIO import StringIO +from functools import partial +from tempfile import NamedTemporaryFile from eventlet import sleep from swift.common import utils +class MockOs(): + def __init__(self, pass_funcs=[], called_funcs=[], raise_funcs=[]): + self.closed_fds = [] + for func in pass_funcs: + setattr(self, func, self.pass_func) + self.called_funcs = {} + for func in called_funcs: + c_func = partial(self.called_func, name) + setattr(self, func, c_func) + for func in raise_funcs: + setattr(self, func, self.raise_func) + + def pass_func(self, *args, **kwargs): + pass + + chdir = setsid = setgid = setuid = umask = pass_func + + def called_func(self, name, *args, **kwargs): + self.called_funcs[name] = True + + def raise_func(self, *args, **kwargs): + raise OSError() + + def dup2(self, source, target): + self.closed_fds.append(target) + + def __getattr__(self, name): + # I only over-ride portions of the os module + try: + return object.__getattr__(self, name) + except AttributeError: + return getattr(os, name) + + +class MockSys(): + + __stderr__ = sys.__stderr__ + + class TestUtils(unittest.TestCase): """ Tests for swift.common.utils """ def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' + self.logger = logging.getLogger() + self.starting_handlers = list(self.logger.handlers) + + def tearDown(self): + # don't let extra handlers pile up redirecting stdio and other stuff... + for handler in self.logger.handlers: + if handler not in self.starting_handlers: + self.logger.removeHandler(handler) def test_normalize_timestamp(self): """ Test swift.common.utils.normalize_timestamp """ @@ -127,6 +176,16 @@ class TestUtils(unittest.TestCase): self.assertEquals(sio.getvalue(), '') def test_LoggerFileObject(self): + if isinstance(sys.stdout, utils.LoggerFileObject): + # This may happen if some other not so nice test allowed stdout to + # be caputred by daemonize w/o cleaning up after itself (i.e. + # test_db_replicator.TestDBReplicator.test_run_once). Normally + # nose would clean this up for us (which works well and is + # probably the best solution). But when running with --nocapture, + # this condition would cause the first print to acctually be + # redirected to a log call and the test would fail - so we have to + # go old school + sys.stdout = sys.__stdout__ orig_stdout = sys.stdout orig_stderr = sys.stderr sio = StringIO() @@ -182,10 +241,63 @@ class TestUtils(unittest.TestCase): self.assertRaises(IOError, lfo.readline, 1024) lfo.tell() - def test_drop_privileges(self): - # Note that this doesn't really drop privileges as it just sets them to - # what they already are; but it exercises the code at least. - utils.drop_privileges(getuser()) + def test_parse_options(self): + # use mkstemp to get a file that is definately on disk + with NamedTemporaryFile() as f: + conf_file = f.name + conf, options = utils.parse_options(test_args=[conf_file]) + self.assertEquals(conf, conf_file) + # assert defaults + self.assertEquals(options['verbose'], False) + self.assert_('once' not in options) + # assert verbose as option + conf, options = utils.parse_options(test_args=[conf_file, '-v']) + self.assertEquals(options['verbose'], True) + # check once option + conf, options = utils.parse_options(test_args=[conf_file], + once=True) + self.assertEquals(options['once'], False) + test_args = [conf_file, '--once'] + conf, options = utils.parse_options(test_args=test_args, once=True) + self.assertEquals(options['once'], True) + # check options as arg parsing + test_args = [conf_file, 'once', 'plugin_name', 'verbose'] + conf, options = utils.parse_options(test_args=test_args, once=True) + self.assertEquals(options['verbose'], True) + self.assertEquals(options['once'], True) + self.assertEquals(options['extra_args'], ['plugin_name']) + + def test_parse_options_errors(self): + orig_stdout = sys.stdout + orig_stderr = sys.stderr + stdo = StringIO() + stde = StringIO() + utils.sys.stdout = stdo + utils.sys.stderr = stde + err_msg = """Usage: test usage + +Error: missing config file argument +""" + test_args = [] + self.assertRaises(SystemExit, utils.parse_options, 'test usage', True, + test_args) + self.assertEquals(stdo.getvalue(), err_msg) + + # verify conf file must exist, context manager will delete temp file + with NamedTemporaryFile() as f: + conf_file = f.name + err_msg += """Usage: test usage + +Error: unable to locate %s +""" % conf_file + test_args = [conf_file] + self.assertRaises(SystemExit, utils.parse_options, 'test usage', True, + test_args) + self.assertEquals(stdo.getvalue(), err_msg) + + # reset stdio + utils.sys.stdout = orig_stdout + utils.sys.stderr = orig_stderr def test_NamedLogger(self): sio = StringIO() @@ -275,5 +387,70 @@ log_name = yarr''' self.assertEquals(result, expected) os.unlink('/tmp/test') + def test_daemonize(self): + # default args + conf = {'user': getuser()} + logger = utils.get_logger(None, 'daemon') + + # over-ride utils system modules with mocks + utils.os = MockOs() + utils.sys = MockSys() + + utils.daemonize(conf, logger) + self.assert_(utils.sys.excepthook is not None) + self.assertEquals(utils.os.closed_fds, [0, 1, 2]) + self.assert_(utils.sys.stdout is not None) + self.assert_(utils.sys.stderr is not None) + + # reset; test same args, OSError trying to get session leader + utils.os = MockOs(raise_funcs=('setsid',)) + utils.sys = MockSys() + + utils.daemonize(conf, logger) + self.assert_(utils.sys.excepthook is not None) + self.assertEquals(utils.os.closed_fds, [0, 1, 2]) + self.assert_(utils.sys.stdout is not None) + self.assert_(utils.sys.stderr is not None) + + # reset; test same args, exc when trying to close stdio + utils.os = MockOs(raise_funcs=('dup2',)) + utils.sys = MockSys() + + utils.daemonize(conf, logger) + self.assert_(utils.sys.excepthook is not None) + # unable to close stdio + self.assertEquals(utils.os.closed_fds, []) + self.assert_(utils.sys.stdout is not None) + self.assert_(utils.sys.stderr is not None) + + # reset; test some other args + utils.os = MockOs() + utils.sys = MockSys() + + conf = {'user': getuser()} + logger = utils.get_logger(None, log_to_console=True) + logger = logging.getLogger() + utils.daemonize(conf, logger, capture_stdout=False, + capture_stderr=False) + self.assert_(utils.sys.excepthook is not None) + # when logging to console, stderr remains open + self.assertEquals(utils.os.closed_fds, [0, 1]) + # stdio not captured + self.assertFalse(hasattr(utils.sys, 'stdout')) + self.assertFalse(hasattr(utils.sys, 'stderr')) + + def test_get_logger_console(self): + reload(utils) # reset get_logger attrs + logger = utils.get_logger(None) + self.assertFalse(hasattr(utils.get_logger, 'console')) + logger = utils.get_logger(None, log_to_console=True) + self.assert_(hasattr(utils.get_logger, 'console')) + self.assert_(isinstance(utils.get_logger.console, + logging.StreamHandler)) + # make sure you can't have two console handlers + old_handler = utils.get_logger.console + logger = utils.get_logger(None, log_to_console=True) + self.assertNotEquals(utils.get_logger.console, old_handler) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/stats/test_log_processor.py b/test/unit/stats/test_log_processor.py index 3cb773e7bb..1231c4a33c 100644 --- a/test/unit/stats/test_log_processor.py +++ b/test/unit/stats/test_log_processor.py @@ -14,25 +14,12 @@ # limitations under the License. import unittest -import os -from contextlib import contextmanager -from tempfile import NamedTemporaryFile +from test.unit import tmpfile from swift.common import internal_proxy from swift.stats import log_processor -@contextmanager -def tmpfile(content): - with NamedTemporaryFile('w', delete=False) as f: - file_name = f.name - f.write(str(content)) - try: - yield file_name - finally: - os.unlink(file_name) - - class FakeUploadApp(object): def __init__(self, *args, **kwargs): pass From 4962f4f58cc1d7cb28f4f81931fd88866d4aa44b Mon Sep 17 00:00:00 2001 From: clayg Date: Thu, 11 Nov 2010 22:57:07 -0600 Subject: [PATCH 2/8] swapped daemonize calls --- swift/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index 7a70acd64d..dafdfb2787 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -428,8 +428,8 @@ def daemonize(conf, logger, **kwargs): :param logger: Logger object to handle stdio redirect and uncaught exc """ - drop_privileges(conf.get('user', 'swift')) capture_stdio(logger, **kwargs) + drop_privileges(conf.get('user', 'swift')) def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None): From 410635485be5c4b02306ef5574b2e4c34a138076 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 15 Nov 2010 16:52:29 -0600 Subject: [PATCH 3/8] fixed typo in doc string for daemonize --- swift/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index dafdfb2787..76897d0f9f 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -424,7 +424,7 @@ def daemonize(conf, logger, **kwargs): """ Perform standard python/linux daemonization operations. - :param user: Configuration dict to read settings from (i.e. user) + :param conf: Configuration dict to read settings from (i.e. user) :param logger: Logger object to handle stdio redirect and uncaught exc """ From 354f7dd2a595093a34b0207d2787db5d79e320a2 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 16 Nov 2010 13:52:05 -0600 Subject: [PATCH 4/8] fixed missing kwargs in bin/swift-log-uploader --- bin/swift-log-uploader | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/swift-log-uploader b/bin/swift-log-uploader index 617380ca9f..e533cad824 100755 --- a/bin/swift-log-uploader +++ b/bin/swift-log-uploader @@ -38,4 +38,4 @@ if __name__ == '__main__': log_to_console=options.get('verbose', False)) # currently LogUploader only supports run_once options['once'] = True - uploader = LogUploader(uploader_conf, plugin).run(once=True) + uploader = LogUploader(uploader_conf, plugin).run(**options) From d583fd9bdbdd381d5ebb2497061afa7588c402c0 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 17 Nov 2010 17:17:05 -0600 Subject: [PATCH 5/8] cleaned up test reloads --- test/unit/common/test_utils.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 81ad0fb06b..5427ba65c5 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -176,16 +176,7 @@ class TestUtils(unittest.TestCase): self.assertEquals(sio.getvalue(), '') def test_LoggerFileObject(self): - if isinstance(sys.stdout, utils.LoggerFileObject): - # This may happen if some other not so nice test allowed stdout to - # be caputred by daemonize w/o cleaning up after itself (i.e. - # test_db_replicator.TestDBReplicator.test_run_once). Normally - # nose would clean this up for us (which works well and is - # probably the best solution). But when running with --nocapture, - # this condition would cause the first print to acctually be - # redirected to a log call and the test would fail - so we have to - # go old school - sys.stdout = sys.__stdout__ + reload(sys) # reset stdio redirection orig_stdout = sys.stdout orig_stderr = sys.stderr sio = StringIO() @@ -439,6 +430,9 @@ log_name = yarr''' self.assertFalse(hasattr(utils.sys, 'stdout')) self.assertFalse(hasattr(utils.sys, 'stderr')) + # reset mocks on utils + reload(utils) + def test_get_logger_console(self): reload(utils) # reset get_logger attrs logger = utils.get_logger(None) From c007d0296e49eef70542526a0b49429287aba8f2 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 19 Nov 2010 12:15:41 -0600 Subject: [PATCH 6/8] removed unneeded daemonize function from utils, pulled get_socket out of run_wsgi, reworked test_utils and test_wsgi --- swift/common/daemon.py | 3 +- swift/common/utils.py | 12 ---- swift/common/wsgi.py | 59 +++++++++++------- test/unit/common/test_daemon.py | 3 +- test/unit/common/test_utils.py | 66 +++++++++++--------- test/unit/common/test_wsgi.py | 104 +++++++++++++++++++++++++++++++- 6 files changed, 181 insertions(+), 66 deletions(-) diff --git a/swift/common/daemon.py b/swift/common/daemon.py index faa4112a1d..d305c247f6 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -38,7 +38,8 @@ class Daemon(object): def run(self, once=False, **kwargs): """Run the daemon""" utils.validate_configuration() - utils.daemonize(self.conf, self.logger, **kwargs) + utils.capture_stdio(self.logger, **kwargs) + utils.drop_privileges(self.conf.get('user', 'swift')) def kill_children(*args): signal.signal(signal.SIGTERM, signal.SIG_IGN) diff --git a/swift/common/utils.py b/swift/common/utils.py index 9311c28fce..1c48c61339 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -419,18 +419,6 @@ def capture_stdio(logger, **kwargs): sys.stderr = LoggerFileObject(logger) -def daemonize(conf, logger, **kwargs): - """ - Perform standard python/linux daemonization operations. - - :param conf: Configuration dict to read settings from (i.e. user) - :param logger: Logger object to handle stdio redirect and uncaught exc - """ - - capture_stdio(logger, **kwargs) - drop_privileges(conf.get('user', 'swift')) - - def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None): """ Parse standard swift server/daemon options with optparse.OptionParser. diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 790de69c37..a93c21aa8a 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -56,6 +56,38 @@ def monkey_patch_mimetools(): mimetools.Message.parsetype = parsetype +def get_socket(conf, default_port=8080): + """Bind socket to bind ip:port in conf + + :param conf: Configuration dict to read settings from + :param default_port: port to use if not specified in conf + + :returns : a socket object as returned from socket.listen or ssl.wrap_socket + if conf specifies cert_file + """ + bind_addr = (conf.get('bind_ip', '0.0.0.0'), + int(conf.get('bind_port', default_port))) + sock = None + retry_until = time.time() + 30 + while not sock and time.time() < retry_until: + try: + sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096))) + if 'cert_file' in conf: + sock = ssl.wrap_socket(sock, certfile=conf['cert_file'], + keyfile=conf['key_file']) + except socket.error, err: + if err.args[0] != errno.EADDRINUSE: + raise + sleep(0.1) + if not sock: + raise Exception('Could not bind to %s:%s after trying for 30 seconds' % + bind_addr) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # in my experience, sockets can hang around forever without keepalive + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) + return sock + # TODO: pull pieces of this out to test def run_wsgi(conf_file, app_section, *args, **kwargs): @@ -84,29 +116,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # redirect errors to logger and close stdio capture_stdio(logger) - - bind_addr = (conf.get('bind_ip', '0.0.0.0'), - int(conf.get('bind_port', kwargs.get('default_port', 8080)))) - sock = None - retry_until = time.time() + 30 - while not sock and time.time() < retry_until: - try: - sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096))) - if 'cert_file' in conf: - sock = ssl.wrap_socket(sock, certfile=conf['cert_file'], - keyfile=conf['key_file']) - except socket.error, err: - if err.args[0] != errno.EADDRINUSE: - raise - sleep(0.1) - if not sock: - raise Exception('Could not bind to %s:%s after trying for 30 seconds' % - bind_addr) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - # in my experience, sockets can hang around forever without keepalive - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) - worker_count = int(conf.get('workers', '1')) + # bind to address and port + sock = get_socket(conf, default_port=kwargs.get('default_port', 8080)) + # remaining tasks should not require elevated privileges drop_privileges(conf.get('user', 'swift')) # finally after binding to ports and privilege drop, run app __init__ code @@ -125,6 +137,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): raise pool.waitall() + worker_count = int(conf.get('workers', '1')) # Useful for profiling [no forks]. if worker_count == 0: run_server() diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index 83bb971907..c5b95a3013 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -62,7 +62,8 @@ class TestRunDaemon(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' - utils.daemonize = lambda *args: None + utils.drop_privileges = lambda *args: None + utils.capture_stdio = lambda *args: None def tearDown(self): reload(utils) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 5427ba65c5..2e9d4b0005 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -40,10 +40,11 @@ class MockOs(): setattr(self, func, self.pass_func) self.called_funcs = {} for func in called_funcs: - c_func = partial(self.called_func, name) + c_func = partial(self.called_func, func) setattr(self, func, c_func) for func in raise_funcs: - setattr(self, func, self.raise_func) + r_func = partial(self.raise_func, func) + setattr(self, func, r_func) def pass_func(self, *args, **kwargs): pass @@ -53,7 +54,8 @@ class MockOs(): def called_func(self, name, *args, **kwargs): self.called_funcs[name] = True - def raise_func(self, *args, **kwargs): + def raise_func(self, name, *args, **kwargs): + self.called_funcs[name] = True raise OSError() def dup2(self, source, target): @@ -378,51 +380,59 @@ log_name = yarr''' self.assertEquals(result, expected) os.unlink('/tmp/test') - def test_daemonize(self): - # default args - conf = {'user': getuser()} - logger = utils.get_logger(None, 'daemon') - - # over-ride utils system modules with mocks - utils.os = MockOs() - utils.sys = MockSys() - - utils.daemonize(conf, logger) - self.assert_(utils.sys.excepthook is not None) - self.assertEquals(utils.os.closed_fds, [0, 1, 2]) - self.assert_(utils.sys.stdout is not None) - self.assert_(utils.sys.stderr is not None) + def test_drop_privileges(self): + user = getuser() + # over-ride os with mock + required_func_calls = ('setgid', 'setuid', 'setsid', 'chdir', 'umask') + utils.os = MockOs(called_funcs=required_func_calls) + # exercise the code + utils.drop_privileges(user) + for func in required_func_calls: + self.assert_(utils.os.called_funcs[func]) # reset; test same args, OSError trying to get session leader - utils.os = MockOs(raise_funcs=('setsid',)) - utils.sys = MockSys() + utils.os = MockOs(called_funcs=required_func_calls, + raise_funcs=('setsid',)) + for func in required_func_calls: + self.assertFalse(utils.os.called_funcs.get(func, False)) + utils.drop_privileges(user) + for func in required_func_calls: + self.assert_(utils.os.called_funcs[func]) - utils.daemonize(conf, logger) + def test_capture_stdio(self): + # stubs + logger = utils.get_logger(None, 'dummy') + + # mock utils system modules + utils.sys = MockSys() + utils.os = MockOs() + + # basic test + utils.capture_stdio(logger) self.assert_(utils.sys.excepthook is not None) self.assertEquals(utils.os.closed_fds, [0, 1, 2]) self.assert_(utils.sys.stdout is not None) self.assert_(utils.sys.stderr is not None) - # reset; test same args, exc when trying to close stdio + # reset; test same args, but exc when trying to close stdio utils.os = MockOs(raise_funcs=('dup2',)) utils.sys = MockSys() - utils.daemonize(conf, logger) + # test unable to close stdio + utils.capture_stdio(logger) self.assert_(utils.sys.excepthook is not None) - # unable to close stdio self.assertEquals(utils.os.closed_fds, []) self.assert_(utils.sys.stdout is not None) self.assert_(utils.sys.stderr is not None) # reset; test some other args + logger = utils.get_logger(None, log_to_console=True) utils.os = MockOs() utils.sys = MockSys() - conf = {'user': getuser()} - logger = utils.get_logger(None, log_to_console=True) - logger = logging.getLogger() - utils.daemonize(conf, logger, capture_stdout=False, - capture_stderr=False) + # test console log + utils.capture_stdio(logger, capture_stdout=False, + capture_stderr=False) self.assert_(utils.sys.excepthook is not None) # when logging to console, stderr remains open self.assertEquals(utils.os.closed_fds, [0, 1]) diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index 1f81962ff3..2df6936a83 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -25,12 +25,12 @@ import unittest from getpass import getuser from shutil import rmtree from StringIO import StringIO +from collections import defaultdict from eventlet import sleep from swift.common import wsgi - class TestWSGI(unittest.TestCase): """ Tests for swift.common.wsgi """ @@ -72,5 +72,107 @@ class TestWSGI(unittest.TestCase): sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).subtype, 'html') + def test_get_socket(self): + # stubs + conf = {} + ssl_conf = { + 'cert_file': '', + 'key_file': '', + } + + # mocks + class MockSocket(): + def __init__(self): + self.opts = defaultdict(dict) + + def setsockopt(self, level, optname, value): + self.opts[level][optname] = value + + def mock_listen(*args, **kwargs): + return MockSocket() + + class MockSsl(): + def __init__(self): + self.wrap_socket_called = [] + + def wrap_socket(self, sock, **kwargs): + self.wrap_socket_called.append(kwargs) + return sock + + # patch + old_listen = wsgi.listen + old_ssl = wsgi.ssl + try: + wsgi.listen = mock_listen + wsgi.ssl = MockSsl() + # test + sock = wsgi.get_socket(conf) + # assert + self.assert_(isinstance(sock, MockSocket)) + expected_socket_opts = { + socket.SOL_SOCKET: { + socket.SO_REUSEADDR: 1, + socket.SO_KEEPALIVE: 1, + }, + socket.IPPROTO_TCP: { + socket.TCP_KEEPIDLE: 600, + }, + } + self.assertEquals(sock.opts, expected_socket_opts) + # test ssl + sock = wsgi.get_socket(ssl_conf) + expected_kwargs = { + 'certfile': '', + 'keyfile': '', + } + self.assertEquals(wsgi.ssl.wrap_socket_called, [expected_kwargs]) + finally: + wsgi.listen = old_listen + wsgi.ssl = old_ssl + + def test_address_in_use(self): + # stubs + conf = {} + + # mocks + def mock_listen(*args, **kwargs): + raise socket.error(errno.EADDRINUSE) + + def value_error_listen(*args, **kwargs): + raise ValueError('fake') + + def mock_sleep(*args): + pass + + class MockTime(): + """Fast clock advances 10 seconds after every call to time + """ + def __init__(self): + self.current_time = old_time.time() + + def time(self, *args, **kwargs): + rv = self.current_time + # advance for next call + self.current_time += 10 + return rv + + old_listen = wsgi.listen + old_sleep = wsgi.sleep + old_time = wsgi.time + try: + wsgi.listen = mock_listen + wsgi.sleep = mock_sleep + wsgi.time = MockTime() + # test error + self.assertRaises(Exception, wsgi.get_socket, conf) + # different error + wsgi.listen = value_error_listen + self.assertRaises(ValueError, wsgi.get_socket, conf) + finally: + wsgi.listen = old_listen + wsgi.sleep = old_sleep + wsgi.time = old_time + + if __name__ == '__main__': unittest.main() From ab53587796e15313048bd7a3bcbde2368eaea30e Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 19 Nov 2010 16:20:17 -0600 Subject: [PATCH 7/8] fixed some calls to get_logger that didn't clean up after themselves --- test/unit/common/test_utils.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 2e9d4b0005..c41d147f79 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -79,14 +79,6 @@ class TestUtils(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' - self.logger = logging.getLogger() - self.starting_handlers = list(self.logger.handlers) - - def tearDown(self): - # don't let extra handlers pile up redirecting stdio and other stuff... - for handler in self.logger.handlers: - if handler not in self.starting_handlers: - self.logger.removeHandler(handler) def test_normalize_timestamp(self): """ Test swift.common.utils.normalize_timestamp """ @@ -178,7 +170,6 @@ class TestUtils(unittest.TestCase): self.assertEquals(sio.getvalue(), '') def test_LoggerFileObject(self): - reload(sys) # reset stdio redirection orig_stdout = sys.stdout orig_stderr = sys.stderr sio = StringIO() @@ -436,13 +427,11 @@ log_name = yarr''' self.assert_(utils.sys.excepthook is not None) # when logging to console, stderr remains open self.assertEquals(utils.os.closed_fds, [0, 1]) + logger.logger.removeHandler(utils.get_logger.console) # stdio not captured self.assertFalse(hasattr(utils.sys, 'stdout')) self.assertFalse(hasattr(utils.sys, 'stderr')) - # reset mocks on utils - reload(utils) - def test_get_logger_console(self): reload(utils) # reset get_logger attrs logger = utils.get_logger(None) @@ -455,6 +444,7 @@ log_name = yarr''' old_handler = utils.get_logger.console logger = utils.get_logger(None, log_to_console=True) self.assertNotEquals(utils.get_logger.console, old_handler) + logger.logger.removeHandler(utils.get_logger.console) if __name__ == '__main__': unittest.main() From 8197c4d7bae40bce9241edd3a1ab62f36e5424c9 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 19 Nov 2010 16:31:11 -0600 Subject: [PATCH 8/8] remove capture_io stuff from db_replicator.__init__ --- swift/common/db_replicator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 81396d1f8f..8519e7128f 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -93,10 +93,6 @@ class Replicator(Daemon): def __init__(self, conf): self.conf = conf self.logger = get_logger(conf) - # log uncaught exceptions - sys.excepthook = lambda * exc_info: \ - self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) - sys.stdout = sys.stderr = LoggerFileObject(self.logger) self.root = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y')