updated daemonize process, added option for servers/daemons to log to console

This commit is contained in:
Clay Gerrard 2010-11-19 22:52:31 +00:00 committed by Tarmac
commit 111ebb5a09
26 changed files with 651 additions and 222 deletions

View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.account.auditor import AccountAuditor from swift.account.auditor import AccountAuditor
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: swift-account-auditor CONFIG_FILE [once]" run_daemon(AccountAuditor, conf_file, **options)
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-auditor')
auditor = AccountAuditor(conf).run(once)

View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.account.reaper import AccountReaper from swift.account.reaper import AccountReaper
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: account-reaper CONFIG_FILE [once]" run_daemon(AccountReaper, conf_file, **options)
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-reaper')
reaper = AccountReaper(conf).run(once)

View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.common import utils
from swift.account.replicator import AccountReplicator from swift.account.replicator import AccountReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: swift-account-replicator CONFIG_FILE [once]" run_daemon(AccountReplicator, conf_file, **options)
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-replicator')
AccountReplicator(conf).run(once)

View File

@ -14,12 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) != 2: conf_file, options = parse_options()
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) run_wsgi(conf_file, 'account-server', default_port=6002, **options)
run_wsgi(sys.argv[1], 'account-server', default_port=6002)

View File

@ -14,14 +14,13 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.stats.account_stats import AccountStat from swift.stats.account_stats import AccountStat
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options()
print "Usage: swift-account-stats-logger CONFIG_FILE" # currently AccountStat only supports run_once
sys.exit() options['once'] = True
stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats') run_daemon(AccountStat, conf_file, section_name='log-processor-stats',
stats = AccountStat(stats_conf).run(once=True) **options)

View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) != 2: conf_file, options = parse_options()
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) run_wsgi(conf_file, 'auth-server', default_port=11000, **options)
run_wsgi(sys.argv[1], 'auth-server', default_port=11000)

View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.container.auditor import ContainerAuditor from swift.container.auditor import ContainerAuditor
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: swift-container-auditor CONFIG_FILE [once]" run_daemon(ContainerAuditor, conf_file, **options)
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-auditor')
ContainerAuditor(conf).run(once)

View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.common import db, utils
from swift.container.replicator import ContainerReplicator from swift.container.replicator import ContainerReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: swift-container-replicator CONFIG_FILE [once]" run_daemon(ContainerReplicator, conf_file, **options)
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-replicator')
ContainerReplicator(conf).run(once)

View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) != 2: conf_file, options = parse_options()
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) run_wsgi(conf_file, 'container-server', default_port=6001, **options)
run_wsgi(sys.argv[1], 'container-server', default_port=6001)

View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.container.updater import ContainerUpdater from swift.container.updater import ContainerUpdater
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: swift-container-updater CONFIG_FILE [once]" run_daemon(ContainerUpdater, conf_file, **options)
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-updater')
ContainerUpdater(conf).run(once)

View File

@ -14,14 +14,13 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.stats.log_processor import LogProcessorDaemon from swift.stats.log_processor import LogProcessorDaemon
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options()
print "Usage: swift-log-stats-collector CONFIG_FILE" # currently the LogProcessorDaemon only supports run_once
sys.exit() options['once'] = True
conf = utils.readconf(sys.argv[1], log_name='log-stats-collector') run_daemon(LogProcessorDaemon, conf_file, section_name=None,
stats = LogProcessorDaemon(conf).run(once=True) log_name='log-stats-collector', **options)

View File

@ -17,15 +17,25 @@
import sys import sys
from swift.stats.log_uploader import LogUploader from swift.stats.log_uploader import LogUploader
from swift.common.utils import parse_options
from swift.common import utils from swift.common import utils
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 3: conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN")
print "Usage: swift-log-uploader CONFIG_FILE plugin" try:
sys.exit() plugin = options['extra_args'][0]
uploader_conf = utils.readconf(sys.argv[1], 'log-processor') except IndexError:
plugin = sys.argv[2] print "Error: missing plugin name"
sys.exit(1)
uploader_conf = utils.readconf(conf_file, 'log-processor')
section_name = 'log-processor-%s' % plugin section_name = 'log-processor-%s' % plugin
plugin_conf = utils.readconf(sys.argv[1], section_name) plugin_conf = utils.readconf(conf_file, section_name)
uploader_conf.update(plugin_conf) uploader_conf.update(plugin_conf)
uploader = LogUploader(uploader_conf, plugin).run(once=True)
# pre-configure logger
logger = utils.get_logger(uploader_conf, plugin,
log_to_console=options.get('verbose', False))
# currently LogUploader only supports run_once
options['once'] = True
uploader = LogUploader(uploader_conf, plugin).run(**options)

View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.obj.auditor import ObjectAuditor from swift.obj.auditor import ObjectAuditor
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: swift-object-auditor CONFIG_FILE [once]" run_daemon(ObjectAuditor, conf_file, **options)
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'object-auditor')
ObjectAuditor(conf).run(once)

View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.obj.replicator import ObjectReplicator from swift.obj.replicator import ObjectReplicator
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: swift-object-replicator CONFIG_FILE [once]" run_daemon(ObjectReplicator, conf_file, **options)
sys.exit()
conf = utils.readconf(sys.argv[1], "object-replicator")
once = (len(sys.argv) > 2 and sys.argv[2] == 'once') or \
conf.get('daemonize', 'true') not in utils.TRUE_VALUES
ObjectReplicator(conf).run(once)

View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) != 2: conf_file, options = parse_options()
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) run_wsgi(conf_file, 'object-server', default_port=6000, **options)
run_wsgi(sys.argv[1], 'object-server', default_port=6000)

View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
from swift.obj.updater import ObjectUpdater from swift.obj.updater import ObjectUpdater
from swift.common import utils from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: conf_file, options = parse_options(once=True)
print "Usage: swift-object-updater CONFIG_FILE [once]" run_daemon(ObjectUpdater, conf_file, **options)
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'object-updater')
ObjectUpdater(conf).run(once)

View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi from swift.common.wsgi import run_wsgi
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) != 2: conf_file, options = parse_options()
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) run_wsgi(conf_file, 'proxy-server', default_port=8080, **options)
run_wsgi(sys.argv[1], 'proxy-server', default_port=8080)

View File

@ -16,6 +16,7 @@
import os import os
import sys import sys
import signal import signal
from re import sub
from swift.common import utils from swift.common import utils
@ -34,23 +35,11 @@ class Daemon(object):
"""Override this to run forever""" """Override this to run forever"""
raise NotImplementedError('run_forever not implemented') raise NotImplementedError('run_forever not implemented')
def run(self, once=False, capture_stdout=True, capture_stderr=True): def run(self, once=False, **kwargs):
"""Run the daemon""" """Run the daemon"""
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
if capture_stdout:
sys.stdout = utils.LoggerFileObject(self.logger)
if capture_stderr:
sys.stderr = utils.LoggerFileObject(self.logger)
utils.drop_privileges(self.conf.get('user', 'swift'))
utils.validate_configuration() utils.validate_configuration()
utils.capture_stdio(self.logger, **kwargs)
try: utils.drop_privileges(self.conf.get('user', 'swift'))
os.setsid()
except OSError:
pass
def kill_children(*args): def kill_children(*args):
signal.signal(signal.SIGTERM, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN)
@ -63,3 +52,40 @@ class Daemon(object):
self.run_once() self.run_once()
else: else:
self.run_forever() self.run_forever()
def run_daemon(klass, conf_file, section_name='',
once=False, **kwargs):
"""
Loads settings from conf, then instantiates daemon "klass" and runs the
daemon with the specified once kwarg. The section_name will be derived
from the daemon "klass" if not provided (e.g. ObjectReplicator =>
object-replicator).
:param klass: Class to instantiate, subclass of common.daemon.Daemon
:param conf_file: Path to configuration file
:param section_name: Section name from conf file to load config from
:param once: Passed to daemon run method
"""
# very often the config section_name is based on the class name
# the None singleton will be passed through to readconf as is
if section_name is '':
section_name = sub(r'([a-z])([A-Z])', r'\1-\2',
klass.__name__).lower()
conf = utils.readconf(conf_file, section_name,
log_name=kwargs.get('log_name'))
# once on command line (i.e. daemonize=false) will over-ride config
once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES
# pre-configure logger
if 'logger' in kwargs:
logger = kwargs.pop('logger')
else:
logger = utils.get_logger(conf, conf.get('log_name', section_name),
log_to_console=kwargs.pop('verbose', False))
try:
klass(conf).run(once=once, **kwargs)
except KeyboardInterrupt:
logger.info('User quit')
logger.info('Exited')

View File

@ -93,10 +93,6 @@ class Replicator(Daemon):
def __init__(self, conf): def __init__(self, conf):
self.conf = conf self.conf = conf
self.logger = get_logger(conf) self.logger = get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = LoggerFileObject(self.logger)
self.root = conf.get('devices', '/srv/node') self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \ self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y') ('true', 't', '1', 'on', 'yes', 'y')

View File

@ -31,6 +31,7 @@ import ctypes
import ctypes.util import ctypes.util
import struct import struct
from ConfigParser import ConfigParser, NoSectionError, NoOptionError from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from optparse import OptionParser
from tempfile import mkstemp from tempfile import mkstemp
import cPickle as pickle import cPickle as pickle
@ -283,17 +284,6 @@ class LoggerFileObject(object):
return self return self
def drop_privileges(user):
"""
Sets the userid of the current process
:param user: User id to change privileges to
"""
user = pwd.getpwnam(user)
os.setgid(user[3])
os.setuid(user[2])
class NamedLogger(object): class NamedLogger(object):
"""Cheesy version of the LoggerAdapter available in Python 3""" """Cheesy version of the LoggerAdapter available in Python 3"""
@ -343,7 +333,7 @@ class NamedLogger(object):
call('%s %s: %s' % (self.server, msg, emsg), *args) call('%s %s: %s' % (self.server, msg, emsg), *args)
def get_logger(conf, name=None): def get_logger(conf, name=None, log_to_console=False):
""" """
Get the current system logger using config settings. Get the current system logger using config settings.
@ -355,11 +345,18 @@ def get_logger(conf, name=None):
:param conf: Configuration dict to read settings from :param conf: Configuration dict to read settings from
:param name: Name of the logger :param name: Name of the logger
:param log_to_console: Add handler which writes to console on stderr
""" """
root_logger = logging.getLogger() root_logger = logging.getLogger()
if hasattr(get_logger, 'handler') and get_logger.handler: if hasattr(get_logger, 'handler') and get_logger.handler:
root_logger.removeHandler(get_logger.handler) root_logger.removeHandler(get_logger.handler)
get_logger.handler = None get_logger.handler = None
if log_to_console:
# check if a previous call to get_logger already added a console logger
if hasattr(get_logger, 'console') and get_logger.console:
root_logger.removeHandler(get_logger.console)
get_logger.console = logging.StreamHandler(sys.__stderr__)
root_logger.addHandler(get_logger.console)
if conf is None: if conf is None:
root_logger.setLevel(logging.INFO) root_logger.setLevel(logging.INFO)
return NamedLogger(root_logger, name) return NamedLogger(root_logger, name)
@ -375,6 +372,99 @@ def get_logger(conf, name=None):
return NamedLogger(root_logger, name) return NamedLogger(root_logger, name)
def drop_privileges(user):
"""
Sets the userid/groupid of the current process, get session leader, etc.
:param user: User name to change privileges to
"""
user = pwd.getpwnam(user)
os.setgid(user[3])
os.setuid(user[2])
try:
os.setsid()
except OSError:
pass
os.chdir('/') # in case you need to rmdir on where you started the daemon
os.umask(0) # ensure files are created with the correct privileges
def capture_stdio(logger, **kwargs):
"""
Log unhandled exceptions, close stdio, capture stdout and stderr.
param logger: Logger object to use
"""
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
# collect stdio file desc not in use for logging
stdio_fds = [0, 1, 2]
if hasattr(get_logger, 'console'):
stdio_fds.remove(get_logger.console.stream.fileno())
with open(os.devnull, 'r+b') as nullfile:
# close stdio (excludes fds open for logging)
for desc in stdio_fds:
try:
os.dup2(nullfile.fileno(), desc)
except OSError:
pass
# redirect stdio
if kwargs.pop('capture_stdout', True):
sys.stdout = LoggerFileObject(logger)
if kwargs.pop('capture_stderr', True):
sys.stderr = LoggerFileObject(logger)
def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None):
"""
Parse standard swift server/daemon options with optparse.OptionParser.
:param usage: String describing usage
:param once: Boolean indicating the "once" option is available
:param test_args: Override sys.argv; used in testing
:returns : Tuple of (config, options); config is an absolute path to the
config file, options is the parser options as a dictionary.
:raises SystemExit: First arg (CONFIG) is required, file must exist
"""
parser = OptionParser(usage)
parser.add_option("-v", "--verbose", default=False, action="store_true",
help="log to console")
if once:
parser.add_option("-o", "--once", default=False, action="store_true",
help="only run one pass of daemon")
# if test_args is None, optparse will use sys.argv[:1]
options, args = parser.parse_args(args=test_args)
if not args:
parser.print_usage()
print "Error: missing config file argument"
sys.exit(1)
config = os.path.abspath(args.pop(0))
if not os.path.exists(config):
parser.print_usage()
print "Error: unable to locate %s" % config
sys.exit(1)
extra_args = []
# if any named options appear in remaining args, set the option to True
for arg in args:
if arg in options.__dict__:
setattr(options, arg, True)
else:
extra_args.append(arg)
options = vars(options)
options['extra_args'] = extra_args
return config, options
def whataremyips(): def whataremyips():
""" """
Get the machine's ip addresses using ifconfig Get the machine's ip addresses using ifconfig

View File

@ -34,7 +34,7 @@ wsgi.ACCEPT_ERRNO.add(ECONNRESET)
from eventlet.green import socket, ssl from eventlet.green import socket, ssl
from swift.common.utils import get_logger, drop_privileges, \ from swift.common.utils import get_logger, drop_privileges, \
validate_configuration, LoggerFileObject, NullLogger validate_configuration, capture_stdio, NullLogger
def monkey_patch_mimetools(): def monkey_patch_mimetools():
@ -56,41 +56,17 @@ def monkey_patch_mimetools():
mimetools.Message.parsetype = parsetype mimetools.Message.parsetype = parsetype
def get_socket(conf, default_port=8080):
"""Bind socket to bind ip:port in conf
# We might be able to pull pieces of this out to test, but right now it seems :param conf: Configuration dict to read settings from
# like more work than it's worth. :param default_port: port to use if not specified in conf
def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
:returns : a socket object as returned from socket.listen or ssl.wrap_socket
if conf specifies cert_file
""" """
Loads common settings from conf, then instantiates app and runs
the server using the specified number of workers.
:param conf_file: Path to paste.deploy style configuration file
:param app_section: App name from conf file to load config from
"""
try:
conf = appconfig('config:%s' % conf_file, name=app_section)
log_name = conf.get('log_name', app_section)
app = loadapp('config:%s' % conf_file,
global_conf={'log_name': log_name})
except Exception, e:
print "Error trying to load config %s: %s" % (conf_file, e)
return
if 'logger' in kwargs:
logger = kwargs['logger']
else:
logger = get_logger(conf, log_name)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = LoggerFileObject(logger)
try:
os.setsid()
except OSError:
no_cover = True # pass
bind_addr = (conf.get('bind_ip', '0.0.0.0'), bind_addr = (conf.get('bind_ip', '0.0.0.0'),
int(conf.get('bind_port', kwargs.get('default_port', 8080)))) int(conf.get('bind_port', default_port)))
sock = None sock = None
retry_until = time.time() + 30 retry_until = time.time() + 30
while not sock and time.time() < retry_until: while not sock and time.time() < retry_until:
@ -110,10 +86,44 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
# in my experience, sockets can hang around forever without keepalive # in my experience, sockets can hang around forever without keepalive
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600)
worker_count = int(conf.get('workers', '1')) return sock
drop_privileges(conf.get('user', 'swift'))
# TODO: pull pieces of this out to test
def run_wsgi(conf_file, app_section, *args, **kwargs):
"""
Loads common settings from conf, then instantiates app and runs
the server using the specified number of workers.
:param conf_file: Path to paste.deploy style configuration file
:param app_section: App name from conf file to load config from
"""
try:
conf = appconfig('config:%s' % conf_file, name=app_section)
except Exception, e:
print "Error trying to load config %s: %s" % (conf_file, e)
return
validate_configuration() validate_configuration()
# pre-configure logger
log_name = conf.get('log_name', app_section)
if 'logger' in kwargs:
logger = kwargs.pop('logger')
else:
logger = get_logger(conf, log_name,
log_to_console=kwargs.pop('verbose', False))
# redirect errors to logger and close stdio
capture_stdio(logger)
# bind to address and port
sock = get_socket(conf, default_port=kwargs.get('default_port', 8080))
# remaining tasks should not require elevated privileges
drop_privileges(conf.get('user', 'swift'))
# finally after binding to ports and privilege drop, run app __init__ code
app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})
def run_server(): def run_server():
wsgi.HttpProtocol.default_request_version = "HTTP/1.0" wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
eventlet.hubs.use_hub('poll') eventlet.hubs.use_hub('poll')
@ -127,6 +137,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
raise raise
pool.waitall() pool.waitall()
worker_count = int(conf.get('workers', '1'))
# Useful for profiling [no forks]. # Useful for profiling [no forks].
if worker_count == 0: if worker_count == 0:
run_server() run_server()
@ -169,6 +180,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
except OSError, err: except OSError, err:
if err.errno not in (errno.EINTR, errno.ECHILD): if err.errno not in (errno.EINTR, errno.ECHILD):
raise raise
except KeyboardInterrupt:
logger.info('User quit')
break
greenio.shutdown_safe(sock) greenio.shutdown_safe(sock)
sock.close() sock.close()
logger.info('Exited') logger.info('Exited')

View File

@ -1,5 +1,8 @@
""" Swift tests """ """ Swift tests """
import os
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from eventlet.green import socket from eventlet.green import socket
@ -23,6 +26,18 @@ def connect_tcp(hostport):
rv.connect(hostport) rv.connect(hostport)
return rv return rv
@contextmanager
def tmpfile(content):
with NamedTemporaryFile('w', delete=False) as f:
file_name = f.name
f.write(str(content))
try:
yield file_name
finally:
os.unlink(file_name)
class MockTrue(object): class MockTrue(object):
""" """
Instances of MockTrue evaluate like True Instances of MockTrue evaluate like True

View File

@ -13,16 +13,95 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# TODO: Tests # TODO: Test kill_children signal handlers
import unittest import unittest
from swift.common import daemon from getpass import getuser
import logging
from StringIO import StringIO
from test.unit import tmpfile
from swift.common import daemon, utils
class MyDaemon(daemon.Daemon):
def __init__(self, conf):
self.conf = conf
self.logger = utils.get_logger(None)
MyDaemon.forever_called = False
MyDaemon.once_called = False
def run_forever(self):
MyDaemon.forever_called = True
def run_once(self):
MyDaemon.once_called = True
def run_raise(self):
raise OSError
def run_quit(self):
raise KeyboardInterrupt
class TestDaemon(unittest.TestCase): class TestDaemon(unittest.TestCase):
def test_placeholder(self): def test_create(self):
pass d = daemon.Daemon({})
self.assertEquals(d.conf, {})
self.assert_(isinstance(d.logger, utils.NamedLogger))
def test_stubs(self):
d = daemon.Daemon({})
self.assertRaises(NotImplementedError, d.run_once)
self.assertRaises(NotImplementedError, d.run_forever)
class TestRunDaemon(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.drop_privileges = lambda *args: None
utils.capture_stdio = lambda *args: None
def tearDown(self):
reload(utils)
def test_run(self):
d = MyDaemon({})
self.assertFalse(MyDaemon.forever_called)
self.assertFalse(MyDaemon.once_called)
# test default
d.run()
self.assertEquals(d.forever_called, True)
# test once
d.run(once=True)
self.assertEquals(d.once_called, True)
def test_run_daemon(self):
sample_conf = """[my-daemon]
user = %s
""" % getuser()
with tmpfile(sample_conf) as conf_file:
daemon.run_daemon(MyDaemon, conf_file)
self.assertEquals(MyDaemon.forever_called, True)
daemon.run_daemon(MyDaemon, conf_file, once=True)
self.assertEquals(MyDaemon.once_called, True)
# test raise in daemon code
MyDaemon.run_once = MyDaemon.run_raise
self.assertRaises(OSError, daemon.run_daemon, MyDaemon,
conf_file, once=True)
# test user quit
MyDaemon.run_forever = MyDaemon.run_quit
sio = StringIO()
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(sio))
logger = utils.get_logger(None, 'server')
daemon.run_daemon(MyDaemon, conf_file, logger=logger)
self.assert_('user quit' in sio.getvalue().lower())
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -25,12 +25,55 @@ import unittest
from getpass import getuser from getpass import getuser
from shutil import rmtree from shutil import rmtree
from StringIO import StringIO from StringIO import StringIO
from functools import partial
from tempfile import NamedTemporaryFile
from eventlet import sleep from eventlet import sleep
from swift.common import utils from swift.common import utils
class MockOs():
def __init__(self, pass_funcs=[], called_funcs=[], raise_funcs=[]):
self.closed_fds = []
for func in pass_funcs:
setattr(self, func, self.pass_func)
self.called_funcs = {}
for func in called_funcs:
c_func = partial(self.called_func, func)
setattr(self, func, c_func)
for func in raise_funcs:
r_func = partial(self.raise_func, func)
setattr(self, func, r_func)
def pass_func(self, *args, **kwargs):
pass
chdir = setsid = setgid = setuid = umask = pass_func
def called_func(self, name, *args, **kwargs):
self.called_funcs[name] = True
def raise_func(self, name, *args, **kwargs):
self.called_funcs[name] = True
raise OSError()
def dup2(self, source, target):
self.closed_fds.append(target)
def __getattr__(self, name):
# I only over-ride portions of the os module
try:
return object.__getattr__(self, name)
except AttributeError:
return getattr(os, name)
class MockSys():
__stderr__ = sys.__stderr__
class TestUtils(unittest.TestCase): class TestUtils(unittest.TestCase):
""" Tests for swift.common.utils """ """ Tests for swift.common.utils """
@ -182,10 +225,63 @@ class TestUtils(unittest.TestCase):
self.assertRaises(IOError, lfo.readline, 1024) self.assertRaises(IOError, lfo.readline, 1024)
lfo.tell() lfo.tell()
def test_drop_privileges(self): def test_parse_options(self):
# Note that this doesn't really drop privileges as it just sets them to # use mkstemp to get a file that is definately on disk
# what they already are; but it exercises the code at least. with NamedTemporaryFile() as f:
utils.drop_privileges(getuser()) conf_file = f.name
conf, options = utils.parse_options(test_args=[conf_file])
self.assertEquals(conf, conf_file)
# assert defaults
self.assertEquals(options['verbose'], False)
self.assert_('once' not in options)
# assert verbose as option
conf, options = utils.parse_options(test_args=[conf_file, '-v'])
self.assertEquals(options['verbose'], True)
# check once option
conf, options = utils.parse_options(test_args=[conf_file],
once=True)
self.assertEquals(options['once'], False)
test_args = [conf_file, '--once']
conf, options = utils.parse_options(test_args=test_args, once=True)
self.assertEquals(options['once'], True)
# check options as arg parsing
test_args = [conf_file, 'once', 'plugin_name', 'verbose']
conf, options = utils.parse_options(test_args=test_args, once=True)
self.assertEquals(options['verbose'], True)
self.assertEquals(options['once'], True)
self.assertEquals(options['extra_args'], ['plugin_name'])
def test_parse_options_errors(self):
orig_stdout = sys.stdout
orig_stderr = sys.stderr
stdo = StringIO()
stde = StringIO()
utils.sys.stdout = stdo
utils.sys.stderr = stde
err_msg = """Usage: test usage
Error: missing config file argument
"""
test_args = []
self.assertRaises(SystemExit, utils.parse_options, 'test usage', True,
test_args)
self.assertEquals(stdo.getvalue(), err_msg)
# verify conf file must exist, context manager will delete temp file
with NamedTemporaryFile() as f:
conf_file = f.name
err_msg += """Usage: test usage
Error: unable to locate %s
""" % conf_file
test_args = [conf_file]
self.assertRaises(SystemExit, utils.parse_options, 'test usage', True,
test_args)
self.assertEquals(stdo.getvalue(), err_msg)
# reset stdio
utils.sys.stdout = orig_stdout
utils.sys.stderr = orig_stderr
def test_NamedLogger(self): def test_NamedLogger(self):
sio = StringIO() sio = StringIO()
@ -275,5 +371,80 @@ log_name = yarr'''
self.assertEquals(result, expected) self.assertEquals(result, expected)
os.unlink('/tmp/test') os.unlink('/tmp/test')
def test_drop_privileges(self):
user = getuser()
# over-ride os with mock
required_func_calls = ('setgid', 'setuid', 'setsid', 'chdir', 'umask')
utils.os = MockOs(called_funcs=required_func_calls)
# exercise the code
utils.drop_privileges(user)
for func in required_func_calls:
self.assert_(utils.os.called_funcs[func])
# reset; test same args, OSError trying to get session leader
utils.os = MockOs(called_funcs=required_func_calls,
raise_funcs=('setsid',))
for func in required_func_calls:
self.assertFalse(utils.os.called_funcs.get(func, False))
utils.drop_privileges(user)
for func in required_func_calls:
self.assert_(utils.os.called_funcs[func])
def test_capture_stdio(self):
# stubs
logger = utils.get_logger(None, 'dummy')
# mock utils system modules
utils.sys = MockSys()
utils.os = MockOs()
# basic test
utils.capture_stdio(logger)
self.assert_(utils.sys.excepthook is not None)
self.assertEquals(utils.os.closed_fds, [0, 1, 2])
self.assert_(utils.sys.stdout is not None)
self.assert_(utils.sys.stderr is not None)
# reset; test same args, but exc when trying to close stdio
utils.os = MockOs(raise_funcs=('dup2',))
utils.sys = MockSys()
# test unable to close stdio
utils.capture_stdio(logger)
self.assert_(utils.sys.excepthook is not None)
self.assertEquals(utils.os.closed_fds, [])
self.assert_(utils.sys.stdout is not None)
self.assert_(utils.sys.stderr is not None)
# reset; test some other args
logger = utils.get_logger(None, log_to_console=True)
utils.os = MockOs()
utils.sys = MockSys()
# test console log
utils.capture_stdio(logger, capture_stdout=False,
capture_stderr=False)
self.assert_(utils.sys.excepthook is not None)
# when logging to console, stderr remains open
self.assertEquals(utils.os.closed_fds, [0, 1])
logger.logger.removeHandler(utils.get_logger.console)
# stdio not captured
self.assertFalse(hasattr(utils.sys, 'stdout'))
self.assertFalse(hasattr(utils.sys, 'stderr'))
def test_get_logger_console(self):
reload(utils) # reset get_logger attrs
logger = utils.get_logger(None)
self.assertFalse(hasattr(utils.get_logger, 'console'))
logger = utils.get_logger(None, log_to_console=True)
self.assert_(hasattr(utils.get_logger, 'console'))
self.assert_(isinstance(utils.get_logger.console,
logging.StreamHandler))
# make sure you can't have two console handlers
old_handler = utils.get_logger.console
logger = utils.get_logger(None, log_to_console=True)
self.assertNotEquals(utils.get_logger.console, old_handler)
logger.logger.removeHandler(utils.get_logger.console)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@ -25,12 +25,12 @@ import unittest
from getpass import getuser from getpass import getuser
from shutil import rmtree from shutil import rmtree
from StringIO import StringIO from StringIO import StringIO
from collections import defaultdict
from eventlet import sleep from eventlet import sleep
from swift.common import wsgi from swift.common import wsgi
class TestWSGI(unittest.TestCase): class TestWSGI(unittest.TestCase):
""" Tests for swift.common.wsgi """ """ Tests for swift.common.wsgi """
@ -72,5 +72,107 @@ class TestWSGI(unittest.TestCase):
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEquals(mimetools.Message(sio).subtype, 'html') self.assertEquals(mimetools.Message(sio).subtype, 'html')
def test_get_socket(self):
# stubs
conf = {}
ssl_conf = {
'cert_file': '',
'key_file': '',
}
# mocks
class MockSocket():
def __init__(self):
self.opts = defaultdict(dict)
def setsockopt(self, level, optname, value):
self.opts[level][optname] = value
def mock_listen(*args, **kwargs):
return MockSocket()
class MockSsl():
def __init__(self):
self.wrap_socket_called = []
def wrap_socket(self, sock, **kwargs):
self.wrap_socket_called.append(kwargs)
return sock
# patch
old_listen = wsgi.listen
old_ssl = wsgi.ssl
try:
wsgi.listen = mock_listen
wsgi.ssl = MockSsl()
# test
sock = wsgi.get_socket(conf)
# assert
self.assert_(isinstance(sock, MockSocket))
expected_socket_opts = {
socket.SOL_SOCKET: {
socket.SO_REUSEADDR: 1,
socket.SO_KEEPALIVE: 1,
},
socket.IPPROTO_TCP: {
socket.TCP_KEEPIDLE: 600,
},
}
self.assertEquals(sock.opts, expected_socket_opts)
# test ssl
sock = wsgi.get_socket(ssl_conf)
expected_kwargs = {
'certfile': '',
'keyfile': '',
}
self.assertEquals(wsgi.ssl.wrap_socket_called, [expected_kwargs])
finally:
wsgi.listen = old_listen
wsgi.ssl = old_ssl
def test_address_in_use(self):
# stubs
conf = {}
# mocks
def mock_listen(*args, **kwargs):
raise socket.error(errno.EADDRINUSE)
def value_error_listen(*args, **kwargs):
raise ValueError('fake')
def mock_sleep(*args):
pass
class MockTime():
"""Fast clock advances 10 seconds after every call to time
"""
def __init__(self):
self.current_time = old_time.time()
def time(self, *args, **kwargs):
rv = self.current_time
# advance for next call
self.current_time += 10
return rv
old_listen = wsgi.listen
old_sleep = wsgi.sleep
old_time = wsgi.time
try:
wsgi.listen = mock_listen
wsgi.sleep = mock_sleep
wsgi.time = MockTime()
# test error
self.assertRaises(Exception, wsgi.get_socket, conf)
# different error
wsgi.listen = value_error_listen
self.assertRaises(ValueError, wsgi.get_socket, conf)
finally:
wsgi.listen = old_listen
wsgi.sleep = old_sleep
wsgi.time = old_time
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@ -14,25 +14,12 @@
# limitations under the License. # limitations under the License.
import unittest import unittest
import os from test.unit import tmpfile
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from swift.common import internal_proxy from swift.common import internal_proxy
from swift.stats import log_processor from swift.stats import log_processor
@contextmanager
def tmpfile(content):
with NamedTemporaryFile('w', delete=False) as f:
file_name = f.name
f.write(str(content))
try:
yield file_name
finally:
os.unlink(file_name)
class FakeUploadApp(object): class FakeUploadApp(object):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
pass pass