openstack/swift
Code Issues Proposed changes

added helper/util to parse command line args; removed some duplicated code in

Browse Source 
server/daemon bin scripts;  more standized python/linux daemonization
procedures; fixed lp:666957 "devauth server creates auth.db with the wrong
privileges"; new run_daemon helper based on run_wsgi simplifies daemon
launching/testing; new - all servers/daemons support verbose option when
started interactivlty which will log to the console; fixed lp:667839 "can't
start servers with relative paths to configs"; added tests
This commit is contained in:
Clay Gerrard 2010-11-11 16:41:07 -06:00
parent ce0a0be664
commit 57a35f0d7c
24 changed files with 535 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/swift-account-auditor
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.account.auditor import AccountAuditor
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-account-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-auditor')
auditor = AccountAuditor(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(AccountAuditor, conf_file, **options)

13
bin/swift-account-reaper
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.account.reaper import AccountReaper
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: account-reaper CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-reaper')
reaper = AccountReaper(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(AccountReaper, conf_file, **options)

13
bin/swift-account-replicator
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common import utils
from swift.account.replicator import AccountReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-account-replicator CONFIG_FILE [once]"
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'account-replicator')
AccountReplicator(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(AccountReplicator, conf_file, **options)

9
bin/swift-account-server
View File

@ -14,12 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'account-server', default_port=6002)
conf_file, options = parse_options()
run_wsgi(conf_file, 'account-server', default_port=6002, **options)

15
bin/swift-account-stats-logger
View File

@ -14,14 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.stats.account_stats import AccountStat
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-account-stats-logger CONFIG_FILE"
sys.exit()
stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats')
stats = AccountStat(stats_conf).run(once=True)
conf_file, options = parse_options()
# currently AccountStat only supports run_once
options['once'] = True
run_daemon(AccountStat, conf_file, section_name='log-processor-stats',
**options)

8
bin/swift-auth-server
View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'auth-server', default_port=11000)
conf_file, options = parse_options()
run_wsgi(conf_file, 'auth-server', default_port=11000, **options)

13
bin/swift-container-auditor
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.container.auditor import ContainerAuditor
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-container-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-auditor')
ContainerAuditor(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ContainerAuditor, conf_file, **options)

14
bin/swift-container-replicator
View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common import db, utils
from swift.container.replicator import ContainerReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-container-replicator CONFIG_FILE [once]"
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-replicator')
ContainerReplicator(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ContainerReplicator, conf_file, **options)

8
bin/swift-container-server
View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'container-server', default_port=6001)
conf_file, options = parse_options()
run_wsgi(conf_file, 'container-server', default_port=6001, **options)

13
bin/swift-container-updater
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.container.updater import ContainerUpdater
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-container-updater CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'container-updater')
ContainerUpdater(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ContainerUpdater, conf_file, **options)

15
bin/swift-log-stats-collector
View File

@ -14,14 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.stats.log_processor import LogProcessorDaemon
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-log-stats-collector CONFIG_FILE"
sys.exit()
conf = utils.readconf(sys.argv[1], log_name='log-stats-collector')
stats = LogProcessorDaemon(conf).run(once=True)
conf_file, options = parse_options()
# currently the LogProcessorDaemon only supports run_once
options['once'] = True
run_daemon(LogProcessorDaemon, conf_file, section_name=None,
log_name='log-stats-collector', **options)

22
bin/swift-log-uploader
View File

@ -17,15 +17,25 @@
import sys
from swift.stats.log_uploader import LogUploader
from swift.common.utils import parse_options
from swift.common import utils
if __name__ == '__main__':
if len(sys.argv) < 3:
print "Usage: swift-log-uploader CONFIG_FILE plugin"
sys.exit()
uploader_conf = utils.readconf(sys.argv[1], 'log-processor')
plugin = sys.argv[2]
conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN")
try:
plugin = options['extra_args'][0]
except IndexError:
print "Error: missing plugin name"
sys.exit(1)
uploader_conf = utils.readconf(conf_file, 'log-processor')
section_name = 'log-processor-%s' % plugin
plugin_conf = utils.readconf(sys.argv[1], section_name)
plugin_conf = utils.readconf(conf_file, section_name)
uploader_conf.update(plugin_conf)
# pre-configure logger
logger = utils.get_logger(uploader_conf, plugin,
log_to_console=options.get('verbose', False))
# currently LogUploader only supports run_once
options['once'] = True
uploader = LogUploader(uploader_conf, plugin).run(once=True)

14
bin/swift-object-auditor
View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.obj.auditor import ObjectAuditor
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-object-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'object-auditor')
ObjectAuditor(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ObjectAuditor, conf_file, **options)

14
bin/swift-object-replicator
View File

@ -14,16 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.obj.replicator import ObjectReplicator
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-object-replicator CONFIG_FILE [once]"
sys.exit()
conf = utils.readconf(sys.argv[1], "object-replicator")
once = (len(sys.argv) > 2 and sys.argv[2] == 'once') or \
conf.get('daemonize', 'true') not in utils.TRUE_VALUES
ObjectReplicator(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ObjectReplicator, conf_file, **options)

8
bin/swift-object-server
View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'object-server', default_port=6000)
conf_file, options = parse_options()
run_wsgi(conf_file, 'object-server', default_port=6000, **options)

13
bin/swift-object-updater
View File

@ -14,15 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.obj.updater import ObjectUpdater
from swift.common import utils
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-object-updater CONFIG_FILE [once]"
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
conf = utils.readconf(sys.argv[1], 'object-updater')
ObjectUpdater(conf).run(once)
conf_file, options = parse_options(once=True)
run_daemon(ObjectUpdater, conf_file, **options)

8
bin/swift-proxy-server
View File

@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'proxy-server', default_port=8080)
conf_file, options = parse_options()
run_wsgi(conf_file, 'proxy-server', default_port=8080, **options)

55
swift/common/daemon.py
View File

@ -16,6 +16,7 @@
import os
import sys
import signal
from re import sub
from swift.common import utils
@ -34,23 +35,10 @@ class Daemon(object):
"""Override this to run forever"""
raise NotImplementedError('run_forever not implemented')
def run(self, once=False, capture_stdout=True, capture_stderr=True):
def run(self, once=False, **kwargs):
"""Run the daemon"""
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
if capture_stdout:
sys.stdout = utils.LoggerFileObject(self.logger)
if capture_stderr:
sys.stderr = utils.LoggerFileObject(self.logger)
utils.drop_privileges(self.conf.get('user', 'swift'))
utils.validate_configuration()
try:
os.setsid()
except OSError:
pass
utils.daemonize(self.conf, self.logger, **kwargs)
def kill_children(*args):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
@ -63,3 +51,40 @@ class Daemon(object):
self.run_once()
else:
self.run_forever()
def run_daemon(klass, conf_file, section_name='',
once=False, **kwargs):
"""
Loads settings from conf, then instantiates daemon "klass" and runs the
daemon with the specified once kwarg. The section_name will be derived
from the daemon "klass" if not provided (e.g. ObjectReplicator =>
object-replicator).
:param klass: Class to instantiate, subclass of common.daemon.Daemon
:param conf_file: Path to configuration file
:param section_name: Section name from conf file to load config from
:param once: Passed to daemon run method
"""
# very often the config section_name is based on the class name
# the None singleton will be passed through to readconf as is
if section_name is '':
section_name = sub(r'([a-z])([A-Z])', r'\1-\2',
klass.__name__).lower()
conf = utils.readconf(conf_file, section_name,
log_name=kwargs.get('log_name'))
# once on command line (i.e. daemonize=false) will over-ride config
once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES
# pre-configure logger
if 'logger' in kwargs:
logger = kwargs.pop('logger')
else:
logger = utils.get_logger(conf, conf.get('log_name', section_name),
log_to_console=kwargs.pop('verbose', False))
try:
klass(conf).run(once=once, **kwargs)
except KeyboardInterrupt:
logger.info('User quit')
logger.info('Exited')

126
swift/common/utils.py
View File

@ -32,6 +32,7 @@ import ctypes.util
import fcntl
import struct
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from optparse import OptionParser
from tempfile import mkstemp
import cPickle as pickle
@ -284,17 +285,6 @@ class LoggerFileObject(object):
return self
def drop_privileges(user):
"""
Sets the userid of the current process
:param user: User id to change privileges to
"""
user = pwd.getpwnam(user)
os.setgid(user[3])
os.setuid(user[2])
class NamedLogger(object):
"""Cheesy version of the LoggerAdapter available in Python 3"""
@ -344,7 +334,7 @@ class NamedLogger(object):
call('%s %s: %s' % (self.server, msg, emsg), *args)
def get_logger(conf, name=None):
def get_logger(conf, name=None, log_to_console=False):
"""
Get the current system logger using config settings.
@ -356,11 +346,18 @@ def get_logger(conf, name=None):
:param conf: Configuration dict to read settings from
:param name: Name of the logger
:param log_to_console: Add handler which writes to console on stderr
"""
root_logger = logging.getLogger()
if hasattr(get_logger, 'handler') and get_logger.handler:
root_logger.removeHandler(get_logger.handler)
get_logger.handler = None
if log_to_console:
# check if a previous call to get_logger already added a console logger
if hasattr(get_logger, 'console') and get_logger.console:
root_logger.removeHandler(get_logger.console)
get_logger.console = logging.StreamHandler(sys.__stderr__)
root_logger.addHandler(get_logger.console)
if conf is None:
root_logger.setLevel(logging.INFO)
return NamedLogger(root_logger, name)
@ -376,6 +373,111 @@ def get_logger(conf, name=None):
return NamedLogger(root_logger, name)
def drop_privileges(user):
"""
Sets the userid/groupid of the current process, get session leader, etc.
:param user: User name to change privileges to
"""
user = pwd.getpwnam(user)
os.setgid(user[3])
os.setuid(user[2])
try:
os.setsid()
except OSError:
pass
os.chdir('/') # in case you need to rmdir on where you started the daemon
os.umask(0) # ensure files are created with the correct privledges
def capture_stdio(logger, **kwargs):
"""
Log unhaneled exceptions, close stdio, capture stdout and stderr.
param logger: Logger object to use
"""
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
# collect stdio file desc not in use for logging
stdio_fds = [0, 1, 2]
if hasattr(get_logger, 'console'):
stdio_fds.remove(get_logger.console.stream.fileno())
with open(os.devnull, 'r+b') as nullfile:
# close stdio (excludes fds open for logging)
for desc in stdio_fds:
try:
os.dup2(nullfile.fileno(), desc)
except OSError:
pass
# redirect stdio
if kwargs.pop('capture_stdout', True):
sys.stdout = LoggerFileObject(logger)
if kwargs.pop('capture_stderr', True):
sys.stderr = LoggerFileObject(logger)
def daemonize(conf, logger, **kwargs):
"""
Perform standard python/linux daemonization operations.
:param user: Configuration dict to read settings from (i.e. user)
:param logger: Logger object to handle stdio redirect and uncaught exc
"""
drop_privileges(conf.get('user', 'swift'))
capture_stdio(logger, **kwargs)
def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None):
"""
Parse standard swift server/daemon options with optparse.OptionParser.
:param usage: String describing usage
:param once: Boolean indicating the "once" option is avaiable
:param test_args: Override sys.argv; used in testing
:returns : Tuple of (config, options); config is an absolute path to the
config file, options is the parser options as a dictionary.
:raises SystemExit: First arg (CONFIG) is required, file must exist
"""
parser = OptionParser(usage)
parser.add_option("-v", "--verbose", default=False, action="store_true",
help="log to console")
if once:
parser.add_option("-o", "--once", default=False, action="store_true",
help="only run one pass of daemon")
# if test_args is None, optparse will use sys.argv[:1]
options, args = parser.parse_args(args=test_args)
if not args:
parser.print_usage()
print "Error: missing config file argument"
sys.exit(1)
config = os.path.abspath(args.pop(0))
if not os.path.exists(config):
parser.print_usage()
print "Error: unable to locate %s" % config
sys.exit(1)
extra_args = []
# if any named options appear in remaining args, set the option to True
for arg in args:
if arg in options.__dict__:
setattr(options, arg, True)
else:
extra_args.append(arg)
options = vars(options)
options['extra_args'] = extra_args
return config, options
def whataremyips():
"""
Get the machine's ip addresses using ifconfig

41
swift/common/wsgi.py
View File

@ -34,7 +34,7 @@ wsgi.ACCEPT_ERRNO.add(ECONNRESET)
from eventlet.green import socket, ssl
from swift.common.utils import get_logger, drop_privileges, \
validate_configuration, LoggerFileObject, NullLogger
validate_configuration, capture_stdio, NullLogger
def monkey_patch_mimetools():
@ -57,9 +57,8 @@ def monkey_patch_mimetools():
mimetools.Message.parsetype = parsetype
# We might be able to pull pieces of this out to test, but right now it seems
# like more work than it's worth.
def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
# TODO: pull pieces of this out to test
def run_wsgi(conf_file, app_section, *args, **kwargs):
"""
Loads common settings from conf, then instantiates app and runs
the server using the specified number of workers.
@ -70,25 +69,22 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
try:
conf = appconfig('config:%s' % conf_file, name=app_section)
log_name = conf.get('log_name', app_section)
app = loadapp('config:%s' % conf_file,
global_conf={'log_name': log_name})
except Exception, e:
print "Error trying to load config %s: %s" % (conf_file, e)
return
if 'logger' in kwargs:
logger = kwargs['logger']
else:
logger = get_logger(conf, log_name)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = LoggerFileObject(logger)
validate_configuration()
# pre-configure logger
log_name = conf.get('log_name', app_section)
if 'logger' in kwargs:
logger = kwargs.pop('logger')
else:
logger = get_logger(conf, log_name,
log_to_console=kwargs.pop('verbose', False))
# redirect errors to logger and close stdio
capture_stdio(logger)
try:
os.setsid()
except OSError:
no_cover = True # pass
bind_addr = (conf.get('bind_ip', '0.0.0.0'),
int(conf.get('bind_port', kwargs.get('default_port', 8080))))
sock = None
@ -112,7 +108,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600)
worker_count = int(conf.get('workers', '1'))
drop_privileges(conf.get('user', 'swift'))
validate_configuration()
# finally after binding to ports and privilege drop, run app __init__ code
app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})
def run_server():
wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
@ -169,6 +167,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
except OSError, err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
except KeyboardInterrupt:
logger.info('User quit')
break
greenio.shutdown_safe(sock)
sock.close()
logger.info('Exited')

15
test/unit/__init__.py
View File

@ -1,5 +1,8 @@
""" Swift tests """
import os
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from eventlet.green import socket
@ -23,6 +26,18 @@ def connect_tcp(hostport):
rv.connect(hostport)
return rv
@contextmanager
def tmpfile(content):
with NamedTemporaryFile('w', delete=False) as f:
file_name = f.name
f.write(str(content))
try:
yield file_name
finally:
os.unlink(file_name)
class MockTrue(object):
"""
Instances of MockTrue evaluate like True

86
test/unit/common/test_daemon.py
View File

@ -13,16 +13,94 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO: Tests
# TODO: Test kill_children signal handlers
import unittest
from swift.common import daemon
from getpass import getuser
import logging
from StringIO import StringIO
from test.unit import tmpfile
from swift.common import daemon, utils
class MyDaemon(daemon.Daemon):
def __init__(self, conf):
self.conf = conf
self.logger = utils.get_logger(None)
MyDaemon.forever_called = False
MyDaemon.once_called = False
def run_forever(self):
MyDaemon.forever_called = True
def run_once(self):
MyDaemon.once_called = True
def run_raise(self):
raise OSError
def run_quit(self):
raise KeyboardInterrupt
class TestDaemon(unittest.TestCase):
def test_placeholder(self):
pass
def test_create(self):
d = daemon.Daemon({})
self.assertEquals(d.conf, {})
self.assert_(isinstance(d.logger, utils.NamedLogger))
def test_stubs(self):
d = daemon.Daemon({})
self.assertRaises(NotImplementedError, d.run_once)
self.assertRaises(NotImplementedError, d.run_forever)
class TestRunDaemon(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.daemonize = lambda *args: None
def tearDown(self):
reload(utils)
def test_run(self):
d = MyDaemon({})
self.assertFalse(MyDaemon.forever_called)
self.assertFalse(MyDaemon.once_called)
# test default
d.run()
self.assertEquals(d.forever_called, True)
# test once
d.run(once=True)
self.assertEquals(d.once_called, True)
def test_run_daemon(self):
sample_conf = """[my-daemon]
user = %s
""" % getuser()
with tmpfile(sample_conf) as conf_file:
daemon.run_daemon(MyDaemon, conf_file)
self.assertEquals(MyDaemon.forever_called, True)
daemon.run_daemon(MyDaemon, conf_file, once=True)
self.assertEquals(MyDaemon.once_called, True)
# test raise in daemon code
MyDaemon.run_once = MyDaemon.run_raise
self.assertRaises(OSError, daemon.run_daemon, MyDaemon,
conf_file, once=True)
# test user quit
MyDaemon.run_forever = MyDaemon.run_quit
sio = StringIO()
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(sio))
logger = utils.get_logger(None, 'server')
daemon.run_daemon(MyDaemon, conf_file, logger=logger)
self.assert_('user quit' in sio.getvalue().lower())
if __name__ == '__main__':

185
test/unit/common/test_utils.py
View File

@ -25,17 +25,66 @@ import unittest
from getpass import getuser
from shutil import rmtree
from StringIO import StringIO
from functools import partial
from tempfile import NamedTemporaryFile
from eventlet import sleep
from swift.common import utils
class MockOs():
def __init__(self, pass_funcs=[], called_funcs=[], raise_funcs=[]):
self.closed_fds = []
for func in pass_funcs:
setattr(self, func, self.pass_func)
self.called_funcs = {}
for func in called_funcs:
c_func = partial(self.called_func, name)
setattr(self, func, c_func)
for func in raise_funcs:
setattr(self, func, self.raise_func)
def pass_func(self, *args, **kwargs):
pass
chdir = setsid = setgid = setuid = umask = pass_func
def called_func(self, name, *args, **kwargs):
self.called_funcs[name] = True
def raise_func(self, *args, **kwargs):
raise OSError()
def dup2(self, source, target):
self.closed_fds.append(target)
def __getattr__(self, name):
# I only over-ride portions of the os module
try:
return object.__getattr__(self, name)
except AttributeError:
return getattr(os, name)
class MockSys():
__stderr__ = sys.__stderr__
class TestUtils(unittest.TestCase):
""" Tests for swift.common.utils """
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
self.logger = logging.getLogger()
self.starting_handlers = list(self.logger.handlers)
def tearDown(self):
# don't let extra handlers pile up redirecting stdio and other stuff...
for handler in self.logger.handlers:
if handler not in self.starting_handlers:
self.logger.removeHandler(handler)
def test_normalize_timestamp(self):
""" Test swift.common.utils.normalize_timestamp """
@ -127,6 +176,16 @@ class TestUtils(unittest.TestCase):
self.assertEquals(sio.getvalue(), '')
def test_LoggerFileObject(self):
if isinstance(sys.stdout, utils.LoggerFileObject):
# This may happen if some other not so nice test allowed stdout to
# be caputred by daemonize w/o cleaning up after itself (i.e.
# test_db_replicator.TestDBReplicator.test_run_once). Normally
# nose would clean this up for us (which works well and is
# probably the best solution). But when running with --nocapture,
# this condition would cause the first print to acctually be
# redirected to a log call and the test would fail - so we have to
# go old school
sys.stdout = sys.__stdout__
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sio = StringIO()
@ -182,10 +241,63 @@ class TestUtils(unittest.TestCase):
self.assertRaises(IOError, lfo.readline, 1024)
lfo.tell()
def test_drop_privileges(self):
# Note that this doesn't really drop privileges as it just sets them to
# what they already are; but it exercises the code at least.
utils.drop_privileges(getuser())
def test_parse_options(self):
# use mkstemp to get a file that is definately on disk
with NamedTemporaryFile() as f:
conf_file = f.name
conf, options = utils.parse_options(test_args=[conf_file])
self.assertEquals(conf, conf_file)
# assert defaults
self.assertEquals(options['verbose'], False)
self.assert_('once' not in options)
# assert verbose as option
conf, options = utils.parse_options(test_args=[conf_file, '-v'])
self.assertEquals(options['verbose'], True)
# check once option
conf, options = utils.parse_options(test_args=[conf_file],
once=True)
self.assertEquals(options['once'], False)
test_args = [conf_file, '--once']
conf, options = utils.parse_options(test_args=test_args, once=True)
self.assertEquals(options['once'], True)
# check options as arg parsing
test_args = [conf_file, 'once', 'plugin_name', 'verbose']
conf, options = utils.parse_options(test_args=test_args, once=True)
self.assertEquals(options['verbose'], True)
self.assertEquals(options['once'], True)
self.assertEquals(options['extra_args'], ['plugin_name'])
def test_parse_options_errors(self):
orig_stdout = sys.stdout
orig_stderr = sys.stderr
stdo = StringIO()
stde = StringIO()
utils.sys.stdout = stdo
utils.sys.stderr = stde
err_msg = """Usage: test usage
Error: missing config file argument
"""
test_args = []
self.assertRaises(SystemExit, utils.parse_options, 'test usage', True,
test_args)
self.assertEquals(stdo.getvalue(), err_msg)
# verify conf file must exist, context manager will delete temp file
with NamedTemporaryFile() as f:
conf_file = f.name
err_msg += """Usage: test usage
Error: unable to locate %s
""" % conf_file
test_args = [conf_file]
self.assertRaises(SystemExit, utils.parse_options, 'test usage', True,
test_args)
self.assertEquals(stdo.getvalue(), err_msg)
# reset stdio
utils.sys.stdout = orig_stdout
utils.sys.stderr = orig_stderr
def test_NamedLogger(self):
sio = StringIO()
@ -275,5 +387,70 @@ log_name = yarr'''
self.assertEquals(result, expected)
os.unlink('/tmp/test')
def test_daemonize(self):
# default args
conf = {'user': getuser()}
logger = utils.get_logger(None, 'daemon')
# over-ride utils system modules with mocks
utils.os = MockOs()
utils.sys = MockSys()
utils.daemonize(conf, logger)
self.assert_(utils.sys.excepthook is not None)
self.assertEquals(utils.os.closed_fds, [0, 1, 2])
self.assert_(utils.sys.stdout is not None)
self.assert_(utils.sys.stderr is not None)
# reset; test same args, OSError trying to get session leader
utils.os = MockOs(raise_funcs=('setsid',))
utils.sys = MockSys()
utils.daemonize(conf, logger)
self.assert_(utils.sys.excepthook is not None)
self.assertEquals(utils.os.closed_fds, [0, 1, 2])
self.assert_(utils.sys.stdout is not None)
self.assert_(utils.sys.stderr is not None)
# reset; test same args, exc when trying to close stdio
utils.os = MockOs(raise_funcs=('dup2',))
utils.sys = MockSys()
utils.daemonize(conf, logger)
self.assert_(utils.sys.excepthook is not None)
# unable to close stdio
self.assertEquals(utils.os.closed_fds, [])
self.assert_(utils.sys.stdout is not None)
self.assert_(utils.sys.stderr is not None)
# reset; test some other args
utils.os = MockOs()
utils.sys = MockSys()
conf = {'user': getuser()}
logger = utils.get_logger(None, log_to_console=True)
logger = logging.getLogger()
utils.daemonize(conf, logger, capture_stdout=False,
capture_stderr=False)
self.assert_(utils.sys.excepthook is not None)
# when logging to console, stderr remains open
self.assertEquals(utils.os.closed_fds, [0, 1])
# stdio not captured
self.assertFalse(hasattr(utils.sys, 'stdout'))
self.assertFalse(hasattr(utils.sys, 'stderr'))
def test_get_logger_console(self):
reload(utils) # reset get_logger attrs
logger = utils.get_logger(None)
self.assertFalse(hasattr(utils.get_logger, 'console'))
logger = utils.get_logger(None, log_to_console=True)
self.assert_(hasattr(utils.get_logger, 'console'))
self.assert_(isinstance(utils.get_logger.console,
logging.StreamHandler))
# make sure you can't have two console handlers
old_handler = utils.get_logger.console
logger = utils.get_logger(None, log_to_console=True)
self.assertNotEquals(utils.get_logger.console, old_handler)
if __name__ == '__main__':
unittest.main()

15
test/unit/stats/test_log_processor.py
View File

@ -14,25 +14,12 @@
# limitations under the License.
import unittest
import os
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from test.unit import tmpfile
from swift.common import internal_proxy
from swift.stats import log_processor
@contextmanager
def tmpfile(content):
with NamedTemporaryFile('w', delete=False) as f:
file_name = f.name
f.write(str(content))
try:
yield file_name
finally:
os.unlink(file_name)
class FakeUploadApp(object):
def __init__(self, *args, **kwargs):
pass