# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function import unittest from test.unit import temptree import os import sys import resource import signal import errno from collections import defaultdict from time import sleep, time from swift.common import manager from swift.common.exceptions import InvalidPidFileException import eventlet threading = eventlet.patcher.original('threading') DUMMY_SIG = 1 class MockOs(object): RAISE_EPERM_SIG = 99 def __init__(self, pids): self.running_pids = pids self.pid_sigs = defaultdict(list) self.closed_fds = [] self.child_pid = 9999 # fork defaults to test parent process path self.execlp_called = False def kill(self, pid, sig): if sig == self.RAISE_EPERM_SIG: raise OSError(errno.EPERM, 'Operation not permitted') if pid not in self.running_pids: raise OSError(3, 'No such process') self.pid_sigs[pid].append(sig) def __getattr__(self, name): # I only over-ride portions of the os module try: return object.__getattr__(self, name) except AttributeError: return getattr(os, name) def pop_stream(f): """read everything out of file from the top and clear it out """ f.flush() f.seek(0) output = f.read() f.seek(0) f.truncate() return output class TestManagerModule(unittest.TestCase): def test_servers(self): main_plus_rest = set(manager.MAIN_SERVERS + manager.REST_SERVERS) self.assertEqual(set(manager.ALL_SERVERS), main_plus_rest) # make sure there's no server listed in both self.assertEqual(len(main_plus_rest), len(manager.MAIN_SERVERS) + len(manager.REST_SERVERS)) def test_setup_env(self): class MockResource(object): def __init__(self, error=None): self.error = error self.called_with_args = [] def setrlimit(self, resource, limits): if self.error: raise self.error self.called_with_args.append((resource, limits)) def __getattr__(self, name): # I only over-ride portions of the resource module try: return object.__getattr__(self, name) except AttributeError: return getattr(resource, name) _orig_resource = manager.resource _orig_environ = os.environ try: manager.resource = MockResource() manager.os.environ = {} manager.setup_env() expected = [ (resource.RLIMIT_NOFILE, (manager.MAX_DESCRIPTORS, manager.MAX_DESCRIPTORS)), (resource.RLIMIT_DATA, (manager.MAX_MEMORY, manager.MAX_MEMORY)), (resource.RLIMIT_NPROC, (manager.MAX_PROCS, manager.MAX_PROCS)), ] self.assertEqual(manager.resource.called_with_args, expected) self.assertTrue( manager.os.environ['PYTHON_EGG_CACHE'].startswith('/tmp')) # test error condition manager.resource = MockResource(error=ValueError()) manager.os.environ = {} manager.setup_env() self.assertEqual(manager.resource.called_with_args, []) self.assertTrue( manager.os.environ['PYTHON_EGG_CACHE'].startswith('/tmp')) manager.resource = MockResource(error=OSError()) manager.os.environ = {} self.assertRaises(OSError, manager.setup_env) self.assertEqual(manager.os.environ.get('PYTHON_EGG_CACHE'), None) finally: manager.resource = _orig_resource os.environ = _orig_environ def test_command_wrapper(self): @manager.command def myfunc(arg1): """test doc """ return arg1 self.assertEqual(myfunc.__doc__.strip(), 'test doc') self.assertEqual(myfunc(1), 1) self.assertEqual(myfunc(0), 0) self.assertEqual(myfunc(True), 1) self.assertEqual(myfunc(False), 0) self.assertTrue(hasattr(myfunc, 'publicly_accessible')) self.assertTrue(myfunc.publicly_accessible) def test_watch_server_pids(self): class MockOs(object): WNOHANG = os.WNOHANG def __init__(self, pid_map=None): if pid_map is None: pid_map = {} self.pid_map = {} for pid, v in pid_map.items(): self.pid_map[pid] = (x for x in v) def waitpid(self, pid, options): try: rv = next(self.pid_map[pid]) except StopIteration: raise OSError(errno.ECHILD, os.strerror(errno.ECHILD)) except KeyError: raise OSError(errno.ESRCH, os.strerror(errno.ESRCH)) if isinstance(rv, Exception): raise rv else: return rv class MockTime(object): def __init__(self, ticks=None): self.tock = time() if not ticks: ticks = [] self.ticks = (t for t in ticks) def time(self): try: self.tock += next(self.ticks) except StopIteration: self.tock += 1 return self.tock def sleep(*args): return class MockServer(object): def __init__(self, pids, run_dir=manager.RUN_DIR, zombie=0): self.heartbeat = (pids for _ in range(zombie)) def get_running_pids(self): try: rv = next(self.heartbeat) return rv except StopIteration: return {} _orig_os = manager.os _orig_time = manager.time _orig_server = manager.Server try: manager.time = MockTime() manager.os = MockOs() # this server always says it's dead when you ask for running pids server = MockServer([1]) # list of pids keyed on servers to watch server_pids = { server: [1], } # basic test, server dies gen = manager.watch_server_pids(server_pids) expected = [(server, 1)] self.assertEqual([x for x in gen], expected) # start long running server and short interval server = MockServer([1], zombie=15) server_pids = { server: [1], } gen = manager.watch_server_pids(server_pids) self.assertEqual([x for x in gen], []) # wait a little longer gen = manager.watch_server_pids(server_pids, interval=15) self.assertEqual([x for x in gen], [(server, 1)]) # zombie process server = MockServer([1], zombie=200) server_pids = { server: [1], } # test weird os error manager.os = MockOs({1: [OSError()]}) gen = manager.watch_server_pids(server_pids) self.assertRaises(OSError, lambda: [x for x in gen]) # test multi-server server1 = MockServer([1, 10], zombie=200) server2 = MockServer([2, 20], zombie=8) server_pids = { server1: [1, 10], server2: [2, 20], } pid_map = { 1: [None for _ in range(10)], 2: [None for _ in range(8)], 20: [None for _ in range(4)], } manager.os = MockOs(pid_map) gen = manager.watch_server_pids(server_pids, interval=manager.KILL_WAIT) expected = [ (server2, 2), (server2, 20), ] self.assertEqual([x for x in gen], expected) finally: manager.os = _orig_os manager.time = _orig_time manager.Server = _orig_server def test_safe_kill(self): manager.os = MockOs([1, 2, 3, 4]) proc_files = ( ('1/cmdline', 'same-procname'), ('2/cmdline', 'another-procname'), ('4/cmdline', 'another-procname'), ) files, contents = zip(*proc_files) with temptree(files, contents) as t: manager.PROC_DIR = t manager.safe_kill(1, signal.SIG_DFL, 'same-procname') self.assertRaises(InvalidPidFileException, manager.safe_kill, 2, signal.SIG_DFL, 'same-procname') manager.safe_kill(3, signal.SIG_DFL, 'same-procname') manager.safe_kill(4, signal.SIGHUP, 'same-procname') def test_exc(self): self.assertTrue(issubclass(manager.UnknownCommandError, Exception)) class TestServer(unittest.TestCase): def tearDown(self): reload(manager) def join_swift_dir(self, path): return os.path.join(manager.SWIFT_DIR, path) def join_run_dir(self, path): return os.path.join(manager.RUN_DIR, path) def test_create_server(self): server = manager.Server('proxy') self.assertEqual(server.server, 'proxy-server') self.assertEqual(server.type, 'proxy') self.assertEqual(server.cmd, 'swift-proxy-server') server = manager.Server('object-replicator') self.assertEqual(server.server, 'object-replicator') self.assertEqual(server.type, 'object') self.assertEqual(server.cmd, 'swift-object-replicator') def test_server_to_string(self): server = manager.Server('Proxy') self.assertEqual(str(server), 'proxy-server') server = manager.Server('object-replicator') self.assertEqual(str(server), 'object-replicator') def test_server_repr(self): server = manager.Server('proxy') self.assertTrue(server.__class__.__name__ in repr(server)) self.assertTrue(str(server) in repr(server)) def test_server_equality(self): server1 = manager.Server('Proxy') server2 = manager.Server('proxy-server') self.assertEqual(server1, server2) # it is NOT a string self.assertNotEqual(server1, 'proxy-server') def test_get_pid_file_name(self): server = manager.Server('proxy') conf_file = self.join_swift_dir('proxy-server.conf') pid_file = self.join_run_dir('proxy-server.pid') self.assertEqual(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('object-replicator') conf_file = self.join_swift_dir('object-server/1.conf') pid_file = self.join_run_dir('object-replicator/1.pid') self.assertEqual(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('container-auditor') conf_file = self.join_swift_dir( 'container-server/1/container-auditor.conf') pid_file = self.join_run_dir( 'container-auditor/1/container-auditor.pid') self.assertEqual(pid_file, server.get_pid_file_name(conf_file)) def test_get_custom_pid_file_name(self): random_run_dir = "/random/dir" get_random_run_dir = lambda x: os.path.join(random_run_dir, x) server = manager.Server('proxy', run_dir=random_run_dir) conf_file = self.join_swift_dir('proxy-server.conf') pid_file = get_random_run_dir('proxy-server.pid') self.assertEqual(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('object-replicator', run_dir=random_run_dir) conf_file = self.join_swift_dir('object-server/1.conf') pid_file = get_random_run_dir('object-replicator/1.pid') self.assertEqual(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('container-auditor', run_dir=random_run_dir) conf_file = self.join_swift_dir( 'container-server/1/container-auditor.conf') pid_file = get_random_run_dir( 'container-auditor/1/container-auditor.pid') self.assertEqual(pid_file, server.get_pid_file_name(conf_file)) def test_get_conf_file_name(self): server = manager.Server('proxy') conf_file = self.join_swift_dir('proxy-server.conf') pid_file = self.join_run_dir('proxy-server.pid') self.assertEqual(conf_file, server.get_conf_file_name(pid_file)) server = manager.Server('object-replicator') conf_file = self.join_swift_dir('object-server/1.conf') pid_file = self.join_run_dir('object-replicator/1.pid') self.assertEqual(conf_file, server.get_conf_file_name(pid_file)) server = manager.Server('container-auditor') conf_file = self.join_swift_dir( 'container-server/1/container-auditor.conf') pid_file = self.join_run_dir( 'container-auditor/1/container-auditor.pid') self.assertEqual(conf_file, server.get_conf_file_name(pid_file)) server_name = manager.STANDALONE_SERVERS[0] server = manager.Server(server_name) conf_file = self.join_swift_dir(server_name + '.conf') pid_file = self.join_run_dir(server_name + '.pid') self.assertEqual(conf_file, server.get_conf_file_name(pid_file)) def test_conf_files(self): # test get single conf file conf_files = ( 'proxy-server.conf', 'proxy-server.ini', 'auth-server.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('proxy') conf_files = server.conf_files() self.assertEqual(len(conf_files), 1) conf_file = conf_files[0] proxy_conf = self.join_swift_dir('proxy-server.conf') self.assertEqual(conf_file, proxy_conf) # test multi server conf files & grouping of server-type config conf_files = ( 'object-server1.conf', 'object-server/2.conf', 'object-server/object3.conf', 'object-server/conf/server4.conf', 'object-server.txt', 'proxy-server.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('object-replicator') conf_files = server.conf_files() self.assertEqual(len(conf_files), 4) c1 = self.join_swift_dir('object-server1.conf') c2 = self.join_swift_dir('object-server/2.conf') c3 = self.join_swift_dir('object-server/object3.conf') c4 = self.join_swift_dir('object-server/conf/server4.conf') for c in [c1, c2, c3, c4]: self.assertTrue(c in conf_files) # test configs returned sorted sorted_confs = sorted([c1, c2, c3, c4]) self.assertEqual(conf_files, sorted_confs) # test get single numbered conf conf_files = ( 'account-server/1.conf', 'account-server/2.conf', 'account-server/3.conf', 'account-server/4.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('account') conf_files = server.conf_files(number=2) self.assertEqual(len(conf_files), 1) conf_file = conf_files[0] self.assertEqual(conf_file, self.join_swift_dir('account-server/2.conf')) # test missing config number conf_files = server.conf_files(number=5) self.assertFalse(conf_files) # test getting specific conf conf_files = ( 'account-server/1.conf', 'account-server/2.conf', 'account-server/3.conf', 'account-server/4.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('account.2') conf_files = server.conf_files() self.assertEqual(len(conf_files), 1) conf_file = conf_files[0] self.assertEqual(conf_file, self.join_swift_dir('account-server/2.conf')) # test verbose & quiet conf_files = ( 'auth-server.ini', 'container-server/1.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t old_stdout = sys.stdout try: with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f server = manager.Server('auth') # check warn "unable to locate" conf_files = server.conf_files() self.assertFalse(conf_files) self.assertTrue('unable to locate config for auth' in pop_stream(f).lower()) # check quiet will silence warning conf_files = server.conf_files(verbose=True, quiet=True) self.assertEqual(pop_stream(f), '') # check found config no warning server = manager.Server('container-auditor') conf_files = server.conf_files() self.assertEqual(pop_stream(f), '') # check missing config number warn "unable to locate" conf_files = server.conf_files(number=2) self.assertTrue( 'unable to locate config number 2 for ' + 'container-auditor' in pop_stream(f).lower()) # check verbose lists configs conf_files = server.conf_files(number=2, verbose=True) c1 = self.join_swift_dir('container-server/1.conf') self.assertTrue(c1 in pop_stream(f)) finally: sys.stdout = old_stdout # test standalone conf file server_name = manager.STANDALONE_SERVERS[0] conf_files = (server_name + '.conf',) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server(server_name) conf_files = server.conf_files() self.assertEqual(len(conf_files), 1) conf_file = conf_files[0] conf = self.join_swift_dir(server_name + '.conf') self.assertEqual(conf_file, conf) def test_proxy_conf_dir(self): conf_files = ( 'proxy-server.conf.d/00.conf', 'proxy-server.conf.d/01.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('proxy') conf_dirs = server.conf_files() self.assertEqual(len(conf_dirs), 1) conf_dir = conf_dirs[0] proxy_conf_dir = self.join_swift_dir('proxy-server.conf.d') self.assertEqual(proxy_conf_dir, conf_dir) def test_named_conf_dir(self): conf_files = ( 'object-server/base.conf-template', 'object-server/object-server.conf.d/00_base.conf', 'object-server/object-server.conf.d/10_server.conf', 'object-server/object-replication.conf.d/00_base.conf', 'object-server/object-replication.conf.d/10_server.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('object.replication') conf_dirs = server.conf_files() self.assertEqual(len(conf_dirs), 1) conf_dir = conf_dirs[0] replication_server_conf_dir = self.join_swift_dir( 'object-server/object-replication.conf.d') self.assertEqual(replication_server_conf_dir, conf_dir) # and again with no named filter server = manager.Server('object') conf_dirs = server.conf_files() self.assertEqual(len(conf_dirs), 2) for named_conf in ('server', 'replication'): conf_dir = self.join_swift_dir( 'object-server/object-%s.conf.d' % named_conf) self.assertTrue(conf_dir in conf_dirs) def test_conf_dir(self): conf_files = ( 'object-server/object-server.conf-base', 'object-server/1.conf.d/base.conf', 'object-server/1.conf.d/1.conf', 'object-server/2.conf.d/base.conf', 'object-server/2.conf.d/2.conf', 'object-server/3.conf.d/base.conf', 'object-server/3.conf.d/3.conf', 'object-server/4.conf.d/base.conf', 'object-server/4.conf.d/4.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('object-replicator') conf_dirs = server.conf_files() self.assertEqual(len(conf_dirs), 4) c1 = self.join_swift_dir('object-server/1.conf.d') c2 = self.join_swift_dir('object-server/2.conf.d') c3 = self.join_swift_dir('object-server/3.conf.d') c4 = self.join_swift_dir('object-server/4.conf.d') for c in [c1, c2, c3, c4]: self.assertTrue(c in conf_dirs) # test configs returned sorted sorted_confs = sorted([c1, c2, c3, c4]) self.assertEqual(conf_dirs, sorted_confs) def test_named_conf_dir_pid_files(self): conf_files = ( 'object-server/object-server.pid.d', 'object-server/object-replication.pid.d', ) with temptree(conf_files) as t: manager.RUN_DIR = t server = manager.Server('object.replication', run_dir=t) pid_files = server.pid_files() self.assertEqual(len(pid_files), 1) pid_file = pid_files[0] replication_server_pid = self.join_run_dir( 'object-server/object-replication.pid.d') self.assertEqual(replication_server_pid, pid_file) # and again with no named filter server = manager.Server('object', run_dir=t) pid_files = server.pid_files() self.assertEqual(len(pid_files), 2) for named_pid in ('server', 'replication'): pid_file = self.join_run_dir( 'object-server/object-%s.pid.d' % named_pid) self.assertTrue(pid_file in pid_files) def test_iter_pid_files(self): """ Server.iter_pid_files is kinda boring, test the Server.pid_files stuff here as well """ pid_files = ( ('proxy-server.pid', 1), ('auth-server.pid', 'blah'), ('object-replicator/1.pid', 11), ('object-replicator/2.pid', 12), ) files, contents = zip(*pid_files) with temptree(files, contents) as t: manager.RUN_DIR = t server = manager.Server('proxy', run_dir=t) # test get one file iter = server.iter_pid_files() pid_file, pid = next(iter) self.assertEqual(pid_file, self.join_run_dir('proxy-server.pid')) self.assertEqual(pid, 1) # ... and only one file self.assertRaises(StopIteration, iter.next) # test invalid value in pid file server = manager.Server('auth', run_dir=t) pid_file, pid = next(server.iter_pid_files()) self.assertIsNone(pid) # test object-server doesn't steal pids from object-replicator server = manager.Server('object', run_dir=t) self.assertRaises(StopIteration, server.iter_pid_files().next) # test multi-pid iter server = manager.Server('object-replicator', run_dir=t) real_map = { 11: self.join_run_dir('object-replicator/1.pid'), 12: self.join_run_dir('object-replicator/2.pid'), } pid_map = {} for pid_file, pid in server.iter_pid_files(): pid_map[pid] = pid_file self.assertEqual(pid_map, real_map) # test get pid_files by number conf_files = ( 'object-server/1.conf', 'object-server/2.conf', 'object-server/3.conf', 'object-server/4.conf', ) pid_files = ( ('object-server/1.pid', 1), ('object-server/2.pid', 2), ('object-server/5.pid', 5), ) with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t server = manager.Server('object', run_dir=t) # test get all pid files real_map = { 1: self.join_run_dir('object-server/1.pid'), 2: self.join_run_dir('object-server/2.pid'), 5: self.join_run_dir('object-server/5.pid'), } pid_map = {} for pid_file, pid in server.iter_pid_files(): pid_map[pid] = pid_file self.assertEqual(pid_map, real_map) # test get pid with matching conf pids = list(server.iter_pid_files(number=2)) self.assertEqual(len(pids), 1) pid_file, pid = pids[0] self.assertEqual(pid, 2) pid_two = self.join_run_dir('object-server/2.pid') self.assertEqual(pid_file, pid_two) # try to iter on a pid number with a matching conf but no pid pids = list(server.iter_pid_files(number=3)) self.assertFalse(pids) # test get pids w/o matching conf pids = list(server.iter_pid_files(number=5)) self.assertFalse(pids) # test get pid_files by conf name conf_files = ( 'object-server/1.conf', 'object-server/2.conf', 'object-server/3.conf', 'object-server/4.conf', ) pid_files = ( ('object-server/1.pid', 1), ('object-server/2.pid', 2), ('object-server/5.pid', 5), ) with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t server = manager.Server('object.2', run_dir=t) # test get pid with matching conf pids = list(server.iter_pid_files()) self.assertEqual(len(pids), 1) pid_file, pid = pids[0] self.assertEqual(pid, 2) pid_two = self.join_run_dir('object-server/2.pid') self.assertEqual(pid_file, pid_two) def test_signal_pids(self): temp_files = ( ('var/run/zero-server.pid', 0), ('var/run/proxy-server.pid', 1), ('var/run/auth-server.pid', 2), ('var/run/one-server.pid', 3), ('var/run/object-server.pid', 4), ('var/run/invalid-server.pid', 'Forty-Two'), ('proc/3/cmdline', 'swift-another-server') ) with temptree(*zip(*temp_files)) as t: manager.RUN_DIR = os.path.join(t, 'var/run') manager.PROC_DIR = os.path.join(t, 'proc') # mock os with so both the first and second are running manager.os = MockOs([1, 2]) server = manager.Server('proxy', run_dir=manager.RUN_DIR) pids = server.signal_pids(DUMMY_SIG) self.assertEqual(len(pids), 1) self.assertTrue(1 in pids) self.assertEqual(manager.os.pid_sigs[1], [DUMMY_SIG]) # make sure other process not signaled self.assertFalse(2 in pids) self.assertFalse(2 in manager.os.pid_sigs) # capture stdio old_stdout = sys.stdout try: with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f # test print details pids = server.signal_pids(DUMMY_SIG) output = pop_stream(f) self.assertTrue('pid: %s' % 1 in output) self.assertTrue('signal: %s' % DUMMY_SIG in output) # test no details on signal.SIG_DFL pids = server.signal_pids(signal.SIG_DFL) self.assertEqual(pop_stream(f), '') # reset mock os so only the second server is running manager.os = MockOs([2]) # test pid not running pids = server.signal_pids(signal.SIG_DFL) self.assertTrue(1 not in pids) self.assertTrue(1 not in manager.os.pid_sigs) # test remove stale pid file self.assertFalse(os.path.exists( self.join_run_dir('proxy-server.pid'))) # reset mock os with no running pids manager.os = MockOs([]) server = manager.Server('auth', run_dir=manager.RUN_DIR) # test verbose warns on removing stale pid file pids = server.signal_pids(signal.SIG_DFL, verbose=True) output = pop_stream(f) self.assertTrue('stale pid' in output.lower()) auth_pid = self.join_run_dir('auth-server.pid') self.assertTrue(auth_pid in output) # reset mock os so only the third server is running manager.os = MockOs([3]) server = manager.Server('one', run_dir=manager.RUN_DIR) # test verbose warns on removing invalid pid file pids = server.signal_pids(signal.SIG_DFL, verbose=True) output = pop_stream(f) old_stdout.write('output %s' % output) self.assertTrue('removing pid file' in output.lower()) one_pid = self.join_run_dir('one-server.pid') self.assertTrue(one_pid in output) server = manager.Server('zero', run_dir=manager.RUN_DIR) self.assertTrue(os.path.exists( self.join_run_dir('zero-server.pid'))) # sanity # test verbose warns on removing pid file with invalid pid pids = server.signal_pids(signal.SIG_DFL, verbose=True) output = pop_stream(f) old_stdout.write('output %s' % output) self.assertTrue('with invalid pid' in output.lower()) self.assertFalse(os.path.exists( self.join_run_dir('zero-server.pid'))) server = manager.Server('invalid-server', run_dir=manager.RUN_DIR) self.assertTrue(os.path.exists( self.join_run_dir('invalid-server.pid'))) # sanity # test verbose warns on removing pid file with invalid pid pids = server.signal_pids(signal.SIG_DFL, verbose=True) output = pop_stream(f) old_stdout.write('output %s' % output) self.assertTrue('with invalid pid' in output.lower()) self.assertFalse(os.path.exists( self.join_run_dir('invalid-server.pid'))) # reset mock os with no running pids manager.os = MockOs([]) # test warning with insufficient permissions server = manager.Server('object', run_dir=manager.RUN_DIR) pids = server.signal_pids(manager.os.RAISE_EPERM_SIG) output = pop_stream(f) self.assertTrue('no permission to signal pid 4' in output.lower(), output) finally: sys.stdout = old_stdout def test_get_running_pids(self): # test only gets running pids temp_files = ( ('var/run/test-server1.pid', 1), ('var/run/test-server2.pid', 2), ('var/run/test-server3.pid', 3), ('proc/1/cmdline', 'swift-test-server'), ('proc/3/cmdline', 'swift-another-server') ) with temptree(*zip(*temp_files)) as t: manager.RUN_DIR = os.path.join(t, 'var/run') manager.PROC_DIR = os.path.join(t, 'proc') server = manager.Server( 'test-server', run_dir=manager.RUN_DIR) # mock os, only pid '1' is running manager.os = MockOs([1, 3]) running_pids = server.get_running_pids() self.assertEqual(len(running_pids), 1) self.assertTrue(1 in running_pids) self.assertTrue(2 not in running_pids) self.assertTrue(3 not in running_pids) # test persistent running pid files self.assertTrue(os.path.exists( os.path.join(manager.RUN_DIR, 'test-server1.pid'))) # test clean up stale pids pid_two = self.join_swift_dir('test-server2.pid') self.assertFalse(os.path.exists(pid_two)) pid_three = self.join_swift_dir('test-server3.pid') self.assertFalse(os.path.exists(pid_three)) # reset mock os, no pids running manager.os = MockOs([]) running_pids = server.get_running_pids() self.assertFalse(running_pids) # and now all pid files are cleaned out pid_one = self.join_run_dir('test-server1.pid') self.assertFalse(os.path.exists(pid_one)) all_pids = os.listdir(manager.RUN_DIR) self.assertEqual(len(all_pids), 0) # test only get pids for right server pid_files = ( ('thing-doer.pid', 1), ('thing-sayer.pid', 2), ('other-doer.pid', 3), ('other-sayer.pid', 4), ) files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t # all pids are running manager.os = MockOs(pids) server = manager.Server('thing-doer', run_dir=t) running_pids = server.get_running_pids() # only thing-doer.pid, 1 self.assertEqual(len(running_pids), 1) self.assertTrue(1 in running_pids) # no other pids returned for n in (2, 3, 4): self.assertTrue(n not in running_pids) # assert stale pids for other servers ignored manager.os = MockOs([1]) # only thing-doer is running running_pids = server.get_running_pids() for f in ('thing-sayer.pid', 'other-doer.pid', 'other-sayer.pid'): # other server pid files persist self.assertTrue(os.path.exists, os.path.join(t, f)) # verify that servers are in fact not running for server_name in ('thing-sayer', 'other-doer', 'other-sayer'): server = manager.Server(server_name, run_dir=t) running_pids = server.get_running_pids() self.assertFalse(running_pids) # and now all OTHER pid files are cleaned out all_pids = os.listdir(t) self.assertEqual(len(all_pids), 1) self.assertTrue(os.path.exists(os.path.join(t, 'thing-doer.pid'))) def test_kill_running_pids(self): pid_files = ( ('object-server.pid', 1), ('object-replicator1.pid', 11), ('object-replicator2.pid', 12), ) files, running_pids = zip(*pid_files) with temptree(files, running_pids) as t: manager.RUN_DIR = t server = manager.Server('object', run_dir=t) # test no servers running manager.os = MockOs([]) pids = server.kill_running_pids() self.assertFalse(pids, pids) files, running_pids = zip(*pid_files) with temptree(files, running_pids) as t: manager.RUN_DIR = t server.run_dir = t # start up pid manager.os = MockOs([1]) server = manager.Server('object', run_dir=t) # test kill one pid pids = server.kill_running_pids() self.assertEqual(len(pids), 1) self.assertTrue(1 in pids) self.assertEqual(manager.os.pid_sigs[1], [signal.SIGTERM]) # reset os mock manager.os = MockOs([1]) # test shutdown self.assertTrue('object-server' in manager.GRACEFUL_SHUTDOWN_SERVERS) pids = server.kill_running_pids(graceful=True) self.assertEqual(len(pids), 1) self.assertTrue(1 in pids) self.assertEqual(manager.os.pid_sigs[1], [signal.SIGHUP]) # start up other servers manager.os = MockOs([11, 12]) # test multi server kill & ignore graceful on unsupported server self.assertFalse('object-replicator' in manager.GRACEFUL_SHUTDOWN_SERVERS) server = manager.Server('object-replicator', run_dir=t) pids = server.kill_running_pids(graceful=True) self.assertEqual(len(pids), 2) for pid in (11, 12): self.assertTrue(pid in pids) self.assertEqual(manager.os.pid_sigs[pid], [signal.SIGTERM]) # and the other pid is of course not signaled self.assertTrue(1 not in manager.os.pid_sigs) def test_status(self): conf_files = ( 'test-server/1.conf', 'test-server/2.conf', 'test-server/3.conf', 'test-server/4.conf', ) pid_files = ( ('test-server/1.pid', 1), ('test-server/2.pid', 2), ('test-server/3.pid', 3), ('test-server/4.pid', 4), ) with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t # setup running servers server = manager.Server('test', run_dir=t) # capture stdio old_stdout = sys.stdout try: with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f # test status for all running manager.os = MockOs(pids) proc_files = ( ('1/cmdline', 'swift-test-server'), ('2/cmdline', 'swift-test-server'), ('3/cmdline', 'swift-test-server'), ('4/cmdline', 'swift-test-server'), ) files, contents = zip(*proc_files) with temptree(files, contents) as t: manager.PROC_DIR = t self.assertEqual(server.status(), 0) output = pop_stream(f).strip().splitlines() self.assertEqual(len(output), 4) for line in output: self.assertTrue('test-server running' in line) # test get single server by number with temptree([], []) as t: manager.PROC_DIR = t self.assertEqual(server.status(number=4), 0) output = pop_stream(f).strip().splitlines() self.assertEqual(len(output), 1) line = output[0] self.assertTrue('test-server running' in line) conf_four = self.join_swift_dir(conf_files[3]) self.assertTrue('4 - %s' % conf_four in line) # test some servers not running manager.os = MockOs([1, 2, 3]) proc_files = ( ('1/cmdline', 'swift-test-server'), ('2/cmdline', 'swift-test-server'), ('3/cmdline', 'swift-test-server'), ) files, contents = zip(*proc_files) with temptree(files, contents) as t: manager.PROC_DIR = t self.assertEqual(server.status(), 0) output = pop_stream(f).strip().splitlines() self.assertEqual(len(output), 3) for line in output: self.assertTrue('test-server running' in line) # test single server not running manager.os = MockOs([1, 2]) proc_files = ( ('1/cmdline', 'swift-test-server'), ('2/cmdline', 'swift-test-server'), ) files, contents = zip(*proc_files) with temptree(files, contents) as t: manager.PROC_DIR = t self.assertEqual(server.status(number=3), 1) output = pop_stream(f).strip().splitlines() self.assertEqual(len(output), 1) line = output[0] self.assertTrue('not running' in line) conf_three = self.join_swift_dir(conf_files[2]) self.assertTrue(conf_three in line) # test no running pids manager.os = MockOs([]) with temptree([], []) as t: manager.PROC_DIR = t self.assertEqual(server.status(), 1) output = pop_stream(f).lower() self.assertTrue('no test-server running' in output) # test use provided pids pids = { 1: '1.pid', 2: '2.pid', } # shouldn't call get_running_pids called = [] def mock(*args, **kwargs): called.append(True) server.get_running_pids = mock status = server.status(pids=pids) self.assertEqual(status, 0) self.assertFalse(called) output = pop_stream(f).strip().splitlines() self.assertEqual(len(output), 2) for line in output: self.assertTrue('test-server running' in line) finally: sys.stdout = old_stdout def test_spawn(self): # mocks class MockProcess(object): NOTHING = 'default besides None' STDOUT = 'stdout' PIPE = 'pipe' def __init__(self, pids=None): if pids is None: pids = [] self.pids = (p for p in pids) def Popen(self, args, **kwargs): return MockProc(next(self.pids), args, **kwargs) class MockProc(object): def __init__(self, pid, args, stdout=MockProcess.NOTHING, stderr=MockProcess.NOTHING): self.pid = pid self.args = args self.stdout = stdout if stderr == MockProcess.STDOUT: self.stderr = self.stdout else: self.stderr = stderr # setup running servers server = manager.Server('test') with temptree(['test-server.conf']) as swift_dir: manager.SWIFT_DIR = swift_dir with temptree([]) as t: manager.RUN_DIR = t server.run_dir = t old_subprocess = manager.subprocess try: # test single server process calls spawn once manager.subprocess = MockProcess([1]) conf_file = self.join_swift_dir('test-server.conf') # spawn server no kwargs server.spawn(conf_file) # test pid file pid_file = self.join_run_dir('test-server.pid') self.assertTrue(os.path.exists(pid_file)) pid_on_disk = int(open(pid_file).read().strip()) self.assertEqual(pid_on_disk, 1) # assert procs args self.assertTrue(server.procs) self.assertEqual(len(server.procs), 1) proc = server.procs[0] expected_args = [ 'swift-test-server', conf_file, ] self.assertEqual(proc.args, expected_args) # assert stdout is piped self.assertEqual(proc.stdout, MockProcess.PIPE) self.assertEqual(proc.stderr, proc.stdout) # test multi server process calls spawn multiple times manager.subprocess = MockProcess([11, 12, 13, 14]) conf1 = self.join_swift_dir('test-server/1.conf') conf2 = self.join_swift_dir('test-server/2.conf') conf3 = self.join_swift_dir('test-server/3.conf') conf4 = self.join_swift_dir('test-server/4.conf') server = manager.Server('test', run_dir=t) # test server run once server.spawn(conf1, once=True) self.assertTrue(server.procs) self.assertEqual(len(server.procs), 1) proc = server.procs[0] expected_args = ['swift-test-server', conf1, 'once'] # assert stdout is piped self.assertEqual(proc.stdout, MockProcess.PIPE) self.assertEqual(proc.stderr, proc.stdout) # test server not daemon server.spawn(conf2, daemon=False) self.assertTrue(server.procs) self.assertEqual(len(server.procs), 2) proc = server.procs[1] expected_args = ['swift-test-server', conf2, 'verbose'] self.assertEqual(proc.args, expected_args) # assert stdout is not changed self.assertEqual(proc.stdout, None) self.assertEqual(proc.stderr, None) # test server wait server.spawn(conf3, wait=False) self.assertTrue(server.procs) self.assertEqual(len(server.procs), 3) proc = server.procs[2] # assert stdout is /dev/null self.assertTrue(isinstance(proc.stdout, file)) self.assertEqual(proc.stdout.name, os.devnull) self.assertEqual(proc.stdout.mode, 'w+b') self.assertEqual(proc.stderr, proc.stdout) # test not daemon over-rides wait server.spawn(conf4, wait=False, daemon=False, once=True) self.assertTrue(server.procs) self.assertEqual(len(server.procs), 4) proc = server.procs[3] expected_args = ['swift-test-server', conf4, 'once', 'verbose'] self.assertEqual(proc.args, expected_args) # daemon behavior should trump wait, once shouldn't matter self.assertEqual(proc.stdout, None) self.assertEqual(proc.stderr, None) # assert pids for i, proc in enumerate(server.procs): pid_file = self.join_run_dir('test-server/%d.pid' % (i + 1)) pid_on_disk = int(open(pid_file).read().strip()) self.assertEqual(pid_on_disk, proc.pid) finally: manager.subprocess = old_subprocess def test_wait(self): server = manager.Server('test') self.assertEqual(server.wait(), 0) class MockProcess(threading.Thread): def __init__(self, delay=0.1, fail_to_start=False): threading.Thread.__init__(self) # setup pipe rfd, wfd = os.pipe() # subprocess connection to read stdout self.stdout = os.fdopen(rfd) # real process connection to write stdout self._stdout = os.fdopen(wfd, 'w') self.delay = delay self.finished = False self.returncode = None if fail_to_start: self._returncode = 1 self.run = self.fail else: self._returncode = 0 def __enter__(self): self.start() return self def __exit__(self, *args): if self.isAlive(): self.join() def close_stdout(self): self._stdout.flush() with open(os.devnull, 'wb') as nullfile: try: os.dup2(nullfile.fileno(), self._stdout.fileno()) except OSError: pass def fail(self): print('mock process started', file=self._stdout) sleep(self.delay) # perform setup processing print('mock process failed to start', file=self._stdout) self.close_stdout() def poll(self): self.returncode = self._returncode return self.returncode or None def run(self): print('mock process started', file=self._stdout) sleep(self.delay) # perform setup processing print('setup complete!', file=self._stdout) self.close_stdout() sleep(self.delay) # do some more processing print('mock process finished', file=self._stdout) self.finished = True class MockTime(object): def time(self): return time() def sleep(self, *args, **kwargs): pass with temptree([]) as t: old_stdout = sys.stdout old_wait = manager.WARNING_WAIT old_time = manager.time try: manager.WARNING_WAIT = 0.01 manager.time = MockTime() with open(os.path.join(t, 'output'), 'w+') as f: # actually capture the read stdout (for prints) sys.stdout = f # test closing pipe in subprocess unblocks read with MockProcess() as proc: server.procs = [proc] status = server.wait() self.assertEqual(status, 0) # wait should return before process exits self.assertTrue(proc.isAlive()) self.assertFalse(proc.finished) self.assertTrue(proc.finished) # make sure it did finish # test output kwarg prints subprocess output with MockProcess() as proc: server.procs = [proc] status = server.wait(output=True) output = pop_stream(f) self.assertTrue('mock process started' in output) self.assertTrue('setup complete' in output) # make sure we don't get prints after stdout was closed self.assertTrue('mock process finished' not in output) # test process which fails to start with MockProcess(fail_to_start=True) as proc: server.procs = [proc] status = server.wait() self.assertEqual(status, 1) self.assertTrue('failed' in pop_stream(f)) # test multiple procs procs = [MockProcess(delay=.5) for i in range(3)] for proc in procs: proc.start() server.procs = procs status = server.wait() self.assertEqual(status, 0) for proc in procs: self.assertTrue(proc.isAlive()) for proc in procs: proc.join() finally: sys.stdout = old_stdout manager.WARNING_WAIT = old_wait manager.time = old_time def test_interact(self): class MockProcess(object): def __init__(self, fail=False): self.returncode = None if fail: self._returncode = 1 else: self._returncode = 0 def communicate(self): self.returncode = self._returncode return '', '' server = manager.Server('test') server.procs = [MockProcess()] self.assertEqual(server.interact(), 0) server.procs = [MockProcess(fail=True)] self.assertEqual(server.interact(), 1) procs = [] for fail in (False, True, True): procs.append(MockProcess(fail=fail)) server.procs = procs self.assertTrue(server.interact() > 0) def test_launch(self): # stubs conf_files = ( 'proxy-server.conf', 'auth-server.conf', 'object-server/1.conf', 'object-server/2.conf', 'object-server/3.conf', 'object-server/4.conf', ) pid_files = ( ('proxy-server.pid', 1), ('proxy-server/2.pid', 2), ) # mocks class MockSpawn(object): def __init__(self, pids=None): self.conf_files = [] self.kwargs = [] if not pids: def one_forever(): while True: yield 1 self.pids = one_forever() else: self.pids = (x for x in pids) def __call__(self, conf_file, **kwargs): self.conf_files.append(conf_file) self.kwargs.append(kwargs) rv = next(self.pids) if isinstance(rv, Exception): raise rv else: return rv with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t old_stdout = sys.stdout try: with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f # can't start server w/o an conf server = manager.Server('test', run_dir=t) self.assertFalse(server.launch()) # start mock os running all pids manager.os = MockOs(pids) proc_files = ( ('1/cmdline', 'swift-proxy-server'), ('2/cmdline', 'swift-proxy-server'), ) files, contents = zip(*proc_files) with temptree(files, contents) as proc_dir: manager.PROC_DIR = proc_dir server = manager.Server('proxy', run_dir=t) # can't start server if it's already running self.assertFalse(server.launch()) output = pop_stream(f) self.assertTrue('running' in output) conf_file = self.join_swift_dir( 'proxy-server.conf') self.assertTrue(conf_file in output) pid_file = self.join_run_dir('proxy-server/2.pid') self.assertTrue(pid_file in output) self.assertTrue('already started' in output) # no running pids manager.os = MockOs([]) with temptree([], []) as proc_dir: manager.PROC_DIR = proc_dir # test ignore once for non-start-once server mock_spawn = MockSpawn([1]) server.spawn = mock_spawn conf_file = self.join_swift_dir( 'proxy-server.conf') expected = { 1: conf_file, } self.assertEqual(server.launch(once=True), expected) self.assertEqual(mock_spawn.conf_files, [conf_file]) expected = { 'once': False, } self.assertEqual(mock_spawn.kwargs, [expected]) output = pop_stream(f) self.assertTrue('Starting' in output) self.assertTrue('once' not in output) # test multi-server kwarg once server = manager.Server('object-replicator') with temptree([], []) as proc_dir: manager.PROC_DIR = proc_dir mock_spawn = MockSpawn([1, 2, 3, 4]) server.spawn = mock_spawn conf1 = self.join_swift_dir('object-server/1.conf') conf2 = self.join_swift_dir('object-server/2.conf') conf3 = self.join_swift_dir('object-server/3.conf') conf4 = self.join_swift_dir('object-server/4.conf') expected = { 1: conf1, 2: conf2, 3: conf3, 4: conf4, } self.assertEqual(server.launch(once=True), expected) self.assertEqual(mock_spawn.conf_files, [ conf1, conf2, conf3, conf4]) expected = { 'once': True, } self.assertEqual(len(mock_spawn.kwargs), 4) for kwargs in mock_spawn.kwargs: self.assertEqual(kwargs, expected) # test number kwarg mock_spawn = MockSpawn([4]) manager.PROC_DIR = proc_dir server.spawn = mock_spawn expected = { 4: conf4, } self.assertEqual(server.launch(number=4), expected) self.assertEqual(mock_spawn.conf_files, [conf4]) expected = { 'number': 4 } self.assertEqual(mock_spawn.kwargs, [expected]) # test cmd does not exist server = manager.Server('auth') with temptree([], []) as proc_dir: manager.PROC_DIR = proc_dir mock_spawn = MockSpawn([OSError(errno.ENOENT, 'blah')]) server.spawn = mock_spawn self.assertEqual(server.launch(), {}) self.assertTrue( 'swift-auth-server does not exist' in pop_stream(f)) finally: sys.stdout = old_stdout def test_stop(self): conf_files = ( 'account-server/1.conf', 'account-server/2.conf', 'account-server/3.conf', 'account-server/4.conf', ) pid_files = ( ('account-reaper/1.pid', 1), ('account-reaper/2.pid', 2), ('account-reaper/3.pid', 3), ('account-reaper/4.pid', 4), ) with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t # start all pids in mock os manager.os = MockOs(pids) server = manager.Server('account-reaper', run_dir=t) # test kill all running pids pids = server.stop() self.assertEqual(len(pids), 4) for pid in (1, 2, 3, 4): self.assertTrue(pid in pids) self.assertEqual(manager.os.pid_sigs[pid], [signal.SIGTERM]) conf1 = self.join_swift_dir('account-reaper/1.conf') conf2 = self.join_swift_dir('account-reaper/2.conf') conf3 = self.join_swift_dir('account-reaper/3.conf') conf4 = self.join_swift_dir('account-reaper/4.conf') # reset mock os with only 2 running pids manager.os = MockOs([3, 4]) pids = server.stop() self.assertEqual(len(pids), 2) for pid in (3, 4): self.assertTrue(pid in pids) self.assertEqual(manager.os.pid_sigs[pid], [signal.SIGTERM]) self.assertFalse(os.path.exists(conf1)) self.assertFalse(os.path.exists(conf2)) # test number kwarg manager.os = MockOs([3, 4]) pids = server.stop(number=3) self.assertEqual(len(pids), 1) expected = { 3: conf3, } self.assertTrue(pids, expected) self.assertEqual(manager.os.pid_sigs[3], [signal.SIGTERM]) self.assertFalse(os.path.exists(conf4)) self.assertFalse(os.path.exists(conf3)) class TestManager(unittest.TestCase): def test_create(self): m = manager.Manager(['test']) self.assertEqual(len(m.servers), 1) server = m.servers.pop() self.assertTrue(isinstance(server, manager.Server)) self.assertEqual(server.server, 'test-server') # test multi-server and simple dedupe servers = ['object-replicator', 'object-auditor', 'object-replicator'] m = manager.Manager(servers) self.assertEqual(len(m.servers), 2) for server in m.servers: self.assertTrue(server.server in servers) # test all m = manager.Manager(['all']) self.assertEqual(len(m.servers), len(manager.ALL_SERVERS)) for server in m.servers: self.assertTrue(server.server in manager.ALL_SERVERS) # test main m = manager.Manager(['main']) self.assertEqual(len(m.servers), len(manager.MAIN_SERVERS)) for server in m.servers: self.assertTrue(server.server in manager.MAIN_SERVERS) # test rest m = manager.Manager(['rest']) self.assertEqual(len(m.servers), len(manager.REST_SERVERS)) for server in m.servers: self.assertTrue(server.server in manager.REST_SERVERS) # test main + rest == all m = manager.Manager(['main', 'rest']) self.assertEqual(len(m.servers), len(manager.ALL_SERVERS)) for server in m.servers: self.assertTrue(server.server in manager.ALL_SERVERS) # test dedupe m = manager.Manager(['main', 'rest', 'proxy', 'object', 'container', 'account']) self.assertEqual(len(m.servers), len(manager.ALL_SERVERS)) for server in m.servers: self.assertTrue(server.server in manager.ALL_SERVERS) # test glob m = manager.Manager(['object-*']) object_servers = [s for s in manager.ALL_SERVERS if s.startswith('object')] self.assertEqual(len(m.servers), len(object_servers)) for s in m.servers: self.assertTrue(str(s) in object_servers) m = manager.Manager(['*-replicator']) replicators = [s for s in manager.ALL_SERVERS if s.endswith('replicator')] for s in m.servers: self.assertTrue(str(s) in replicators) def test_iter(self): m = manager.Manager(['all']) self.assertEqual(len(list(m)), len(manager.ALL_SERVERS)) for server in m: self.assertTrue(server.server in manager.ALL_SERVERS) def test_default_strict(self): # test default strict m = manager.Manager(['proxy']) self.assertEqual(m._default_strict, True) # aliases m = manager.Manager(['main']) self.assertEqual(m._default_strict, False) m = manager.Manager(['proxy*']) self.assertEqual(m._default_strict, False) def test_status(self): class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called_kwargs = [] def status(self, **kwargs): self.called_kwargs.append(kwargs) if 'error' in self.server: return 1 else: return 0 old_server_class = manager.Server try: manager.Server = MockServer m = manager.Manager(['test']) status = m.status() self.assertEqual(status, 0) m = manager.Manager(['error']) status = m.status() self.assertEqual(status, 1) # test multi-server m = manager.Manager(['test', 'error']) kwargs = {'key': 'value'} status = m.status(**kwargs) self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called_kwargs, [kwargs]) finally: manager.Server = old_server_class def test_start(self): def mock_setup_env(): getattr(mock_setup_env, 'called', []).append(True) class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called = defaultdict(list) def launch(self, **kwargs): self.called['launch'].append(kwargs) if 'noconfig' in self.server: return {} elif 'somerunning' in self.server: return {} else: return {1: self.server[0]} def wait(self, **kwargs): self.called['wait'].append(kwargs) return int('error' in self.server) def stop(self, **kwargs): self.called['stop'].append(kwargs) def interact(self, **kwargs): self.called['interact'].append(kwargs) if 'raise' in self.server: raise KeyboardInterrupt elif 'error' in self.server: return 1 else: return 0 old_setup_env = manager.setup_env old_swift_server = manager.Server try: manager.setup_env = mock_setup_env manager.Server = MockServer # test no errors on launch m = manager.Manager(['proxy']) status = m.start() self.assertEqual(status, 0) for server in m.servers: self.assertEqual(server.called['launch'], [{}]) # test error on launch m = manager.Manager(['proxy', 'error']) status = m.start() self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called['launch'], [{}]) self.assertEqual(server.called['wait'], [{}]) # test interact m = manager.Manager(['proxy', 'error']) kwargs = {'daemon': False} status = m.start(**kwargs) self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called['launch'], [kwargs]) self.assertEqual(server.called['interact'], [kwargs]) m = manager.Manager(['raise']) kwargs = {'daemon': False} status = m.start(**kwargs) # test no config m = manager.Manager(['proxy', 'noconfig']) status = m.start() self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called['launch'], [{}]) self.assertEqual(server.called['wait'], [{}]) # test no config with --non-strict m = manager.Manager(['proxy', 'noconfig']) status = m.start(strict=False) self.assertEqual(status, 0) for server in m.servers: self.assertEqual(server.called['launch'], [{'strict': False}]) self.assertEqual(server.called['wait'], [{'strict': False}]) # test no config --strict m = manager.Manager(['proxy', 'noconfig']) status = m.start(strict=True) self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called['launch'], [{'strict': True}]) self.assertEqual(server.called['wait'], [{'strict': True}]) # test no config with alias m = manager.Manager(['main', 'noconfig']) status = m.start() self.assertEqual(status, 0) for server in m.servers: self.assertEqual(server.called['launch'], [{}]) self.assertEqual(server.called['wait'], [{}]) # test no config with alias and --non-strict m = manager.Manager(['main', 'noconfig']) status = m.start(strict=False) self.assertEqual(status, 0) for server in m.servers: self.assertEqual(server.called['launch'], [{'strict': False}]) self.assertEqual(server.called['wait'], [{'strict': False}]) # test no config with alias and --strict m = manager.Manager(['main', 'noconfig']) status = m.start(strict=True) self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called['launch'], [{'strict': True}]) self.assertEqual(server.called['wait'], [{'strict': True}]) # test already all running m = manager.Manager(['proxy', 'somerunning']) status = m.start() self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called['launch'], [{}]) self.assertEqual(server.called['wait'], [{}]) # test already all running --non-strict m = manager.Manager(['proxy', 'somerunning']) status = m.start(strict=False) self.assertEqual(status, 0) for server in m.servers: self.assertEqual(server.called['launch'], [{'strict': False}]) self.assertEqual(server.called['wait'], [{'strict': False}]) # test already all running --strict m = manager.Manager(['proxy', 'somerunning']) status = m.start(strict=True) self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called['launch'], [{'strict': True}]) self.assertEqual(server.called['wait'], [{'strict': True}]) # test already all running with alias m = manager.Manager(['main', 'somerunning']) status = m.start() self.assertEqual(status, 0) for server in m.servers: self.assertEqual(server.called['launch'], [{}]) self.assertEqual(server.called['wait'], [{}]) # test already all running with alias and --non-strict m = manager.Manager(['main', 'somerunning']) status = m.start(strict=False) self.assertEqual(status, 0) for server in m.servers: self.assertEqual(server.called['launch'], [{'strict': False}]) self.assertEqual(server.called['wait'], [{'strict': False}]) # test already all running with alias and --strict m = manager.Manager(['main', 'somerunning']) status = m.start(strict=True) self.assertEqual(status, 1) for server in m.servers: self.assertEqual(server.called['launch'], [{'strict': True}]) self.assertEqual(server.called['wait'], [{'strict': True}]) finally: manager.setup_env = old_setup_env manager.Server = old_swift_server def test_no_wait(self): class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called = defaultdict(list) def launch(self, **kwargs): self.called['launch'].append(kwargs) # must return non-empty dict if launch succeeded return {1: self.server[0]} def wait(self, **kwargs): self.called['wait'].append(kwargs) return int('error' in self.server) orig_swift_server = manager.Server try: manager.Server = MockServer # test success init = manager.Manager(['proxy']) status = init.no_wait() self.assertEqual(status, 0) for server in init.servers: self.assertEqual(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assertFalse(called_kwargs['wait']) self.assertFalse(server.called['wait']) # test no errocode status even on error init = manager.Manager(['error']) status = init.no_wait() self.assertEqual(status, 0) for server in init.servers: self.assertEqual(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assertTrue('wait' in called_kwargs) self.assertFalse(called_kwargs['wait']) self.assertFalse(server.called['wait']) # test wait with once option init = manager.Manager(['updater', 'replicator-error']) status = init.no_wait(once=True) self.assertEqual(status, 0) for server in init.servers: self.assertEqual(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assertTrue('wait' in called_kwargs) self.assertFalse(called_kwargs['wait']) self.assertTrue('once' in called_kwargs) self.assertTrue(called_kwargs['once']) self.assertFalse(server.called['wait']) finally: manager.Server = orig_swift_server def test_no_daemon(self): class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called = defaultdict(list) def launch(self, **kwargs): self.called['launch'].append(kwargs) # must return non-empty dict if launch succeeded return {1: self.server[0]} def interact(self, **kwargs): self.called['interact'].append(kwargs) return int('error' in self.server) orig_swift_server = manager.Server try: manager.Server = MockServer # test success init = manager.Manager(['proxy']) stats = init.no_daemon() self.assertEqual(stats, 0) # test error init = manager.Manager(['proxy', 'object-error']) stats = init.no_daemon() self.assertEqual(stats, 1) # test once init = manager.Manager(['proxy', 'object-error']) stats = init.no_daemon() for server in init.servers: self.assertEqual(len(server.called['launch']), 1) self.assertEqual(len(server.called['wait']), 0) self.assertEqual(len(server.called['interact']), 1) finally: manager.Server = orig_swift_server def test_once(self): class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called = defaultdict(list) def wait(self, **kwargs): self.called['wait'].append(kwargs) if 'error' in self.server: return 1 else: return 0 def launch(self, **kwargs): self.called['launch'].append(kwargs) return {1: 'account-reaper'} orig_swift_server = manager.Server try: manager.Server = MockServer # test no errors init = manager.Manager(['account-reaper']) status = init.once() self.assertEqual(status, 0) # test error code on error init = manager.Manager(['error-reaper']) status = init.once() self.assertEqual(status, 1) for server in init.servers: self.assertEqual(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assertEqual(called_kwargs, {'once': True}) self.assertEqual(len(server.called['wait']), 1) self.assertEqual(len(server.called['interact']), 0) finally: manager.Server = orig_swift_server def test_stop(self): class MockServerFactory(object): class MockServer(object): def __init__(self, pids, run_dir=manager.RUN_DIR): self.pids = pids def stop(self, **kwargs): return self.pids def status(self, **kwargs): return not self.pids def __init__(self, server_pids, run_dir=manager.RUN_DIR): self.server_pids = server_pids def __call__(self, server, run_dir=manager.RUN_DIR): return MockServerFactory.MockServer(self.server_pids[server]) def mock_watch_server_pids(server_pids, **kwargs): for server, pids in server_pids.items(): for pid in pids: if pid is None: continue yield server, pid def mock_kill_group(pid, sig): self.fail('kill_group should not be called') _orig_server = manager.Server _orig_watch_server_pids = manager.watch_server_pids _orig_kill_group = manager.kill_group try: manager.watch_server_pids = mock_watch_server_pids manager.kill_group = mock_kill_group # test stop one server server_pids = { 'test': {1: "dummy.pid"} } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.stop() self.assertEqual(status, 0) # test not running server_pids = { 'test': {} } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.stop() self.assertEqual(status, 1) # test kill not running server_pids = { 'test': {} } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.kill() self.assertEqual(status, 0) # test won't die server_pids = { 'test': {None: None} } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.stop() self.assertEqual(status, 1) finally: manager.Server = _orig_server manager.watch_server_pids = _orig_watch_server_pids manager.kill_group = _orig_kill_group def test_stop_kill_after_timeout(self): class MockServerFactory(object): class MockServer(object): def __init__(self, pids, run_dir=manager.RUN_DIR): self.pids = pids def stop(self, **kwargs): return self.pids def status(self, **kwargs): return not self.pids def __init__(self, server_pids, run_dir=manager.RUN_DIR): self.server_pids = server_pids def __call__(self, server, run_dir=manager.RUN_DIR): return MockServerFactory.MockServer(self.server_pids[server]) def mock_watch_server_pids(server_pids, **kwargs): for server, pids in server_pids.items(): for pid in pids: if pid is None: continue yield server, pid mock_kill_group_called = [] def mock_kill_group(*args): mock_kill_group_called.append(args) def mock_kill_group_oserr(*args): raise OSError() def mock_kill_group_oserr_ESRCH(*args): raise OSError(errno.ESRCH, 'No such process') _orig_server = manager.Server _orig_watch_server_pids = manager.watch_server_pids _orig_kill_group = manager.kill_group try: manager.watch_server_pids = mock_watch_server_pids manager.kill_group = mock_kill_group # test stop one server server_pids = { 'test': {None: None} } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.stop(kill_after_timeout=True) self.assertEqual(status, 1) self.assertEqual(mock_kill_group_called, [(None, 9)]) manager.kill_group = mock_kill_group_oserr # test stop one server - OSError server_pids = { 'test': {None: None} } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) with self.assertRaises(OSError): status = m.stop(kill_after_timeout=True) manager.kill_group = mock_kill_group_oserr_ESRCH # test stop one server - OSError: No such process server_pids = { 'test': {None: None} } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.stop(kill_after_timeout=True) self.assertEqual(status, 1) finally: manager.Server = _orig_server manager.watch_server_pids = _orig_watch_server_pids manager.kill_group = _orig_kill_group # TODO(clayg): more tests def test_shutdown(self): m = manager.Manager(['test']) m.stop_was_called = False def mock_stop(*args, **kwargs): m.stop_was_called = True expected = {'graceful': True} self.assertEqual(kwargs, expected) return 0 m.stop = mock_stop status = m.shutdown() self.assertEqual(status, 0) self.assertEqual(m.stop_was_called, True) def test_restart(self): m = manager.Manager(['test']) m.stop_was_called = False def mock_stop(*args, **kwargs): m.stop_was_called = True return 0 m.start_was_called = False def mock_start(*args, **kwargs): m.start_was_called = True return 0 m.stop = mock_stop m.start = mock_start status = m.restart() self.assertEqual(status, 0) self.assertEqual(m.stop_was_called, True) self.assertEqual(m.start_was_called, True) def test_reload(self): class MockManager(object): called = defaultdict(list) def __init__(self, servers): pass @classmethod def reset_called(cls): cls.called = defaultdict(list) def stop(self, **kwargs): MockManager.called['stop'].append(kwargs) return 0 def start(self, **kwargs): MockManager.called['start'].append(kwargs) return 0 _orig_manager = manager.Manager try: m = _orig_manager(['auth']) for server in m.servers: self.assertTrue(server.server in manager.GRACEFUL_SHUTDOWN_SERVERS) manager.Manager = MockManager status = m.reload() self.assertEqual(status, 0) expected = { 'start': [{'graceful': True}], 'stop': [{'graceful': True}], } self.assertEqual(MockManager.called, expected) # test force graceful MockManager.reset_called() m = _orig_manager(['*-server']) self.assertEqual(len(m.servers), 4) for server in m.servers: self.assertTrue(server.server in manager.GRACEFUL_SHUTDOWN_SERVERS) manager.Manager = MockManager status = m.reload(graceful=False) self.assertEqual(status, 0) expected = { 'start': [{'graceful': True}] * 4, 'stop': [{'graceful': True}] * 4, } self.assertEqual(MockManager.called, expected) finally: manager.Manager = _orig_manager def test_force_reload(self): m = manager.Manager(['test']) m.reload_was_called = False def mock_reload(*args, **kwargs): m.reload_was_called = True return 0 m.reload = mock_reload status = m.force_reload() self.assertEqual(status, 0) self.assertEqual(m.reload_was_called, True) def test_get_command(self): m = manager.Manager(['test']) self.assertEqual(m.start, m.get_command('start')) self.assertEqual(m.force_reload, m.get_command('force-reload')) self.assertEqual(m.get_command('force-reload'), m.get_command('force_reload')) self.assertRaises(manager.UnknownCommandError, m.get_command, 'no_command') self.assertRaises(manager.UnknownCommandError, m.get_command, '__init__') def test_list_commands(self): for cmd, help in manager.Manager.list_commands(): method = getattr(manager.Manager, cmd.replace('-', '_'), None) self.assertTrue(method, '%s is not a command' % cmd) self.assertTrue(getattr(method, 'publicly_accessible', False)) self.assertEqual(method.__doc__.strip(), help) def test_run_command(self): m = manager.Manager(['test']) m.cmd_was_called = False def mock_cmd(*args, **kwargs): m.cmd_was_called = True expected = {'kw1': True, 'kw2': False} self.assertEqual(kwargs, expected) return 0 mock_cmd.publicly_accessible = True m.mock_cmd = mock_cmd kwargs = {'kw1': True, 'kw2': False} status = m.run_command('mock_cmd', **kwargs) self.assertEqual(status, 0) self.assertEqual(m.cmd_was_called, True) if __name__ == '__main__': unittest.main()