openstack/swift
Code Issues Proposed changes

Merge "Add process name checking into swift-init"

Browse Source
This commit is contained in:
Jenkins 2015-05-28 14:43:55 +00:00 committed by Gerrit Code Review
commit c716b170b3
3 changed files with 241 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
swift/common/exceptions.py
View File

@ -256,3 +256,7 @@ class ClientException(Exception):
b += ' [first 60 chars of response] %s' \ b += ' [first 60 chars of response] %s' \
% self.http_response_content[:60] % self.http_response_content[:60]
return b and '%s: %s' % (a, b) or a return b and '%s: %s' % (a, b) or a
class InvalidPidFileException(Exception):
pass

32
swift/common/manager.py
View File

@ -24,9 +24,11 @@ import re
from swift import gettext_ as _ from swift import gettext_ as _
from swift.common.utils import search_tree, remove_file, write_file from swift.common.utils import search_tree, remove_file, write_file
from swift.common.exceptions import InvalidPidFileException
SWIFT_DIR = '/etc/swift' SWIFT_DIR = '/etc/swift'
RUN_DIR = '/var/run/swift' RUN_DIR = '/var/run/swift'
PROC_DIR = '/proc'
# auth-server has been removed from ALL_SERVERS, start it explicitly # auth-server has been removed from ALL_SERVERS, start it explicitly
ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor',
@ -134,6 +136,29 @@ def watch_server_pids(server_pids, interval=1, **kwargs):
time.sleep(0.1) time.sleep(0.1)
def safe_kill(pid, sig, name):
"""Send signal to process and check process name
: param pid: process id
: param sig: signal to send
: param name: name to ensure target process
"""
# check process name for SIG_DFL
if sig == signal.SIG_DFL:
try:
proc_file = '%s/%d/cmdline' % (PROC_DIR, pid)
if os.path.exists(proc_file):
with open(proc_file, 'r') as fd:
if name not in fd.read():
# unknown process is using the pid
raise InvalidPidFileException()
except IOError:
pass
os.kill(pid, sig)
class UnknownCommandError(Exception): class UnknownCommandError(Exception):
pass pass
@ -488,7 +513,12 @@ class Server(object):
if sig != signal.SIG_DFL: if sig != signal.SIG_DFL:
print _('Signal %s pid: %s signal: %s') % (self.server, print _('Signal %s pid: %s signal: %s') % (self.server,
pid, sig) pid, sig)
os.kill(pid, sig) safe_kill(pid, sig, 'swift-%s' % self.server)
except InvalidPidFileException as e:
if kwargs.get('verbose'):
print _('Removing pid file %s with wrong pid %d') \
% (pid_file, pid)
remove_file(pid_file)
except OSError as e: except OSError as e:
if e.errno == errno.ESRCH: if e.errno == errno.ESRCH:
# pid does not exist # pid does not exist

323
test/unit/common/test_manager.py
View File

@ -26,6 +26,7 @@ from threading import Thread
from time import sleep, time from time import sleep, time
from swift.common import manager from swift.common import manager
from swift.common.exceptions import InvalidPidFileException
DUMMY_SIG = 1 DUMMY_SIG = 1
@ -63,7 +64,6 @@ def pop_stream(f):
output = f.read() output = f.read()
f.seek(0) f.seek(0)
f.truncate() f.truncate()
#print >> sys.stderr, output
return output return output
@ -257,6 +257,23 @@ class TestManagerModule(unittest.TestCase):
manager.time = _orig_time manager.time = _orig_time
manager.Server = _orig_server manager.Server = _orig_server
def test_safe_kill(self):
manager.os = MockOs([1, 2, 3, 4])
proc_files = (
('1/cmdline', 'same-procname'),
('2/cmdline', 'another-procname'),
('4/cmdline', 'another-procname'),
)
files, contents = zip(*proc_files)
with temptree(files, contents) as t:
manager.PROC_DIR = t
manager.safe_kill(1, signal.SIG_DFL, 'same-procname')
self.assertRaises(InvalidPidFileException, manager.safe_kill,
2, signal.SIG_DFL, 'same-procname')
manager.safe_kill(3, signal.SIG_DFL, 'same-procname')
manager.safe_kill(4, signal.SIGHUP, 'same-procname')
def test_exc(self): def test_exc(self):
self.assert_(issubclass(manager.UnknownCommandError, Exception)) self.assert_(issubclass(manager.UnknownCommandError, Exception))
@ -680,17 +697,19 @@ class TestServer(unittest.TestCase):
self.assertEquals(pid_file, pid_two) self.assertEquals(pid_file, pid_two)
def test_signal_pids(self): def test_signal_pids(self):
pid_files = ( temp_files = (
('proxy-server.pid', 1), ('var/run/proxy-server.pid', 1),
('auth-server.pid', 2), ('var/run/auth-server.pid', 2),
('object-server.pid', 3), ('var/run/one-server.pid', 3),
('var/run/object-server.pid', 4),
('proc/3/cmdline', 'swift-another-server')
) )
files, pids = zip(*pid_files) with temptree(*zip(*temp_files)) as t:
with temptree(files, pids) as t: manager.RUN_DIR = os.path.join(t, 'var/run')
manager.RUN_DIR = t manager.PROC_DIR = os.path.join(t, 'proc')
# mock os with both pids running # mock os with so both the first and second are running
manager.os = MockOs([1, 2]) manager.os = MockOs([1, 2])
server = manager.Server('proxy', run_dir=t) server = manager.Server('proxy', run_dir=manager.RUN_DIR)
pids = server.signal_pids(DUMMY_SIG) pids = server.signal_pids(DUMMY_SIG)
self.assertEquals(len(pids), 1) self.assertEquals(len(pids), 1)
self.assert_(1 in pids) self.assert_(1 in pids)
@ -703,7 +722,7 @@ class TestServer(unittest.TestCase):
try: try:
with open(os.path.join(t, 'output'), 'w+') as f: with open(os.path.join(t, 'output'), 'w+') as f:
sys.stdout = f sys.stdout = f
#test print details # test print details
pids = server.signal_pids(DUMMY_SIG) pids = server.signal_pids(DUMMY_SIG)
output = pop_stream(f) output = pop_stream(f)
self.assert_('pid: %s' % 1 in output) self.assert_('pid: %s' % 1 in output)
@ -711,7 +730,7 @@ class TestServer(unittest.TestCase):
# test no details on signal.SIG_DFL # test no details on signal.SIG_DFL
pids = server.signal_pids(signal.SIG_DFL) pids = server.signal_pids(signal.SIG_DFL)
self.assertEquals(pop_stream(f), '') self.assertEquals(pop_stream(f), '')
# reset mock os so only the other server is running # reset mock os so only the second server is running
manager.os = MockOs([2]) manager.os = MockOs([2])
# test pid not running # test pid not running
pids = server.signal_pids(signal.SIG_DFL) pids = server.signal_pids(signal.SIG_DFL)
@ -722,42 +741,63 @@ class TestServer(unittest.TestCase):
self.join_run_dir('proxy-server.pid'))) self.join_run_dir('proxy-server.pid')))
# reset mock os with no running pids # reset mock os with no running pids
manager.os = MockOs([]) manager.os = MockOs([])
server = manager.Server('auth', run_dir=t) server = manager.Server('auth', run_dir=manager.RUN_DIR)
# test verbose warns on removing pid file # test verbose warns on removing stale pid file
pids = server.signal_pids(signal.SIG_DFL, verbose=True) pids = server.signal_pids(signal.SIG_DFL, verbose=True)
output = pop_stream(f) output = pop_stream(f)
self.assert_('stale pid' in output.lower()) self.assert_('stale pid' in output.lower())
auth_pid = self.join_run_dir('auth-server.pid') auth_pid = self.join_run_dir('auth-server.pid')
self.assert_(auth_pid in output) self.assert_(auth_pid in output)
# reset mock os so only the third server is running
manager.os = MockOs([3])
server = manager.Server('one', run_dir=manager.RUN_DIR)
# test verbose warns on removing invalid pid file
pids = server.signal_pids(signal.SIG_DFL, verbose=True)
output = pop_stream(f)
old_stdout.write('output %s' % output)
self.assert_('removing pid file' in output.lower())
one_pid = self.join_run_dir('one-server.pid')
self.assert_(one_pid in output)
# reset mock os with no running pids
manager.os = MockOs([])
# test warning with insufficient permissions # test warning with insufficient permissions
server = manager.Server('object', run_dir=t) server = manager.Server('object', run_dir=manager.RUN_DIR)
pids = server.signal_pids(manager.os.RAISE_EPERM_SIG) pids = server.signal_pids(manager.os.RAISE_EPERM_SIG)
output = pop_stream(f) output = pop_stream(f)
self.assert_('no permission to signal pid 3' in self.assert_('no permission to signal pid 4' in
output.lower(), output) output.lower(), output)
finally: finally:
sys.stdout = old_stdout sys.stdout = old_stdout
def test_get_running_pids(self): def test_get_running_pids(self):
# test only gets running pids # test only gets running pids
pid_files = ( temp_files = (
('test-server1.pid', 1), ('var/run/test-server1.pid', 1),
('test-server2.pid', 2), ('var/run/test-server2.pid', 2),
('var/run/test-server3.pid', 3),
('proc/1/cmdline', 'swift-test-server'),
('proc/3/cmdline', 'swift-another-server')
) )
with temptree(*zip(*pid_files)) as t: with temptree(*zip(*temp_files)) as t:
manager.RUN_DIR = t manager.RUN_DIR = os.path.join(t, 'var/run')
server = manager.Server('test-server', run_dir=t) manager.PROC_DIR = os.path.join(t, 'proc')
server = manager.Server(
'test-server', run_dir=manager.RUN_DIR)
# mock os, only pid '1' is running # mock os, only pid '1' is running
manager.os = MockOs([1]) manager.os = MockOs([1, 3])
running_pids = server.get_running_pids() running_pids = server.get_running_pids()
self.assertEquals(len(running_pids), 1) self.assertEquals(len(running_pids), 1)
self.assert_(1 in running_pids) self.assert_(1 in running_pids)
self.assert_(2 not in running_pids) self.assert_(2 not in running_pids)
self.assert_(3 not in running_pids)
# test persistent running pid files # test persistent running pid files
self.assert_(os.path.exists(os.path.join(t, 'test-server1.pid'))) self.assert_(os.path.exists(
os.path.join(manager.RUN_DIR, 'test-server1.pid')))
# test clean up stale pids # test clean up stale pids
pid_two = self.join_swift_dir('test-server2.pid') pid_two = self.join_swift_dir('test-server2.pid')
self.assertFalse(os.path.exists(pid_two)) self.assertFalse(os.path.exists(pid_two))
pid_three = self.join_swift_dir('test-server3.pid')
self.assertFalse(os.path.exists(pid_three))
# reset mock os, no pids running # reset mock os, no pids running
manager.os = MockOs([]) manager.os = MockOs([])
running_pids = server.get_running_pids() running_pids = server.get_running_pids()
@ -765,7 +805,7 @@ class TestServer(unittest.TestCase):
# and now all pid files are cleaned out # and now all pid files are cleaned out
pid_one = self.join_run_dir('test-server1.pid') pid_one = self.join_run_dir('test-server1.pid')
self.assertFalse(os.path.exists(pid_one)) self.assertFalse(os.path.exists(pid_one))
all_pids = os.listdir(t) all_pids = os.listdir(manager.RUN_DIR)
self.assertEquals(len(all_pids), 0) self.assertEquals(len(all_pids), 0)
# test only get pids for right server # test only get pids for right server
@ -883,40 +923,68 @@ class TestServer(unittest.TestCase):
sys.stdout = f sys.stdout = f
# test status for all running # test status for all running
manager.os = MockOs(pids) manager.os = MockOs(pids)
self.assertEquals(server.status(), 0) proc_files = (
output = pop_stream(f).strip().splitlines() ('1/cmdline', 'swift-test-server'),
self.assertEquals(len(output), 4) ('2/cmdline', 'swift-test-server'),
for line in output: ('3/cmdline', 'swift-test-server'),
self.assert_('test-server running' in line) ('4/cmdline', 'swift-test-server'),
)
files, contents = zip(*proc_files)
with temptree(files, contents) as t:
manager.PROC_DIR = t
self.assertEquals(server.status(), 0)
output = pop_stream(f).strip().splitlines()
self.assertEquals(len(output), 4)
for line in output:
self.assert_('test-server running' in line)
# test get single server by number # test get single server by number
self.assertEquals(server.status(number=4), 0) with temptree([], []) as t:
output = pop_stream(f).strip().splitlines() manager.PROC_DIR = t
self.assertEquals(len(output), 1) self.assertEquals(server.status(number=4), 0)
line = output[0] output = pop_stream(f).strip().splitlines()
self.assert_('test-server running' in line) self.assertEquals(len(output), 1)
conf_four = self.join_swift_dir(conf_files[3]) line = output[0]
self.assert_('4 - %s' % conf_four in line) self.assert_('test-server running' in line)
conf_four = self.join_swift_dir(conf_files[3])
self.assert_('4 - %s' % conf_four in line)
# test some servers not running # test some servers not running
manager.os = MockOs([1, 2, 3]) manager.os = MockOs([1, 2, 3])
self.assertEquals(server.status(), 0) proc_files = (
output = pop_stream(f).strip().splitlines() ('1/cmdline', 'swift-test-server'),
self.assertEquals(len(output), 3) ('2/cmdline', 'swift-test-server'),
for line in output: ('3/cmdline', 'swift-test-server'),
self.assert_('test-server running' in line) )
files, contents = zip(*proc_files)
with temptree(files, contents) as t:
manager.PROC_DIR = t
self.assertEquals(server.status(), 0)
output = pop_stream(f).strip().splitlines()
self.assertEquals(len(output), 3)
for line in output:
self.assert_('test-server running' in line)
# test single server not running # test single server not running
manager.os = MockOs([1, 2]) manager.os = MockOs([1, 2])
self.assertEquals(server.status(number=3), 1) proc_files = (
output = pop_stream(f).strip().splitlines() ('1/cmdline', 'swift-test-server'),
self.assertEquals(len(output), 1) ('2/cmdline', 'swift-test-server'),
line = output[0] )
self.assert_('not running' in line) files, contents = zip(*proc_files)
conf_three = self.join_swift_dir(conf_files[2]) with temptree(files, contents) as t:
self.assert_(conf_three in line) manager.PROC_DIR = t
self.assertEquals(server.status(number=3), 1)
output = pop_stream(f).strip().splitlines()
self.assertEquals(len(output), 1)
line = output[0]
self.assert_('not running' in line)
conf_three = self.join_swift_dir(conf_files[2])
self.assert_(conf_three in line)
# test no running pids # test no running pids
manager.os = MockOs([]) manager.os = MockOs([])
self.assertEquals(server.status(), 1) with temptree([], []) as t:
output = pop_stream(f).lower() manager.PROC_DIR = t
self.assert_('no test-server running' in output) self.assertEquals(server.status(), 1)
output = pop_stream(f).lower()
self.assert_('no test-server running' in output)
# test use provided pids # test use provided pids
pids = { pids = {
1: '1.pid', 1: '1.pid',
@ -1210,7 +1278,7 @@ class TestServer(unittest.TestCase):
('proxy-server/2.pid', 2), ('proxy-server/2.pid', 2),
) )
#mocks # mocks
class MockSpawn(object): class MockSpawn(object):
def __init__(self, pids=None): def __init__(self, pids=None):
@ -1247,76 +1315,97 @@ class TestServer(unittest.TestCase):
self.assertFalse(server.launch()) self.assertFalse(server.launch())
# start mock os running all pids # start mock os running all pids
manager.os = MockOs(pids) manager.os = MockOs(pids)
server = manager.Server('proxy', run_dir=t) proc_files = (
# can't start server if it's already running ('1/cmdline', 'swift-proxy-server'),
self.assertFalse(server.launch()) ('2/cmdline', 'swift-proxy-server'),
output = pop_stream(f) )
self.assert_('running' in output) files, contents = zip(*proc_files)
conf_file = self.join_swift_dir('proxy-server.conf') with temptree(files, contents) as proc_dir:
self.assert_(conf_file in output) manager.PROC_DIR = proc_dir
pid_file = self.join_run_dir('proxy-server/2.pid') server = manager.Server('proxy', run_dir=t)
self.assert_(pid_file in output) # can't start server if it's already running
self.assert_('already started' in output) self.assertFalse(server.launch())
output = pop_stream(f)
self.assert_('running' in output)
conf_file = self.join_swift_dir(
'proxy-server.conf')
self.assert_(conf_file in output)
pid_file = self.join_run_dir('proxy-server/2.pid')
self.assert_(pid_file in output)
self.assert_('already started' in output)
# no running pids # no running pids
manager.os = MockOs([]) manager.os = MockOs([])
# test ignore once for non-start-once server with temptree([], []) as proc_dir:
mock_spawn = MockSpawn([1]) manager.PROC_DIR = proc_dir
server.spawn = mock_spawn # test ignore once for non-start-once server
conf_file = self.join_swift_dir('proxy-server.conf') mock_spawn = MockSpawn([1])
expected = { server.spawn = mock_spawn
1: conf_file, conf_file = self.join_swift_dir(
} 'proxy-server.conf')
self.assertEquals(server.launch(once=True), expected) expected = {
self.assertEquals(mock_spawn.conf_files, [conf_file]) 1: conf_file,
expected = { }
'once': False, self.assertEquals(server.launch(once=True),
} expected)
self.assertEquals(mock_spawn.kwargs, [expected]) self.assertEquals(mock_spawn.conf_files,
output = pop_stream(f) [conf_file])
self.assert_('Starting' in output) expected = {
self.assert_('once' not in output) 'once': False,
}
self.assertEquals(mock_spawn.kwargs, [expected])
output = pop_stream(f)
self.assert_('Starting' in output)
self.assert_('once' not in output)
# test multi-server kwarg once # test multi-server kwarg once
server = manager.Server('object-replicator') server = manager.Server('object-replicator')
mock_spawn = MockSpawn([1, 2, 3, 4]) with temptree([], []) as proc_dir:
server.spawn = mock_spawn manager.PROC_DIR = proc_dir
conf1 = self.join_swift_dir('object-server/1.conf') mock_spawn = MockSpawn([1, 2, 3, 4])
conf2 = self.join_swift_dir('object-server/2.conf') server.spawn = mock_spawn
conf3 = self.join_swift_dir('object-server/3.conf') conf1 = self.join_swift_dir('object-server/1.conf')
conf4 = self.join_swift_dir('object-server/4.conf') conf2 = self.join_swift_dir('object-server/2.conf')
expected = { conf3 = self.join_swift_dir('object-server/3.conf')
1: conf1, conf4 = self.join_swift_dir('object-server/4.conf')
2: conf2, expected = {
3: conf3, 1: conf1,
4: conf4, 2: conf2,
} 3: conf3,
self.assertEquals(server.launch(once=True), expected) 4: conf4,
self.assertEquals(mock_spawn.conf_files, [ }
conf1, conf2, conf3, conf4]) self.assertEquals(server.launch(once=True),
expected = { expected)
'once': True, self.assertEquals(mock_spawn.conf_files, [
} conf1, conf2, conf3, conf4])
self.assertEquals(len(mock_spawn.kwargs), 4) expected = {
for kwargs in mock_spawn.kwargs: 'once': True,
self.assertEquals(kwargs, expected) }
# test number kwarg self.assertEquals(len(mock_spawn.kwargs), 4)
mock_spawn = MockSpawn([4]) for kwargs in mock_spawn.kwargs:
server.spawn = mock_spawn self.assertEquals(kwargs, expected)
expected = { # test number kwarg
4: conf4, mock_spawn = MockSpawn([4])
} manager.PROC_DIR = proc_dir
self.assertEquals(server.launch(number=4), expected) server.spawn = mock_spawn
self.assertEquals(mock_spawn.conf_files, [conf4]) expected = {
expected = { 4: conf4,
'number': 4 }
} self.assertEquals(server.launch(number=4),
self.assertEquals(mock_spawn.kwargs, [expected]) expected)
self.assertEquals(mock_spawn.conf_files, [conf4])
expected = {
'number': 4
}
self.assertEquals(mock_spawn.kwargs, [expected])
# test cmd does not exist # test cmd does not exist
server = manager.Server('auth') server = manager.Server('auth')
mock_spawn = MockSpawn([OSError(errno.ENOENT, 'blah')]) with temptree([], []) as proc_dir:
server.spawn = mock_spawn manager.PROC_DIR = proc_dir
self.assertEquals(server.launch(), {}) mock_spawn = MockSpawn([OSError(errno.ENOENT,
self.assert_('swift-auth-server does not exist' in 'blah')])
pop_stream(f)) server.spawn = mock_spawn
self.assertEquals(server.launch(), {})
self.assert_('swift-auth-server does not exist' in
pop_stream(f))
finally: finally:
sys.stdout = old_stdout sys.stdout = old_stdout