openstack/swift
Code Issues Proposed changes

Merge "Add process name checking into swift-init"

Browse Source
This commit is contained in:
Jenkins 2015-05-28 14:43:55 +00:00 committed by Gerrit Code Review
commit c716b170b3
3 changed files with 241 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
swift/common/exceptions.py
View File

@ -256,3 +256,7 @@ class ClientException(Exception):
b += ' [first 60 chars of response] %s' \ b += ' [first 60 chars of response] %s' \
% self.http_response_content[:60] % self.http_response_content[:60]
return b and '%s: %s' % (a, b) or a return b and '%s: %s' % (a, b) or a
class InvalidPidFileException(Exception):
pass

32
swift/common/manager.py
View File

@ -24,9 +24,11 @@ import re
from swift import gettext_ as _ from swift import gettext_ as _
from swift.common.utils import search_tree, remove_file, write_file from swift.common.utils import search_tree, remove_file, write_file
from swift.common.exceptions import InvalidPidFileException
SWIFT_DIR = '/etc/swift' SWIFT_DIR = '/etc/swift'
RUN_DIR = '/var/run/swift' RUN_DIR = '/var/run/swift'
PROC_DIR = '/proc'
# auth-server has been removed from ALL_SERVERS, start it explicitly # auth-server has been removed from ALL_SERVERS, start it explicitly
ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor',
@ -134,6 +136,29 @@ def watch_server_pids(server_pids, interval=1, **kwargs):
time.sleep(0.1) time.sleep(0.1)
def safe_kill(pid, sig, name):
"""Send signal to process and check process name
: param pid: process id
: param sig: signal to send
: param name: name to ensure target process
"""
# check process name for SIG_DFL
if sig == signal.SIG_DFL:
try:
proc_file = '%s/%d/cmdline' % (PROC_DIR, pid)
if os.path.exists(proc_file):
with open(proc_file, 'r') as fd:
if name not in fd.read():
# unknown process is using the pid
raise InvalidPidFileException()
except IOError:
pass
os.kill(pid, sig)
class UnknownCommandError(Exception): class UnknownCommandError(Exception):
pass pass
@ -488,7 +513,12 @@ class Server(object):
if sig != signal.SIG_DFL: if sig != signal.SIG_DFL:
print _('Signal %s pid: %s signal: %s') % (self.server, print _('Signal %s pid: %s signal: %s') % (self.server,
pid, sig) pid, sig)
os.kill(pid, sig) safe_kill(pid, sig, 'swift-%s' % self.server)
except InvalidPidFileException as e:
if kwargs.get('verbose'):
print _('Removing pid file %s with wrong pid %d') \
% (pid_file, pid)
remove_file(pid_file)
except OSError as e: except OSError as e:
if e.errno == errno.ESRCH: if e.errno == errno.ESRCH:
# pid does not exist # pid does not exist

151
test/unit/common/test_manager.py
View File

@ -26,6 +26,7 @@ from threading import Thread
from time import sleep, time from time import sleep, time
from swift.common import manager from swift.common import manager
from swift.common.exceptions import InvalidPidFileException
DUMMY_SIG = 1 DUMMY_SIG = 1
@ -63,7 +64,6 @@ def pop_stream(f):
output = f.read() output = f.read()
f.seek(0) f.seek(0)
f.truncate() f.truncate()
#print >> sys.stderr, output
return output return output
@ -257,6 +257,23 @@ class TestManagerModule(unittest.TestCase):
manager.time = _orig_time manager.time = _orig_time
manager.Server = _orig_server manager.Server = _orig_server
def test_safe_kill(self):
manager.os = MockOs([1, 2, 3, 4])
proc_files = (
('1/cmdline', 'same-procname'),
('2/cmdline', 'another-procname'),
('4/cmdline', 'another-procname'),
)
files, contents = zip(*proc_files)
with temptree(files, contents) as t:
manager.PROC_DIR = t
manager.safe_kill(1, signal.SIG_DFL, 'same-procname')
self.assertRaises(InvalidPidFileException, manager.safe_kill,
2, signal.SIG_DFL, 'same-procname')
manager.safe_kill(3, signal.SIG_DFL, 'same-procname')
manager.safe_kill(4, signal.SIGHUP, 'same-procname')
def test_exc(self): def test_exc(self):
self.assert_(issubclass(manager.UnknownCommandError, Exception)) self.assert_(issubclass(manager.UnknownCommandError, Exception))
@ -680,17 +697,19 @@ class TestServer(unittest.TestCase):
self.assertEquals(pid_file, pid_two) self.assertEquals(pid_file, pid_two)
def test_signal_pids(self): def test_signal_pids(self):
pid_files = ( temp_files = (
('proxy-server.pid', 1), ('var/run/proxy-server.pid', 1),
('auth-server.pid', 2), ('var/run/auth-server.pid', 2),
('object-server.pid', 3), ('var/run/one-server.pid', 3),
('var/run/object-server.pid', 4),
('proc/3/cmdline', 'swift-another-server')
) )
files, pids = zip(*pid_files) with temptree(*zip(*temp_files)) as t:
with temptree(files, pids) as t: manager.RUN_DIR = os.path.join(t, 'var/run')
manager.RUN_DIR = t manager.PROC_DIR = os.path.join(t, 'proc')
# mock os with both pids running # mock os with so both the first and second are running
manager.os = MockOs([1, 2]) manager.os = MockOs([1, 2])
server = manager.Server('proxy', run_dir=t) server = manager.Server('proxy', run_dir=manager.RUN_DIR)
pids = server.signal_pids(DUMMY_SIG) pids = server.signal_pids(DUMMY_SIG)
self.assertEquals(len(pids), 1) self.assertEquals(len(pids), 1)
self.assert_(1 in pids) self.assert_(1 in pids)
@ -711,7 +730,7 @@ class TestServer(unittest.TestCase):
# test no details on signal.SIG_DFL # test no details on signal.SIG_DFL
pids = server.signal_pids(signal.SIG_DFL) pids = server.signal_pids(signal.SIG_DFL)
self.assertEquals(pop_stream(f), '') self.assertEquals(pop_stream(f), '')
# reset mock os so only the other server is running # reset mock os so only the second server is running
manager.os = MockOs([2]) manager.os = MockOs([2])
# test pid not running # test pid not running
pids = server.signal_pids(signal.SIG_DFL) pids = server.signal_pids(signal.SIG_DFL)
@ -722,42 +741,63 @@ class TestServer(unittest.TestCase):
self.join_run_dir('proxy-server.pid'))) self.join_run_dir('proxy-server.pid')))
# reset mock os with no running pids # reset mock os with no running pids
manager.os = MockOs([]) manager.os = MockOs([])
server = manager.Server('auth', run_dir=t) server = manager.Server('auth', run_dir=manager.RUN_DIR)
# test verbose warns on removing pid file # test verbose warns on removing stale pid file
pids = server.signal_pids(signal.SIG_DFL, verbose=True) pids = server.signal_pids(signal.SIG_DFL, verbose=True)
output = pop_stream(f) output = pop_stream(f)
self.assert_('stale pid' in output.lower()) self.assert_('stale pid' in output.lower())
auth_pid = self.join_run_dir('auth-server.pid') auth_pid = self.join_run_dir('auth-server.pid')
self.assert_(auth_pid in output) self.assert_(auth_pid in output)
# reset mock os so only the third server is running
manager.os = MockOs([3])
server = manager.Server('one', run_dir=manager.RUN_DIR)
# test verbose warns on removing invalid pid file
pids = server.signal_pids(signal.SIG_DFL, verbose=True)
output = pop_stream(f)
old_stdout.write('output %s' % output)
self.assert_('removing pid file' in output.lower())
one_pid = self.join_run_dir('one-server.pid')
self.assert_(one_pid in output)
# reset mock os with no running pids
manager.os = MockOs([])
# test warning with insufficient permissions # test warning with insufficient permissions
server = manager.Server('object', run_dir=t) server = manager.Server('object', run_dir=manager.RUN_DIR)
pids = server.signal_pids(manager.os.RAISE_EPERM_SIG) pids = server.signal_pids(manager.os.RAISE_EPERM_SIG)
output = pop_stream(f) output = pop_stream(f)
self.assert_('no permission to signal pid 3' in self.assert_('no permission to signal pid 4' in
output.lower(), output) output.lower(), output)
finally: finally:
sys.stdout = old_stdout sys.stdout = old_stdout
def test_get_running_pids(self): def test_get_running_pids(self):
# test only gets running pids # test only gets running pids
pid_files = ( temp_files = (
('test-server1.pid', 1), ('var/run/test-server1.pid', 1),
('test-server2.pid', 2), ('var/run/test-server2.pid', 2),
('var/run/test-server3.pid', 3),
('proc/1/cmdline', 'swift-test-server'),
('proc/3/cmdline', 'swift-another-server')
) )
with temptree(*zip(*pid_files)) as t: with temptree(*zip(*temp_files)) as t:
manager.RUN_DIR = t manager.RUN_DIR = os.path.join(t, 'var/run')
server = manager.Server('test-server', run_dir=t) manager.PROC_DIR = os.path.join(t, 'proc')
server = manager.Server(
'test-server', run_dir=manager.RUN_DIR)
# mock os, only pid '1' is running # mock os, only pid '1' is running
manager.os = MockOs([1]) manager.os = MockOs([1, 3])
running_pids = server.get_running_pids() running_pids = server.get_running_pids()
self.assertEquals(len(running_pids), 1) self.assertEquals(len(running_pids), 1)
self.assert_(1 in running_pids) self.assert_(1 in running_pids)
self.assert_(2 not in running_pids) self.assert_(2 not in running_pids)
self.assert_(3 not in running_pids)
# test persistent running pid files # test persistent running pid files
self.assert_(os.path.exists(os.path.join(t, 'test-server1.pid'))) self.assert_(os.path.exists(
os.path.join(manager.RUN_DIR, 'test-server1.pid')))
# test clean up stale pids # test clean up stale pids
pid_two = self.join_swift_dir('test-server2.pid') pid_two = self.join_swift_dir('test-server2.pid')
self.assertFalse(os.path.exists(pid_two)) self.assertFalse(os.path.exists(pid_two))
pid_three = self.join_swift_dir('test-server3.pid')
self.assertFalse(os.path.exists(pid_three))
# reset mock os, no pids running # reset mock os, no pids running
manager.os = MockOs([]) manager.os = MockOs([])
running_pids = server.get_running_pids() running_pids = server.get_running_pids()
@ -765,7 +805,7 @@ class TestServer(unittest.TestCase):
# and now all pid files are cleaned out # and now all pid files are cleaned out
pid_one = self.join_run_dir('test-server1.pid') pid_one = self.join_run_dir('test-server1.pid')
self.assertFalse(os.path.exists(pid_one)) self.assertFalse(os.path.exists(pid_one))
all_pids = os.listdir(t) all_pids = os.listdir(manager.RUN_DIR)
self.assertEquals(len(all_pids), 0) self.assertEquals(len(all_pids), 0)
# test only get pids for right server # test only get pids for right server
@ -883,12 +923,23 @@ class TestServer(unittest.TestCase):
sys.stdout = f sys.stdout = f
# test status for all running # test status for all running
manager.os = MockOs(pids) manager.os = MockOs(pids)
proc_files = (
('1/cmdline', 'swift-test-server'),
('2/cmdline', 'swift-test-server'),
('3/cmdline', 'swift-test-server'),
('4/cmdline', 'swift-test-server'),
)
files, contents = zip(*proc_files)
with temptree(files, contents) as t:
manager.PROC_DIR = t
self.assertEquals(server.status(), 0) self.assertEquals(server.status(), 0)
output = pop_stream(f).strip().splitlines() output = pop_stream(f).strip().splitlines()
self.assertEquals(len(output), 4) self.assertEquals(len(output), 4)
for line in output: for line in output:
self.assert_('test-server running' in line) self.assert_('test-server running' in line)
# test get single server by number # test get single server by number
with temptree([], []) as t:
manager.PROC_DIR = t
self.assertEquals(server.status(number=4), 0) self.assertEquals(server.status(number=4), 0)
output = pop_stream(f).strip().splitlines() output = pop_stream(f).strip().splitlines()
self.assertEquals(len(output), 1) self.assertEquals(len(output), 1)
@ -898,6 +949,14 @@ class TestServer(unittest.TestCase):
self.assert_('4 - %s' % conf_four in line) self.assert_('4 - %s' % conf_four in line)
# test some servers not running # test some servers not running
manager.os = MockOs([1, 2, 3]) manager.os = MockOs([1, 2, 3])
proc_files = (
('1/cmdline', 'swift-test-server'),
('2/cmdline', 'swift-test-server'),
('3/cmdline', 'swift-test-server'),
)
files, contents = zip(*proc_files)
with temptree(files, contents) as t:
manager.PROC_DIR = t
self.assertEquals(server.status(), 0) self.assertEquals(server.status(), 0)
output = pop_stream(f).strip().splitlines() output = pop_stream(f).strip().splitlines()
self.assertEquals(len(output), 3) self.assertEquals(len(output), 3)
@ -905,6 +964,13 @@ class TestServer(unittest.TestCase):
self.assert_('test-server running' in line) self.assert_('test-server running' in line)
# test single server not running # test single server not running
manager.os = MockOs([1, 2]) manager.os = MockOs([1, 2])
proc_files = (
('1/cmdline', 'swift-test-server'),
('2/cmdline', 'swift-test-server'),
)
files, contents = zip(*proc_files)
with temptree(files, contents) as t:
manager.PROC_DIR = t
self.assertEquals(server.status(number=3), 1) self.assertEquals(server.status(number=3), 1)
output = pop_stream(f).strip().splitlines() output = pop_stream(f).strip().splitlines()
self.assertEquals(len(output), 1) self.assertEquals(len(output), 1)
@ -914,6 +980,8 @@ class TestServer(unittest.TestCase):
self.assert_(conf_three in line) self.assert_(conf_three in line)
# test no running pids # test no running pids
manager.os = MockOs([]) manager.os = MockOs([])
with temptree([], []) as t:
manager.PROC_DIR = t
self.assertEquals(server.status(), 1) self.assertEquals(server.status(), 1)
output = pop_stream(f).lower() output = pop_stream(f).lower()
self.assert_('no test-server running' in output) self.assert_('no test-server running' in output)
@ -1247,27 +1315,40 @@ class TestServer(unittest.TestCase):
self.assertFalse(server.launch()) self.assertFalse(server.launch())
# start mock os running all pids # start mock os running all pids
manager.os = MockOs(pids) manager.os = MockOs(pids)
proc_files = (
('1/cmdline', 'swift-proxy-server'),
('2/cmdline', 'swift-proxy-server'),
)
files, contents = zip(*proc_files)
with temptree(files, contents) as proc_dir:
manager.PROC_DIR = proc_dir
server = manager.Server('proxy', run_dir=t) server = manager.Server('proxy', run_dir=t)
# can't start server if it's already running # can't start server if it's already running
self.assertFalse(server.launch()) self.assertFalse(server.launch())
output = pop_stream(f) output = pop_stream(f)
self.assert_('running' in output) self.assert_('running' in output)
conf_file = self.join_swift_dir('proxy-server.conf') conf_file = self.join_swift_dir(
'proxy-server.conf')
self.assert_(conf_file in output) self.assert_(conf_file in output)
pid_file = self.join_run_dir('proxy-server/2.pid') pid_file = self.join_run_dir('proxy-server/2.pid')
self.assert_(pid_file in output) self.assert_(pid_file in output)
self.assert_('already started' in output) self.assert_('already started' in output)
# no running pids # no running pids
manager.os = MockOs([]) manager.os = MockOs([])
with temptree([], []) as proc_dir:
manager.PROC_DIR = proc_dir
# test ignore once for non-start-once server # test ignore once for non-start-once server
mock_spawn = MockSpawn([1]) mock_spawn = MockSpawn([1])
server.spawn = mock_spawn server.spawn = mock_spawn
conf_file = self.join_swift_dir('proxy-server.conf') conf_file = self.join_swift_dir(
'proxy-server.conf')
expected = { expected = {
1: conf_file, 1: conf_file,
} }
self.assertEquals(server.launch(once=True), expected) self.assertEquals(server.launch(once=True),
self.assertEquals(mock_spawn.conf_files, [conf_file]) expected)
self.assertEquals(mock_spawn.conf_files,
[conf_file])
expected = { expected = {
'once': False, 'once': False,
} }
@ -1277,6 +1358,8 @@ class TestServer(unittest.TestCase):
self.assert_('once' not in output) self.assert_('once' not in output)
# test multi-server kwarg once # test multi-server kwarg once
server = manager.Server('object-replicator') server = manager.Server('object-replicator')
with temptree([], []) as proc_dir:
manager.PROC_DIR = proc_dir
mock_spawn = MockSpawn([1, 2, 3, 4]) mock_spawn = MockSpawn([1, 2, 3, 4])
server.spawn = mock_spawn server.spawn = mock_spawn
conf1 = self.join_swift_dir('object-server/1.conf') conf1 = self.join_swift_dir('object-server/1.conf')
@ -1289,7 +1372,8 @@ class TestServer(unittest.TestCase):
3: conf3, 3: conf3,
4: conf4, 4: conf4,
} }
self.assertEquals(server.launch(once=True), expected) self.assertEquals(server.launch(once=True),
expected)
self.assertEquals(mock_spawn.conf_files, [ self.assertEquals(mock_spawn.conf_files, [
conf1, conf2, conf3, conf4]) conf1, conf2, conf3, conf4])
expected = { expected = {
@ -1300,11 +1384,13 @@ class TestServer(unittest.TestCase):
self.assertEquals(kwargs, expected) self.assertEquals(kwargs, expected)
# test number kwarg # test number kwarg
mock_spawn = MockSpawn([4]) mock_spawn = MockSpawn([4])
manager.PROC_DIR = proc_dir
server.spawn = mock_spawn server.spawn = mock_spawn
expected = { expected = {
4: conf4, 4: conf4,
} }
self.assertEquals(server.launch(number=4), expected) self.assertEquals(server.launch(number=4),
expected)
self.assertEquals(mock_spawn.conf_files, [conf4]) self.assertEquals(mock_spawn.conf_files, [conf4])
expected = { expected = {
'number': 4 'number': 4
@ -1312,7 +1398,10 @@ class TestServer(unittest.TestCase):
self.assertEquals(mock_spawn.kwargs, [expected]) self.assertEquals(mock_spawn.kwargs, [expected])
# test cmd does not exist # test cmd does not exist
server = manager.Server('auth') server = manager.Server('auth')
mock_spawn = MockSpawn([OSError(errno.ENOENT, 'blah')]) with temptree([], []) as proc_dir:
manager.PROC_DIR = proc_dir
mock_spawn = MockSpawn([OSError(errno.ENOENT,
'blah')])
server.spawn = mock_spawn server.spawn = mock_spawn
self.assertEquals(server.launch(), {}) self.assertEquals(server.launch(), {})
self.assert_('swift-auth-server does not exist' in self.assert_('swift-auth-server does not exist' in