1f43ee050b
Currently probetests take advantage of a number of assumptions about the SUT.
Unfortunately after some time with a working SAIO, configuration drift may
result in a system that is no longer compatible with these assumptions. To
help weary developers more quickly identify the changes they've made since
they last ran probetests successfully, some handy validators have been added
to test.probe.common
Additionally a new option 'validate_rsync' in test.conf, when enabled, will
run a series of up front validations during the setup of each probetest by
inspecting the ring, the mounted devices, and the rsync exports ("modules") in
order to ensure that when probetests fail the do so early and with specific
complaints.
To preserve existing failures, the option is disabled by default.
Change-Id: I2be11c7e67ccd0bc0589c360c170049b6288c152
263 lines
9.6 KiB
Python
263 lines
9.6 KiB
Python
# Copyright (c) 2010-2012 OpenStack, LLC.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from httplib import HTTPConnection
|
|
import os
|
|
from os import kill, path
|
|
from signal import SIGTERM
|
|
from subprocess import Popen, PIPE
|
|
import sys
|
|
from time import sleep, time
|
|
|
|
from swiftclient import get_auth, head_account
|
|
|
|
from swift.common.ring import Ring
|
|
from swift.common.utils import readconf
|
|
|
|
from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC
|
|
|
|
|
|
def start_server(port, port2server, pids, check=True):
|
|
server = port2server[port]
|
|
if server[:-1] in ('account', 'container', 'object'):
|
|
if not path.exists('/etc/swift/%s-server/%s.conf' %
|
|
(server[:-1], server[-1])):
|
|
return None
|
|
pids[server] = Popen([
|
|
'swift-%s-server' % server[:-1],
|
|
'/etc/swift/%s-server/%s.conf' % (server[:-1], server[-1])]).pid
|
|
if check:
|
|
return check_server(port, port2server, pids)
|
|
else:
|
|
pids[server] = Popen(['swift-%s-server' % server,
|
|
'/etc/swift/%s-server.conf' % server]).pid
|
|
if check:
|
|
return check_server(port, port2server, pids)
|
|
return None
|
|
|
|
|
|
def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT):
|
|
server = port2server[port]
|
|
if server[:-1] in ('account', 'container', 'object'):
|
|
if int(server[-1]) > 4:
|
|
return None
|
|
path = '/connect/1/2'
|
|
if server[:-1] == 'container':
|
|
path += '/3'
|
|
elif server[:-1] == 'object':
|
|
path += '/3/4'
|
|
try_until = time() + timeout
|
|
while True:
|
|
try:
|
|
conn = HTTPConnection('127.0.0.1', port)
|
|
conn.request('GET', path)
|
|
resp = conn.getresponse()
|
|
# 404 because it's a nonsense path (and mount_check is false)
|
|
# 507 in case the test target is a VM using mount_check
|
|
if resp.status not in (404, 507):
|
|
raise Exception(
|
|
'Unexpected status %s' % resp.status)
|
|
break
|
|
except Exception, err:
|
|
if time() > try_until:
|
|
print err
|
|
print 'Giving up on %s:%s after %s seconds.' % (
|
|
server, port, timeout)
|
|
raise err
|
|
sleep(0.1)
|
|
else:
|
|
try_until = time() + timeout
|
|
while True:
|
|
try:
|
|
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
|
|
'test:tester', 'testing')
|
|
account = url.split('/')[-1]
|
|
head_account(url, token)
|
|
return url, token, account
|
|
except Exception, err:
|
|
if time() > try_until:
|
|
print err
|
|
print 'Giving up on proxy:8080 after 30 seconds.'
|
|
raise err
|
|
sleep(0.1)
|
|
return None
|
|
|
|
|
|
def kill_server(port, port2server, pids):
|
|
try:
|
|
kill(pids[port2server[port]], SIGTERM)
|
|
except Exception, err:
|
|
print err
|
|
try_until = time() + 30
|
|
while True:
|
|
try:
|
|
conn = HTTPConnection('127.0.0.1', port)
|
|
conn.request('GET', '/')
|
|
conn.getresponse()
|
|
except Exception, err:
|
|
break
|
|
if time() > try_until:
|
|
raise Exception(
|
|
'Still answering on port %s after 30 seconds' % port)
|
|
sleep(0.1)
|
|
|
|
|
|
def kill_servers(port2server, pids):
|
|
for port in port2server:
|
|
kill_server(port, port2server, pids)
|
|
|
|
|
|
def kill_nonprimary_server(primary_nodes, port2server, pids):
|
|
primary_ports = [n['port'] for n in primary_nodes]
|
|
for port, server in port2server.iteritems():
|
|
if port in primary_ports:
|
|
server_type = server[:-1]
|
|
break
|
|
else:
|
|
raise Exception('Cannot figure out server type for %r' % primary_nodes)
|
|
for port, server in list(port2server.iteritems()):
|
|
if server[:-1] == server_type and port not in primary_ports:
|
|
kill_server(port, port2server, pids)
|
|
return port
|
|
|
|
|
|
def get_ring(server, force_validate=None):
|
|
ring = Ring('/etc/swift/%s.ring.gz' % server)
|
|
if not VALIDATE_RSYNC and not force_validate:
|
|
return ring
|
|
# easy sanity checks
|
|
assert 3 == ring.replica_count, '%s has %s replicas instead of 3' % (
|
|
ring.serialized_path, ring.replica_count)
|
|
assert 4 == len(ring.devs), '%s has %s devices instead of 4' % (
|
|
ring.serialized_path, len(ring.devs))
|
|
# map server to config by port
|
|
port_to_config = {}
|
|
for node_id in range(1,5):
|
|
conf = readconf('/etc/swift/%s-server/%d.conf' % (server, node_id),
|
|
section_name='%s-replicator' % server)
|
|
port_to_config[int(conf['bind_port'])] = conf
|
|
for dev in ring.devs:
|
|
# verify server is exposing mounted device
|
|
conf = port_to_config[dev['port']]
|
|
for device in os.listdir(conf['devices']):
|
|
if device == dev['device']:
|
|
full_path = path.realpath(path.join(conf['devices'], device))
|
|
assert path.ismount(full_path), 'device %s in %s was not ' \
|
|
'mounted (%s)' % (device, conf['devices'], full_path)
|
|
break
|
|
else:
|
|
assert False, "unable to find ring device %s " \
|
|
"under %s's devices (%s)" % (
|
|
dev['device'], server, conf['devices'])
|
|
# verify server is exposing rsync device
|
|
rsync_export = '%s%s' % (server, dev['replication_port'])
|
|
cmd = "rsync rsync://localhost/%s" % rsync_export
|
|
p = Popen(cmd, shell=True, stdout=PIPE)
|
|
stdout, _stderr = p.communicate()
|
|
if p.returncode:
|
|
raise AssertionError('unable to connect to rsync '
|
|
'export %s (%s)' % (rsync_export, cmd))
|
|
for line in stdout.splitlines():
|
|
if line.rsplit(None, 1)[-1] == dev['device']:
|
|
break
|
|
else:
|
|
assert False, "unable to find ring device %s under rsync's " \
|
|
"exported devices for %s (%s)" % (
|
|
dev['device'], rsync_export, cmd)
|
|
return ring
|
|
|
|
|
|
def reset_environment():
|
|
p = Popen("resetswift 2>&1", shell=True, stdout=PIPE)
|
|
stdout, _stderr = p.communicate()
|
|
print stdout
|
|
pids = {}
|
|
try:
|
|
account_ring = get_ring('account')
|
|
container_ring = get_ring('container')
|
|
object_ring = get_ring('object')
|
|
port2server = {}
|
|
config_dict = {}
|
|
for server, port in [('account', 6002), ('container', 6001),
|
|
('object', 6000)]:
|
|
for number in xrange(1, 9):
|
|
port2server[port + (number * 10)] = '%s%d' % (server, number)
|
|
for port in port2server:
|
|
start_server(port, port2server, pids, check=False)
|
|
for port in port2server:
|
|
check_server(port, port2server, pids)
|
|
port2server[8080] = 'proxy'
|
|
url, token, account = start_server(8080, port2server, pids)
|
|
for name in ('account', 'container', 'object'):
|
|
for server in (name, '%s-replicator' % name):
|
|
config_dict[server] = '/etc/swift/%s-server/%%d.conf' % name
|
|
except BaseException:
|
|
try:
|
|
raise
|
|
except AssertionError, e:
|
|
print >>sys.stderr, 'ERROR: %s' % e
|
|
os._exit(1)
|
|
finally:
|
|
try:
|
|
kill_servers(port2server, pids)
|
|
except Exception:
|
|
pass
|
|
return pids, port2server, account_ring, container_ring, object_ring, url, \
|
|
token, account, config_dict
|
|
|
|
|
|
def get_to_final_state():
|
|
processes = []
|
|
for job in ('account-replicator', 'container-replicator',
|
|
'object-replicator'):
|
|
for number in xrange(1, 9):
|
|
if not path.exists('/etc/swift/%s-server/%d.conf' %
|
|
(job.split('-')[0], number)):
|
|
continue
|
|
processes.append(Popen([
|
|
'swift-%s' % job,
|
|
'/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number),
|
|
'once']))
|
|
for process in processes:
|
|
process.wait()
|
|
processes = []
|
|
for job in ('container-updater', 'object-updater'):
|
|
for number in xrange(1, 5):
|
|
processes.append(Popen([
|
|
'swift-%s' % job,
|
|
'/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number),
|
|
'once']))
|
|
for process in processes:
|
|
process.wait()
|
|
processes = []
|
|
for job in ('account-replicator', 'container-replicator',
|
|
'object-replicator'):
|
|
for number in xrange(1, 9):
|
|
if not path.exists('/etc/swift/%s-server/%d.conf' %
|
|
(job.split('-')[0], number)):
|
|
continue
|
|
processes.append(Popen([
|
|
'swift-%s' % job,
|
|
'/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number),
|
|
'once']))
|
|
for process in processes:
|
|
process.wait()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
for server in ('account', 'container', 'object'):
|
|
get_ring(server, force_validate=True)
|
|
print '%s OK' % server
|