openstack/swift
Code Issues Proposed changes

Update FakeRing and FakeLogger

Browse Source 
FakeLogger gets better log level handling

Parameterize logger on some daemons which were previously
unparameterized and try and use the interface in tests.

FakeRing use more real code

The existing FakeRing mock's implementation bit me on some pretty subtle
character encoding issue by-passing the hash_path code that is normally
part of get_part_nodes.  This change tries to exercise more of the real
ring code paths when it makes sense and provide a better Fake for use in
testing.

Add write_fake_ring helper to test.unit for when you need a real ring.

DocImpact
Implements: blueprint storage-policies
Change-Id: Id2e3740b1dd569050f4e083617e7dd6a4249027e
This commit is contained in:
Clay Gerrard 2014-04-28 19:22:51 -07:00
parent 46c68aebd1
commit 7624b198cf
8 changed files with 199 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
swift/common/db_replicator.py
View File

@ -146,9 +146,9 @@ class Replicator(Daemon):
Implements the logic for directing db replication. Implements the logic for directing db replication.
""" """
def __init__(self, conf): def __init__(self, conf, logger=None):
self.conf = conf self.conf = conf
self.logger = get_logger(conf, log_route='replicator') self.logger = logger or get_logger(conf, log_route='replicator')
self.root = conf.get('devices', '/srv/node') self.root = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.port = int(conf.get('bind_port', self.default_port)) self.port = int(conf.get('bind_port', self.default_port))

4
swift/obj/updater.py
View File

@ -37,9 +37,9 @@ from swift.common.http import is_success, HTTP_NOT_FOUND, \
class ObjectUpdater(Daemon): class ObjectUpdater(Daemon):
"""Update object information in container listings.""" """Update object information in container listings."""
def __init__(self, conf): def __init__(self, conf, logger=None):
self.conf = conf self.conf = conf
self.logger = get_logger(conf, log_route='object-updater') self.logger = logger or get_logger(conf, log_route='object-updater')
self.devices = conf.get('devices', '/srv/node') self.devices = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.swift_dir = conf.get('swift_dir', '/etc/swift') self.swift_dir = conf.get('swift_dir', '/etc/swift')

152
test/unit/__init__.py
View File

@ -20,7 +20,7 @@ import copy
import logging import logging
import errno import errno
import sys import sys
from contextlib import contextmanager from contextlib import contextmanager, closing
from collections import defaultdict, Iterable from collections import defaultdict, Iterable
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
import time import time
@ -29,6 +29,7 @@ from tempfile import mkdtemp
from shutil import rmtree from shutil import rmtree
from test import get_config from test import get_config
from swift.common.utils import config_true_value, LogAdapter from swift.common.utils import config_true_value, LogAdapter
from swift.common.ring import Ring, RingData
from hashlib import md5 from hashlib import md5
from eventlet import sleep, Timeout from eventlet import sleep, Timeout
import logging.handlers import logging.handlers
@ -36,6 +37,9 @@ from httplib import HTTPException
from numbers import Number from numbers import Number
from swift.common import storage_policy from swift.common import storage_policy
import functools import functools
import cPickle as pickle
from gzip import GzipFile
import mock as mocklib
DEFAULT_PATCH_POLICIES = [storage_policy.StoragePolicy(0, 'nulo', True), DEFAULT_PATCH_POLICIES = [storage_policy.StoragePolicy(0, 'nulo', True),
storage_policy.StoragePolicy(1, 'unu')] storage_policy.StoragePolicy(1, 'unu')]
@ -116,42 +120,63 @@ class PatchPolicies(object):
storage_policy._POLICIES = self._orig_POLICIES storage_policy._POLICIES = self._orig_POLICIES
class FakeRing(object): class FakeRing(Ring):
def __init__(self, replicas=3, max_more_nodes=0): def __init__(self, replicas=3, max_more_nodes=0, part_power=0):
"""
:param part_power: make part calculation based on the path
If you set a part_power when you setup your FakeRing the parts you get
out of ring methods will actually be based on the path - otherwise we
exercise the real ring code, but ignore the result and return 1.
"""
# 9 total nodes (6 more past the initial 3) is the cap, no matter if # 9 total nodes (6 more past the initial 3) is the cap, no matter if
# this is set higher, or R^2 for R replicas # this is set higher, or R^2 for R replicas
self.replicas = replicas self.set_replicas(replicas)
self.max_more_nodes = max_more_nodes self.max_more_nodes = max_more_nodes
self.devs = {} self._part_shift = 32 - part_power
self._reload()
def get_part(self, *args, **kwargs):
real_part = super(FakeRing, self).get_part(*args, **kwargs)
if self._part_shift == 32:
return 1
return real_part
def _reload(self):
self._rtime = time.time()
def clear_errors(self):
for dev in self.devs:
for key in ('errors', 'last_error'):
try:
del dev[key]
except KeyError:
pass
def set_replicas(self, replicas): def set_replicas(self, replicas):
self.replicas = replicas self.replicas = replicas
self.devs = {} self._devs = []
for x in range(self.replicas):
ip = '10.0.0.%s' % x
port = 1000 + x
self._devs.append({
'ip': ip,
'replication_ip': ip,
'port': port,
'replication_port': port,
'device': 'sd' + (chr(ord('a') + x)),
'zone': x % 3,
'region': x % 2,
'id': x,
})
@property @property
def replica_count(self): def replica_count(self):
return self.replicas return self.replicas
def get_part(self, account, container=None, obj=None): def _get_part_nodes(self, part):
return 1 return list(self._devs)
def get_nodes(self, account, container=None, obj=None):
devs = []
for x in xrange(self.replicas):
devs.append(self.devs.get(x))
if devs[x] is None:
self.devs[x] = devs[x] = \
{'ip': '10.0.0.%s' % x,
'port': 1000 + x,
'device': 'sd' + (chr(ord('a') + x)),
'zone': x % 3,
'region': x % 2,
'id': x}
return 1, devs
def get_part_nodes(self, part):
return self.get_nodes('blah')[1]
def get_more_nodes(self, part): def get_more_nodes(self, part):
# replicas^2 is the true cap # replicas^2 is the true cap
@ -165,6 +190,27 @@ class FakeRing(object):
'id': x} 'id': x}
def write_fake_ring(path, *devs):
"""
Pretty much just a two node, two replica, 2 part power ring...
"""
dev1 = {'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6000}
dev2 = {'id': 0, 'zone': 0, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6000}
dev1_updates, dev2_updates = devs or ({}, {})
dev1.update(dev1_updates)
dev2.update(dev2_updates)
replica2part2dev_id = [[0, 1, 0, 1], [1, 0, 1, 0]]
devs = [dev1, dev2]
part_shift = 30
with closing(GzipFile(path, 'wb')) as f:
pickle.dump(RingData(replica2part2dev_id, devs, part_shift), f)
class FakeMemcache(object): class FakeMemcache(object):
def __init__(self): def __init__(self):
@ -317,6 +363,7 @@ class FakeLogger(logging.Logger):
self.facility = kwargs['facility'] self.facility = kwargs['facility']
self.statsd_client = None self.statsd_client = None
self.thread_locals = None self.thread_locals = None
self.parent = None
def _clear(self): def _clear(self):
self.log_dict = defaultdict(list) self.log_dict = defaultdict(list)
@ -327,20 +374,20 @@ class FakeLogger(logging.Logger):
self.log_dict[store_name].append((args, kwargs)) self.log_dict[store_name].append((args, kwargs))
return stub_fn return stub_fn
def _store_and_log_in(store_name): def _store_and_log_in(store_name, level):
def stub_fn(self, *args, **kwargs): def stub_fn(self, *args, **kwargs):
self.log_dict[store_name].append((args, kwargs)) self.log_dict[store_name].append((args, kwargs))
self._log(store_name, args[0], args[1:], **kwargs) self._log(level, args[0], args[1:], **kwargs)
return stub_fn return stub_fn
def get_lines_for_level(self, level): def get_lines_for_level(self, level):
return self.lines_dict[level] return self.lines_dict[level]
error = _store_and_log_in('error') error = _store_and_log_in('error', logging.ERROR)
info = _store_and_log_in('info') info = _store_and_log_in('info', logging.INFO)
warning = _store_and_log_in('warning') warning = _store_and_log_in('warning', logging.WARNING)
warn = _store_and_log_in('warning') warn = _store_and_log_in('warning', logging.WARNING)
debug = _store_and_log_in('debug') debug = _store_and_log_in('debug', logging.DEBUG)
def exception(self, *args, **kwargs): def exception(self, *args, **kwargs):
self.log_dict['exception'].append((args, kwargs, self.log_dict['exception'].append((args, kwargs,
@ -348,11 +395,12 @@ class FakeLogger(logging.Logger):
print 'FakeLogger Exception: %s' % self.log_dict print 'FakeLogger Exception: %s' % self.log_dict
# mock out the StatsD logging methods: # mock out the StatsD logging methods:
update_stats = _store_in('update_stats')
increment = _store_in('increment') increment = _store_in('increment')
decrement = _store_in('decrement') decrement = _store_in('decrement')
timing = _store_in('timing') timing = _store_in('timing')
timing_since = _store_in('timing_since') timing_since = _store_in('timing_since')
update_stats = _store_in('update_stats') transfer_rate = _store_in('transfer_rate')
set_statsd_prefix = _store_in('set_statsd_prefix') set_statsd_prefix = _store_in('set_statsd_prefix')
def get_increments(self): def get_increments(self):
@ -395,7 +443,7 @@ class FakeLogger(logging.Logger):
print 'WARNING: unable to format log message %r %% %r' % ( print 'WARNING: unable to format log message %r %% %r' % (
record.msg, record.args) record.msg, record.args)
raise raise
self.lines_dict[record.levelno].append(line) self.lines_dict[record.levelname.lower()].append(line)
def handle(self, record): def handle(self, record):
self._handle(record) self._handle(record)
@ -412,16 +460,40 @@ class DebugLogger(FakeLogger):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
FakeLogger.__init__(self, *args, **kwargs) FakeLogger.__init__(self, *args, **kwargs)
self.formatter = logging.Formatter("%(server)s: %(message)s") self.formatter = logging.Formatter(
"%(server)s %(levelname)s: %(message)s")
def handle(self, record): def handle(self, record):
self._handle(record) self._handle(record)
print self.formatter.format(record) print self.formatter.format(record)
class DebugLogAdapter(LogAdapter):
def _send_to_logger(name):
def stub_fn(self, *args, **kwargs):
return getattr(self.logger, name)(*args, **kwargs)
return stub_fn
# delegate to FakeLogger's mocks
update_stats = _send_to_logger('update_stats')
increment = _send_to_logger('increment')
decrement = _send_to_logger('decrement')
timing = _send_to_logger('timing')
timing_since = _send_to_logger('timing_since')
transfer_rate = _send_to_logger('transfer_rate')
set_statsd_prefix = _send_to_logger('set_statsd_prefix')
def __getattribute__(self, name):
try:
return object.__getattribute__(self, name)
except AttributeError:
return getattr(self.__dict__['logger'], name)
def debug_logger(name='test'): def debug_logger(name='test'):
"""get a named adapted debug logger""" """get a named adapted debug logger"""
return LogAdapter(DebugLogger(), name) return DebugLogAdapter(DebugLogger(), name)
original_syslog_handler = logging.handlers.SysLogHandler original_syslog_handler = logging.handlers.SysLogHandler
@ -666,3 +738,11 @@ def fake_http_connect(*code_iter, **kwargs):
connect.code_iter = code_iter connect.code_iter = code_iter
return connect return connect
@contextmanager
def mocked_http_conn(*args, **kwargs):
fake_conn = fake_http_connect(*args, **kwargs)
with mocklib.patch('swift.common.bufferedhttp.http_connect_raw',
new=fake_conn):
yield fake_conn

24
test/unit/cli/test_info.py
View File

@ -14,14 +14,12 @@
import os import os
import unittest import unittest
import cPickle as pickle
import mock import mock
from cStringIO import StringIO from cStringIO import StringIO
from contextlib import closing
from gzip import GzipFile
from shutil import rmtree from shutil import rmtree
from tempfile import mkdtemp from tempfile import mkdtemp
from test.unit import write_fake_ring
from swift.common import ring, utils from swift.common import ring, utils
from swift.common.swob import Request from swift.common.swob import Request
from swift.cli.info import print_db_info_metadata, print_ring_locations, \ from swift.cli.info import print_db_info_metadata, print_ring_locations, \
@ -44,22 +42,14 @@ class TestCliInfo(unittest.TestCase):
utils.mkdirs(os.path.join(self.testdir, 'sdb1')) utils.mkdirs(os.path.join(self.testdir, 'sdb1'))
utils.mkdirs(os.path.join(self.testdir, 'sdb1', 'tmp')) utils.mkdirs(os.path.join(self.testdir, 'sdb1', 'tmp'))
self.account_ring_path = os.path.join(self.testdir, 'account.ring.gz') self.account_ring_path = os.path.join(self.testdir, 'account.ring.gz')
with closing(GzipFile(self.account_ring_path, 'wb')) as f: account_devs = [{'ip': '127.0.0.1', 'port': 42},
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], {'ip': '127.0.0.2', 'port': 43}]
[{'id': 0, 'zone': 0, 'device': 'sda1', write_fake_ring(self.account_ring_path, *account_devs)
'ip': '127.0.0.1', 'port': 42},
{'id': 1, 'zone': 1, 'device': 'sdb1',
'ip': '127.0.0.2', 'port': 43}], 30),
f)
self.container_ring_path = os.path.join(self.testdir, self.container_ring_path = os.path.join(self.testdir,
'container.ring.gz') 'container.ring.gz')
with closing(GzipFile(self.container_ring_path, 'wb')) as f: container_devs = [{'ip': '127.0.0.3', 'port': 42},
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], {'ip': '127.0.0.4', 'port': 43}]
[{'id': 0, 'zone': 0, 'device': 'sda1', write_fake_ring(self.container_ring_path, *container_devs)
'ip': '127.0.0.3', 'port': 42},
{'id': 1, 'zone': 1, 'device': 'sdb1',
'ip': '127.0.0.4', 'port': 43}], 30),
f)
def tearDown(self): def tearDown(self):
utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = self.orig_hp utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = self.orig_hp

32
test/unit/common/test_db_replicator.py
View File

@ -32,7 +32,7 @@ from swift.common.utils import normalize_timestamp
from swift.common.exceptions import DriveNotMounted from swift.common.exceptions import DriveNotMounted
from swift.common.swob import HTTPException from swift.common.swob import HTTPException
from test.unit import FakeLogger from test import unit
TEST_ACCOUNT_NAME = 'a c t' TEST_ACCOUNT_NAME = 'a c t'
@ -218,7 +218,7 @@ class FakeBroker(object):
if self.stub_replication_info: if self.stub_replication_info:
return self.stub_replication_info return self.stub_replication_info
return {'delete_timestamp': 0, 'put_timestamp': 1, 'count': 0, return {'delete_timestamp': 0, 'put_timestamp': 1, 'count': 0,
'hash': 12345} 'hash': 12345, 'created_at': 1}
def reclaim(self, item_timestamp, sync_timestamp): def reclaim(self, item_timestamp, sync_timestamp):
pass pass
@ -447,8 +447,7 @@ class TestDBReplicator(unittest.TestCase):
replicator.run_once() replicator.run_once()
def test_run_once_no_ips(self): def test_run_once_no_ips(self):
replicator = TestReplicator({}) replicator = TestReplicator({}, logger=unit.FakeLogger())
replicator.logger = FakeLogger()
self._patch(patch.object, db_replicator, 'whataremyips', self._patch(patch.object, db_replicator, 'whataremyips',
lambda *args: []) lambda *args: [])
@ -460,10 +459,10 @@ class TestDBReplicator(unittest.TestCase):
def test_run_once_node_is_not_mounted(self): def test_run_once_node_is_not_mounted(self):
db_replicator.ring = FakeRingWithSingleNode() db_replicator.ring = FakeRingWithSingleNode()
replicator = TestReplicator({}) conf = {'mount_check': 'true', 'bind_port': 6000}
replicator.logger = FakeLogger() replicator = TestReplicator(conf, logger=unit.FakeLogger())
replicator.mount_check = True self.assertEqual(replicator.mount_check, True)
replicator.port = 6000 self.assertEqual(replicator.port, 6000)
def mock_ismount(path): def mock_ismount(path):
self.assertEquals(path, self.assertEquals(path,
@ -483,10 +482,10 @@ class TestDBReplicator(unittest.TestCase):
def test_run_once_node_is_mounted(self): def test_run_once_node_is_mounted(self):
db_replicator.ring = FakeRingWithSingleNode() db_replicator.ring = FakeRingWithSingleNode()
replicator = TestReplicator({}) conf = {'mount_check': 'true', 'bind_port': 6000}
replicator.logger = FakeLogger() replicator = TestReplicator(conf, logger=unit.FakeLogger())
replicator.mount_check = True self.assertEqual(replicator.mount_check, True)
replicator.port = 6000 self.assertEqual(replicator.port, 6000)
def mock_unlink_older_than(path, mtime): def mock_unlink_older_than(path, mtime):
self.assertEquals(path, self.assertEquals(path,
@ -592,12 +591,11 @@ class TestDBReplicator(unittest.TestCase):
self.assertEquals(['/path/to/file'], self.delete_db_calls) self.assertEquals(['/path/to/file'], self.delete_db_calls)
def test_replicate_account_out_of_place(self): def test_replicate_account_out_of_place(self):
replicator = TestReplicator({}) replicator = TestReplicator({}, logger=unit.FakeLogger())
replicator.ring = FakeRingWithNodes().Ring('path') replicator.ring = FakeRingWithNodes().Ring('path')
replicator.brokerclass = FakeAccountBroker replicator.brokerclass = FakeAccountBroker
replicator._repl_to_node = lambda *args: True replicator._repl_to_node = lambda *args: True
replicator.delete_db = self.stub_delete_db replicator.delete_db = self.stub_delete_db
replicator.logger = FakeLogger()
# Correct node_id, wrong part # Correct node_id, wrong part
part = replicator.ring.get_part(TEST_ACCOUNT_NAME) + 1 part = replicator.ring.get_part(TEST_ACCOUNT_NAME) + 1
node_id = replicator.ring.get_part_nodes(part)[0]['id'] node_id = replicator.ring.get_part_nodes(part)[0]['id']
@ -609,11 +607,10 @@ class TestDBReplicator(unittest.TestCase):
'partition 0; will replicate out and remove.',), {})]) 'partition 0; will replicate out and remove.',), {})])
def test_replicate_container_out_of_place(self): def test_replicate_container_out_of_place(self):
replicator = TestReplicator({}) replicator = TestReplicator({}, logger=unit.FakeLogger())
replicator.ring = FakeRingWithNodes().Ring('path') replicator.ring = FakeRingWithNodes().Ring('path')
replicator._repl_to_node = lambda *args: True replicator._repl_to_node = lambda *args: True
replicator.delete_db = self.stub_delete_db replicator.delete_db = self.stub_delete_db
replicator.logger = FakeLogger()
# Correct node_id, wrong part # Correct node_id, wrong part
part = replicator.ring.get_part( part = replicator.ring.get_part(
TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1 TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1
@ -627,10 +624,9 @@ class TestDBReplicator(unittest.TestCase):
def test_delete_db(self): def test_delete_db(self):
db_replicator.lock_parent_directory = lock_parent_directory db_replicator.lock_parent_directory = lock_parent_directory
replicator = TestReplicator({}) replicator = TestReplicator({}, logger=unit.FakeLogger())
replicator._zero_stats() replicator._zero_stats()
replicator.extract_device = lambda _: 'some_device' replicator.extract_device = lambda _: 'some_device'
replicator.logger = FakeLogger()
temp_dir = mkdtemp() temp_dir = mkdtemp()
try: try:

34
test/unit/common/test_wsgi.py
View File

@ -21,13 +21,10 @@ import mimetools
import socket import socket
import unittest import unittest
import os import os
import pickle
from textwrap import dedent from textwrap import dedent
from gzip import GzipFile
from contextlib import nested from contextlib import nested
from StringIO import StringIO from StringIO import StringIO
from collections import defaultdict from collections import defaultdict
from contextlib import closing
from urllib import quote from urllib import quote
from eventlet import listen from eventlet import listen
@ -39,38 +36,17 @@ import swift.common.middleware.gatekeeper
import swift.proxy.server import swift.proxy.server
from swift.common.swob import Request from swift.common.swob import Request
from swift.common import wsgi, utils, ring from swift.common import wsgi, utils
from test.unit import temptree from test.unit import temptree, write_fake_ring
from paste.deploy import loadwsgi from paste.deploy import loadwsgi
def _fake_rings(tmpdir): def _fake_rings(tmpdir):
account_ring_path = os.path.join(tmpdir, 'account.ring.gz') write_fake_ring(os.path.join(tmpdir, 'account.ring.gz'))
with closing(GzipFile(account_ring_path, 'wb')) as f: write_fake_ring(os.path.join(tmpdir, 'container.ring.gz'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], write_fake_ring(os.path.join(tmpdir, 'object.ring.gz'))
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6012},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6022}], 30),
f)
container_ring_path = os.path.join(tmpdir, 'container.ring.gz')
with closing(GzipFile(container_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6011},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6021}], 30),
f)
object_ring_path = os.path.join(tmpdir, 'object.ring.gz')
with closing(GzipFile(object_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6010},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6020}], 30),
f)
class TestWSGI(unittest.TestCase): class TestWSGI(unittest.TestCase):

1
test/unit/proxy/controllers/test_obj.py
View File

@ -78,6 +78,7 @@ class TestObjControllerWriteAffinity(unittest.TestCase):
self.assertEqual([1, 1, 1, 1], self.assertEqual([1, 1, 1, 1],
[node['region'] for node in local_first_nodes[:4]]) [node['region'] for node in local_first_nodes[:4]])
# we don't skip any nodes # we don't skip any nodes
self.assertEqual(len(all_nodes), len(local_first_nodes))
self.assertEqual(sorted(all_nodes), sorted(local_first_nodes)) self.assertEqual(sorted(all_nodes), sorted(local_first_nodes))
def test_connect_put_node_timeout(self): def test_connect_put_node_timeout(self):

107
test/unit/proxy/test_server.py
View File

@ -13,13 +13,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import cPickle as pickle
import logging import logging
import os import os
import sys import sys
import unittest import unittest
from contextlib import contextmanager, nested, closing from contextlib import contextmanager, nested
from gzip import GzipFile
from shutil import rmtree from shutil import rmtree
import gc import gc
import time import time
@ -34,14 +32,13 @@ import mock
from eventlet import sleep, spawn, wsgi, listen from eventlet import sleep, spawn, wsgi, listen
import simplejson import simplejson
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \ from test.unit import (
fake_http_connect, FakeRing, FakeMemcache, debug_logger, \ connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect, FakeRing,
patch_policies FakeMemcache, debug_logger, patch_policies, write_fake_ring)
from swift.proxy import server as proxy_server from swift.proxy import server as proxy_server
from swift.account import server as account_server from swift.account import server as account_server
from swift.container import server as container_server from swift.container import server as container_server
from swift.obj import server as object_server from swift.obj import server as object_server
from swift.common import ring
from swift.common.middleware import proxy_logging from swift.common.middleware import proxy_logging
from swift.common.middleware.acl import parse_acl, format_acl from swift.common.middleware.acl import parse_acl, format_acl
from swift.common.exceptions import ChunkReadTimeout from swift.common.exceptions import ChunkReadTimeout
@ -98,29 +95,23 @@ def do_setup(the_object_server):
_test_sockets = \ _test_sockets = \
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis)
account_ring_path = os.path.join(_testdir, 'account.ring.gz') account_ring_path = os.path.join(_testdir, 'account.ring.gz')
with closing(GzipFile(account_ring_path, 'wb')) as f: account_devs = [
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], {'port': acc1lis.getsockname()[1]},
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', {'port': acc2lis.getsockname()[1]},
'port': acc1lis.getsockname()[1]}, ]
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', write_fake_ring(account_ring_path, *account_devs)
'port': acc2lis.getsockname()[1]}], 30),
f)
container_ring_path = os.path.join(_testdir, 'container.ring.gz') container_ring_path = os.path.join(_testdir, 'container.ring.gz')
with closing(GzipFile(container_ring_path, 'wb')) as f: container_devs = [
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], {'port': con1lis.getsockname()[1]},
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', {'port': con2lis.getsockname()[1]},
'port': con1lis.getsockname()[1]}, ]
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', write_fake_ring(container_ring_path, *container_devs)
'port': con2lis.getsockname()[1]}], 30),
f)
object_ring_path = os.path.join(_testdir, 'object.ring.gz') object_ring_path = os.path.join(_testdir, 'object.ring.gz')
with closing(GzipFile(object_ring_path, 'wb')) as f: object_devs = [
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], {'port': obj1lis.getsockname()[1]},
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', {'port': obj2lis.getsockname()[1]},
'port': obj1lis.getsockname()[1]}, ]
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', write_fake_ring(object_ring_path, *object_devs)
'port': obj2lis.getsockname()[1]}], 30),
f)
prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone(), prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone(),
logger=debug_logger('proxy')) logger=debug_logger('proxy'))
acc1srv = account_server.AccountController( acc1srv = account_server.AccountController(
@ -1614,15 +1605,15 @@ class TestObjectController(unittest.TestCase):
def test_client_timeout(self): def test_client_timeout(self):
with save_globals(): with save_globals():
self.app.account_ring.get_nodes('account') self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
self.app.container_ring.get_nodes('account') self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs.values(): for dev in self.app.container_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
self.app.object_ring.get_nodes('account') self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs.values(): for dev in self.app.object_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
@ -1663,15 +1654,15 @@ class TestObjectController(unittest.TestCase):
def test_client_disconnect(self): def test_client_disconnect(self):
with save_globals(): with save_globals():
self.app.account_ring.get_nodes('account') self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
self.app.container_ring.get_nodes('account') self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs.values(): for dev in self.app.container_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
self.app.object_ring.get_nodes('account') self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs.values(): for dev in self.app.object_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
@ -1697,15 +1688,15 @@ class TestObjectController(unittest.TestCase):
def test_node_read_timeout(self): def test_node_read_timeout(self):
with save_globals(): with save_globals():
self.app.account_ring.get_nodes('account') self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
self.app.container_ring.get_nodes('account') self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs.values(): for dev in self.app.container_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
self.app.object_ring.get_nodes('account') self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs.values(): for dev in self.app.object_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
@ -1731,6 +1722,18 @@ class TestObjectController(unittest.TestCase):
def test_node_read_timeout_retry(self): def test_node_read_timeout_retry(self):
with save_globals(): with save_globals():
self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs:
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs:
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs:
dev['ip'] = '127.0.0.1'
dev['port'] = 1
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.app.update_request(req) self.app.update_request(req)
@ -1788,15 +1791,15 @@ class TestObjectController(unittest.TestCase):
def test_node_write_timeout(self): def test_node_write_timeout(self):
with save_globals(): with save_globals():
self.app.account_ring.get_nodes('account') self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
self.app.container_ring.get_nodes('account') self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs.values(): for dev in self.app.container_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
self.app.object_ring.get_nodes('account') self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs.values(): for dev in self.app.object_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
req = Request.blank('/v1/a/c/o', req = Request.blank('/v1/a/c/o',
@ -2023,12 +2026,8 @@ class TestObjectController(unittest.TestCase):
def test_acc_or_con_missing_returns_404(self): def test_acc_or_con_missing_returns_404(self):
with save_globals(): with save_globals():
self.app.memcache = FakeMemcacheReturnsNone() self.app.memcache = FakeMemcacheReturnsNone()
for dev in self.app.account_ring.devs.values(): self.app.account_ring.clear_errors()
del dev['errors'] self.app.container_ring.clear_errors()
del dev['last_error']
for dev in self.app.container_ring.devs.values():
del dev['errors']
del dev['last_error']
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container', 'object')
set_http_connect(200, 200, 200, 200, 200, 200) set_http_connect(200, 200, 200, 200, 200, 200)
@ -2094,7 +2093,7 @@ class TestObjectController(unittest.TestCase):
resp = getattr(controller, 'DELETE')(req) resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404) self.assertEquals(resp.status_int, 404)
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['errors'] = self.app.error_suppression_limit + 1 dev['errors'] = self.app.error_suppression_limit + 1
dev['last_error'] = time.time() dev['last_error'] = time.time()
set_http_connect(200) set_http_connect(200)
@ -2106,9 +2105,9 @@ class TestObjectController(unittest.TestCase):
resp = getattr(controller, 'DELETE')(req) resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404) self.assertEquals(resp.status_int, 404)
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['errors'] = 0 dev['errors'] = 0
for dev in self.app.container_ring.devs.values(): for dev in self.app.container_ring.devs:
dev['errors'] = self.app.error_suppression_limit + 1 dev['errors'] = self.app.error_suppression_limit + 1
dev['last_error'] = time.time() dev['last_error'] = time.time()
set_http_connect(200, 200) set_http_connect(200, 200)
@ -4421,9 +4420,7 @@ class TestContainerController(unittest.TestCase):
for meth in ('DELETE', 'PUT'): for meth in ('DELETE', 'PUT'):
with save_globals(): with save_globals():
self.app.memcache = FakeMemcacheReturnsNone() self.app.memcache = FakeMemcacheReturnsNone()
for dev in self.app.account_ring.devs.values(): self.app.account_ring.clear_errors()
del dev['errors']
del dev['last_error']
controller = proxy_server.ContainerController(self.app, controller = proxy_server.ContainerController(self.app,
'account', 'account',
'container') 'container')
@ -4460,7 +4457,7 @@ class TestContainerController(unittest.TestCase):
resp = getattr(controller, meth)(req) resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404) self.assertEquals(resp.status_int, 404)
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['errors'] = self.app.error_suppression_limit + 1 dev['errors'] = self.app.error_suppression_limit + 1
dev['last_error'] = time.time() dev['last_error'] = time.time()
set_http_connect(200, 200, 200, 200, 200, 200) set_http_connect(200, 200, 200, 200, 200, 200)
@ -5338,7 +5335,7 @@ class TestAccountController(unittest.TestCase):
def test_connection_refused(self): def test_connection_refused(self):
self.app.account_ring.get_nodes('account') self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 # can't connect on this port dev['port'] = 1 # can't connect on this port
controller = proxy_server.AccountController(self.app, 'account') controller = proxy_server.AccountController(self.app, 'account')
@ -5349,7 +5346,7 @@ class TestAccountController(unittest.TestCase):
def test_other_socket_error(self): def test_other_socket_error(self):
self.app.account_ring.get_nodes('account') self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values(): for dev in self.app.account_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = -1 # invalid port number dev['port'] = -1 # invalid port number
controller = proxy_server.AccountController(self.app, 'account') controller = proxy_server.AccountController(self.app, 'account')