diff --git a/swift/proxy/controllers/account.py b/swift/proxy/controllers/account.py index bebae114b6..0e1a5c7981 100644 --- a/swift/proxy/controllers/account.py +++ b/swift/proxy/controllers/account.py @@ -21,7 +21,7 @@ from swift.common.utils import public from swift.common.constraints import check_metadata from swift.common.http import HTTP_NOT_FOUND, HTTP_GONE from swift.proxy.controllers.base import Controller, clear_info_cache, \ - set_info_cache + set_info_cache, NodeIter from swift.common.middleware import listing_formats from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed from swift.common.request_helpers import get_sys_meta_prefix @@ -63,8 +63,8 @@ class AccountController(Controller): partition = self.app.account_ring.get_part(self.account_name) concurrency = self.app.account_ring.replica_count \ if self.app.get_policy_options(None).concurrent_gets else 1 - node_iter = self.app.iter_nodes(self.app.account_ring, partition, - self.logger, req) + node_iter = NodeIter(self.app, self.app.account_ring, partition, + self.logger, req) params = req.params params['format'] = 'json' req.params = params diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index b2c93a30df..e774eeb382 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -2007,7 +2007,7 @@ class Controller(object): :returns: a swob.Response object """ nodes = GreenthreadSafeIterator( - node_iterator or self.app.iter_nodes(ring, part, self.logger, req) + node_iterator or NodeIter(self.app, ring, part, self.logger, req) ) node_number = node_count or len(ring.get_part_nodes(part)) pile = GreenAsyncPile(node_number) diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index fa7efe6d4c..032037b10b 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -27,7 +27,7 @@ from swift.common.constraints import check_metadata, CONTAINER_LISTING_LIMIT from swift.common.http import HTTP_ACCEPTED, is_success from swift.common.request_helpers import get_sys_meta_prefix, get_param, \ constrain_req_limit, validate_container_params -from swift.proxy.controllers.base import Controller, delay_denial, \ +from swift.proxy.controllers.base import Controller, delay_denial, NodeIter, \ cors_validation, set_info_cache, clear_info_cache, get_container_info, \ record_cache_op_metrics, get_cache_key, headers_from_container_info, \ update_headers @@ -103,8 +103,8 @@ class ContainerController(Controller): self.account_name, self.container_name) concurrency = self.app.container_ring.replica_count \ if self.app.get_policy_options(None).concurrent_gets else 1 - node_iter = self.app.iter_nodes(self.app.container_ring, part, - self.logger, req) + node_iter = NodeIter(self.app, self.app.container_ring, part, + self.logger, req) resp = self.GETorHEAD_base( req, 'Container', node_iter, part, req.swift_entity_path, concurrency) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index c149623526..bef377955e 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -70,7 +70,7 @@ from swift.common.storage_policy import (POLICIES, REPL_POLICY, EC_POLICY, from swift.proxy.controllers.base import Controller, delay_denial, \ cors_validation, update_headers, bytes_to_skip, ByteCountEnforcer, \ record_cache_op_metrics, get_cache_key, GetterBase, GetterSource, \ - is_good_source + is_good_source, NodeIter from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \ HTTPServerError, HTTPServiceUnavailable, HTTPClientDisconnect, \ @@ -201,8 +201,8 @@ class BaseObjectController(Controller): policy_options = self.app.get_policy_options(policy) is_local = policy_options.write_affinity_is_local_fn if is_local is None: - return self.app.iter_nodes(ring, partition, self.logger, request, - policy=policy) + return NodeIter(self.app, ring, partition, self.logger, request, + policy=policy) primary_nodes = ring.get_part_nodes(partition) handoff_nodes = ring.get_more_nodes(partition) @@ -235,8 +235,8 @@ class BaseObjectController(Controller): (node for node in all_nodes if node not in preferred_nodes) ) - return self.app.iter_nodes(ring, partition, self.logger, request, - node_iter=node_iter, policy=policy) + return NodeIter(self.app, ring, partition, self.logger, request, + node_iter=node_iter, policy=policy) def GETorHEAD(self, req): """Handle HTTP GET or HEAD requests.""" @@ -255,8 +255,8 @@ class BaseObjectController(Controller): return aresp partition = obj_ring.get_part( self.account_name, self.container_name, self.object_name) - node_iter = self.app.iter_nodes(obj_ring, partition, self.logger, req, - policy=policy) + node_iter = NodeIter(self.app, obj_ring, partition, self.logger, req, + policy=policy) resp = self._get_or_head_response(req, node_iter, partition, policy) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 64a14ff1d5..e4f82d61c5 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -41,7 +41,7 @@ from swift.common.registry import register_swift_info from swift.common.constraints import check_utf8, valid_api_version from swift.proxy.controllers import AccountController, ContainerController, \ ObjectControllerRouter, InfoController -from swift.proxy.controllers.base import get_container_info, NodeIter, \ +from swift.proxy.controllers.base import get_container_info, \ DEFAULT_RECHECK_CONTAINER_EXISTENCE, DEFAULT_RECHECK_ACCOUNT_EXISTENCE, \ DEFAULT_RECHECK_UPDATING_SHARD_RANGES, DEFAULT_RECHECK_LISTING_SHARD_RANGES from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ @@ -743,11 +743,6 @@ class Application(object): return ok - def iter_nodes(self, ring, partition, logger, request, node_iter=None, - policy=None): - return NodeIter(self, ring, partition, logger, request=request, - node_iter=node_iter, policy=policy, ) - def exception_occurred(self, node, typ, additional_info, **kwargs): """ diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index 1dc333c656..22350f9ba1 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -403,8 +403,9 @@ class TestContainerController(TestRingBase): def __iter__(self): return iter([]) - with mocked_http_conn(), mock.patch.object(self.app, 'iter_nodes', - return_value=FakeIter()): + with mocked_http_conn(), mock.patch( + 'swift.proxy.controllers.container.NodeIter', + return_value=FakeIter()): req = Request.blank('/v1/a/c') resp = req.get_response(self.app) self.assertEqual(resp.status_int, 503) diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 6252c40944..09b19a8192 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -45,7 +45,8 @@ from swift.common.utils import Timestamp, list_from_csv, md5, FileLikeIter from swift.proxy import server as proxy_server from swift.proxy.controllers import obj from swift.proxy.controllers.base import \ - get_container_info as _real_get_container_info, GetterSource + get_container_info as _real_get_container_info, GetterSource, \ + NodeIter from swift.common.storage_policy import POLICIES, ECDriverError, \ StoragePolicy, ECStoragePolicy from swift.common.swob import Request @@ -2775,9 +2776,9 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase): def test_feed_remaining_primaries(self): controller = self.controller_cls( self.app, 'a', 'c', 'o') - safe_iter = utils.GreenthreadSafeIterator(self.app.iter_nodes( - self.policy.object_ring, 0, self.logger, policy=self.policy, - request=Request.blank(''))) + safe_iter = utils.GreenthreadSafeIterator(NodeIter( + self.app, self.policy.object_ring, 0, self.logger, + policy=self.policy, request=Request.blank(''))) controller._fragment_GET_request = lambda *a, **k: next(safe_iter) pile = utils.GreenAsyncPile(self.policy.ec_ndata) for i in range(self.policy.ec_ndata): diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index f015a7d2bd..cbe79f434e 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1097,7 +1097,7 @@ class TestProxyServer(unittest.TestCase): conn = FakeConn(ip, *args, **kargs) return conn - with mock.patch('swift.proxy.server.Application.iter_nodes', + with mock.patch('swift.proxy.controllers.account.NodeIter', fake_iter_nodes): with mock.patch('swift.common.bufferedhttp.http_connect_raw', myfake_http_connect_raw): @@ -5472,9 +5472,9 @@ class TestReplicatedObjectController( 'container', 'object') collected_nodes = [] - for node in self.app.iter_nodes(object_ring, partition, - self.logger, - request=Request.blank('')): + for node in proxy_base.NodeIter( + self.app, object_ring, partition, self.logger, + request=Request.blank('')): collected_nodes.append(node) self.assertEqual(len(collected_nodes), 5) @@ -5484,9 +5484,9 @@ class TestReplicatedObjectController( 'container', 'object') collected_nodes = [] - for node in self.app.iter_nodes(object_ring, partition, - self.logger, - request=Request.blank('')): + for node in proxy_base.NodeIter( + self.app, object_ring, partition, self.logger, + request=Request.blank('')): collected_nodes.append(node) self.assertEqual(len(collected_nodes), 9) @@ -5499,9 +5499,9 @@ class TestReplicatedObjectController( 'container', 'object') collected_nodes = [] - for node in self.app.iter_nodes(object_ring, partition, - self.logger, - request=Request.blank('')): + for node in proxy_base.NodeIter( + self.app, object_ring, partition, self.logger, + request=Request.blank('')): collected_nodes.append(node) self.assertEqual(len(collected_nodes), 7) self.assertEqual(self.app.logger.log_dict['warning'], []) @@ -5517,9 +5517,9 @@ class TestReplicatedObjectController( last_error=(2 ** 63 - 1)) collected_nodes = [] - for node in self.app.iter_nodes(object_ring, partition, - self.logger, - request=Request.blank('')): + for node in proxy_base.NodeIter( + self.app, object_ring, partition, self.logger, + request=Request.blank('')): collected_nodes.append(node) self.assertEqual(len(collected_nodes), 7) self.assertEqual( @@ -5539,9 +5539,9 @@ class TestReplicatedObjectController( last_error=(2 ** 63 - 1)) collected_nodes = [] - for node in self.app.iter_nodes(object_ring, partition, - self.logger, - request=Request.blank('')): + for node in proxy_base.NodeIter( + self.app, object_ring, partition, self.logger, + request=Request.blank('')): collected_nodes.append(node) self.assertEqual(len(collected_nodes), 7) self.assertEqual( @@ -5565,9 +5565,9 @@ class TestReplicatedObjectController( last_error=(2 ** 63 - 1)) collected_nodes = [] - for node in self.app.iter_nodes(object_ring, partition, - self.logger, - request=Request.blank('')): + for node in proxy_base.NodeIter( + self.app, object_ring, partition, self.logger, + request=Request.blank('')): collected_nodes.append(node) self.assertEqual(len(collected_nodes), 10) self.assertEqual( @@ -5596,8 +5596,9 @@ class TestReplicatedObjectController( with mock.patch.object(self.app, 'sort_nodes', side_effect=fake_sort_nodes): object_ring = self.app.get_object_ring(None) - for node in self.app.iter_nodes(object_ring, 0, self.logger, - request=Request.blank('')): + for node in proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, + request=Request.blank('')): pass self.assertEqual(called, [ mock.call(object_ring.get_part_nodes(0), policy=None) @@ -5607,10 +5608,12 @@ class TestReplicatedObjectController( with mock.patch.object(self.app, 'sort_nodes', lambda n, *args, **kwargs: n): object_ring = self.app.get_object_ring(None) - first_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, request=Request.blank(''))) - second_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, request=Request.blank(''))) + first_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, + request=Request.blank(''))) + second_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, + request=Request.blank(''))) self.assertIn(first_nodes[0], second_nodes) self.assertEqual( @@ -5629,14 +5632,16 @@ class TestReplicatedObjectController( ('Node will be error limited for 60.00s: %s, error: %s' % (node_to_string(first_nodes[0]), 'test')), line) - second_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, request=Request.blank(''))) + second_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, + request=Request.blank(''))) self.assertNotIn(first_nodes[0], second_nodes) self.assertEqual( 1, self.logger.statsd_client.get_increment_counts().get( 'error_limiter.is_limited', 0)) - third_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, request=Request.blank(''))) + third_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, + request=Request.blank(''))) self.assertNotIn(first_nodes[0], third_nodes) self.assertEqual( 2, self.logger.statsd_client.get_increment_counts().get( @@ -5649,11 +5654,13 @@ class TestReplicatedObjectController( mock.patch.object(self.app, 'request_node_count', lambda r: 6), \ mock.patch.object(object_ring, 'max_more_nodes', 99): - first_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, request=Request.blank(''))) + first_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, + request=Request.blank(''))) second_nodes = [] - for node in self.app.iter_nodes(object_ring, 0, self.logger, - request=Request.blank('')): + for node in proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, + request=Request.blank('')): if not second_nodes: self.app.error_limit(node, 'test') second_nodes.append(node) @@ -5670,8 +5677,8 @@ class TestReplicatedObjectController( lambda n, *args, **kwargs: n), \ mock.patch.object(self.app, 'request_node_count', lambda r: 3): - got_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, Request.blank(''), + got_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, Request.blank(''), node_iter=iter(node_list))) self.assertEqual(expected[:3], got_nodes) @@ -5682,8 +5689,9 @@ class TestReplicatedObjectController( lambda n, *args, **kwargs: n), \ mock.patch.object(self.app, 'request_node_count', lambda r: 1000000): - got_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, req, node_iter=iter(node_list))) + got_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, req, + node_iter=iter(node_list))) self.assertEqual(expected, got_nodes) def test_iter_nodes_with_replication_network(self): @@ -5697,8 +5705,9 @@ class TestReplicatedObjectController( lambda n, *args, **kwargs: n), \ mock.patch.object(self.app, 'request_node_count', lambda r: 3): - got_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, req, node_iter=iter(node_list))) + got_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, req, + node_iter=iter(node_list))) expected = [dict(n, use_replication=True) for n in node_list] self.assertEqual(expected[:3], got_nodes) req = Request.blank( @@ -5708,8 +5717,9 @@ class TestReplicatedObjectController( lambda n, *args, **kwargs: n), \ mock.patch.object(self.app, 'request_node_count', lambda r: 13): - got_nodes = list(self.app.iter_nodes( - object_ring, 0, self.logger, req, node_iter=iter(node_list))) + got_nodes = list(proxy_base.NodeIter( + self.app, object_ring, 0, self.logger, req, + node_iter=iter(node_list))) self.assertEqual(expected, got_nodes) def test_best_response_sets_headers(self):