openstack/swift
Code Issues Proposed changes

Proxy: override user_agent with backend_user_agent

Browse Source 
The proxy currently _ALWAYS_ overrides any user_agent in requests with
`proxy-server <pid>`. This is what we want for any normal request. But
when using the internal client, which itself uses it's own proxy server,
we'd rather use the internal client's defined user_agent, so we can
track the source of the request in logs.

This patch adds a backend_user_agent to the proxy, that when set a proxy
will use this as a user_agent in a request.

Every member of the pipeline has a `_pipeline_final_app` to point to the
app (proxy), so uses this to directly set the proxies
backend_user_agent.

Closes-Bug: #1951304
Change-Id: I5f7bf65b4e747b5a93b79328fac640b791ecdc95
This commit is contained in:
Matthew Oliver 2021-11-26 10:54:44 +11:00 committed by Clay Gerrard
parent 9bc1c008a5
commit b0245f4eb5
6 changed files with 177 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
swift/common/internal_client.py
View File

@ -157,7 +157,8 @@ class InternalClient(object):
raise ValueError('request_tries must be positive') raise ValueError('request_tries must be positive')
self.app = app or loadapp(conf_path, global_conf=global_conf, self.app = app or loadapp(conf_path, global_conf=global_conf,
allow_modify_pipeline=allow_modify_pipeline,) allow_modify_pipeline=allow_modify_pipeline,)
self.user_agent = user_agent self.user_agent = \
self.app._pipeline_final_app.backend_user_agent = user_agent
self.request_tries = request_tries self.request_tries = request_tries
self.use_replication_network = use_replication_network self.use_replication_network = use_replication_network
self.get_object_ring = self.app._pipeline_final_app.get_object_ring self.get_object_ring = self.app._pipeline_final_app.get_object_ring

3
swift/proxy/controllers/base.py
View File

@ -26,7 +26,6 @@
from six.moves.urllib.parse import quote from six.moves.urllib.parse import quote
import os
import time import time
import json import json
import functools import functools
@ -1776,7 +1775,7 @@ class Controller(object):
referer = '' referer = ''
headers['x-trans-id'] = self.trans_id headers['x-trans-id'] = self.trans_id
headers['connection'] = 'close' headers['connection'] = 'close'
headers['user-agent'] = 'proxy-server %s' % os.getpid() headers['user-agent'] = self.app.backend_user_agent
headers['referer'] = referer headers['referer'] = referer
return headers return headers

1
swift/proxy/server.py
View File

@ -199,6 +199,7 @@ class Application(object):
else: else:
self.logger = logger self.logger = logger
self._error_limiting = {} self._error_limiting = {}
self.backend_user_agent = 'proxy-server %s' % os.getpid()
swift_dir = conf.get('swift_dir', '/etc/swift') swift_dir = conf.get('swift_dir', '/etc/swift')
self.swift_dir = swift_dir self.swift_dir = swift_dir

1
test/unit/common/middleware/helpers.py
View File

@ -94,6 +94,7 @@ class FakeSwift(object):
self.container_ring = FakeRing() self.container_ring = FakeRing()
self.get_object_ring = lambda policy_index: FakeRing() self.get_object_ring = lambda policy_index: FakeRing()
self.auto_create_account_prefix = '.' self.auto_create_account_prefix = '.'
self.backend_user_agent = "fake_swift"
self._pipeline_final_app = self self._pipeline_final_app = self
def _find_response(self, method, path): def _find_response(self, method, path):

301
test/unit/common/test_internal_client.py
View File

@ -421,51 +421,52 @@ class TestInternalClient(unittest.TestCase):
self.assertTrue(client.use_replication_network) self.assertTrue(client.use_replication_network)
def test_make_request_sets_user_agent(self): def test_make_request_sets_user_agent(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self, test): def __init__(self, test):
super(FakeApp, self).__init__()
self.test = test self.test = test
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 1
self.use_replication_network = False
def fake_app(self, env, start_response): def __call__(self, env, start_response):
self.test.assertNotIn( self.test.assertNotIn(
'HTTP_X_BACKEND_USE_REPLICATION_NETWORK', env) 'HTTP_X_BACKEND_USE_REPLICATION_NETWORK', env)
self.test.assertEqual(self.user_agent, env['HTTP_USER_AGENT']) self.test.assertEqual(self.backend_user_agent,
"some_agent")
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return [] return []
client = InternalClient(self) client = internal_client.InternalClient(
None, 'some_agent', 1, use_replication_network=False,
app=FakeApp(self))
client.make_request('GET', '/', {}, (200,)) client.make_request('GET', '/', {}, (200,))
def test_make_request_defaults_replication_network_header(self): def test_make_request_defaults_replication_network_header(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self, test): def __init__(self, test):
super(FakeApp, self).__init__()
self.test = test self.test = test
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 1
self.use_replication_network = False
self.expected_header_value = None self.expected_header_value = None
def fake_app(self, env, start_response): def __call__(self, env, start_response):
if self.expected_header_value is None: if self.expected_header_value is None:
self.test.assertNotIn( self.test.assertNotIn(
'HTTP_X_BACKEND_USE_REPLICATION_NETWORK', env) 'HTTP_X_BACKEND_USE_REPLICATION_NETWORK', env)
else: else:
hdr_val = env['HTTP_X_BACKEND_USE_REPLICATION_NETWORK'] hdr_val = env['HTTP_X_BACKEND_USE_REPLICATION_NETWORK']
self.test.assertEqual(self.expected_header_value, hdr_val) self.test.assertEqual(self.expected_header_value, hdr_val)
self.test.assertEqual(self.backend_user_agent,
'some_agent')
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return [] return []
client = InternalClient(self) client = internal_client.InternalClient(
None, 'some_agent', 1, use_replication_network=False,
app=FakeApp(self))
client.make_request('GET', '/', {}, (200,)) client.make_request('GET', '/', {}, (200,))
# Caller can still override # Caller can still override
client.expected_header_value = 'false' client.app.expected_header_value = 'false'
client.make_request('GET', '/', { client.make_request('GET', '/', {
request_helpers.USE_REPLICATION_NETWORK_HEADER: 'false'}, (200,)) request_helpers.USE_REPLICATION_NETWORK_HEADER: 'false'}, (200,))
client.expected_header_value = 'true' client.app.expected_header_value = 'true'
client.make_request('GET', '/', { client.make_request('GET', '/', {
request_helpers.USE_REPLICATION_NETWORK_HEADER: 'true'}, (200,)) request_helpers.USE_REPLICATION_NETWORK_HEADER: 'true'}, (200,))
@ -473,49 +474,42 @@ class TestInternalClient(unittest.TestCase):
client.use_replication_network = True client.use_replication_network = True
client.make_request('GET', '/', {}, (200,)) client.make_request('GET', '/', {}, (200,))
client.expected_header_value = 'false' client.app.expected_header_value = 'false'
client.make_request('GET', '/', { client.make_request('GET', '/', {
request_helpers.USE_REPLICATION_NETWORK_HEADER: 'false'}, (200,)) request_helpers.USE_REPLICATION_NETWORK_HEADER: 'false'}, (200,))
client.expected_header_value = 'on' client.app.expected_header_value = 'on'
client.make_request('GET', '/', { client.make_request('GET', '/', {
request_helpers.USE_REPLICATION_NETWORK_HEADER: 'on'}, (200,)) request_helpers.USE_REPLICATION_NETWORK_HEADER: 'on'}, (200,))
def test_make_request_sets_query_string(self): def test_make_request_sets_query_string(self):
captured_envs = [] captured_envs = []
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self, test): def __call__(self, env, start_response):
self.test = test
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 1
self.use_replication_network = False
def fake_app(self, env, start_response):
captured_envs.append(env) captured_envs.append(env)
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return [] return []
client = InternalClient(self) client = internal_client.InternalClient(
None, 'some_agent', 1, use_replication_network=False,
app=FakeApp())
params = {'param1': 'p1', 'tasty': 'soup'} params = {'param1': 'p1', 'tasty': 'soup'}
client.make_request('GET', '/', {}, (200,), params=params) client.make_request('GET', '/', {}, (200,), params=params)
actual_params = dict(parse_qsl(captured_envs[0]['QUERY_STRING'], actual_params = dict(parse_qsl(captured_envs[0]['QUERY_STRING'],
keep_blank_values=True, keep_blank_values=True,
strict_parsing=True)) strict_parsing=True))
self.assertEqual(params, actual_params) self.assertEqual(params, actual_params)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
def test_make_request_retries(self): def test_make_request_retries(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self, test): def __init__(self):
self.test = test super(FakeApp, self).__init__()
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 4 self.request_tries = 4
self.use_replication_network = False
self.tries = 0 self.tries = 0
self.sleep_called = 0
def fake_app(self, env, start_response): def __call__(self, env, start_response):
self.tries += 1 self.tries += 1
if self.tries < self.request_tries: if self.tries < self.request_tries:
start_response( start_response(
@ -524,11 +518,19 @@ class TestInternalClient(unittest.TestCase):
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return [] return []
class InternalClient(internal_client.InternalClient):
def __init__(self, *args, **kwargs):
self.test = kwargs.pop('test')
super(InternalClient, self).__init__(*args, **kwargs)
self.sleep_called = 0
def sleep(self, seconds): def sleep(self, seconds):
self.sleep_called += 1 self.sleep_called += 1
self.test.assertEqual(2 ** (self.sleep_called), seconds) self.test.assertEqual(2 ** (self.sleep_called), seconds)
client = InternalClient(self) client = InternalClient(
None, 'some_agent', 4, use_replication_network=False,
app=FakeApp(), test=self)
old_sleep = internal_client.sleep old_sleep = internal_client.sleep
internal_client.sleep = client.sleep internal_client.sleep = client.sleep
@ -539,7 +541,9 @@ class TestInternalClient(unittest.TestCase):
internal_client.sleep = old_sleep internal_client.sleep = old_sleep
self.assertEqual(3, client.sleep_called) self.assertEqual(3, client.sleep_called)
self.assertEqual(4, client.tries) self.assertEqual(4, client.app.tries)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
def test_base_request_timeout(self): def test_base_request_timeout(self):
# verify that base_request passes timeout arg on to urlopen # verify that base_request passes timeout arg on to urlopen
@ -586,48 +590,46 @@ class TestInternalClient(unittest.TestCase):
'/?format=json&marker=d', actual_requests[2].selector) '/?format=json&marker=d', actual_requests[2].selector)
def test_make_request_method_path_headers(self): def test_make_request_method_path_headers(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self): def __init__(self):
self.app = self.fake_app super(FakeApp, self).__init__()
self.user_agent = 'some_agent'
self.request_tries = 3
self.use_replication_network = False
self.env = None self.env = None
def fake_app(self, env, start_response): def __call__(self, env, start_response):
self.env = env self.env = env
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return [] return []
client = InternalClient() client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp())
for method in 'GET PUT HEAD'.split(): for method in 'GET PUT HEAD'.split():
client.make_request(method, '/', {}, (200,)) client.make_request(method, '/', {}, (200,))
self.assertEqual(client.env['REQUEST_METHOD'], method) self.assertEqual(client.app.env['REQUEST_METHOD'], method)
for path in '/one /two/three'.split(): for path in '/one /two/three'.split():
client.make_request('GET', path, {'X-Test': path}, (200,)) client.make_request('GET', path, {'X-Test': path}, (200,))
self.assertEqual(client.env['PATH_INFO'], path) self.assertEqual(client.app.env['PATH_INFO'], path)
self.assertEqual(client.env['HTTP_X_TEST'], path) self.assertEqual(client.app.env['HTTP_X_TEST'], path)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
def test_make_request_error_case(self): def test_make_request_error_case(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self): def __call__(self, env, start_response):
self.logger = debug_logger('test-ic')
# wrap the fake app with ProxyLoggingMiddleware
self.app = ProxyLoggingMiddleware(
self.fake_app, {}, self.logger)
self.user_agent = 'some_agent'
self.request_tries = 3
self.use_replication_network = False
def fake_app(self, env, start_response):
body = b'fake error response' body = b'fake error response'
start_response('409 Conflict', start_response('409 Conflict',
[('Content-Length', str(len(body)))]) [('Content-Length', str(len(body)))])
return [body] return [body]
client = InternalClient() final_fake_app = FakeApp()
fake_app = ProxyLoggingMiddleware(
final_fake_app, {}, final_fake_app.logger)
fake_app._pipeline_final_app = final_fake_app
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False, app=fake_app)
with self.assertRaises(internal_client.UnexpectedResponse), \ with self.assertRaises(internal_client.UnexpectedResponse), \
mock.patch('swift.common.internal_client.sleep'): mock.patch('swift.common.internal_client.sleep'):
client.make_request('DELETE', '/container', {}, (200,)) client.make_request('DELETE', '/container', {}, (200,))
@ -635,27 +637,23 @@ class TestInternalClient(unittest.TestCase):
# Since we didn't provide an X-Timestamp, retrying gives us a chance to # Since we didn't provide an X-Timestamp, retrying gives us a chance to
# succeed (assuming the failure was due to clock skew between servers) # succeed (assuming the failure was due to clock skew between servers)
expected = (' HTTP/1.0 409 ',) expected = (' HTTP/1.0 409 ',)
loglines = client.logger.get_lines_for_level('info') logger = client.app._pipeline_final_app.logger
loglines = logger.get_lines_for_level('info')
for expected, logline in zip_longest(expected, loglines): for expected, logline in zip_longest(expected, loglines):
if not expected: if not expected:
self.fail('Unexpected extra log line: %r' % logline) self.fail('Unexpected extra log line: %r' % logline)
self.assertIn(expected, logline) self.assertIn(expected, logline)
self.assertEqual(client.app.app.backend_user_agent, 'some_agent')
def test_make_request_acceptable_status_not_2xx(self): def test_make_request_acceptable_status_not_2xx(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self, resp_status): def __init__(self):
self.logger = debug_logger('test-ic') super(FakeApp, self).__init__()
# wrap the fake app with ProxyLoggingMiddleware
self.app = ProxyLoggingMiddleware(
self.fake_app, {}, self.logger)
self.user_agent = 'some_agent'
self.resp_status = resp_status
self.request_tries = 3
self.use_replication_network = False
self.closed_paths = [] self.closed_paths = []
self.fully_read_paths = [] self.fully_read_paths = []
self.resp_status = None
def fake_app(self, env, start_response): def __call__(self, env, start_response):
body = b'fake error response' body = b'fake error response'
start_response(self.resp_status, start_response(self.resp_status,
[('Content-Length', str(len(body)))]) [('Content-Length', str(len(body)))])
@ -664,15 +662,25 @@ class TestInternalClient(unittest.TestCase):
env['PATH_INFO']) env['PATH_INFO'])
def do_test(resp_status): def do_test(resp_status):
client = InternalClient(resp_status) final_fake_app = FakeApp()
fake_app = ProxyLoggingMiddleware(
final_fake_app, {}, final_fake_app.logger)
fake_app._pipeline_final_app = final_fake_app
final_fake_app.resp_status = resp_status
client = internal_client.InternalClient(
None, "some_agent", 3, use_replication_network=False,
app=fake_app)
with self.assertRaises(internal_client.UnexpectedResponse) as ctx,\ with self.assertRaises(internal_client.UnexpectedResponse) as ctx,\
mock.patch('swift.common.internal_client.sleep'): mock.patch('swift.common.internal_client.sleep'):
# This is obvious strange tests to expect only 400 Bad Request # This is obvious strange tests to expect only 400 Bad Request
# but this test intended to avoid extra body drain if it's # but this test intended to avoid extra body drain if it's
# correct object body with 2xx. # correct object body with 2xx.
client.make_request('GET', '/cont/obj', {}, (400,)) client.make_request('GET', '/cont/obj', {}, (400,))
loglines = client.logger.get_lines_for_level('info') logger = client.app._pipeline_final_app.logger
return (client.fully_read_paths, client.closed_paths, loglines = logger.get_lines_for_level('info')
self.assertEqual(client.app.app.backend_user_agent, 'some_agent')
return (client.app._pipeline_final_app.fully_read_paths,
client.app._pipeline_final_app.closed_paths,
ctx.exception.resp, loglines) ctx.exception.resp, loglines)
fully_read_paths, closed_paths, resp, loglines = do_test('200 OK') fully_read_paths, closed_paths, resp, loglines = do_test('200 OK')
@ -705,18 +713,14 @@ class TestInternalClient(unittest.TestCase):
self.assertIn(expected, logline) self.assertIn(expected, logline)
def test_make_request_codes(self): def test_make_request_codes(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self): def __call__(self, env, start_response):
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 3
self.use_replication_network = False
def fake_app(self, env, start_response):
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return [] return []
client = InternalClient() client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp())
try: try:
old_sleep = internal_client.sleep old_sleep = internal_client.sleep
@ -742,6 +746,8 @@ class TestInternalClient(unittest.TestCase):
client.make_request('GET', '/', {}, (111,)) client.make_request('GET', '/', {}, (111,))
self.assertTrue(str(raised.exception).startswith( self.assertTrue(str(raised.exception).startswith(
'Unexpected response')) 'Unexpected response'))
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
finally: finally:
internal_client.sleep = old_sleep internal_client.sleep = old_sleep
@ -756,23 +762,21 @@ class TestInternalClient(unittest.TestCase):
self.test.assertEqual(0, offset) self.test.assertEqual(0, offset)
self.test.assertEqual(0, whence) self.test.assertEqual(0, whence)
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self, status): def __init__(self, status):
self.app = self.fake_app super(FakeApp, self).__init__()
self.user_agent = 'some_agent'
self.request_tries = 3
self.use_replication_network = False
self.status = status self.status = status
self.call_count = 0
def fake_app(self, env, start_response): def __call__(self, env, start_response):
self.call_count += 1
start_response(self.status, [('Content-Length', '0')]) start_response(self.status, [('Content-Length', '0')])
self._calls.append('')
return [] return []
def do_test(status, expected_calls): def do_test(status, expected_calls):
fobj = FileObject(self) fobj = FileObject(self)
client = InternalClient(status) client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp(status))
with mock.patch.object(internal_client, 'sleep', not_sleep): with mock.patch.object(internal_client, 'sleep', not_sleep):
with self.assertRaises(Exception) as exc_mgr: with self.assertRaises(Exception) as exc_mgr:
@ -780,23 +784,23 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(int(status[:3]), self.assertEqual(int(status[:3]),
exc_mgr.exception.resp.status_int) exc_mgr.exception.resp.status_int)
self.assertEqual(client.call_count, fobj.seek_called) self.assertEqual(client.app.call_count, fobj.seek_called)
self.assertEqual(client.call_count, expected_calls) self.assertEqual(client.app.call_count, expected_calls)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
do_test('404 Not Found', 1) do_test('404 Not Found', 1)
do_test('503 Service Unavailable', 3) do_test('503 Service Unavailable', 3)
def test_make_request_request_exception(self): def test_make_request_request_exception(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self): def __call__(self, env, start_response):
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 3
def fake_app(self, env, start_response):
raise Exception() raise Exception()
client = InternalClient() client = internal_client.InternalClient(
None, 'some_agent', 3, app=FakeApp())
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
try: try:
old_sleep = internal_client.sleep old_sleep = internal_client.sleep
internal_client.sleep = not_sleep internal_client.sleep = not_sleep
@ -849,23 +853,21 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(1, client.make_request_called) self.assertEqual(1, client.make_request_called)
def test_get_metadata_invalid_status(self): def test_get_metadata_invalid_status(self):
class InternalClient(internal_client.InternalClient):
def __init__(self):
self.user_agent = 'test'
self.request_tries = 1
self.use_replication_network = False
self.app = self.fake_app
def fake_app(self, environ, start_response): class FakeApp(FakeSwift):
def __call__(self, environ, start_response):
start_response('404 Not Found', [('x-foo', 'bar')]) start_response('404 Not Found', [('x-foo', 'bar')])
return [b'nope'] return [b'nope']
client = InternalClient() client = internal_client.InternalClient(
None, 'test', 1, use_replication_network=False, app=FakeApp())
self.assertRaises(internal_client.UnexpectedResponse, self.assertRaises(internal_client.UnexpectedResponse,
client._get_metadata, 'path') client._get_metadata, 'path')
metadata = client._get_metadata('path', metadata_prefix='x-', metadata = client._get_metadata('path', metadata_prefix='x-',
acceptable_statuses=(4,)) acceptable_statuses=(4,))
self.assertEqual(metadata, {'foo': 'bar'}) self.assertEqual(metadata, {'foo': 'bar'})
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'test')
def test_make_path(self): def test_make_path(self):
account, container, obj = path_parts() account, container, obj = path_parts()
@ -1122,6 +1124,7 @@ class TestInternalClient(unittest.TestCase):
'Host': 'localhost:80', 'Host': 'localhost:80',
'User-Agent': 'test' 'User-Agent': 'test'
})], app._calls) })], app._calls)
self.assertEqual(app.backend_user_agent, 'test')
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
@ -1134,6 +1137,7 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(1, len(app._calls)) self.assertEqual(1, len(app._calls))
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_get_account_info(self): def test_get_account_info(self):
class Response(object): class Response(object):
@ -1233,10 +1237,11 @@ class TestInternalClient(unittest.TestCase):
'X-Backend-Allow-Reserved-Names': 'true', 'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80', 'Host': 'localhost:80',
'X-Account-Meta-Color': 'Blue', 'X-Account-Meta-Color': 'Blue',
'User-Agent': 'test', 'User-Agent': 'test'
})], app._calls) })], app._calls)
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_set_account_metadata_plumbing(self): def test_set_account_metadata_plumbing(self):
account, container, obj = path_parts() account, container, obj = path_parts()
@ -1295,6 +1300,7 @@ class TestInternalClient(unittest.TestCase):
'Host': 'localhost:80', 'Host': 'localhost:80',
'User-Agent': 'test' 'User-Agent': 'test'
})], app._calls) })], app._calls)
self.assertEqual(app.backend_user_agent, 'test')
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
@ -1332,6 +1338,7 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(1, len(app._calls)) self.assertEqual(1, len(app._calls))
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_delete_container_plumbing(self): def test_delete_container_plumbing(self):
class InternalClient(internal_client.InternalClient): class InternalClient(internal_client.InternalClient):
@ -1396,10 +1403,11 @@ class TestInternalClient(unittest.TestCase):
'X-Backend-Allow-Reserved-Names': 'true', 'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80', 'Host': 'localhost:80',
'X-Container-Meta-Color': 'Blue', 'X-Container-Meta-Color': 'Blue',
'User-Agent': 'test', 'User-Agent': 'test'
})], app._calls) })], app._calls)
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_set_container_metadata_plumbing(self): def test_set_container_metadata_plumbing(self):
account, container, obj = path_parts() account, container, obj = path_parts()
@ -1423,6 +1431,7 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(1, len(app._calls)) self.assertEqual(1, len(app._calls))
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
app.register('DELETE', path, swob.HTTPNotFound, {}) app.register('DELETE', path, swob.HTTPNotFound, {})
client.delete_object(account, container, obj) client.delete_object(account, container, obj)
@ -1445,23 +1454,19 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(1, client.get_metadata_called) self.assertEqual(1, client.get_metadata_called)
def test_get_metadata_extra_headers(self): def test_get_metadata_extra_headers(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self): def __call__(self, env, start_response):
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 3
self.use_replication_network = False
def fake_app(self, env, start_response):
self.req_env = env self.req_env = env
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return [] return []
client = InternalClient() client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp())
headers = {'X-Foo': 'bar'} headers = {'X-Foo': 'bar'}
client.get_object_metadata('account', 'container', 'obj', client.get_object_metadata('account', 'container', 'obj',
headers=headers) headers=headers)
self.assertEqual(client.req_env['HTTP_X_FOO'], 'bar') self.assertEqual(client.app.req_env['HTTP_X_FOO'], 'bar')
def test_get_object(self): def test_get_object(self):
account, container, obj = path_parts() account, container, obj = path_parts()
@ -1482,49 +1487,49 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(app.call_count, 1) self.assertEqual(app.call_count, 1)
req_headers.update({ req_headers.update({
'host': 'localhost:80', # from swob.Request.blank 'host': 'localhost:80', # from swob.Request.blank
'user-agent': 'test', # from InternalClient.make_request
'x-backend-allow-reserved-names': 'true', # also from IC 'x-backend-allow-reserved-names': 'true', # also from IC
'x-backend-storage-policy-index': '2', # from proxy-server app 'x-backend-storage-policy-index': '2', # from proxy-server app
'user-agent': 'test',
}) })
self.assertEqual(app.calls_with_headers, [( self.assertEqual(app.calls_with_headers, [(
'GET', path_info + '?symlink=get', HeaderKeyDict(req_headers))]) 'GET', path_info + '?symlink=get', HeaderKeyDict(req_headers))])
def test_iter_object_lines(self): def test_iter_object_lines(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self, lines): def __init__(self, lines):
super(FakeApp, self).__init__()
self.lines = lines self.lines = lines
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 3
self.use_replication_network = False
def fake_app(self, env, start_response): def __call__(self, env, start_response):
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return [b'%s\n' % x for x in self.lines] return [b'%s\n' % x for x in self.lines]
lines = b'line1 line2 line3'.split() lines = b'line1 line2 line3'.split()
client = InternalClient(lines) client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp(lines))
ret_lines = [] ret_lines = []
for line in client.iter_object_lines('account', 'container', 'object'): for line in client.iter_object_lines('account', 'container', 'object'):
ret_lines.append(line) ret_lines.append(line)
self.assertEqual(lines, ret_lines) self.assertEqual(lines, ret_lines)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
def test_iter_object_lines_compressed_object(self): def test_iter_object_lines_compressed_object(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self, lines): def __init__(self, lines):
super(FakeApp, self).__init__()
self.lines = lines self.lines = lines
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 3
self.use_replication_network = False
def fake_app(self, env, start_response): def __call__(self, env, start_response):
start_response('200 Ok', [('Content-Length', '0')]) start_response('200 Ok', [('Content-Length', '0')])
return internal_client.CompressingFileReader( return internal_client.CompressingFileReader(
BytesIO(b'\n'.join(self.lines))) BytesIO(b'\n'.join(self.lines)))
lines = b'line1 line2 line3'.split() lines = b'line1 line2 line3'.split()
client = InternalClient(lines) client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp(lines))
ret_lines = [] ret_lines = []
for line in client.iter_object_lines( for line in client.iter_object_lines(
'account', 'container', 'object.gz'): 'account', 'container', 'object.gz'):
@ -1532,18 +1537,14 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(lines, ret_lines) self.assertEqual(lines, ret_lines)
def test_iter_object_lines_404(self): def test_iter_object_lines_404(self):
class InternalClient(internal_client.InternalClient): class FakeApp(FakeSwift):
def __init__(self): def __call__(self, env, start_response):
self.app = self.fake_app
self.user_agent = 'some_agent'
self.request_tries = 3
self.use_replication_network = False
def fake_app(self, env, start_response):
start_response('404 Not Found', []) start_response('404 Not Found', [])
return [b'one\ntwo\nthree'] return [b'one\ntwo\nthree']
client = InternalClient() client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp())
lines = [] lines = []
for line in client.iter_object_lines( for line in client.iter_object_lines(
'some_account', 'some_container', 'some_object', 'some_account', 'some_container', 'some_object',
@ -1562,10 +1563,11 @@ class TestInternalClient(unittest.TestCase):
'X-Backend-Allow-Reserved-Names': 'true', 'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80', 'Host': 'localhost:80',
'X-Object-Meta-Color': 'Blue', 'X-Object-Meta-Color': 'Blue',
'User-Agent': 'test', 'User-Agent': 'test'
})], app._calls) })], app._calls)
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_set_object_metadata_plumbing(self): def test_set_object_metadata_plumbing(self):
account, container, obj = path_parts() account, container, obj = path_parts()
@ -1594,6 +1596,7 @@ class TestInternalClient(unittest.TestCase):
})], app._calls) })], app._calls)
self.assertEqual({}, app.unread_requests) self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests) self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_upload_object_plumbing(self): def test_upload_object_plumbing(self):
class InternalClient(internal_client.InternalClient): class InternalClient(internal_client.InternalClient):

22
test/unit/proxy/controllers/test_base.py
View File

@ -12,7 +12,7 @@
# implied. # implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
from argparse import Namespace from argparse import Namespace
import itertools import itertools
import json import json
@ -1082,7 +1082,25 @@ class TestFuncs(BaseTest):
dst_headers = base.generate_request_headers(req, transfer=True) dst_headers = base.generate_request_headers(req, transfer=True)
expected_headers = {'x-base-meta-owner': '', expected_headers = {'x-base-meta-owner': '',
'x-base-meta-size': '151M', 'x-base-meta-size': '151M',
'connection': 'close'} 'connection': 'close',
'user-agent': 'proxy-server %d' % os.getpid()}
for k, v in expected_headers.items():
self.assertIn(k, dst_headers)
self.assertEqual(v, dst_headers[k])
self.assertNotIn('new-owner', dst_headers)
def test_generate_request_headers_change_backend_user_agent(self):
base = Controller(self.app)
self.app.backend_user_agent = "swift-flux-capacitor"
src_headers = {'x-remove-base-meta-owner': 'x',
'x-base-meta-size': '151M',
'new-owner': 'Kun'}
req = Request.blank('/v1/a/c/o', headers=src_headers)
dst_headers = base.generate_request_headers(req, transfer=True)
expected_headers = {'x-base-meta-owner': '',
'x-base-meta-size': '151M',
'connection': 'close',
'user-agent': 'swift-flux-capacitor'}
for k, v in expected_headers.items(): for k, v in expected_headers.items():
self.assertIn(k, dst_headers) self.assertIn(k, dst_headers)
self.assertEqual(v, dst_headers[k]) self.assertEqual(v, dst_headers[k])