From 630c9ef809efc8fc05823ad66acac3b293db4f24 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 3 Feb 2020 13:02:58 -0800 Subject: [PATCH] probe tests: Work when fronted by a TLS terminator * Add a new config option, proxy_base_url * Support HTTPS as well as HTTP connections * Monkey-patch eventlet early so we never import an unpatched version from swiftclient Change-Id: I4945d512966d3666f2738058f15a916c65ad4a6b --- test/probe/__init__.py | 8 ++++++++ test/probe/brain.py | 3 ++- test/probe/common.py | 10 +++++----- .../test_account_get_fake_responses_match.py | 12 +++++++++--- test/probe/test_sharder.py | 3 ++- test/probe/test_signals.py | 18 +++++++++++++----- test/sample.conf | 1 + 7 files changed, 40 insertions(+), 15 deletions(-) diff --git a/test/probe/__init__.py b/test/probe/__init__.py index 7c0928b207..fb8af91d09 100644 --- a/test/probe/__init__.py +++ b/test/probe/__init__.py @@ -14,6 +14,9 @@ # limitations under the License. +import eventlet +eventlet.monkey_patch() + from test import get_config from swift.common.utils import config_true_value @@ -21,3 +24,8 @@ from swift.common.utils import config_true_value config = get_config('probe_test') CHECK_SERVER_TIMEOUT = int(config.get('check_server_timeout', 30)) VALIDATE_RSYNC = config_true_value(config.get('validate_rsync', False)) +PROXY_BASE_URL = config.get('proxy_base_url') +if PROXY_BASE_URL is None: + # TODO: find and load an "appropriate" proxy-server.conf(.d), piece + # something together from bind_ip, bind_port, and cert_file + PROXY_BASE_URL = 'http://127.0.0.1:8080' diff --git a/test/probe/brain.py b/test/probe/brain.py index e8eb8ca547..3ab80305f3 100644 --- a/test/probe/brain.py +++ b/test/probe/brain.py @@ -31,6 +31,7 @@ from swift.common.http import HTTP_NOT_FOUND from swiftclient import client, get_auth, ClientException +from test.probe import PROXY_BASE_URL from test.probe.common import ENABLED_POLICIES TIMEOUT = 60 @@ -340,7 +341,7 @@ def main(): if cmd not in BrainSplitter.__commands__: parser.print_help() return 'ERROR: unknown command %s' % cmd - url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', + url, token = get_auth(PROXY_BASE_URL + '/auth/v1.0', 'test:tester', 'testing') if options.server_type == 'object' and not options.policy_name: options.policy_name = POLICIES.default.name diff --git a/test/probe/common.py b/test/probe/common.py index f494e26b34..00a21ec48f 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -39,7 +39,7 @@ from swift.common.manager import Manager from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY from swift.obj.diskfile import get_data_dir -from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC +from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC, PROXY_BASE_URL ENABLED_POLICIES = [p for p in POLICIES if not p.is_deprecated] @@ -80,8 +80,8 @@ def _check_storage(ipport, path): return resp -def _check_proxy(ipport, user, key): - url, token = get_auth('http://%s:%d/auth/v1.0' % ipport, +def _check_proxy(user, key): + url, token = get_auth(PROXY_BASE_URL + '/auth/v1.0', user, key) account = url.split('/')[-1] head_account(url, token) @@ -118,7 +118,7 @@ def check_server(ipport, ipport2server): rv = _retry_timeout(_check_storage, args=(ipport, path)) else: rv = _retry_timeout(_check_proxy, args=( - ipport, 'test:tester', 'testing')) + 'test:tester', 'testing')) return rv @@ -392,7 +392,7 @@ class ProbeTest(unittest.TestCase): 'url': self.url, 'token': self.token, 'account': self.account} rv = _retry_timeout(_check_proxy, args=( - proxy_ipport, 'test2:tester2', 'testing2')) + 'test2:tester2', 'testing2')) self.account_2 = { k: v for (k, v) in zip(('url', 'token', 'account'), rv)} diff --git a/test/probe/test_account_get_fake_responses_match.py b/test/probe/test_account_get_fake_responses_match.py index ba664518b5..a0d58ffd56 100644 --- a/test/probe/test_account_get_fake_responses_match.py +++ b/test/probe/test_account_get_fake_responses_match.py @@ -20,6 +20,7 @@ import unittest from six.moves import http_client from six.moves.urllib.parse import urlparse from swiftclient import get_auth +from test.probe import PROXY_BASE_URL from test.probe.common import ReplProbeTest @@ -28,7 +29,7 @@ class TestAccountGetFakeResponsesMatch(ReplProbeTest): def setUp(self): super(TestAccountGetFakeResponsesMatch, self).setUp() self.url, self.token = get_auth( - 'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin') + PROXY_BASE_URL + '/auth/v1.0', 'admin:admin', 'admin') def _account_path(self, account): _, _, path, _, _, _ = urlparse(self.url) @@ -46,10 +47,15 @@ class TestAccountGetFakeResponsesMatch(ReplProbeTest): headers['X-Auth-Token'] = self.token scheme, netloc, path, _, _, _ = urlparse(self.url) - host, port = netloc.split(':') + host, port = netloc.partition(':')[::2] + if not port: + port = '443' if scheme == 'https' else '80' port = int(port) - conn = http_client.HTTPConnection(host, port) + if scheme == 'https': + conn = http_client.HTTPSConnection(host, port) + else: + conn = http_client.HTTPConnection(host, port) conn.request(method, self._account_path(account), headers=headers) resp = conn.getresponse() if resp.status // 100 != 2: diff --git a/test/probe/test_sharder.py b/test/probe/test_sharder.py index 3d06b0bc18..f39fbb8c5d 100644 --- a/test/probe/test_sharder.py +++ b/test/probe/test_sharder.py @@ -34,6 +34,7 @@ from swiftclient import client, get_auth, ClientException from swift.proxy.controllers.base import get_cache_key from swift.proxy.controllers.obj import num_container_updates from test import annotate_failure +from test.probe import PROXY_BASE_URL from test.probe.brain import BrainSplitter from test.probe.common import ReplProbeTest, get_server_number, \ wait_for_server_to_hangup @@ -115,7 +116,7 @@ class BaseTestContainerSharding(ReplProbeTest): client.requests.logging.WARNING) super(BaseTestContainerSharding, self).setUp() _, self.admin_token = get_auth( - 'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin') + PROXY_BASE_URL + '/auth/v1.0', 'admin:admin', 'admin') self._setup_container_name() self.init_brain(self.container_name) self.sharders = Manager(['container-sharder']) diff --git a/test/probe/test_signals.py b/test/probe/test_signals.py index c787265f5b..2023bfcf19 100644 --- a/test/probe/test_signals.py +++ b/test/probe/test_signals.py @@ -26,10 +26,12 @@ import time from uuid import uuid4 from six.moves import http_client as httplib +from six.moves.urllib.parse import urlparse from swift.common.ring import Ring from swift.common.manager import Manager +from test.probe import PROXY_BASE_URL from test.probe.common import resetswift, ReplProbeTest, client @@ -109,7 +111,9 @@ class TestWSGIServerProcessHandling(ReplProbeTest): self.assertIn('Invalid path: blah', got_body) def get_conn(self): - ip, port = self.get_ip_port() + scheme, ip, port = self.get_scheme_ip_port() + if scheme == 'https': + return httplib.HTTPSConnection('%s:%s' % (ip, port)) return httplib.HTTPConnection('%s:%s' % (ip, port)) def _check_reload(self): @@ -228,11 +232,11 @@ class TestObjectServerReloadBase(TestWSGIServerProcessHandling): SERVER_NAME = 'object' PID_TIMEOUT = 35 - def get_ip_port(self): + def get_scheme_ip_port(self): self.policy.load_ring('/etc/swift') self.ring_node = random.choice( self.policy.object_ring.get_part_nodes(1)) - return self.ring_node['ip'], self.ring_node['port'] + return 'http', self.ring_node['ip'], self.ring_node['port'] def start_write_req(self, conn, suffix): putrequest(conn, 'PUT', '/%s/123/%s/%s/blah-%s' % ( @@ -296,8 +300,12 @@ class TestProxyServerReloadBase(TestWSGIServerProcessHandling): def swap_configs(self): shutil.copy(self.new_swift_conf_path, self.swift_conf_path) - def get_ip_port(self): - return 'localhost', 8080 + def get_scheme_ip_port(self): + parsed = urlparse(PROXY_BASE_URL) + host, port = parsed.netloc.partition(':')[::2] + if not port: + port = '443' if parsed.scheme == 'https' else '80' + return parsed.scheme, host, int(port) def assertMaxHeaderSize(self, resp, exp_max_header_size): self.assertEqual(resp.status // 100, 2) diff --git a/test/sample.conf b/test/sample.conf index 38235ce6fc..6b6a7ac4d8 100644 --- a/test/sample.conf +++ b/test/sample.conf @@ -101,6 +101,7 @@ fake_syslog = False [probe_test] # check_server_timeout = 30 # validate_rsync = false +# proxy_base_url = http://localhost:8080 [swift-constraints] # The functional test runner will try to use the constraint values provided in