SimpleClient http proxying
Previously, this code was attempting to set up http proxying but it wasn't working. We noticed after a while when we saw traffic going through an alternate route instead of our set of http proxies with container sync. Additional work and testing by clayg; thanks! Change-Id: I840b8e55a80c13ae85c65bf68de261d735685b27
This commit is contained in:
gholt
parent
177e9a36fc
commit
7fcbbebbeb
@ -728,14 +728,14 @@ class SimpleClient(object):
|
||||
max_backoff=5, retries=5):
|
||||
self.url = url
|
||||
self.token = token
|
||||
self.attempts = 0
|
||||
self.starting_backoff = starting_backoff
|
||||
self.max_backoff = max_backoff
|
||||
self.retries = retries
|
||||
|
||||
def base_request(self, method, container=None, name=None, prefix=None,
|
||||
headers=None, proxy=None, contents=None,
|
||||
full_listing=None, logger=None, additional_info=None):
|
||||
full_listing=None, logger=None, additional_info=None,
|
||||
timeout=None):
|
||||
# Common request method
|
||||
trans_start = time()
|
||||
url = self.url
|
||||
@ -756,15 +756,12 @@ class SimpleClient(object):
|
||||
if prefix:
|
||||
url += '&prefix=%s' % prefix
|
||||
|
||||
req = urllib2.Request(url, headers=headers, data=contents)
|
||||
if proxy:
|
||||
proxy = urlparse.urlparse(proxy)
|
||||
proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc})
|
||||
opener = urllib2.build_opener(proxy)
|
||||
urllib2.install_opener(opener)
|
||||
|
||||
req = urllib2.Request(url, headers=headers, data=contents)
|
||||
req.set_proxy(proxy.netloc, proxy.scheme)
|
||||
req.get_method = lambda: method
|
||||
conn = urllib2.urlopen(req)
|
||||
conn = urllib2.urlopen(req, timeout=timeout)
|
||||
body = conn.read()
|
||||
try:
|
||||
body_data = json.loads(body)
|
||||
@ -798,14 +795,15 @@ class SimpleClient(object):
|
||||
return [None, body_data]
|
||||
|
||||
def retry_request(self, method, **kwargs):
|
||||
self.attempts = 0
|
||||
retries = kwargs.pop('retries', self.retries)
|
||||
attempts = 0
|
||||
backoff = self.starting_backoff
|
||||
while self.attempts <= self.retries:
|
||||
self.attempts += 1
|
||||
while attempts <= retries:
|
||||
attempts += 1
|
||||
try:
|
||||
return self.base_request(method, **kwargs)
|
||||
except (socket.error, httplib.HTTPException, urllib2.URLError):
|
||||
if self.attempts > self.retries:
|
||||
if attempts > retries:
|
||||
raise
|
||||
sleep(backoff)
|
||||
backoff = min(backoff * 2, self.max_backoff)
|
||||
|
||||
@ -23,6 +23,7 @@ from textwrap import dedent
|
||||
import os
|
||||
|
||||
from test.unit import FakeLogger
|
||||
import eventlet
|
||||
from eventlet.green import urllib2
|
||||
from swift.common import internal_client
|
||||
from swift.common import swob
|
||||
@ -1210,6 +1211,79 @@ class TestSimpleClient(unittest.TestCase):
|
||||
headers={'X-Auth-Token': 'token'})
|
||||
self.assertEqual([None, None], retval)
|
||||
|
||||
@mock.patch('eventlet.green.urllib2.urlopen')
|
||||
def test_get_with_retries_param(self, mock_urlopen):
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.read.return_value = ''
|
||||
mock_urlopen.side_effect = internal_client.httplib.BadStatusLine('')
|
||||
c = internal_client.SimpleClient(url='http://127.0.0.1', token='token')
|
||||
self.assertEqual(c.retries, 5)
|
||||
|
||||
# first without retries param
|
||||
with mock.patch('swift.common.internal_client.sleep') as mock_sleep:
|
||||
self.assertRaises(internal_client.httplib.BadStatusLine,
|
||||
c.retry_request, 'GET')
|
||||
self.assertEqual(mock_sleep.call_count, 5)
|
||||
self.assertEqual(mock_urlopen.call_count, 6)
|
||||
# then with retries param
|
||||
mock_urlopen.reset_mock()
|
||||
with mock.patch('swift.common.internal_client.sleep') as mock_sleep:
|
||||
self.assertRaises(internal_client.httplib.BadStatusLine,
|
||||
c.retry_request, 'GET', retries=2)
|
||||
self.assertEqual(mock_sleep.call_count, 2)
|
||||
self.assertEqual(mock_urlopen.call_count, 3)
|
||||
# and this time with a real response
|
||||
mock_urlopen.reset_mock()
|
||||
mock_urlopen.side_effect = [internal_client.httplib.BadStatusLine(''),
|
||||
mock_response]
|
||||
with mock.patch('swift.common.internal_client.sleep') as mock_sleep:
|
||||
retval = c.retry_request('GET', retries=1)
|
||||
self.assertEqual(mock_sleep.call_count, 1)
|
||||
self.assertEqual(mock_urlopen.call_count, 2)
|
||||
self.assertEqual([None, None], retval)
|
||||
|
||||
def test_proxy(self):
|
||||
running = True
|
||||
|
||||
def handle(sock):
|
||||
while running:
|
||||
try:
|
||||
with eventlet.Timeout(0.1):
|
||||
(conn, addr) = sock.accept()
|
||||
except eventlet.Timeout:
|
||||
continue
|
||||
else:
|
||||
conn.send('HTTP/1.1 503 Server Error')
|
||||
conn.close()
|
||||
sock.close()
|
||||
|
||||
sock = eventlet.listen(('', 0))
|
||||
port = sock.getsockname()[1]
|
||||
proxy = 'http://127.0.0.1:%s' % port
|
||||
url = 'https://127.0.0.1:1/a'
|
||||
server = eventlet.spawn(handle, sock)
|
||||
try:
|
||||
headers = {'Content-Length': '0'}
|
||||
with mock.patch('swift.common.internal_client.sleep'):
|
||||
try:
|
||||
internal_client.put_object(
|
||||
url, container='c', name='o1', headers=headers,
|
||||
contents='', proxy=proxy, timeout=0.1, retries=0)
|
||||
except urllib2.HTTPError as e:
|
||||
self.assertEqual(e.code, 503)
|
||||
except urllib2.URLError as e:
|
||||
if 'ECONNREFUSED' in str(e):
|
||||
self.fail(
|
||||
"Got %s which probably means the http proxy "
|
||||
"settings were not used" % e)
|
||||
else:
|
||||
raise e
|
||||
else:
|
||||
self.fail('Unexpected successful response')
|
||||
finally:
|
||||
running = False
|
||||
server.wait()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user