From 69d331d0d65d2f316bfdace8b9e018631ef47eba Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 16 Jan 2014 01:58:49 +0000 Subject: [PATCH] Container Sync: Simple HTTP Proxy load balancing Change-Id: I021b043b927153bacff48cae648d4d8c5bbad765 --- etc/container-server.conf-sample | 4 +++- swift/container/sync.py | 14 ++++++++++---- test/unit/container/test_sync.py | 25 +++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index da56b03ff0..460b35f704 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -153,7 +153,9 @@ use = egg:swift#recon # log_address = /dev/log # # If you need to use an HTTP Proxy, set it here; defaults to no proxy. -# sync_proxy = http://127.0.0.1:8888 +# You can also set this to a comma separated list of HTTP Proxies and they will +# be randomly used (simple load balancing). +# sync_proxy = http://10.1.1.1:8888,http://10.1.1.2:8888 # # Will sync each container at most once per interval # interval = 300 diff --git a/swift/container/sync.py b/swift/container/sync.py index 402f602dfd..41302bfb77 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -17,7 +17,7 @@ import os import uuid from swift import gettext_ as _ from time import ctime, time -from random import random, shuffle +from random import choice, random, shuffle from struct import unpack_from from eventlet import sleep, Timeout @@ -133,7 +133,10 @@ class ContainerSync(Daemon): h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip()] - self.proxy = conf.get('sync_proxy') + self.http_proxies = [ + a.strip() + for a in conf.get('sync_proxy', '').split(',') + if a.strip()] #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 @@ -352,7 +355,7 @@ class ContainerSync(Daemon): else: headers['x-container-sync-key'] = user_key delete_object(sync_to, name=row['name'], headers=headers, - proxy=self.proxy) + proxy=self.select_http_proxy()) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise @@ -415,7 +418,7 @@ class ContainerSync(Daemon): headers['x-container-sync-key'] = user_key put_object(sync_to, name=row['name'], headers=headers, contents=FileLikeIter(body), - proxy=self.proxy) + proxy=self.select_http_proxy()) self.container_puts += 1 self.logger.increment('puts') self.logger.timing_since('puts.timing', start_time) @@ -448,3 +451,6 @@ class ContainerSync(Daemon): self.logger.increment('failures') return False return True + + def select_http_proxy(self): + return choice(self.http_proxies) if self.http_proxies else None diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py index 7c6b752557..883d200f11 100644 --- a/test/unit/container/test_sync.py +++ b/test/unit/container/test_sync.py @@ -658,7 +658,7 @@ class TestContainerSync(unittest.TestCase): sync.delete_object = fake_delete_object cs = sync.ContainerSync({}, container_ring=FakeRing(), object_ring=FakeRing()) - cs.proxy = 'http://proxy' + cs.http_proxies = ['http://proxy'] # Success self.assertTrue(cs.container_sync_row( {'deleted': True, @@ -765,7 +765,7 @@ class TestContainerSync(unittest.TestCase): cs = sync.ContainerSync({}, container_ring=FakeRing(), object_ring=FakeRing()) - cs.proxy = 'http://proxy' + cs.http_proxies = ['http://proxy'] def fake_direct_get_object(node, part, account, container, obj, resp_chunk_size=1): @@ -912,6 +912,27 @@ class TestContainerSync(unittest.TestCase): sync.put_object = orig_put_object sync.direct_get_object = orig_direct_get_object + def test_select_http_proxy_None(self): + cs = sync.ContainerSync( + {'sync_proxy': ''}, container_ring=FakeRing(), + object_ring=FakeRing()) + self.assertEqual(cs.select_http_proxy(), None) + + def test_select_http_proxy_one(self): + cs = sync.ContainerSync( + {'sync_proxy': 'http://one'}, container_ring=FakeRing(), + object_ring=FakeRing()) + self.assertEqual(cs.select_http_proxy(), 'http://one') + + def test_select_http_proxy_multiple(self): + cs = sync.ContainerSync( + {'sync_proxy': 'http://one,http://two,http://three'}, + container_ring=FakeRing(), + object_ring=FakeRing()) + self.assertEqual( + set(cs.http_proxies), + set(['http://one', 'http://two', 'http://three'])) + if __name__ == '__main__': unittest.main()