diff --git a/doc/source/container.rst b/doc/source/container.rst index ca6d16c91c..d80adcaa32 100644 --- a/doc/source/container.rst +++ b/doc/source/container.rst @@ -34,3 +34,10 @@ Container Auditor :undoc-members: :show-inheritance: +Container Sync +============== + +.. automodule:: swift.container.sync + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/index.rst b/doc/source/index.rst index 3c23140e72..341079cdbc 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -45,6 +45,7 @@ Overview and Concepts overview_stats ratelimit overview_large_objects + overview_container_sync Developer Documentation ======================= diff --git a/doc/source/overview_container_sync.rst b/doc/source/overview_container_sync.rst new file mode 100644 index 0000000000..f6c99aeeb5 --- /dev/null +++ b/doc/source/overview_container_sync.rst @@ -0,0 +1,178 @@ +====================================== +Container to Container Synchronization +====================================== + +-------- +Overview +-------- + +Swift has a feature where all the contents of a container can be mirrored to +another container through background synchronization. Swift cluster operators +configure their cluster to allow/accept sync requests to/from other clusters, +and the user specifies where to sync their container to along with a secret +synchronization key. + +.. note:: + + This does not sync standard object POSTs, as those do not cause container + updates. A workaround is to do X-Copy-From POSTs. We're considering + solutions to this limitation but leaving it as is for now since POSTs are + fairly uncommon. + +-------------------------------------------- +Configuring a Cluster's Allowable Sync Hosts +-------------------------------------------- + +The Swift cluster operator must allow synchronization with a set of hosts +before the user can enable container synchronization. First, the backend +container server needs to be given this list of hosts in the +container-server.conf file:: + + [DEFAULT] + # This is a comma separated list of hosts allowed in the + # X-Container-Sync-To field for containers. + # allowed_sync_hosts = 127.0.0.1 + allowed_sync_hosts = host1,host2,etc. + ... + + [container-sync] + # You can override the default log routing for this app here (don't + # use set!): + # log_name = container-sync + # log_facility = LOG_LOCAL0 + # log_level = INFO + # Will sync, at most, each container once per interval + # interval = 300 + # Maximum amount of time to spend syncing each container + # container_time = 60 + +The authentication system also needs to be configured to allow synchronization +requests. Here are examples with DevAuth and Swauth:: + + [filter:auth] + # This is a comma separated list of hosts allowed to send + # X-Container-Sync-Key requests. + # allowed_sync_hosts = 127.0.0.1 + allowed_sync_hosts = host1,host2,etc. + + [filter:swauth] + # This is a comma separated list of hosts allowed to send + # X-Container-Sync-Key requests. + # allowed_sync_hosts = 127.0.0.1 + allowed_sync_hosts = host1,host2,etc. + +The default of 127.0.0.1 is just so no configuration is required for SAIO +setups -- for testing. + +---------------------------------------------- +Using ``st`` to set up synchronized containers +---------------------------------------------- + +.. note:: + + You must be the account admin on the account to set synchronization targets + and keys. + +You simply tell each container where to sync to and give it a secret +synchronization key. First, let's get the account details for our two cluster +accounts:: + + $ st -A http://cluster1/auth/v1.0 -U test:tester -K testing stat -v + StorageURL: http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e + Auth Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19 + Account: AUTH_208d1854-e475-4500-b315-81de645d060e + Containers: 0 + Objects: 0 + Bytes: 0 + + $ st -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 stat -v + StorageURL: http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c + Auth Token: AUTH_tk816a1aaf403c49adb92ecfca2f88e430 + Account: AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c + Containers: 0 + Objects: 0 + Bytes: 0 + +Now, let's make our first container and tell it to synchronize to a second +we'll make next:: + + $ st -A http://cluster1/auth/v1.0 -U test:tester -K testing post \ + -t 'http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \ + -k 'secret' container1 + +The ``-t`` indicates the URL to sync to, which is the ``StorageURL`` from +cluster2 we retrieved above plus the container name. The ``-k`` specifies the +secret key the two containers will share for synchronization. Now, we'll do +something similar for the second cluster's container:: + + $ st -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 post \ + -t 'http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e/container1' \ + -k 'secret' container2 + +That's it. Now we can upload a bunch of stuff to the first container and watch +as it gets synchronized over to the second:: + + $ st -A http://cluster1/auth/v1.0 -U test:tester -K testing \ + upload container1 . + photo002.png + photo004.png + photo001.png + photo003.png + + $ st -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \ + list container2 + + [Nothing there yet, so we wait a bit...] + [If you're an operator running SAIO and just testing, you may need to + run 'swift-init container-sync once' to perform a sync scan.] + + $ st -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \ + list container2 + photo001.png + photo002.png + photo003.png + photo004.png + +You can also set up a chain of synced containers if you want more than two. +You'd point 1 -> 2, then 2 -> 3, and finally 3 -> 1 for three containers. +They'd all need to share the same secret synchronization key. + +----------------------------------- +Using curl (or other tools) instead +----------------------------------- + +So what's ``st`` doing behind the scenes? Nothing overly complicated. It +translates the ``-t `` option into an ``X-Container-Sync-To: `` +header and the ``-k `` option into an ``X-Container-Sync-Key: `` +header. + +For instance, when we created the first container above and told it to +synchronize to the second, we could have used this curl command:: + + $ curl -i -X POST -H 'X-Auth-Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19' \ + -H 'X-Container-Sync-To: http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \ + -H 'X-Container-Sync-Key: secret' \ + 'http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e/container1' + HTTP/1.1 204 No Content + Content-Length: 0 + Content-Type: text/plain; charset=UTF-8 + Date: Thu, 24 Feb 2011 22:39:14 GMT + +-------------------------------------------------- +What's going on behind the scenes, in the cluster? +-------------------------------------------------- + +The swift-container-sync does the job of sending updates to the remote +container. + +This is done by scanning the local devices for container databases and checking +for x-container-sync-to and x-container-sync-key metadata values. If they +exist, the last known synced ROWID is retreived and all newer rows trigger PUTs +or DELETEs to the other container. + +.. note:: + + This does not sync standard object POSTs, as those do not cause container + row updates. A workaround is to do X-Copy-From POSTs. We're considering + solutions to this limitation but leaving it as is for now since POSTs are + fairly uncommon. diff --git a/swift/container/sync.py b/swift/container/sync.py index bb50515887..5dcaf7ab65 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -15,7 +15,7 @@ import os import time -from random import random, shuffle +import random from swift.container import server as container_server from swift.common import client, direct_client @@ -26,13 +26,24 @@ from swift.common.utils import audit_location_generator, get_logger, \ from swift.common.daemon import Daemon -class Iter2FileLikeObject(object): +class _Iter2FileLikeObject(object): + """ + Returns an iterator's contents via :func:`read`, making it look like a file + object. + """ def __init__(self, iterator): self.iterator = iterator self._chunk = '' def read(self, size=-1): + """ + read([size]) -> read at most size bytes, returned as a string. + + If the size argument is negative or omitted, read until EOF is reached. + Notice that when in non-blocking mode, less data than what was + requested may be returned, even if no size parameter was given. + """ if size < 0: chunk = self._chunk self._chunk = '' @@ -49,32 +60,74 @@ class Iter2FileLikeObject(object): class ContainerSync(Daemon): - """Sync syncable containers.""" + """ + Daemon to sync syncable containers. + + This is done by scanning the local devices for container databases and + checking for x-container-sync-to and x-container-sync-key metadata values. + If they exist, the last known synced ROWID is retreived from the container + broker via get_info()['x_container_sync_row']. All newer rows trigger PUTs + or DELETEs to the other container. + + .. note:: + + This does not sync standard object POSTs, as those do not cause + container row updates. A workaround is to do X-Copy-From POSTs. We're + considering solutions to this limitation but leaving it as is for now + since POSTs are fairly uncommon. + + :param conf: The dict of configuration values from the [container-sync] + section of the container-server.conf + :param object_ring: If None, the /object.ring.gz will be loaded. + This is overridden by unit tests. + """ def __init__(self, conf, object_ring=None): + #: The dict of configuration values from the [container-sync] section + #: of the container-server.conf. self.conf = conf + #: Logger to use for container-sync log lines. self.logger = get_logger(conf, log_route='container-sync') + #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') + #: Indicates whether mount points should be verified as actual mount + #: points (normally true, false for tests and SAIO). self.mount_check = \ conf.get('mount_check', 'true').lower() in TRUE_VALUES + #: Minimum time between full scans. This is to keep the daemon from + #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) + #: Maximum amount of time to spend syncing a container before moving on + #: to the next one. If a conatiner sync hasn't finished in this time, + #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) + #: The list of hosts we're allowed to send syncs to. self.allowed_sync_hosts = [h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip()] + #: Number of containers with sync turned on that were successfully + #: synced. self.container_syncs = 0 + #: Number of successful DELETEs triggered. self.container_deletes = 0 + #: Number of successful PUTs triggered. self.container_puts = 0 + #: Number of containers that didn't have sync turned on. self.container_skips = 0 + #: Number of containers that had a failure of some type. self.container_failures = 0 + #: Time of last stats report. self.reported = time.time() swift_dir = conf.get('swift_dir', '/etc/swift') + #: swift.common.ring.Ring for locating objects. self.object_ring = object_ring or \ Ring(os.path.join(swift_dir, 'object.ring.gz')) def run_forever(self): - """Run the container sync until stopped.""" - time.sleep(random() * self.interval) + """ + Runs container sync scans until stopped. + """ + time.sleep(random.random() * self.interval) while True: begin = time.time() all_locs = audit_location_generator(self.devices, @@ -84,13 +137,15 @@ class ContainerSync(Daemon): for path, device, partition in all_locs: self.container_sync(path) if time.time() - self.reported >= 3600: # once an hour - self._report() + self.report() elapsed = time.time() - begin if elapsed < self.interval: time.sleep(self.interval - elapsed) def run_once(self): - """Run the container sync once.""" + """ + Runs a single container sync scan. + """ self.logger.info(_('Begin container sync "once" mode')) begin = time.time() all_locs = audit_location_generator(self.devices, @@ -100,13 +155,17 @@ class ContainerSync(Daemon): for path, device, partition in all_locs: self.container_sync(path) if time.time() - self.reported >= 3600: # once an hour - self._report() - self._report() + self.report() + self.report() elapsed = time.time() - begin self.logger.info( _('Container sync "once" mode completed: %.02fs'), elapsed) - def _report(self): + def report(self): + """ + Writes a report of the stats to the logger and resets the stats for the + next report. + """ self.logger.info( _('Since %(time)s: %(sync)s synced [%(delete)s deletes, %(put)s ' 'puts], %(skip)s skipped, %(fail)s failed'), @@ -125,7 +184,9 @@ class ContainerSync(Daemon): def container_sync(self, path): """ - Syncs the given container path + Checks the given path for a container database, determines if syncing + is turned on for that database and, if so, sends any updates to the + other container. :param path: the path to a container db """ @@ -171,6 +232,19 @@ class ContainerSync(Daemon): self.logger.exception(_('ERROR Syncing %s'), (broker.db_file)) def container_sync_row(self, row, sync_to, sync_key, broker, info): + """ + Sends the update the row indicates to the sync_to container. + + :param row: The updated row in the local database triggering the sync + update. + :param sync_to: The URL to the remote container. + :param sync_key: The X-Container-Sync-Key to use when sending requests + to the other container. + :param broker: The local container database broker. + :param info: The get_info result from the local container database + broker. + :returns: True on success + """ try: if row['deleted']: try: @@ -185,7 +259,7 @@ class ContainerSync(Daemon): part, nodes = self.object_ring.get_nodes( info['account'], info['container'], row['name']) - shuffle(nodes) + random.shuffle(nodes) exc = None for node in nodes: try: @@ -214,7 +288,7 @@ class ContainerSync(Daemon): headers['X-Container-Sync-Key'] = sync_key client.put_object(sync_to, name=row['name'], headers=headers, - contents=Iter2FileLikeObject(body)) + contents=_Iter2FileLikeObject(body)) self.container_puts += 1 except client.ClientException, err: if err.http_status == 401: diff --git a/test/functionalnosetests/test_object.py b/test/functionalnosetests/test_object.py index 8e46715ae3..5975cf16a2 100644 --- a/test/functionalnosetests/test_object.py +++ b/test/functionalnosetests/test_object.py @@ -215,7 +215,7 @@ class TestObject(unittest.TestCase): conn.request('PUT', '%s/%s/%s' % (parsed.path, shared_container, 'private_object'), - '', {'User-Agent': 'GLHUA', 'X-Auth-Token': token, + '', {'X-Auth-Token': token, 'Content-Length': '0', 'X-Copy-From': '%s/%s' % (self.container, self.obj)})