diff --git a/bin/swift b/bin/swift index c5b22b95ef..339e992401 100755 --- a/bin/swift +++ b/bin/swift @@ -155,6 +155,8 @@ def http_connection(url, proxy=None): Make an HTTPConnection or HTTPSConnection :param url: url to connect to + :param proxy: proxy to connect through, if any; None by default; str of the + format 'http://127.0.0.1:8888' to set one :returns: tuple of (parsed url, connection object) :raises ClientException: Unable to handle protocol scheme """ @@ -587,18 +589,27 @@ def put_object(url, token=None, container=None, name=None, contents=None, Put an object :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to put - :param contents: a string or a file like object to read object data from + :param token: auth token; if None, no token will be sent + :param container: container name that the object is in; if None, the + container name is expected to be part of the url + :param name: object name to put; if None, the object name is expected to be + part of the url + :param contents: a string or a file like object to read object data from; + if None, a zero-byte put will be done :param content_length: value to send as content-length header; also limits - the amount read from contents - :param etag: etag of contents - :param chunk_size: chunk size of data to write - :param content_type: value to send as content-type header - :param headers: additional headers to include in the request + the amount read from contents; if None, it will be + computed via the contents or chunked transfer + encoding will be used + :param etag: etag of contents; if None, no etag will be sent + :param chunk_size: chunk size of data to write; default 65536 + :param content_type: value to send as content-type header; if None, no + content-type will be set (remote end will likely try + to auto-detect it) + :param headers: additional headers to include in the request, if any :param http_conn: HTTP connection object (If None, it will create the conn object) + :param proxy: proxy to connect through, if any; None by default; str of the + format 'http://127.0.0.1:8888' to set one :returns: etag from server response :raises ClientException: HTTP PUT request failed """ @@ -698,11 +709,16 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, Delete object :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to delete + :param token: auth token; if None, no token will be sent + :param container: container name that the object is in; if None, the + container name is expected to be part of the url + :param name: object name to delete; if None, the object name is expected to + be part of the url :param http_conn: HTTP connection object (If None, it will create the conn object) + :param headers: additional headers to include in the request + :param proxy: proxy to connect through, if any; None by default; str of the + format 'http://127.0.0.1:8888' to set one :raises ClientException: HTTP DELETE request failed """ if http_conn: diff --git a/doc/source/overview_container_sync.rst b/doc/source/overview_container_sync.rst index c98bb0e674..b18ccf3465 100644 --- a/doc/source/overview_container_sync.rst +++ b/doc/source/overview_container_sync.rst @@ -14,10 +14,10 @@ synchronization key. .. note:: - This does not sync standard object POSTs, as those do not cause container - updates. A workaround is to do X-Copy-From POSTs. We're considering - solutions to this limitation but leaving it as is for now since POSTs are - fairly uncommon. + Container sync will sync object POSTs only if the proxy server is set to + use "object_post_as_copy = true" which is the default. So-called fast + object posts, "object_post_as_copy = false" do not update the container + listings and therefore can't be detected for synchronization. -------------------------------------------- Configuring a Cluster's Allowable Sync Hosts @@ -64,9 +64,9 @@ requests. Here are examples with DevAuth and Swauth:: The default of 127.0.0.1 is just so no configuration is required for SAIO setups -- for testing. ----------------------------------------------- -Using ``st`` to set up synchronized containers ----------------------------------------------- +---------------------------------------------------------- +Using the ``swift`` tool to set up synchronized containers +---------------------------------------------------------- .. note:: @@ -77,7 +77,7 @@ You simply tell each container where to sync to and give it a secret synchronization key. First, let's get the account details for our two cluster accounts:: - $ st -A http://cluster1/auth/v1.0 -U test:tester -K testing stat -v + $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing stat -v StorageURL: http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e Auth Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19 Account: AUTH_208d1854-e475-4500-b315-81de645d060e @@ -85,7 +85,7 @@ accounts:: Objects: 0 Bytes: 0 - $ st -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 stat -v + $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 stat -v StorageURL: http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c Auth Token: AUTH_tk816a1aaf403c49adb92ecfca2f88e430 Account: AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c @@ -96,7 +96,7 @@ accounts:: Now, let's make our first container and tell it to synchronize to a second we'll make next:: - $ st -A http://cluster1/auth/v1.0 -U test:tester -K testing post \ + $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing post \ -t 'http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \ -k 'secret' container1 @@ -105,28 +105,28 @@ cluster2 we retrieved above plus the container name. The ``-k`` specifies the secret key the two containers will share for synchronization. Now, we'll do something similar for the second cluster's container:: - $ st -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 post \ + $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 post \ -t 'http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e/container1' \ -k 'secret' container2 That's it. Now we can upload a bunch of stuff to the first container and watch as it gets synchronized over to the second:: - $ st -A http://cluster1/auth/v1.0 -U test:tester -K testing \ + $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing \ upload container1 . photo002.png photo004.png photo001.png photo003.png - $ st -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \ + $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \ list container2 [Nothing there yet, so we wait a bit...] [If you're an operator running SAIO and just testing, you may need to run 'swift-init container-sync once' to perform a sync scan.] - $ st -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \ + $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \ list container2 photo001.png photo002.png @@ -141,7 +141,7 @@ They'd all need to share the same secret synchronization key. Using curl (or other tools) instead ----------------------------------- -So what's ``st`` doing behind the scenes? Nothing overly complicated. It +So what's ``swift`` doing behind the scenes? Nothing overly complicated. It translates the ``-t `` option into an ``X-Container-Sync-To: `` header and the ``-k `` option into an ``X-Container-Sync-Key: `` header. @@ -172,10 +172,10 @@ to the other container. .. note:: - This does not sync standard object POSTs, as those do not cause - container row updates. A workaround is to do X-Copy-From POSTs. We're - considering solutions to this limitation but leaving it as is for now - since POSTs are fairly uncommon. + Container sync will sync object POSTs only if the proxy server is set to + use "object_post_as_copy = true" which is the default. So-called fast + object posts, "object_post_as_copy = false" do not update the container + listings and therefore can't be detected for synchronization. The actual syncing is slightly more complicated to make use of the three (or number-of-replicas) main nodes for a container without each trying to diff --git a/doc/source/overview_large_objects.rst b/doc/source/overview_large_objects.rst index 923afb872a..e500e35713 100644 --- a/doc/source/overview_large_objects.rst +++ b/doc/source/overview_large_objects.rst @@ -13,9 +13,9 @@ special manifest file is created that, when downloaded, sends all the segments concatenated as a single object. This also offers much greater upload speed with the possibility of parallel uploads of the segments. ----------------------------------- +------------------------------------- Using ``swift`` for Segmented Objects ----------------------------------- +------------------------------------- The quickest way to try out this feature is use the included ``swift`` Swift Tool. You can use the ``-S`` option to specify the segment size to use when splitting diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 5937ee6561..4c488c25c1 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -73,5 +73,5 @@ use = egg:swift#container # sync_proxy = http://127.0.0.1:8888 # Will sync, at most, each container once per interval # interval = 300 -# Maximum amount of time to spend syncing each container +# Maximum amount of time to spend syncing each container per pass # container_time = 60 diff --git a/swift/common/client.py b/swift/common/client.py index 92bd6b5982..c95020336c 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -142,6 +142,8 @@ def http_connection(url, proxy=None): Make an HTTPConnection or HTTPSConnection :param url: url to connect to + :param proxy: proxy to connect through, if any; None by default; str of the + format 'http://127.0.0.1:8888' to set one :returns: tuple of (parsed url, connection object) :raises ClientException: Unable to handle protocol scheme """ @@ -574,18 +576,27 @@ def put_object(url, token=None, container=None, name=None, contents=None, Put an object :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to put - :param contents: a string or a file like object to read object data from + :param token: auth token; if None, no token will be sent + :param container: container name that the object is in; if None, the + container name is expected to be part of the url + :param name: object name to put; if None, the object name is expected to be + part of the url + :param contents: a string or a file like object to read object data from; + if None, a zero-byte put will be done :param content_length: value to send as content-length header; also limits - the amount read from contents - :param etag: etag of contents - :param chunk_size: chunk size of data to write - :param content_type: value to send as content-type header - :param headers: additional headers to include in the request + the amount read from contents; if None, it will be + computed via the contents or chunked transfer + encoding will be used + :param etag: etag of contents; if None, no etag will be sent + :param chunk_size: chunk size of data to write; default 65536 + :param content_type: value to send as content-type header; if None, no + content-type will be set (remote end will likely try + to auto-detect it) + :param headers: additional headers to include in the request, if any :param http_conn: HTTP connection object (If None, it will create the conn object) + :param proxy: proxy to connect through, if any; None by default; str of the + format 'http://127.0.0.1:8888' to set one :returns: etag from server response :raises ClientException: HTTP PUT request failed """ @@ -685,11 +696,16 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, Delete object :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to delete + :param token: auth token; if None, no token will be sent + :param container: container name that the object is in; if None, the + container name is expected to be part of the url + :param name: object name to delete; if None, the object name is expected to + be part of the url :param http_conn: HTTP connection object (If None, it will create the conn object) + :param headers: additional headers to include in the request + :param proxy: proxy to connect through, if any; None by default; str of the + format 'http://127.0.0.1:8888' to set one :raises ClientException: HTTP DELETE request failed """ if http_conn: