From 1a2b54fc0a22713ff0ca575e0a22834682670c55 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 19 Oct 2015 13:55:02 +0100 Subject: [PATCH] Fix missing *-replicator conf sections in deployment guide The doc for these sections was missing because of an rst error - the source is there in rst file but didn't make it into the html output. Add doc for per_diff and max_diffs in account and container doc sections. Also, fix a bunch of other sphinx build errors and most of the warnings. Change-Id: If9ed2619b2f92c6c65a94f41d8819db8726d3893 --- doc/manpages/account-server.conf.5 | 2 +- doc/manpages/container-server.conf.5 | 2 +- doc/source/api/form_post_middleware.rst | 2 - doc/source/api/large_objects.rst | 2 + doc/source/deployment_guide.rst | 249 ++++++++++++++---------- doc/source/development_guidelines.rst | 10 +- doc/source/overview_auth.rst | 2 + doc/source/overview_backing_store.rst | 3 +- etc/account-server.conf-sample | 11 ++ etc/container-server.conf-sample | 11 ++ swift/common/middleware/tempauth.py | 10 +- swift/common/utils.py | 12 +- swift/common/wsgi.py | 4 + swift/obj/diskfile.py | 10 +- swift/proxy/controllers/base.py | 1 + 15 files changed, 203 insertions(+), 128 deletions(-) diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5 index 955eedfa76..dc69c45114 100644 --- a/doc/manpages/account-server.conf.5 +++ b/doc/manpages/account-server.conf.5 @@ -177,7 +177,7 @@ Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBper_diff\fR -The default is 1000. +Maximum number of database rows that will be sync'd in a single HTTP replication request. The default is 1000. .IP \fBmax_diffs\fR This caps how long the replicator will spend trying to sync a given database per pass so the other databases don't get starved. The default is 100. .IP \fBconcurrency\fR diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index 65981308bd..060c95180b 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -183,7 +183,7 @@ Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBer_diff\fR -The default is 1000. +Maximum number of database rows that will be sync'd in a single HTTP replication request. The default is 1000. .IP \fBmax_diffs\fR This caps how long the replicator will spend trying to sync a given database per pass so the other databases don't get starved. The default is 100. .IP \fBconcurrency\fR diff --git a/doc/source/api/form_post_middleware.rst b/doc/source/api/form_post_middleware.rst index 7e72ead68e..1e62086a57 100644 --- a/doc/source/api/form_post_middleware.rst +++ b/doc/source/api/form_post_middleware.rst @@ -1,5 +1,3 @@ -.. _formpost: - ==================== Form POST middleware ==================== diff --git a/doc/source/api/large_objects.rst b/doc/source/api/large_objects.rst index aafdc9507f..144605f117 100644 --- a/doc/source/api/large_objects.rst +++ b/doc/source/api/large_objects.rst @@ -293,10 +293,12 @@ a manifest object but a normal object with content same as what you would get on a **GET** request to original manifest object. To duplicate a manifest object: + * Use the **GET** operation to read the value of ``X-Object-Manifest`` and use this value in the ``X-Object-Manifest`` request header in a **PUT** operation. * Alternatively, you can include *``?multipart-manifest=get``* query string in the **COPY** request. + This creates a new manifest object that shares the same set of segment objects as the original manifest object. diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 78502d0a8a..7649504383 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -559,53 +559,60 @@ replication_failure_ratio 1.0 If the value of failures / [object-replicator] -================== ================= ======================================= -Option Default Description ------------------- ----------------- --------------------------------------- -log_name object-replicator Label used when logging -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level -daemonize yes Whether or not to run replication as a - daemon -interval 30 Time in seconds to wait between - replication passes -concurrency 1 Number of replication workers to spawn -timeout 5 Timeout value sent to rsync --timeout - and --contimeout options -stats_interval 3600 Interval in seconds between logging - replication statistics -reclaim_age 604800 Time elapsed in seconds before an - object can be reclaimed -handoffs_first false If set to True, partitions that are - not supposed to be on the node will be - replicated first. The default setting - should not be changed, except for - extreme situations. -handoff_delete auto By default handoff partitions will be - removed when it has successfully - replicated to all the canonical nodes. - If set to an integer n, it will remove - the partition if it is successfully - replicated to n nodes. The default - setting should not be changed, except - for extreme situations. -node_timeout DEFAULT or 10 Request timeout to external services. - This uses what's set here, or what's set - in the DEFAULT section, or 10 (though - other sections use 3 as the final - default). -rsync_module {replication_ip}::object - Format of the rsync module where the - replicator will send data. The - configuration value can include some - variables that will be extracted from - the ring. Variables must follow the - format {NAME} where NAME is one of: - ip, port, replication_ip, - replication_port, region, zone, device, - meta. See etc/rsyncd.conf-sample for - some examples. -================== ================= ======================================= +================== ======================== ================================ +Option Default Description +------------------ ------------------------ -------------------------------- +log_name object-replicator Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +daemonize yes Whether or not to run replication + as a daemon +interval 30 Time in seconds to wait between + replication passes +concurrency 1 Number of replication workers to + spawn +timeout 5 Timeout value sent to rsync + --timeout and --contimeout + options +stats_interval 3600 Interval in seconds between + logging replication statistics +reclaim_age 604800 Time elapsed in seconds before an + object can be reclaimed +handoffs_first false If set to True, partitions that + are not supposed to be on the + node will be replicated first. + The default setting should not be + changed, except for extreme + situations. +handoff_delete auto By default handoff partitions + will be removed when it has + successfully replicated to all + the canonical nodes. If set to an + integer n, it will remove the + partition if it is successfully + replicated to n nodes. The + default setting should not be + changed, except for extreme + situations. +node_timeout DEFAULT or 10 Request timeout to external + services. This uses what's set + here, or what's set in the + DEFAULT section, or 10 (though + other sections use 3 as the final + default). +rsync_module {replication_ip}::object Format of the rsync module where + the replicator will send data. + The configuration value can + include some variables that will + be extracted from the ring. + Variables must follow the format + {NAME} where NAME is one of: ip, + port, replication_ip, + replication_port, region, zone, + device, meta. See + etc/rsyncd.conf-sample for some + examples. +================== ======================== ================================ [object-updater] @@ -718,35 +725,53 @@ allow_versions false Enable/Disable object versioning feature [container-replicator] -================== ==================== ==================================== -Option Default Description ------------------- -------------------- ------------------------------------ -log_name container-replicator Label used when logging -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level -per_diff 1000 -concurrency 8 Number of replication workers to - spawn -interval 30 Time in seconds to wait between - replication passes -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external - services -reclaim_age 604800 Time elapsed in seconds before a - container can be reclaimed -rsync_module {replication_ip}::container - Format of the rsync module where the - replicator will send data. The - configuration value can include some - variables that will be extracted from - the ring. Variables must follow the - format {NAME} where NAME is one of: - ip, port, replication_ip, - replication_port, region, zone, - device, meta. See - etc/rsyncd.conf-sample for some - examples. -================== ==================== ==================================== +================== =========================== ============================= +Option Default Description +------------------ --------------------------- ----------------------------- +log_name container-replicator Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +per_diff 1000 Maximum number of database + rows that will be sync'd in a + single HTTP replication + request. Databases with less + than or equal to this number + of differing rows will always + be sync'd using an HTTP + replication request rather + than using rsync. +max_diffs 100 Maximum number of HTTP + replication requests attempted + on each replication pass for + any one container. This caps + how long the replicator will + spend trying to sync a given + database per pass so the other + databases don't get starved. +concurrency 8 Number of replication workers + to spawn +interval 30 Time in seconds to wait + between replication passes +node_timeout 10 Request timeout to external + services +conn_timeout 0.5 Connection timeout to external + services +reclaim_age 604800 Time elapsed in seconds before + a container can be reclaimed +rsync_module {replication_ip}::container Format of the rsync module + where the replicator will send + data. The configuration value + can include some variables + that will be extracted from + the ring. Variables must + follow the format {NAME} where + NAME is one of: ip, port, + replication_ip, + replication_port, region, + zone, device, meta. See + etc/rsyncd.conf-sample for + some examples. +================== =========================== ============================= [container-updater] @@ -859,33 +884,51 @@ set log_level INFO Logging level [account-replicator] -================== ================== ====================================== -Option Default Description ------------------- ------------------ -------------------------------------- -log_name account-replicator Label used when logging -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level -per_diff 1000 -concurrency 8 Number of replication workers to spawn -interval 30 Time in seconds to wait between - replication passes -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -reclaim_age 604800 Time elapsed in seconds before an - account can be reclaimed -rsync_module {replication_ip}::account - Format of the rsync module where the - replicator will send data. The - configuration value can include some - variables that will be extracted from - the ring. Variables must follow the - format {NAME} where NAME is one of: - ip, port, replication_ip, - replication_port, region, zone, - device, meta. See - etc/rsyncd.conf-sample for some - examples. -================== ================== ====================================== +================== ========================= =============================== +Option Default Description +------------------ ------------------------- ------------------------------- +log_name account-replicator Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +per_diff 1000 Maximum number of database rows + that will be sync'd in a single + HTTP replication request. + Databases with less than or + equal to this number of + differing rows will always be + sync'd using an HTTP replication + request rather than using rsync. +max_diffs 100 Maximum number of HTTP + replication requests attempted + on each replication pass for any + one container. This caps how + long the replicator will spend + trying to sync a given database + per pass so the other databases + don't get starved. +concurrency 8 Number of replication workers + to spawn +interval 30 Time in seconds to wait between + replication passes +node_timeout 10 Request timeout to external + services +conn_timeout 0.5 Connection timeout to external + services +reclaim_age 604800 Time elapsed in seconds before + an account can be reclaimed +rsync_module {replication_ip}::account Format of the rsync module where + the replicator will send data. + The configuration value can + include some variables that will + be extracted from the ring. + Variables must follow the format + {NAME} where NAME is one of: ip, + port, replication_ip, + replication_port, region, zone, + device, meta. See + etc/rsyncd.conf-sample for some + examples. +================== ========================= =============================== [account-auditor] diff --git a/doc/source/development_guidelines.rst b/doc/source/development_guidelines.rst index a3528e7714..ec2c45c5ba 100644 --- a/doc/source/development_guidelines.rst +++ b/doc/source/development_guidelines.rst @@ -51,16 +51,16 @@ To execute the unit tests: .. note:: As of tox version 2.0.0, most environment variables are not automatically - passed to the test environment. Swift's tox.ini overrides this default - behavior so that variable names matching SWIFT_* and *_proxy will be passed, - but you may need to run tox --recreate for this to take effect after - upgrading from tox<2.0.0. + passed to the test environment. Swift's `tox.ini` overrides this default + behavior so that variable names matching ``SWIFT_*`` and ``*_proxy`` will be + passed, but you may need to run `tox --recreate` for this to take effect + after upgrading from tox<2.0.0. Conversely, if you do not want those environment variables to be passed to the test environment then you will need to unset them before calling tox. Also, if you ever encounter DistributionNotFound, try to use `tox --recreate` - or remove the .tox directory to force tox to recreate the dependency list. + or remove the `.tox` directory to force tox to recreate the dependency list. The functional tests may be executed against a :doc:`development_saio` or other running Swift cluster using the command: diff --git a/doc/source/overview_auth.rst b/doc/source/overview_auth.rst index fb6272952b..42e1ad029e 100644 --- a/doc/source/overview_auth.rst +++ b/doc/source/overview_auth.rst @@ -254,9 +254,11 @@ This configuration works as follows: ``admin`` or ``swiftoperator`` role(s). When validated, the service token gives the ``service`` role. * Swift interprets the above configuration as follows: + * Did the user token provide one of the roles listed in operator_roles? * Did the service token have the ``service`` role as described by the ``SERVICE_service_roles`` options. + * If both conditions are met, the request is granted. Otherwise, Swift rejects the request. diff --git a/doc/source/overview_backing_store.rst b/doc/source/overview_backing_store.rst index ad29f2575b..57f7f37123 100644 --- a/doc/source/overview_backing_store.rst +++ b/doc/source/overview_backing_store.rst @@ -171,6 +171,7 @@ The sequence of events and actions are as follows: a copy of the . In the X-Service-Token header, place your Service's token. If you use python-swiftclient you can achieve this by: + * Putting the URL in the ``preauthurl`` parameter * Putting the in ``preauthtoken`` paramater * Adding the X-Service-Token to the ``headers`` parameter @@ -251,7 +252,7 @@ However, if one Service is compromised, that Service can access data created by another Service. To prevent this, multiple Service Prefixes may be used. This also requires that the operator configure multiple service roles. For example, in a system that has Glance and Cinder, the following -Swift configuration could be used: +Swift configuration could be used:: [keystoneauth] reseller_prefix = AUTH_, IMAGE_, BLOCK_ diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 9eb61a9d96..ea5f1a3846 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -90,8 +90,19 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # +# Maximum number of database rows that will be sync'd in a single HTTP +# replication request. Databases with less than or equal to this number of +# differing rows will always be sync'd using an HTTP replication request rather +# than using rsync. # per_diff = 1000 +# +# Maximum number of HTTP replication requests attempted on each replication +# pass for any one container. This caps how long the replicator will spend +# trying to sync a given database per pass so the other databases don't get +# starved. # max_diffs = 100 +# +# Number of replication workers to spawn. # concurrency = 8 # # Time in seconds to wait between replication passes diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 0eebf366e0..b525b7927e 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -99,8 +99,19 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # +# Maximum number of database rows that will be sync'd in a single HTTP +# replication request. Databases with less than or equal to this number of +# differing rows will always be sync'd using an HTTP replication request rather +# than using rsync. # per_diff = 1000 +# +# Maximum number of HTTP replication requests attempted on each replication +# pass for any one container. This caps how long the replicator will spend +# trying to sync a given database per pass so the other databases don't get +# starved. # max_diffs = 100 +# +# Number of replication workers to spawn. # concurrency = 8 # # Time in seconds to wait between replication passes diff --git a/swift/common/middleware/tempauth.py b/swift/common/middleware/tempauth.py index 4a3b2b0ec8..fc2e23a4a7 100644 --- a/swift/common/middleware/tempauth.py +++ b/swift/common/middleware/tempauth.py @@ -71,16 +71,16 @@ class TempAuth(object): The reseller prefix specifies which parts of the account namespace this middleware is responsible for managing authentication and authorization. - By default, the prefix is AUTH so accounts and tokens are prefixed - by AUTH_. When a request's token and/or path start with AUTH_, this + By default, the prefix is 'AUTH' so accounts and tokens are prefixed + by 'AUTH\_'. When a request's token and/or path start with 'AUTH\_', this middleware knows it is responsible. We allow the reseller prefix to be a list. In tempauth, the first item in the list is used as the prefix for tokens and user groups. The other prefixes provide alternate accounts that user's can access. For example if the reseller prefix list is 'AUTH, OTHER', a user with - admin access to AUTH_account also has admin access to - OTHER_account. + admin access to 'AUTH_account' also has admin access to + 'OTHER_account'. Required Group: @@ -98,7 +98,7 @@ class TempAuth(object): is not processed. The X-Service-Token is useful when combined with multiple reseller prefix - items. In the following configuration, accounts prefixed SERVICE_ + items. In the following configuration, accounts prefixed 'SERVICE\_' are only accessible if X-Auth-Token is from the end-user and X-Service-Token is from the ``glance`` user:: diff --git a/swift/common/utils.py b/swift/common/utils.py index ce28cea9a7..da1a609b16 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -1739,7 +1739,7 @@ def expand_ipv6(address): def whataremyips(bind_ip=None): """ Get "our" IP addresses ("us" being the set of services configured by - one *.conf file). If our REST listens on a specific address, return it. + one `*.conf` file). If our REST listens on a specific address, return it. Otherwise, if listen on '0.0.0.0' or '::' return all addresses, including the loopback. @@ -3078,15 +3078,15 @@ class ThreadPool(object): def run_in_thread(self, func, *args, **kwargs): """ - Runs func(*args, **kwargs) in a thread. Blocks the current greenlet + Runs ``func(*args, **kwargs)`` in a thread. Blocks the current greenlet until results are available. Exceptions thrown will be reraised in the calling thread. If the threadpool was initialized with nthreads=0, it invokes - func(*args, **kwargs) directly, followed by eventlet.sleep() to ensure - the eventlet hub has a chance to execute. It is more likely the hub - will be invoked when queuing operations to an external thread. + ``func(*args, **kwargs)`` directly, followed by eventlet.sleep() to + ensure the eventlet hub has a chance to execute. It is more likely the + hub will be invoked when queuing operations to an external thread. :returns: result of calling func :raises: whatever func raises @@ -3126,7 +3126,7 @@ class ThreadPool(object): def force_run_in_thread(self, func, *args, **kwargs): """ - Runs func(*args, **kwargs) in a thread. Blocks the current greenlet + Runs ``func(*args, **kwargs)`` in a thread. Blocks the current greenlet until results are available. Exceptions thrown will be reraised in the calling thread. diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 5d8ded023a..5ca14dfcdc 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -597,6 +597,8 @@ class PortPidState(object): def port_index_pairs(self): """ + Returns current (port, server index) pairs. + :returns: A set of (port, server_idx) tuples for currently-tracked ports, sockets, and PIDs. """ @@ -711,6 +713,8 @@ class ServersPerPortStrategy(object): def loop_timeout(self): """ + Return timeout before checking for reloaded rings. + :returns: The time to wait for a child to exit before checking for reloaded rings (new ports). """ diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index 5660b588b6..f8b7b72a7c 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -447,7 +447,7 @@ class BaseDiskFileManager(object): Parse an on disk file name. :param filename: the data file name including extension - :returns: a dict, with keys for timestamp, and ext:: + :returns: a dict, with keys for timestamp, and ext: * timestamp is a :class:`~swift.common.utils.Timestamp` * ext is a string, the file extension including the leading dot or @@ -895,8 +895,10 @@ class BaseDiskFileManager(object): be yielded. timestamps is a dict which may contain items mapping: + ts_data -> timestamp of data or tombstone file, ts_meta -> timestamp of meta file, if one exists + where timestamps are instances of :class:`~swift.common.utils.Timestamp` """ @@ -1961,7 +1963,7 @@ class DiskFileManager(BaseDiskFileManager): Returns the timestamp extracted .data file name. :param filename: the data file name including extension - :returns: a dict, with keys for timestamp, and ext:: + :returns: a dict, with keys for timestamp, and ext: * timestamp is a :class:`~swift.common.utils.Timestamp` * ext is a string, the file extension including the leading dot or @@ -2241,12 +2243,12 @@ class ECDiskFileManager(BaseDiskFileManager): be stripped off to retrieve the timestamp. :param filename: the data file name including extension - :returns: a dict, with keys for timestamp, frag_index, and ext:: + :returns: a dict, with keys for timestamp, frag_index, and ext: * timestamp is a :class:`~swift.common.utils.Timestamp` * frag_index is an int or None * ext is a string, the file extension including the leading dot or - the empty string if the filename has no extenstion. + the empty string if the filename has no extension. :raises DiskFileError: if any part of the filename is not able to be validated. diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index f064d011c4..aa3f4c1736 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -317,6 +317,7 @@ def get_account_info(env, app, swift_source=None): This call bypasses auth. Success does not imply that the request has authorization to the account. + :raises ValueError: when path can't be split(path, 2, 4) """ (version, account, _junk, _junk) = \