From 6987d222de0c39137dcbaf4c1577862963cd75e3 Mon Sep 17 00:00:00 2001 From: Greg Lange Date: Thu, 7 Jun 2012 20:19:31 +0000 Subject: [PATCH] Added config options for rate limiting of large object downloads. bug 786197 Change-Id: Idd73986ca4550b20dff340df4acd12f3bfc01075 --- doc/manpages/proxy-server.conf.5 | 243 ++++++++++++++++--------------- doc/source/deployment_guide.rst | 45 +++--- swift/proxy/server.py | 9 +- test/unit/proxy/test_server.py | 4 +- 4 files changed, 159 insertions(+), 142 deletions(-) diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index af2f648ea1..913b4a149f 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -14,13 +14,13 @@ .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. -.\" +.\" .TH proxy-server.conf 5 "8/26/2011" "Linux" "OpenStack Swift" -.SH NAME +.SH NAME .LP .B proxy-server.conf -\- configuration file for the openstack-swift proxy server +\- configuration file for the openstack-swift proxy server @@ -30,51 +30,51 @@ -.SH DESCRIPTION +.SH DESCRIPTION .PP This is the configuration file used by the proxy server and other proxy middlewares. The configuration file follows the python-pastedeploy syntax. The file is divided -into sections, which are enclosed by square brackets. Each section will contain a -certain number of key/value parameters which are described later. +into sections, which are enclosed by square brackets. Each section will contain a +certain number of key/value parameters which are described later. -Any line that begins with a '#' symbol is ignored. +Any line that begins with a '#' symbol is ignored. -You can find more information about python-pastedeploy configuration format at +You can find more information about python-pastedeploy configuration format at \fIhttp://pythonpaste.org/deploy/#config-format\fR .SH GLOBAL SECTION -.PD 1 +.PD 1 .RS 0 -This is indicated by section named [DEFAULT]. Below are the parameters that -are acceptable within this section. +This is indicated by section named [DEFAULT]. Below are the parameters that +are acceptable within this section. .IP "\fBbind_ip\fR" -IP address the proxy server should bind to. The default is 0.0.0.0 which will make +IP address the proxy server should bind to. The default is 0.0.0.0 which will make it bind to all available addresses. -.IP "\fBbind_port\fR" -TCP port the proxy server should bind to. The default is 80. -.IP \fBbacklog\fR -TCP backlog. Maximum number of allowed pending connections. The default value is 4096. -.IP \fBworkers\fR -Number of container server workers to fork. The default is 1. -.IP \fBuser\fR -The system user that the container server will run as. The default is swift. -.IP \fBswift_dir\fR +.IP "\fBbind_port\fR" +TCP port the proxy server should bind to. The default is 80. +.IP \fBbacklog\fR +TCP backlog. Maximum number of allowed pending connections. The default value is 4096. +.IP \fBworkers\fR +Number of container server workers to fork. The default is 1. +.IP \fBuser\fR +The system user that the container server will run as. The default is swift. +.IP \fBswift_dir\fR Swift configuration directory. The default is /etc/swift. -.IP \fBcert_file\fR -Location of the SSL certificate file. The default path is /etc/swift/proxy.crt. This is -disabled by default. -.IP \fBkey_file\fR -Location of the SSL certificate key file. The default path is /etc/swift/proxy.key. This is -disabled by default. -.IP \fBlog_name\fR +.IP \fBcert_file\fR +Location of the SSL certificate file. The default path is /etc/swift/proxy.crt. This is +disabled by default. +.IP \fBkey_file\fR +Location of the SSL certificate key file. The default path is /etc/swift/proxy.key. This is +disabled by default. +.IP \fBlog_name\fR Label used when logging. The default is swift. -.IP \fBlog_facility\fR +.IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. -.IP \fBlog_level\fR +.IP \fBlog_level\fR Logging level. The default is INFO. .RE .PD @@ -82,14 +82,14 @@ Logging level. The default is INFO. .SH PIPELINE SECTION -.PD 1 +.PD 1 .RS 0 This is indicated by section name [pipeline:main]. Below are the parameters that -are acceptable within this section. +are acceptable within this section. .IP "\fBpipeline\fR" -It is used when you need apply a number of filters. It is a list of filters -ended by an application. The default should be \fB"catch_errors healthcheck +It is used when you need apply a number of filters. It is a list of filters +ended by an application. The default should be \fB"catch_errors healthcheck cache ratelimit tempauth proxy-server"\fR .RE .PD @@ -97,16 +97,16 @@ cache ratelimit tempauth proxy-server"\fR .SH FILTER SECTION -.PD 1 +.PD 1 .RS 0 Any section that has its name prefixed by "filter:" indicates a filter section. Filters are used to specify configuration parameters for specific swift middlewares. -Below are the filters available and respective acceptable parameters. +Below are the filters available and respective acceptable parameters. .IP "\fB[filter:healthcheck]\fR" .RE .RS 3 .IP "\fBuse\fR" -Entry point for paste.deploy for the healthcheck middleware. This is the reference to the installed python egg. +Entry point for paste.deploy for the healthcheck middleware. This is the reference to the installed python egg. The default is \fBegg:swift#healthcheck\fR. .RE @@ -115,8 +115,8 @@ The default is \fBegg:swift#healthcheck\fR. .IP "\fB[filter:tempauth]\fR" .RE .RS 3 -.IP \fBuse\fR -Entry point for paste.deploy for the tempauth middleware. This is the reference to the installed python egg. +.IP \fBuse\fR +Entry point for paste.deploy for the tempauth middleware. This is the reference to the installed python egg. The default is \fBegg:swift#tempauth\fR. .IP "\fBset log_name\fR" Label used when logging. The default is tempauth. @@ -128,31 +128,31 @@ Logging level. The default is INFO. Enables the ability to log request headers. The default is False. .IP \fBreseller_prefix\fR The reseller prefix will verify a token begins with this prefix before even -attempting to validate it. Also, with authorization, only Swift storage accounts -with this prefix will be authorized by this middleware. Useful if multiple auth +attempting to validate it. Also, with authorization, only Swift storage accounts +with this prefix will be authorized by this middleware. Useful if multiple auth systems are in use for one Swift cluster. The default is AUTH. -.IP \fBauth_prefix\fR -The auth prefix will cause requests beginning with this prefix to be routed +.IP \fBauth_prefix\fR +The auth prefix will cause requests beginning with this prefix to be routed to the auth subsystem, for granting tokens, etc. The default is /auth/. -.IP \fBtoken_life\fR +.IP \fBtoken_life\fR This is the time in seconds before the token expires. The default is 86400. -.IP \fBallowed_sync_hosts\fR +.IP \fBallowed_sync_hosts\fR This is a comma separated list of hosts allowed to send X-Container-Sync-Key requests. -.IP \fBuser__\fR -Lastly, you need to list all the accounts/users you want here. The format is: -user__ = [group] [group] [...] [storage_url] +.IP \fBuser__\fR +Lastly, you need to list all the accounts/users you want here. The format is: +user__ = [group] [group] [...] [storage_url] There are special groups of: \fI.reseller_admin\fR who can do anything to any account for this auth and also \fI.admin\fR who can do anything within the account. -If neither of these groups are specified, the user can only access containers that +If neither of these groups are specified, the user can only access containers that have been explicitly allowed for them by a \fI.admin\fR or \fI.reseller_admin\fR. -The trailing optional storage_url allows you to specify an alternate url to hand -back to the user upon authentication. If not specified, this defaults to -\fIhttp[s]://:/v1/_\fR where http or https depends -on whether cert_file is specified in the [DEFAULT] section, and are based -on the [DEFAULT] section's bind_ip and bind_port (falling back to 127.0.0.1 and 8080), - is from this section, and is from the user__ name. +The trailing optional storage_url allows you to specify an alternate url to hand +back to the user upon authentication. If not specified, this defaults to +\fIhttp[s]://:/v1/_\fR where http or https depends +on whether cert_file is specified in the [DEFAULT] section, and are based +on the [DEFAULT] section's bind_ip and bind_port (falling back to 127.0.0.1 and 8080), + is from this section, and is from the user__ name. Here are example entries, required for running the tests: .RE @@ -164,14 +164,14 @@ Here are example entries, required for running the tests: .IP "user_test2_tester2 = testing2 .admin" .IP "user_test_tester3 = testing3" .RE -.PD +.PD .RS 0 .IP "\fB[filter:healthcheck]\fR" .RE .RS 3 .IP \fBuse\fR -Entry point for paste.deploy for the healthcheck middleware. This is the reference to the installed python egg. +Entry point for paste.deploy for the healthcheck middleware. This is the reference to the installed python egg. The default is \fBegg:swift#healthcheck\fR. .IP "\fBset log_name\fR" Label used when logging. The default is healthcheck. @@ -193,7 +193,7 @@ Caching middleware that manages caching in swift. .RS 3 .IP \fBuse\fR -Entry point for paste.deploy for the memcache middleware. This is the reference to the installed python egg. +Entry point for paste.deploy for the memcache middleware. This is the reference to the installed python egg. The default is \fBegg:swift#memcache\fR. .IP "\fBset log_name\fR" Label used when logging. The default is memcache. @@ -217,7 +217,7 @@ Rate limits requests on both an Account and Container level. Limits are configu .RS 3 .IP \fBuse\fR -Entry point for paste.deploy for the ratelimit middleware. This is the reference to the installed python egg. +Entry point for paste.deploy for the ratelimit middleware. This is the reference to the installed python egg. The default is \fBegg:swift#ratelimit\fR. .IP "\fBset log_name\fR" Label used when logging. The default is ratelimit. @@ -228,21 +228,21 @@ Logging level. The default is INFO. .IP "\fBset log_headers\fR " Enables the ability to log request headers. The default is False. .IP \fBclock_accuracy\fR -This should represent how accurate the proxy servers' system clocks are with each other. -1000 means that all the proxies' clock are accurate to each other within 1 millisecond. +This should represent how accurate the proxy servers' system clocks are with each other. +1000 means that all the proxies' clock are accurate to each other within 1 millisecond. No ratelimit should be higher than the clock accuracy. The default is 1000. .IP \fBmax_sleep_time_seconds\fR -App will immediately return a 498 response if the necessary sleep time ever exceeds +App will immediately return a 498 response if the necessary sleep time ever exceeds the given max_sleep_time_seconds. The default is 60 seconds. .IP \fBlog_sleep_time_seconds\fR -To allow visibility into rate limiting set this value > 0 and all sleeps greater than +To allow visibility into rate limiting set this value > 0 and all sleeps greater than the number will be logged. If set to 0 means disabled. The default is 0. .IP \fBrate_buffer_seconds\fR -Number of seconds the rate counter can drop and be allowed to catch up -(at a faster than listed rate). A larger number will result in larger spikes in +Number of seconds the rate counter can drop and be allowed to catch up +(at a faster than listed rate). A larger number will result in larger spikes in rate but better average accuracy. The default is 5. .IP \fBaccount_ratelimit\fR -If set, will limit PUT and DELETE requests to /account_name/container_name. Number is +If set, will limit PUT and DELETE requests to /account_name/container_name. Number is in requests per second. If set to 0 means disabled. The default is 0. .IP \fBaccount_whitelist\fR Comma separated lists of account names that will not be rate limited. The default is ''. @@ -250,7 +250,7 @@ Comma separated lists of account names that will not be rate limited. The defaul Comma separated lists of account names that will not be allowed. Returns a 497 response. The default is ''. .IP \fBcontainer_ratelimit_size\fR -When set with container_limit_x = r: for containers of size x, limit requests per second +When set with container_limit_x = r: for containers of size x, limit requests per second to r. Will limit PUT, DELETE, and POST requests to /a/c/o. The default is ''. .RE @@ -263,34 +263,34 @@ Middleware that translates container and account parts of a domain to path param .RS 3 .IP \fBuse\fR -Entry point for paste.deploy for the domain_remap middleware. This is the reference to the installed python egg. +Entry point for paste.deploy for the domain_remap middleware. This is the reference to the installed python egg. The default is \fBegg:swift#domain_remap\fR. .IP "\fBset log_name\fR" Label used when logging. The default is domain_remap. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. -.IP \fBstorage_domain\fR +.IP \fBstorage_domain\fR The domain to be used by the middleware. .IP \fBpath_root\fR The path root value for the storage URL. The default is v1. .IP \fBreseller_prefixes\fR -Browsers can convert a host header to lowercase, so check that reseller -prefix on the account is the correct case. This is done by comparing the -items in the reseller_prefixes config option to the found prefix. If they -match except for case, the item from reseller_prefixes will be used -instead of the found reseller prefix. The reseller_prefixes list is exclusive. -If defined, any request with an account prefix not in that list will be ignored +Browsers can convert a host header to lowercase, so check that reseller +prefix on the account is the correct case. This is done by comparing the +items in the reseller_prefixes config option to the found prefix. If they +match except for case, the item from reseller_prefixes will be used +instead of the found reseller prefix. The reseller_prefixes list is exclusive. +If defined, any request with an account prefix not in that list will be ignored by this middleware. Defaults to 'AUTH'. .RE -.RS 0 +.RS 0 .IP "\fB[filter:catch_errors]\fR" .RE .RS 3 .IP \fBuse\fR -Entry point for paste.deploy for the catch_errors middleware. This is the reference to the installed python egg. +Entry point for paste.deploy for the catch_errors middleware. This is the reference to the installed python egg. The default is \fBegg:swift#catch_errors\fR. .IP "\fBset log_name\fR" Label used when logging. The default is catch_errors. @@ -304,7 +304,7 @@ Enables the ability to log request headers. The default is False. -.RS 0 +.RS 0 .IP "\fB[filter:cname_lookup]\fR" .RE @@ -312,7 +312,7 @@ Note: this middleware requires python-dnspython .RS 3 .IP \fBuse\fR -Entry point for paste.deploy for the cname_lookup middleware. This is the reference to the installed python egg. +Entry point for paste.deploy for the cname_lookup middleware. This is the reference to the installed python egg. The default is \fBegg:swift#cname_lookup\fR. .IP "\fBset log_name\fR" Label used when logging. The default is cname_lookup. @@ -322,16 +322,16 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. -.IP \fBstorage_domain\fR +.IP \fBstorage_domain\fR The domain to be used by the middleware. .IP \fBlookup_depth\fR -How deep in the CNAME chain to look for something that matches the storage domain. +How deep in the CNAME chain to look for something that matches the storage domain. The default is 1. .RE -.RS 0 +.RS 0 .IP "\fB[filter:staticweb]\fR" .RE @@ -339,7 +339,7 @@ Note: Put staticweb just after your auth filter(s) in the pipeline .RS 3 .IP \fBuse\fR -Entry point for paste.deploy for the staticweb middleware. This is the reference to the installed python egg. +Entry point for paste.deploy for the staticweb middleware. This is the reference to the installed python egg. The default is \fBegg:swift#staticweb\fR. .IP \fBcache_timeout\fR Seconds to cache container x-container-meta-web-* header values. The default is 300 seconds. @@ -361,7 +361,7 @@ Logging level. The default is INFO. -.RS 0 +.RS 0 .IP "\fB[filter:tempurl]\fR" .RE @@ -381,25 +381,25 @@ The headers allowed as exceptions to outgoing_remove_headers. Simply a whitespac -.RS 0 +.RS 0 .IP "\fB[filter:formpost]\fR" .RE -Note: Put formpost just before your auth filter(s) in the pipeline +Note: Put formpost just before your auth filter(s) in the pipeline .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the formpost middleware. This is the reference to the installed python egg. The default is \fBegg:swift#formpost\fR. -.RE +.RE -.RS 0 +.RS 0 .IP "\fB[filter:name_check]\fR" .RE -Note: Just needs to be placed before the proxy-server in the pipeline. +Note: Just needs to be placed before the proxy-server in the pipeline. .RS 3 .IP \fBuse\fR @@ -409,7 +409,7 @@ The default is \fBegg:swift#name_check\fR. Characters that will not be allowed in a name. .IP \fBmaximum_length\fR Maximum number of characters that can be in the name. -.RE +.RE .PD @@ -418,18 +418,18 @@ Maximum number of characters that can be in the name. .SH APP SECTION -.PD 1 +.PD 1 .RS 0 This is indicated by section name [app:proxy-server]. Below are the parameters that are acceptable within this section. .IP \fBuse\fR -Entry point for paste.deploy for the proxy server. This is the reference to the installed python egg. +Entry point for paste.deploy for the proxy server. This is the reference to the installed python egg. The default is \fBegg:swift#proxy\fR. -.IP "\fBset log_name\fR +.IP "\fBset log_name\fR Label used when logging. The default is proxy-server. -.IP "\fBset log_facility\fR +.IP "\fBset log_facility\fR Syslog log facility. The default is LOG_LOCAL0. -.IP "\fB set log_level\fR +.IP "\fB set log_level\fR Logging level. The default is INFO. .IP "\fBset access_log_name\fR" Label used when logging. The default is proxy-server. @@ -437,7 +437,7 @@ Label used when logging. The default is proxy-server. Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset access_log_level\fR " Logging level. The default is INFO. -.IP "\fB set log_requests\fR +.IP "\fB set log_requests\fR Enables request logging. The default is False. .IP \fBrecheck_account_existence\fR Cache timeout in seconds to send memcached for account existence. The default is 60 seconds. @@ -447,29 +447,34 @@ Cache timeout in seconds to send memcached for container existence. The default Chunk size to read from object servers. The default is 8192. .IP \fBclient_chunk_size\fR Chunk size to read from clients. The default is 8192. -.IP \fBnode_timeout\fR -Request timeout to external services. The default is 10 seconds. -.IP \fBclient_timeoutt\fR -Timeout to read one chunk from a client. The default is 60 seconds. -.IP \fBconn_timeout\fR -Connection timeout to external services. The default is 0.5 seconds. -.IP \fBerror_suppression_interval\fR -Time in seconds that must elapse since the last error for a node to +.IP \fBnode_timeout\fR +Request timeout to external services. The default is 10 seconds. +.IP \fBclient_timeoutt\fR +Timeout to read one chunk from a client. The default is 60 seconds. +.IP \fBconn_timeout\fR +Connection timeout to external services. The default is 0.5 seconds. +.IP \fBerror_suppression_interval\fR +Time in seconds that must elapse since the last error for a node to be considered no longer error limited. The default is 60 seconds. -.IP \fBerror_suppression_limit\fR -Error count to consider a node error limited. The default is 10. -.IP \fBallow_account_management\fR -Whether account PUTs and DELETEs are even callable. If set to 'true' any authorized -user may create and delete accounts; if 'false' no one, even authorized, can. The default -is false. -.IP \fBobject_post_as_copy\fR -Set object_post_as_copy = false to turn on fast posts where only the metadata changes -are stored as new and the original data file is kept in place. This makes for quicker -posts; but since the container metadata isn't updated in this mode, features like +.IP \fBerror_suppression_limit\fR +Error count to consider a node error limited. The default is 10. +.IP \fBallow_account_management\fR +Whether account PUTs and DELETEs are even callable. If set to 'true' any authorized +user may create and delete accounts; if 'false' no one, even authorized, can. The default +is false. +.IP \fBobject_post_as_copy\fR +Set object_post_as_copy = false to turn on fast posts where only the metadata changes +are stored as new and the original data file is kept in place. This makes for quicker +posts; but since the container metadata isn't updated in this mode, features like container sync won't be able to sync posts. The default is True. -.IP \fBaccount_autocreate\fR -If set to 'true' authorized accounts that do not yet exist within the Swift cluster +.IP \fBaccount_autocreate\fR +If set to 'true' authorized accounts that do not yet exist within the Swift cluster will be automatically created. The default is set to false. +.IP \fBrate_limit_after_segment\fR +Rate limit the download of large object segments after this segment is +downloaded. The default is 10 segments. +.IP \fBrate_limit_segments_per_sec\fR +Rate limit large object downlods at this rate. The default is 1. .RE .PD @@ -478,9 +483,9 @@ will be automatically created. The default is set to false. .SH DOCUMENTATION .LP More in depth documentation about the swift-proxy-server and -also Openstack-Swift as a whole can be found at -.BI http://swift.openstack.org/admin_guide.html -and +also Openstack-Swift as a whole can be found at +.BI http://swift.openstack.org/admin_guide.html +and .BI http://swift.openstack.org diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 0d520d6960..b980185961 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -94,7 +94,7 @@ You can now start building the ring with:: swift-ring-builder create -This will start the ring build process creating the with +This will start the ring build process creating the with 2^ partitions. is the time in hours before a specific partition can be moved in succession (24 is a good value for this). @@ -213,7 +213,7 @@ The main rule to remember when working with Swift configuration files is: Object Server Configuration --------------------------- -An Example Object Server configuration can be found at +An Example Object Server configuration can be found at etc/object-server.conf-sample in the source code repository. The following configuration options are available: @@ -266,14 +266,14 @@ log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level daemonize yes Whether or not to run replication as a daemon -run_pause 30 Time in seconds to wait between +run_pause 30 Time in seconds to wait between replication passes concurrency 1 Number of replication workers to spawn -timeout 5 Timeout value sent to rsync --timeout +timeout 5 Timeout value sent to rsync --timeout and --contimeout options stats_interval 3600 Interval in seconds between logging replication statistics -reclaim_age 604800 Time elapsed in seconds before an +reclaim_age 604800 Time elapsed in seconds before an object can be reclaimed ================== ================= ======================================= @@ -313,7 +313,7 @@ bytes_per_second 10000000 Maximum bytes audited per second. Should Container Server Configuration ------------------------------ -An example Container Server configuration can be found at +An example Container Server configuration can be found at etc/container-server.conf-sample in the source code repository. The following configuration options are available: @@ -339,8 +339,8 @@ user swift User to run as ================== ================ ======================================== Option Default Description ------------------ ---------------- ---------------------------------------- -use paste.deploy entry point for the - container server. For most cases, this +use paste.deploy entry point for the + container server. For most cases, this should be `egg:swift#container`. set log_name container-server Label used when logging set log_facility LOG_LOCAL0 Syslog log facility @@ -358,14 +358,14 @@ log_name container-replicator Label used when logging log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level per_diff 1000 -concurrency 8 Number of replication workers to +concurrency 8 Number of replication workers to spawn -run_pause 30 Time in seconds to wait between +run_pause 30 Time in seconds to wait between replication passes node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external +conn_timeout 0.5 Connection timeout to external services -reclaim_age 604800 Time elapsed in seconds before a +reclaim_age 604800 Time elapsed in seconds before a container can be reclaimed ================== ==================== ==================================== @@ -383,7 +383,7 @@ node_timeout 3 Request timeout to external services conn_timeout 0.5 Connection timeout to external services -slowdown 0.01 Time in seconds to wait between +slowdown 0.01 Time in seconds to wait between containers account_suppression_time 60 Seconds to suppress updating an account that has generated an @@ -406,7 +406,7 @@ interval 1800 Minimum time for a pass to take Account Server Configuration ---------------------------- -An example Account Server configuration can be found at +An example Account Server configuration can be found at etc/account-server.conf-sample in the source code repository. The following configuration options are available: @@ -454,11 +454,11 @@ log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level per_diff 1000 concurrency 8 Number of replication workers to spawn -run_pause 30 Time in seconds to wait between +run_pause 30 Time in seconds to wait between replication passes node_timeout 10 Request timeout to external services conn_timeout 0.5 Connection timeout to external services -reclaim_age 604800 Time elapsed in seconds before an +reclaim_age 604800 Time elapsed in seconds before an account can be reclaimed ================== ================== ====================================== @@ -496,7 +496,7 @@ delay_reaping 0 Normally, the reaper begins deleting Proxy Server Configuration -------------------------- -An example Proxy Server configuration can be found at +An example Proxy Server configuration can be found at etc/proxy-server.conf-sample in the source code repository. The following configuration options are available: @@ -588,6 +588,11 @@ max_containers_whitelist This is a comma separated list of account hashes that ignore the max_containers_per_account cap. +rate_limit_after_segment 10 Rate limit the download of + large object segments after + this segment is downloaded. +rate_limit_segments_per_sec 1 Rate limit large object + downloads at this rate. ============================ =============== ============================= [tempauth] @@ -657,7 +662,7 @@ Memcached Considerations Several of the Services rely on Memcached for caching certain types of lookups, such as auth tokens, and container/account existence. Swift does not do any caching of actual object data. Memcached should be able to run -on any servers that have available RAM and CPU. At Rackspace, we run +on any servers that have available RAM and CPU. At Rackspace, we run Memcached on the proxy servers. The `memcache_servers` config option in the `proxy-server.conf` should contain all memcached servers. @@ -709,7 +714,7 @@ the best all-around choice. If you decide to use a filesystem other than XFS, we highly recommend thorough testing. If you are using XFS, some settings that can dramatically impact -performance. We recommend the following when creating the XFS +performance. We recommend the following when creating the XFS partition:: mkfs.xfs -i size=1024 -f /dev/sda1 @@ -730,7 +735,7 @@ For a standard swift install, all data drives are mounted directly under /srv/node (as can be seen in the above example of mounting /def/sda1 as /srv/node/sda). If you choose to mount the drives in another directory, be sure to set the `devices` config option in all of the server configs to -point to the correct directory. +point to the correct directory. --------------------- General System Tuning diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 38e9e66e8f..f4c0d832db 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -163,9 +163,10 @@ class SegmentedIterable(object): if self.seek: req.range = 'bytes=%s-' % self.seek self.seek = 0 - if self.segment > 10: + if self.segment > self.controller.app.rate_limit_after_segment: sleep(max(self.next_get_time - time.time(), 0)) - self.next_get_time = time.time() + 1 + self.next_get_time = time.time() + \ + 1.0 / self.controller.app.rate_limit_segments_per_sec shuffle(nodes) resp = self.controller.GETorHEAD_base(req, _('Object'), partition, self.controller.iter_nodes(partition, nodes, @@ -1917,6 +1918,10 @@ class BaseApplication(object): if a.strip()] self.deny_host_headers = [host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip()] + self.rate_limit_after_segment = \ + int(conf.get('rate_limit_after_segment', 10)) + self.rate_limit_segments_per_sec = \ + int(conf.get('rate_limit_segments_per_sec', 1)) def get_controller(self, path): """ diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 6187502ea7..0cd85b632d 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -2879,7 +2879,7 @@ class TestObjectController(unittest.TestCase): self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status, '202 Fake') - self.assertEquals(req.headers.get('x-delete-at'), + self.assertEquals(req.headers.get('x-delete-at'), str(int(t + 60))) self.app.object_post_as_copy = False @@ -3889,6 +3889,8 @@ class FakeObjectController(object): self.trans_id = 'tx1' self.object_ring = FakeRing() self.node_timeout = 1 + self.rate_limit_after_segment = 10 + self.rate_limit_segments_per_sec = 1 def exception(self, *args): self.exception_args = args