Merge "Adding speed limit options for DB auditor"

doc/manpages/account-server.conf.5

@@ -210,6 +210,8 @@ Logging level. The default is INFO.
 Logging address. The default is /dev/log.
 .IP \fBinterval\fR
 Will audit, at most, 1 account per device per interval. The default is 1800 seconds.
+.IP \fBaccounts_per_second\fR
+Maximum accounts audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 200.
 .RE
 
 

doc/manpages/container-server.conf.5

@@ -237,6 +237,8 @@ Logging level. The default is INFO.
 Logging address. The default is /dev/log.
 .IP \fBinterval\fR
 Will audit, at most, 1 container per device per interval. The default is 1800 seconds.
+.IP \fBcontainers_per_second\fR
+Maximum containers audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 200.
 .RE
 
 

doc/source/deployment_guide.rst

@@ -424,14 +424,17 @@ account_suppression_time  60                 Seconds to suppress updating an
 
 [container-auditor]
 
-==================  =================  =======================================
+=====================  =================  =======================================
 Option                 Default            Description
-------------------  -----------------  ---------------------------------------
+---------------------  -----------------  ---------------------------------------
 log_name               container-auditor  Label used when logging
 log_facility           LOG_LOCAL0         Syslog log facility
 log_level              INFO               Logging level
 interval               1800               Minimum time for a pass to take
-==================  =================  =======================================
+containers_per_second  200                Maximum containers audited per second. 
+                                          Should be tuned according to individual
+                                          system specs. 0 is unlimited.
+=====================  =================  =======================================
 
 ----------------------------
 Account Server Configuration
@@ -515,6 +518,9 @@ log_name              account-auditor  Label used when logging
 log_facility          LOG_LOCAL0       Syslog log facility
 log_level             INFO             Logging level
 interval              1800             Minimum time for a pass to take
+accounts_per_second   200              Maximum accounts audited per second. 
+                                       Should be tuned according to individual
+                                       system specs. 0 is unlimited. 
 ====================  ===============  =======================================
 
 [account-reaper]

etc/account-server.conf-sample

@@ -92,6 +92,7 @@ use = egg:swift#recon
 # interval = 1800
 # log_facility = LOG_LOCAL0
 # log_level = INFO
+# accounts_per_second = 200
 # recon_cache_path = /var/cache/swift
 
 [account-reaper]

etc/container-server.conf-sample

@@ -107,6 +107,7 @@ use = egg:swift#recon
 # log_address = /dev/log
 # Will audit each container at most once per interval
 # interval = 1800
+# containers_per_second = 200
 # recon_cache_path = /var/cache/swift
 
 [container-sync]

swift/account/auditor.py

@@ -21,7 +21,7 @@ import swift.common.db
 from swift.account import server as account_server
 from swift.common.db import AccountBroker
 from swift.common.utils import get_logger, audit_location_generator, \
-    config_true_value, dump_recon_cache
+    config_true_value, dump_recon_cache, ratelimit_sleep
 from swift.common.daemon import Daemon
 
 from eventlet import Timeout
@@ -38,6 +38,9 @@ class AccountAuditor(Daemon):
         self.interval = int(conf.get('interval', 1800))
         self.account_passes = 0
         self.account_failures = 0
+        self.accounts_running_time = 0
+        self.max_accounts_per_second = \
+            float(conf.get('accounts_per_second', 200))
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
         self.recon_cache_path = conf.get('recon_cache_path',
@@ -67,6 +70,8 @@ class AccountAuditor(Daemon):
                 reported = time.time()
                 self.account_passes = 0
                 self.account_failures = 0
+            self.accounts_running_time = ratelimit_sleep(
+                self.accounts_running_time, self.max_accounts_per_second)
         return reported
 
     def run_forever(self, *args, **kwargs):

swift/container/auditor.py

@@ -23,7 +23,7 @@ import swift.common.db
 from swift.container import server as container_server
 from swift.common.db import ContainerBroker
 from swift.common.utils import get_logger, audit_location_generator, \
-    config_true_value, dump_recon_cache
+    config_true_value, dump_recon_cache, ratelimit_sleep
 from swift.common.daemon import Daemon
 
 
@@ -38,6 +38,9 @@ class ContainerAuditor(Daemon):
         self.interval = int(conf.get('interval', 1800))
         self.container_passes = 0
         self.container_failures = 0
+        self.containers_running_time = 0
+        self.max_containers_per_second = \
+            float(conf.get('containers_per_second', 200))
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
         self.recon_cache_path = conf.get('recon_cache_path',
@@ -66,6 +69,8 @@ class ContainerAuditor(Daemon):
                 reported = time.time()
                 self.container_passes = 0
                 self.container_failures = 0
+            self.containers_running_time = ratelimit_sleep(
+                self.containers_running_time, self.max_containers_per_second)
         return reported
 
     def run_forever(self, *args, **kwargs):