Merge "Add option for debug query logging"

etc/account-server.conf-sample

@@ -57,6 +57,9 @@ bind_port = 6202
 # on to preallocate disk space with SQLite databases to decrease fragmentation.
 # db_preallocation = off
 #
+# Enable this option to log all sqlite3 queries (requires python >=3.3)
+# db_query_logging = off
+#
 # eventlet_debug = false
 #
 # You can set fallocate_reserve to the number of bytes or percentage of disk

etc/container-server.conf-sample

@@ -63,6 +63,9 @@ bind_port = 6201
 # on to preallocate disk space with SQLite databases to decrease fragmentation.
 # db_preallocation = off
 #
+# Enable this option to log all sqlite3 queries (requires python >=3.3)
+# db_query_logging = off
+#
 # eventlet_debug = false
 #
 # You can set fallocate_reserve to the number of bytes or percentage of disk

swift/account/server.py

@@ -89,6 +89,8 @@ class AccountController(BaseStorageServer):
             conf.get('auto_create_account_prefix') or '.'
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
+        swift.common.db.QUERY_LOGGING = \
+            config_true_value(conf.get('db_query_logging', 'f'))
         self.fallocate_reserve, self.fallocate_is_percent = \
             config_fallocate_value(conf.get('fallocate_reserve', '1%'))
 

swift/common/db.py

@@ -43,6 +43,8 @@ from swift.common.swob import HTTPBadRequest
 
 #: Whether calls will be made to preallocate disk space for database files.
 DB_PREALLOCATION = False
+#: Whether calls will be made to log queries (py3 only)
+QUERY_LOGGING = False
 #: Timeout for trying to connect to a DB
 BROKER_TIMEOUT = 25
 #: Pickle protocol to use
@@ -181,7 +183,7 @@ def chexor(old, name, timestamp):
     return '%032x' % (int(old, 16) ^ int(new, 16))
 
 
-def get_db_connection(path, timeout=30, okay_to_create=False):
+def get_db_connection(path, timeout=30, logger=None, okay_to_create=False):
     """
     Returns a properly configured SQLite database connection.
 
@@ -194,6 +196,8 @@ def get_db_connection(path, timeout=30, okay_to_create=False):
         connect_time = time.time()
         conn = sqlite3.connect(path, check_same_thread=False,
                                factory=GreenDBConnection, timeout=timeout)
+        if QUERY_LOGGING and logger and not six.PY2:
+            conn.set_trace_callback(logger.debug)
         if path != ':memory:' and not okay_to_create:
             # attempt to detect and fail when connect creates the db file
             stat = os.stat(path)
@@ -272,13 +276,15 @@ class DatabaseBroker(object):
         """
         if self._db_file == ':memory:':
             tmp_db_file = None
-            conn = get_db_connection(self._db_file, self.timeout)
+            conn = get_db_connection(self._db_file, self.timeout, self.logger)
         else:
             mkdirs(self.db_dir)
             fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
             os.close(fd)
             conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
                                    factory=GreenDBConnection, timeout=0)
+            if QUERY_LOGGING and not six.PY2:
+                conn.set_trace_callback(self.logger.debug)
         # creating dbs implicitly does a lot of transactions, so we
         # pick fast, unsafe options here and do a big fsync at the end.
         with closing(conn.cursor()) as cur:
@@ -339,7 +345,8 @@ class DatabaseBroker(object):
                     # of the system were "racing" each other.
                     raise DatabaseAlreadyExists(self.db_file)
                 renamer(tmp_db_file, self.db_file)
-            self.conn = get_db_connection(self.db_file, self.timeout)
+            self.conn = get_db_connection(self.db_file, self.timeout,
+                                          self.logger)
         else:
             self.conn = conn
 
@@ -449,7 +456,8 @@ class DatabaseBroker(object):
         if not self.conn:
             if self.db_file != ':memory:' and os.path.exists(self.db_file):
                 try:
-                    self.conn = get_db_connection(self.db_file, self.timeout)
+                    self.conn = get_db_connection(self.db_file, self.timeout,
+                                                  self.logger)
                 except (sqlite3.DatabaseError, DatabaseConnectionError):
                     self.possibly_quarantine(*sys.exc_info())
             else:
@@ -475,7 +483,8 @@ class DatabaseBroker(object):
         """Use with the "with" statement; locks a database."""
         if not self.conn:
             if self.db_file != ':memory:' and os.path.exists(self.db_file):
-                self.conn = get_db_connection(self.db_file, self.timeout)
+                self.conn = get_db_connection(self.db_file, self.timeout,
+                                              self.logger)
             else:
                 raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
         conn = self.conn

swift/common/db_replicator.py

@@ -217,6 +217,8 @@ class Replicator(Daemon):
         self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
+        swift.common.db.QUERY_LOGGING = \
+            config_true_value(conf.get('db_query_logging', 'f'))
         self._zero_stats()
         self.recon_cache_path = conf.get('recon_cache_path',
                                          '/var/cache/swift')
@@ -579,7 +581,8 @@ class Replicator(Daemon):
         shouldbehere = True
         responses = []
         try:
-            broker = self.brokerclass(object_file, pending_timeout=30)
+            broker = self.brokerclass(object_file, pending_timeout=30,
+                                      logger=self.logger)
             broker.reclaim(now - self.reclaim_age,
                            now - (self.reclaim_age * 2))
             info = broker.get_replication_info()

swift/container/server.py

@@ -153,6 +153,8 @@ class ContainerController(BaseStorageServer):
                                 'be ignored in a future release.')
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
+        swift.common.db.QUERY_LOGGING = \
+            config_true_value(conf.get('db_query_logging', 'f'))
         self.sync_store = ContainerSyncStore(self.root,
                                              self.logger,
                                              self.mount_check)