openstack/swift
Code Issues Proposed changes

Add option for debug query logging

Browse Source 
Change-Id: Ic16b505a37748f50dc155212671efb45e2c5051f
This commit is contained in:
Clay Gerrard 2019-10-25 12:30:13 -05:00
parent 739c370410
commit e7cd8df5e9
6 changed files with 28 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/account-server.conf-sample
View File

@ -57,6 +57,9 @@ bind_port = 6202
# on to preallocate disk space with SQLite databases to decrease fragmentation.
# db_preallocation = off
#
# Enable this option to log all sqlite3 queries (requires python >=3.3)
# db_query_logging = off
#
# eventlet_debug = false
#
# You can set fallocate_reserve to the number of bytes or percentage of disk

3
etc/container-server.conf-sample
View File

@ -63,6 +63,9 @@ bind_port = 6201
# on to preallocate disk space with SQLite databases to decrease fragmentation.
# db_preallocation = off
#
# Enable this option to log all sqlite3 queries (requires python >=3.3)
# db_query_logging = off
#
# eventlet_debug = false
#
# You can set fallocate_reserve to the number of bytes or percentage of disk

2
swift/account/server.py
View File

@ -62,6 +62,8 @@ class AccountController(BaseStorageServer):
conf.get('auto_create_account_prefix') or '.'
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
swift.common.db.QUERY_LOGGING = \
config_true_value(conf.get('db_query_logging', 'f'))
self.fallocate_reserve, self.fallocate_is_percent = \
config_fallocate_value(conf.get('fallocate_reserve', '1%'))

19
swift/common/db.py
View File

@ -43,6 +43,8 @@ from swift.common.swob import HTTPBadRequest
#: Whether calls will be made to preallocate disk space for database files.
DB_PREALLOCATION = False
#: Whether calls will be made to log queries (py3 only)
QUERY_LOGGING = False
#: Timeout for trying to connect to a DB
BROKER_TIMEOUT = 25
#: Pickle protocol to use
@ -181,7 +183,7 @@ def chexor(old, name, timestamp):
return '%032x' % (int(old, 16) ^ int(new, 16))
def get_db_connection(path, timeout=30, okay_to_create=False):
def get_db_connection(path, timeout=30, logger=None, okay_to_create=False):
"""
Returns a properly configured SQLite database connection.
@ -194,6 +196,8 @@ def get_db_connection(path, timeout=30, okay_to_create=False):
connect_time = time.time()
conn = sqlite3.connect(path, check_same_thread=False,
factory=GreenDBConnection, timeout=timeout)
if QUERY_LOGGING and logger and not six.PY2:
conn.set_trace_callback(logger.debug)
if path != ':memory:' and not okay_to_create:
# attempt to detect and fail when connect creates the db file
stat = os.stat(path)
@ -272,13 +276,15 @@ class DatabaseBroker(object):
"""
if self._db_file == ':memory:':
tmp_db_file = None
conn = get_db_connection(self._db_file, self.timeout)
conn = get_db_connection(self._db_file, self.timeout, self.logger)
else:
mkdirs(self.db_dir)
fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
os.close(fd)
conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
factory=GreenDBConnection, timeout=0)
if QUERY_LOGGING and not six.PY2:
conn.set_trace_callback(self.logger.debug)
# creating dbs implicitly does a lot of transactions, so we
# pick fast, unsafe options here and do a big fsync at the end.
with closing(conn.cursor()) as cur:
@ -339,7 +345,8 @@ class DatabaseBroker(object):
# of the system were "racing" each other.
raise DatabaseAlreadyExists(self.db_file)
renamer(tmp_db_file, self.db_file)
self.conn = get_db_connection(self.db_file, self.timeout)
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
else:
self.conn = conn
@ -442,7 +449,8 @@ class DatabaseBroker(object):
if not self.conn:
if self.db_file != ':memory:' and os.path.exists(self.db_file):
try:
self.conn = get_db_connection(self.db_file, self.timeout)
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
except (sqlite3.DatabaseError, DatabaseConnectionError):
self.possibly_quarantine(*sys.exc_info())
else:
@ -468,7 +476,8 @@ class DatabaseBroker(object):
"""Use with the "with" statement; locks a database."""
if not self.conn:
if self.db_file != ':memory:' and os.path.exists(self.db_file):
self.conn = get_db_connection(self.db_file, self.timeout)
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
else:
raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
conn = self.conn

5
swift/common/db_replicator.py
View File

@ -217,6 +217,8 @@ class Replicator(Daemon):
self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
swift.common.db.QUERY_LOGGING = \
config_true_value(conf.get('db_query_logging', 'f'))
self._zero_stats()
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
@ -579,7 +581,8 @@ class Replicator(Daemon):
shouldbehere = True
responses = []
try:
broker = self.brokerclass(object_file, pending_timeout=30)
broker = self.brokerclass(object_file, pending_timeout=30,
logger=self.logger)
broker.reclaim(now - self.reclaim_age,
now - (self.reclaim_age * 2))
info = broker.get_replication_info()

2
swift/container/server.py
View File

@ -125,6 +125,8 @@ class ContainerController(BaseStorageServer):
'be ignored in a future release.')
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
swift.common.db.QUERY_LOGGING = \
config_true_value(conf.get('db_query_logging', 'f'))
self.sync_store = ContainerSyncStore(self.root,
self.logger,
self.mount_check)