From e4c4907b321b5bf280b9e4b08562462c100ae40e Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 18 Feb 2011 08:07:03 -0800 Subject: [PATCH 1/3] Readd new index changes --- swift/common/db.py | 134 ++++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index be96411619..282a7201aa 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -165,6 +165,7 @@ class DatabaseBroker(object): self.logger = logger or logging.getLogger() self.account = account self.container = container + self._db_version = -1 def initialize(self, put_timestamp=None): """ @@ -607,7 +608,7 @@ class ContainerBroker(DatabaseBroker): conn.executescript(""" CREATE TABLE object ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT UNIQUE, + name TEXT, created_at TEXT, size INTEGER, content_type TEXT, @@ -615,7 +616,7 @@ class ContainerBroker(DatabaseBroker): deleted INTEGER DEFAULT 0 ); - CREATE INDEX ix_object_deleted ON object (deleted); + CREATE INDEX ix_object_deleted_name ON object (deleted, name); CREATE TRIGGER object_insert AFTER INSERT ON object BEGIN @@ -678,6 +679,15 @@ class ContainerBroker(DatabaseBroker): ''', (self.account, self.container, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) + def _get_db_version(self, conn): + if self._db_version == -1: + self._db_version = 0 + for row in conn.execute(''' + SELECT name FROM sqlite_master + WHERE name = 'ix_object_deleted_name' '''): + self._db_version = 1 + return self._db_version + def _newid(self, conn): conn.execute(''' UPDATE container_stat @@ -919,11 +929,6 @@ class ContainerBroker(DatabaseBroker): :returns: list of object names """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise rv = [] with self.get() as conn: row = conn.execute(''' @@ -988,7 +993,10 @@ class ContainerBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - query += ' +deleted = 0 ORDER BY name LIMIT ?' + if self._get_db_version(conn) < 1: + query += ' +deleted = 0 ORDER BY name LIMIT ?' + else: + query += ' deleted = 0 ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None @@ -1036,18 +1044,19 @@ class ContainerBroker(DatabaseBroker): max_rowid = -1 for rec in item_list: conn.execute(''' - DELETE FROM object WHERE name = ? AND - (created_at < ?) + DELETE FROM object WHERE name = ? AND created_at < ? AND + deleted IN (0, 1) ''', (rec['name'], rec['created_at'])) - try: + if not conn.execute(''' + SELECT name FROM object WHERE name = ? AND + deleted IN (0, 1) + ''', (rec['name'],)).fetchall(): conn.execute(''' INSERT INTO object (name, created_at, size, content_type, etag, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ([rec['name'], rec['created_at'], rec['size'], rec['content_type'], rec['etag'], rec['deleted']])) - except sqlite3.IntegrityError: - pass if source: max_rowid = max(max_rowid, rec['ROWID']) if source: @@ -1091,7 +1100,7 @@ class AccountBroker(DatabaseBroker): conn.executescript(""" CREATE TABLE container ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT UNIQUE, + name TEXT, put_timestamp TEXT, delete_timestamp TEXT, object_count INTEGER, @@ -1099,8 +1108,9 @@ class AccountBroker(DatabaseBroker): deleted INTEGER DEFAULT 0 ); - CREATE INDEX ix_container_deleted ON container (deleted); - CREATE INDEX ix_container_name ON container (name); + CREATE INDEX ix_container_deleted_name ON + container (deleted, name); + CREATE TRIGGER container_insert AFTER INSERT ON container BEGIN UPDATE account_stat @@ -1164,6 +1174,15 @@ class AccountBroker(DatabaseBroker): ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) + def _get_db_version(self, conn): + if self._db_version == -1: + self._db_version = 0 + for row in conn.execute(''' + SELECT name FROM sqlite_master + WHERE name = 'ix_container_deleted_name' '''): + self._db_version = 1 + return self._db_version + def update_put_timestamp(self, timestamp): """ Update the put_timestamp. Only modifies it if it is greater than @@ -1485,7 +1504,10 @@ class AccountBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - query += ' +deleted = 0 ORDER BY name LIMIT ?' + if self._get_db_version(conn) < 1: + query += ' +deleted = 0 ORDER BY name LIMIT ?' + else: + query += ' deleted = 0 ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None @@ -1529,51 +1551,39 @@ class AccountBroker(DatabaseBroker): record = [rec['name'], rec['put_timestamp'], rec['delete_timestamp'], rec['object_count'], rec['bytes_used'], rec['deleted']] - try: - conn.execute(''' - INSERT INTO container (name, put_timestamp, - delete_timestamp, object_count, bytes_used, - deleted) - VALUES (?, ?, ?, ?, ?, ?) - ''', record) - except sqlite3.IntegrityError: - curs = conn.execute(''' - SELECT name, put_timestamp, delete_timestamp, - object_count, bytes_used, deleted - FROM container WHERE name = ? AND - (put_timestamp < ? OR delete_timestamp < ? OR - object_count != ? OR bytes_used != ?)''', - (rec['name'], rec['put_timestamp'], - rec['delete_timestamp'], rec['object_count'], - rec['bytes_used'])) - curs.row_factory = None - row = curs.fetchone() - if row: - row = list(row) - for i in xrange(5): - if record[i] is None and row[i] is not None: - record[i] = row[i] - if row[1] > record[1]: # Keep newest put_timestamp - record[1] = row[1] - if row[2] > record[2]: # Keep newest delete_timestamp - record[2] = row[2] - conn.execute('DELETE FROM container WHERE name = ?', - (record[0],)) - # If deleted, mark as such - if record[2] > record[1] and \ - record[3] in (None, '', 0, '0'): - record[5] = 1 - else: - record[5] = 0 - try: - conn.execute(''' - INSERT INTO container (name, put_timestamp, - delete_timestamp, object_count, bytes_used, - deleted) - VALUES (?, ?, ?, ?, ?, ?) - ''', record) - except sqlite3.IntegrityError: - continue + curs = conn.execute(''' + SELECT name, put_timestamp, delete_timestamp, + object_count, bytes_used, deleted + FROM container WHERE name = ? AND + deleted IN (0, 1) + ''', (rec['name'],)) + curs.row_factory = None + row = curs.fetchone() + if row: + row = list(row) + for i in xrange(5): + if record[i] is None and row[i] is not None: + record[i] = row[i] + if row[1] > record[1]: # Keep newest put_timestamp + record[1] = row[1] + if row[2] > record[2]: # Keep newest delete_timestamp + record[2] = row[2] + # If deleted, mark as such + if record[2] > record[1] and \ + record[3] in (None, '', 0, '0'): + record[5] = 1 + else: + record[5] = 0 + conn.execute(''' + DELETE FROM container WHERE name = ? AND + deleted IN (0, 1) + ''', (record[0],)) + conn.execute(''' + INSERT INTO container (name, put_timestamp, + delete_timestamp, object_count, bytes_used, + deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', record) if source: max_rowid = max(max_rowid, rec['ROWID']) if source: From 1172f9136d680d93b17817ba6f382d9bcde1fb15 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 18 Feb 2011 08:30:08 -0800 Subject: [PATCH 2/3] Updated new index code --- swift/common/db.py | 105 ++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 78 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 282a7201aa..0182d837be 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -679,7 +679,7 @@ class ContainerBroker(DatabaseBroker): ''', (self.account, self.container, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) - def _get_db_version(self, conn): + def get_db_version(self, conn): if self._db_version == -1: self._db_version = 0 for row in conn.execute(''' @@ -920,32 +920,6 @@ class ContainerBroker(DatabaseBroker): ''', (put_timestamp, delete_timestamp, object_count, bytes_used)) conn.commit() - def get_random_objects(self, max_count=100): - """ - Get random objects from the DB. This is used by the container_auditor - when testing random objects for existence. - - :param max_count: maximum number of objects to get - - :returns: list of object names - """ - rv = [] - with self.get() as conn: - row = conn.execute(''' - SELECT ROWID FROM object ORDER BY ROWID DESC LIMIT 1 - ''').fetchone() - if not row: - return [] - max_rowid = row['ROWID'] - for _junk in xrange(min(max_count, max_rowid)): - row = conn.execute(''' - SELECT name FROM object WHERE ROWID >= ? AND +deleted = 0 - LIMIT 1 - ''', (randint(0, max_rowid),)).fetchone() - if row: - rv.append(row['name']) - return list(set(rv)) - def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None, format=None): """ @@ -993,10 +967,11 @@ class ContainerBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - if self._get_db_version(conn) < 1: - query += ' +deleted = 0 ORDER BY name LIMIT ?' + if self.get_db_version(conn) < 1: + query += ' +deleted = 0' else: - query += ' deleted = 0 ORDER BY name LIMIT ?' + query += ' deleted = 0' + query += ' ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None @@ -1043,14 +1018,17 @@ class ContainerBroker(DatabaseBroker): with self.get() as conn: max_rowid = -1 for rec in item_list: - conn.execute(''' - DELETE FROM object WHERE name = ? AND created_at < ? AND - deleted IN (0, 1) - ''', (rec['name'], rec['created_at'])) - if not conn.execute(''' - SELECT name FROM object WHERE name = ? AND - deleted IN (0, 1) - ''', (rec['name'],)).fetchall(): + query = ''' + DELETE FROM object + WHERE name = ? AND (created_at < ?) + ''' + if self.get_db_version(conn) >= 1: + query += ' AND deleted IN (0, 1)' + conn.execute(query, (rec['name'], rec['created_at'])) + query = 'SELECT name FROM object WHERE name = ?' + if self.get_db_version(conn) >= 1: + query += ' AND deleted IN (0, 1)' + if not conn.execute(query, (rec['name'],)).fetchall(): conn.execute(''' INSERT INTO object (name, created_at, size, content_type, etag, deleted) @@ -1174,7 +1152,7 @@ class AccountBroker(DatabaseBroker): ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) - def _get_db_version(self, conn): + def get_db_version(self, conn): if self._db_version == -1: self._db_version = 0 for row in conn.execute(''' @@ -1432,38 +1410,6 @@ class AccountBroker(DatabaseBroker): FROM account_stat ''').fetchone() - def get_random_containers(self, max_count=100): - """ - Get random containers from the DB. This is used by the - account_auditor when testing random containerss for existence. - - :param max_count: maximum number of containers to get - - :returns: list of container names - """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise - rv = [] - with self.get() as conn: - row = conn.execute(''' - SELECT ROWID FROM container ORDER BY ROWID DESC LIMIT 1 - ''').fetchone() - if not row: - return [] - max_rowid = row['ROWID'] - for _junk in xrange(min(max_count, max_rowid)): - row = conn.execute(''' - SELECT name FROM container WHERE - ROWID >= ? AND +deleted = 0 - LIMIT 1 - ''', (randint(0, max_rowid),)).fetchone() - if row: - rv.append(row['name']) - return list(set(rv)) - def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter): """ @@ -1504,10 +1450,11 @@ class AccountBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - if self._get_db_version(conn) < 1: - query += ' +deleted = 0 ORDER BY name LIMIT ?' + if self.get_db_version(conn) < 1: + query += ' +deleted = 0' else: - query += ' deleted = 0 ORDER BY name LIMIT ?' + query += ' deleted = 0' + query += ' ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None @@ -1551,12 +1498,14 @@ class AccountBroker(DatabaseBroker): record = [rec['name'], rec['put_timestamp'], rec['delete_timestamp'], rec['object_count'], rec['bytes_used'], rec['deleted']] - curs = conn.execute(''' + query = ''' SELECT name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted - FROM container WHERE name = ? AND - deleted IN (0, 1) - ''', (rec['name'],)) + FROM container WHERE name = ? + ''' + if self.get_db_version(conn) >= 1: + query += ' AND deleted IN (0, 1)' + curs = conn.execute(query, (rec['name'],)) curs.row_factory = None row = curs.fetchone() if row: From 0a170f268019709d76d684fbcbe5fd21f07d88e6 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 18 Feb 2011 08:43:19 -0800 Subject: [PATCH 3/3] Possibly speed improvement on existence-select --- swift/common/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/common/db.py b/swift/common/db.py index 0182d837be..693a281e20 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -1025,7 +1025,7 @@ class ContainerBroker(DatabaseBroker): if self.get_db_version(conn) >= 1: query += ' AND deleted IN (0, 1)' conn.execute(query, (rec['name'], rec['created_at'])) - query = 'SELECT name FROM object WHERE name = ?' + query = 'SELECT 1 FROM object WHERE name = ?' if self.get_db_version(conn) >= 1: query += ' AND deleted IN (0, 1)' if not conn.execute(query, (rec['name'],)).fetchall():