swift-[account|container]-info when disk is full

Extended the use of the DatabaseBroker "stale_reads_ok" flag to the
AccountBroker and ContainerBroker.  Now checks for an sqlite3 error
from the _commit_puts call that processes the pending files.

If this error is raised, then the stale_reads_ok flag will be checked
to determine how to proceed as opposed to simply raising.

The first time that print_info is attempted, the flag will be
false, but swift-[account|container]-info will check for the
raised exception.  If it was raised, then a warning is reported
that the data may be stale, and another attempt will be
made using the stale_reads_ok=True flag.

Change-Id: I761526eef62327888c865d87a9caafa3e7eabab6
Closes-Bug: 1531302

bin/swift-account-info

@@ -11,12 +11,28 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import sqlite3
 import sys
 from optparse import OptionParser
 
 from swift.cli.info import print_info, InfoSystemExit
 
 
+def run_print_info(args, opts):
+    try:
+        print_info('account', *args, **opts)
+    except InfoSystemExit:
+        sys.exit(1)
+    except sqlite3.OperationalError as e:
+        if not opts.get('stale_reads_ok'):
+            opts['stale_reads_ok'] = True
+            print('Warning: Possibly Stale Data')
+            run_print_info(args, opts)
+            sys.exit(2)
+        else:
+            print('Account info failed: %s' % e)
+            sys.exit(1)
+
 if __name__ == '__main__':
     parser = OptionParser('%prog [options] ACCOUNT_DB_FILE')
     parser.add_option(
@@ -28,7 +44,4 @@ if __name__ == '__main__':
     if len(args) != 1:
         sys.exit(parser.print_help())
 
-    try:
+    run_print_info(args, vars(options))
-        print_info('account', *args, **vars(options))
-    except InfoSystemExit:
-        sys.exit(1)

bin/swift-container-info

@@ -11,12 +11,28 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import sqlite3
 import sys
 from optparse import OptionParser
 
 from swift.cli.info import print_info, InfoSystemExit
 
 
+def run_print_info(args, opts):
+    try:
+        print_info('container', *args, **opts)
+    except InfoSystemExit:
+        sys.exit(1)
+    except sqlite3.OperationalError as e:
+        if not opts.get('stale_reads_ok'):
+            opts['stale_reads_ok'] = True
+            print('Warning: Possibly Stale Data')
+            run_print_info(args, opts)
+            sys.exit(2)
+        else:
+            print('Container info failed: %s' % e)
+            sys.exit(1)
+
 if __name__ == '__main__':
     parser = OptionParser('%prog [options] CONTAINER_DB_FILE')
     parser.add_option(
@@ -28,7 +44,4 @@ if __name__ == '__main__':
     if len(args) != 1:
         sys.exit(parser.print_help())
 
-    try:
+    run_print_info(args, vars(options))
-        print_info('container', *args, **vars(options))
-    except InfoSystemExit:
-        sys.exit(1)

swift/cli/info.py

@@ -308,7 +308,7 @@ def print_obj_metadata(metadata):
     print_metadata('Other Metadata:', other_metadata)
 
 
-def print_info(db_type, db_file, swift_dir='/etc/swift'):
+def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False):
     if db_type not in ('account', 'container'):
         print("Unrecognized DB type: internal error")
         raise InfoSystemExit()
@@ -318,10 +318,10 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'):
     if not db_file.startswith(('/', './')):
         db_file = './' + db_file  # don't break if the bare db file is given
     if db_type == 'account':
-        broker = AccountBroker(db_file)
+        broker = AccountBroker(db_file, stale_reads_ok=stale_reads_ok)
         datadir = ABDATADIR
     else:
-        broker = ContainerBroker(db_file)
+        broker = ContainerBroker(db_file, stale_reads_ok=stale_reads_ok)
         datadir = CBDATADIR
     try:
         info = broker.get_info()

swift/common/db.py

@@ -628,7 +628,7 @@ class DatabaseBroker(object):
             with lock_parent_directory(self.pending_file,
                                        self.pending_timeout):
                 self._commit_puts()
-        except LockTimeout:
+        except (LockTimeout, sqlite3.OperationalError):
             if not self.stale_reads_ok:
                 raise
 

test/unit/account/test_backend.py

@@ -793,15 +793,14 @@ class TestAccountBroker(unittest.TestCase):
         self.assertEqual(items_by_name['b']['object_count'], 0)
         self.assertEqual(items_by_name['b']['bytes_used'], 0)
 
-    def test_load_old_pending_puts(self):
+    @with_tempdir
+    def test_load_old_pending_puts(self, tempdir):
         # pending puts from pre-storage-policy account brokers won't contain
         # the storage policy index
-        tempdir = mkdtemp()
         broker_path = os.path.join(tempdir, 'test-load-old.db')
-        try:
         broker = AccountBroker(broker_path, account='real')
         broker.initialize(Timestamp(1).internal)
-            with open(broker_path + '.pending', 'a+b') as pending:
+        with open(broker.pending_file, 'a+b') as pending:
             pending.write(':')
             pending.write(pickle.dumps(
                 # name, put_timestamp, delete_timestamp, object_count,
@@ -818,8 +817,55 @@ class TestAccountBroker(unittest.TestCase):
         self.assertEqual(len(results), 1)
         self.assertEqual(dict(results[0]),
                          {'name': 'oldcon', 'storage_policy_index': 0})
-        finally:
+
-            rmtree(tempdir)
+    @with_tempdir
+    def test_get_info_stale_read_ok(self, tempdir):
+        # test getting a stale read from the db
+        broker_path = os.path.join(tempdir, 'test-load-old.db')
+
+        def mock_commit_puts():
+            raise sqlite3.OperationalError('unable to open database file')
+
+        broker = AccountBroker(broker_path, account='real',
+                               stale_reads_ok=True)
+        broker.initialize(Timestamp(1).internal)
+        with open(broker.pending_file, 'a+b') as pending:
+            pending.write(':')
+            pending.write(pickle.dumps(
+                # name, put_timestamp, delete_timestamp, object_count,
+                # bytes_used, deleted
+                ('oldcon', Timestamp(200).internal,
+                 Timestamp(0).internal,
+                 896, 9216695, 0)).encode('base64'))
+
+        broker._commit_puts = mock_commit_puts
+        broker.get_info()
+
+    @with_tempdir
+    def test_get_info_no_stale_reads(self, tempdir):
+        broker_path = os.path.join(tempdir, 'test-load-old.db')
+
+        def mock_commit_puts():
+            raise sqlite3.OperationalError('unable to open database file')
+
+        broker = AccountBroker(broker_path, account='real',
+                               stale_reads_ok=False)
+        broker.initialize(Timestamp(1).internal)
+        with open(broker.pending_file, 'a+b') as pending:
+            pending.write(':')
+            pending.write(pickle.dumps(
+                # name, put_timestamp, delete_timestamp, object_count,
+                # bytes_used, deleted
+                ('oldcon', Timestamp(200).internal,
+                 Timestamp(0).internal,
+                 896, 9216695, 0)).encode('base64'))
+
+        broker._commit_puts = mock_commit_puts
+
+        with self.assertRaises(sqlite3.OperationalError) as exc_context:
+            broker.get_info()
+        self.assertIn('unable to open database file',
+                      str(exc_context.exception))
 
     @patch_policies([StoragePolicy(0, 'zero', False),
                      StoragePolicy(1, 'one', True),

test/unit/container/test_backend.py

@@ -1679,6 +1679,63 @@ class TestContainerBroker(unittest.TestCase):
         }
         self.assertEqual(broker.get_policy_stats(), expected)
 
+    @with_tempdir
+    def test_get_info_no_stale_reads(self, tempdir):
+        ts = (Timestamp(t).internal for t in
+              itertools.count(int(time())))
+        db_path = os.path.join(tempdir, 'container.db')
+
+        def mock_commit_puts():
+            raise sqlite3.OperationalError('unable to open database file')
+
+        broker = ContainerBroker(db_path, account='a', container='c',
+                                 stale_reads_ok=False)
+        broker.initialize(next(ts), 1)
+
+        # manually make some pending entries
+        with open(broker.pending_file, 'a+b') as fp:
+            for i in range(10):
+                name, timestamp, size, content_type, etag, deleted = (
+                    'o%s' % i, next(ts), 0, 'c', 'e', 0)
+                fp.write(':')
+                fp.write(pickle.dumps(
+                    (name, timestamp, size, content_type, etag, deleted),
+                    protocol=2).encode('base64'))
+                fp.flush()
+
+        broker._commit_puts = mock_commit_puts
+        with self.assertRaises(sqlite3.OperationalError) as exc_context:
+            broker.get_info()
+        self.assertIn('unable to open database file',
+                      str(exc_context.exception))
+
+    @with_tempdir
+    def test_get_info_stale_read_ok(self, tempdir):
+        ts = (Timestamp(t).internal for t in
+              itertools.count(int(time())))
+        db_path = os.path.join(tempdir, 'container.db')
+
+        def mock_commit_puts():
+            raise sqlite3.OperationalError('unable to open database file')
+
+        broker = ContainerBroker(db_path, account='a', container='c',
+                                 stale_reads_ok=True)
+        broker.initialize(next(ts), 1)
+
+        # manually make some pending entries
+        with open(broker.pending_file, 'a+b') as fp:
+            for i in range(10):
+                name, timestamp, size, content_type, etag, deleted = (
+                    'o%s' % i, next(ts), 0, 'c', 'e', 0)
+                fp.write(':')
+                fp.write(pickle.dumps(
+                    (name, timestamp, size, content_type, etag, deleted),
+                    protocol=2).encode('base64'))
+                fp.flush()
+
+        broker._commit_puts = mock_commit_puts
+        broker.get_info()
+
 
 class TestCommonContainerBroker(test_db.TestExampleBroker):