cli: add --sync to db info to show syncs

When looking at containers and accounts it's sometimes nice to know who
they've been replicating with. This patch adds a `--sync|-s` option to
swift-{container|account}-info which will also dump the incoming and
outgoing sync tables:

  $ swift-container-info /srv/node3/sdb3/containers/294/624/49b9ff074c502ec5e429e7af99a30624/49b9ff074c502ec5e429e7af99a30624.db -s
  Path: /AUTH_test/new
    Account: AUTH_test
    Container: new
    Deleted: False
    Container Hash: 49b9ff074c502ec5e429e7af99a30624
  Metadata:
    Created at: 2022-02-16T05:34:05.988480 (1644989645.98848)
    Put Timestamp: 2022-02-16T05:34:05.981320 (1644989645.98132)
    Delete Timestamp: 1970-01-01T00:00:00.000000 (0)
    Status Timestamp: 2022-02-16T05:34:05.981320 (1644989645.98132)
    Object Count: 1
    Bytes Used: 7
    Storage Policy: default (0)
    Reported Put Timestamp: 1970-01-01T00:00:00.000000 (0)
    Reported Delete Timestamp: 1970-01-01T00:00:00.000000 (0)
    Reported Object Count: 0
    Reported Bytes Used: 0
    Chexor: 962368324c2ca023c56669d03ed92807
    UUID: f33184e7-56d5-4c74-9d2e-5417c187d722-sdb3
    X-Container-Sync-Point2: -1
    X-Container-Sync-Point1: -1
  No system metadata found in db file
  No user metadata found in db file
  Sharding Metadata:
    Type: root
    State: unsharded
  Incoming Syncs:
    Sync Point	Remote ID                                	Updated At
    1         	ce7268a1-f5d0-4b83-b993-af17b602a0ff-sdb1	2022-02-16T05:38:22.000000 (1644989902)
    1         	2af5abc0-7f70-4e2f-8f94-737aeaada7f4-sdb4	2022-02-16T05:38:22.000000 (1644989902)
  Outgoing Syncs:
    Sync Point	Remote ID	Updated At
  Partition	294
  Hash     	49b9ff074c502ec5e429e7af99a30624

As a follow up to the device in DB ID patch we can see that the replicas
at sdb1 and sdb4 have replicated with this node.

Change-Id: I23d786e82c6710bea7660a9acf8bbbd113b5b727

bin/swift-container-info

@@ -47,6 +47,9 @@ if __name__ == '__main__':
         '-v', '--verbose', default=False, action="store_true",
         help="Show all shard ranges. By default, only the number of shard "
              "ranges is displayed if there are many shards.")
+    parser.add_option(
+        '--sync', '-s', default=False, action="store_true",
+        help="Output the contents of the incoming/outging sync tables")
 
     options, args = parser.parse_args()
 

swift/cli/info.py

@@ -198,6 +198,28 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
           'real value is set in the config file on each storage node.')
 
 
+def get_max_len_sync_item(syncs, item, title):
+    def map_func(element):
+        return str(element[item])
+    return max(list(map(len, map(map_func, syncs))) + [len(title)])
+
+
+def print_db_syncs(incoming, syncs):
+    max_sync_point_len = get_max_len_sync_item(syncs, 'sync_point',
+                                               "Sync Point")
+    max_remote_len = get_max_len_sync_item(syncs, 'remote_id', "Remote ID")
+    print('%s Syncs:' % ('Incoming' if incoming else 'Outgoing'))
+    print('  %s\t%s\t%s' % ("Sync Point".ljust(max_sync_point_len),
+                            "Remote ID".ljust(max_remote_len),
+                            "Updated At"))
+    for sync in syncs:
+        print('  %s\t%s\t%s (%s)' % (
+            str(sync['sync_point']).ljust(max_sync_point_len),
+            sync['remote_id'].ljust(max_remote_len),
+            Timestamp(sync['updated_at']).isoformat,
+            sync['updated_at']))
+
+
 def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False,
                            verbose=False):
     """
@@ -439,7 +461,7 @@ def print_obj_metadata(metadata, drop_prefixes=False):
 
 
 def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
-               drop_prefixes=False, verbose=False):
+               drop_prefixes=False, verbose=False, sync=False):
     if db_type not in ('account', 'container'):
         print("Unrecognized DB type: internal error")
         raise InfoSystemExit()
@@ -473,6 +495,11 @@ def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
             info['shard_ranges'] = sranges
     print_db_info_metadata(
         db_type, info, broker.metadata, drop_prefixes, verbose)
+    if sync:
+        # Print incoming / outgoing sync tables.
+        for incoming in (True, False):
+            print_db_syncs(incoming, broker.get_syncs(incoming,
+                                                      include_timestamp=True))
     try:
         ring = Ring(swift_dir, ring_name=db_type)
     except Exception:

swift/common/db.py

@@ -724,22 +724,26 @@ class DatabaseBroker(object):
                 return -1
             return row['sync_point']
 
-    def get_syncs(self, incoming=True):
+    def get_syncs(self, incoming=True, include_timestamp=False):
         """
         Get a serialized copy of the sync table.
 
         :param incoming: if True, get the last incoming sync, otherwise get
                          the last outgoing sync
-        :returns: list of {'remote_id', 'sync_point'}
+        :param include_timestamp: If True include the updated_at timestamp
+        :returns: list of {'remote_id', 'sync_point'} or
+                 {'remote_id', 'sync_point', 'updated_at'}
+                 if include_timestamp is True.
         """
         with self.get() as conn:
+            columns = 'remote_id, sync_point'
+            if include_timestamp:
+                columns += ', updated_at'
             curs = conn.execute('''
-                SELECT remote_id, sync_point FROM %s_sync
+                SELECT %s FROM %s_sync
-            ''' % ('incoming' if incoming else 'outgoing'))
+            ''' % (columns, 'incoming' if incoming else 'outgoing'))
-            result = []
+            curs.row_factory = dict_factory
-            for row in curs:
+            return [r for r in curs]
-                result.append({'remote_id': row[0], 'sync_point': row[1]})
-            return result
 
     def get_max_row(self, table=None):
         if not table:

test/unit/cli/test_info.py

@@ -29,7 +29,7 @@ from swift.common.storage_policy import StoragePolicy, POLICIES
 from swift.cli.info import (print_db_info_metadata, print_ring_locations,
                             print_info, print_obj_metadata, print_obj,
                             InfoSystemExit, print_item_locations,
-                            parse_get_node_args)
+                            parse_get_node_args, print_db_syncs)
 from swift.account.server import AccountController
 from swift.container.server import ContainerController
 from swift.container.backend import UNSHARDED, SHARDED
@@ -666,6 +666,40 @@ Shard Ranges (3):
         self.assertIn(exp_cont_msg, out.getvalue())
         self.assertIn(exp_obj_msg, out.getvalue())
 
+    def test_print_db_syncs(self):
+        # first the empty case
+        for incoming in (True, False):
+            out = StringIO()
+            with mock.patch('sys.stdout', out):
+                print_db_syncs(incoming, [])
+            if incoming:
+                exp_heading = 'Incoming Syncs:'
+            else:
+                exp_heading = 'Outgoing Syncs:'
+            exp_heading += '\n  Sync Point\tRemote ID\tUpdated At'
+            self.assertIn(exp_heading, out.getvalue())
+
+        # now add some syncs
+        ts0 = utils.Timestamp(1)
+        ts1 = utils.Timestamp(2)
+        syncs = [{'sync_point': 0, 'remote_id': 'remote_0',
+                  'updated_at': str(int(ts0))},
+                 {'sync_point': 1, 'remote_id': 'remote_1',
+                  'updated_at': str(int(ts1))}]
+
+        template_output = """%s:\n  Sync Point\tRemote ID\tUpdated At
+  0         \tremote_0 \t%s (%s)
+  1         \tremote_1 \t%s (%s)
+"""
+        for incoming in (True, False):
+            out = StringIO()
+            with mock.patch('sys.stdout', out):
+                print_db_syncs(incoming, syncs)
+            output = template_output % (
+                'Incoming Syncs' if incoming else 'Outgoing Syncs',
+                ts0.isoformat, str(int(ts0)), ts1.isoformat, str(int(ts1)))
+            self.assertEqual(output, out.getvalue())
+
     def test_print_item_locations_account_container_object_dashed_ring(self):
         out = StringIO()
         account = 'account'

test/unit/common/test_db.py

@@ -21,6 +21,8 @@ import unittest
 from tempfile import mkdtemp
 from shutil import rmtree, copy
 from uuid import uuid4
+
+import mock
 import six.moves.cPickle as pickle
 
 import base64
@@ -1043,6 +1045,55 @@ class TestDatabaseBroker(TestDbBase):
         self.assertEqual(broker.get_items_since(3, 2), [])
         self.assertEqual(broker.get_items_since(999, 2), [])
 
+    def test_get_syncs(self):
+        broker = DatabaseBroker(self.db_path)
+        broker.db_type = 'test'
+        broker.db_contains_type = 'test'
+        uuid1 = str(uuid4())
+
+        def _initialize(conn, timestamp, **kwargs):
+            conn.execute('CREATE TABLE test (one TEXT)')
+            conn.execute('CREATE TABLE test_stat (id TEXT)')
+            conn.execute('INSERT INTO test_stat (id) VALUES (?)', (uuid1,))
+            conn.execute('INSERT INTO test (one) VALUES ("1")')
+            conn.commit()
+            pass
+        broker._initialize = _initialize
+        broker.initialize(normalize_timestamp('1'))
+
+        for incoming in (True, False):
+            # Can't mock out timestamp now, because the update_at in the sync
+            # tables are cuase by a trigger inside sqlite which uses it's own
+            # now method. So instead track the time before and after to make
+            # sure we're getting the right timestamps.
+            ts0 = Timestamp.now()
+            broker.merge_syncs([
+                {'sync_point': 0, 'remote_id': 'remote_0'},
+                {'sync_point': 1, 'remote_id': 'remote_1'}], incoming)
+
+            time.sleep(0.005)
+            broker.merge_syncs([
+                {'sync_point': 2, 'remote_id': 'remote_2'}], incoming)
+
+            ts1 = Timestamp.now()
+            expected_syncs = [{'sync_point': 0, 'remote_id': 'remote_0'},
+                              {'sync_point': 1, 'remote_id': 'remote_1'},
+                              {'sync_point': 2, 'remote_id': 'remote_2'}]
+
+            self.assertEqual(expected_syncs, broker.get_syncs(incoming))
+
+            # if we want the updated_at timestamps too then:
+            expected_syncs[0]['updated_at'] = mock.ANY
+            expected_syncs[1]['updated_at'] = mock.ANY
+            expected_syncs[2]['updated_at'] = mock.ANY
+            actual_syncs = broker.get_syncs(incoming, include_timestamp=True)
+            self.assertEqual(expected_syncs, actual_syncs)
+            # Note that most of the time, we expect these all to be ==
+            # but we've been known to see sizeable delays in the gate at times
+            self.assertTrue(all([
+                str(int(ts0)) <= s['updated_at'] <= str(int(ts1))
+                for s in actual_syncs]))
+
     def test_get_sync(self):
         broker = DatabaseBroker(self.db_path)
         broker.db_type = 'test'