Show each policy's information on quarantined files in recon

After the release of Swift ver. 2.0.0, some recon responses do not
show each policy's information yet. To make things worse, some recon
results only count on policy-0's score, therefore the total is not
shown in the recon results.

This patch makes the count of quarantined files policy-aware for recon
requests. Suppose a number of quarantined objects for policy-0 is 2
and a number for policy-1 is 3, recon sums up every policy's amount
and shows information for each policy as follows.

$ curl http://<host>:<port>/recon/quarantined
{"accounts": 0, "containers": 0, "objects": 5, "policies": {"0":
{"objects": 2}, "1": {"objects": 3}}}

Moreover, this patch adds stats for each policy in CLI output.

Change-Id: I07217c635f6fc4ea809ddbc3d859c4e81c4fde37
Related-Bug: 1375327
Related-Bug: 1375332

swift/cli/recon.py

@@ -692,6 +692,7 @@ class SwiftRecon(object):
         objq = {}
         conq = {}
         acctq = {}
+        stats = {}
         recon = Scout("quarantined", self.verbose, self.suppress_errors,
                       self.timeout)
         print("[%s] Checking quarantine" % self._ptime())
@@ -700,7 +701,12 @@ class SwiftRecon(object):
                 objq[url] = response['objects']
                 conq[url] = response['containers']
                 acctq[url] = response['accounts']
-        stats = {"objects": objq, "containers": conq, "accounts": acctq}
+                if response['policies']:
+                    for key in response['policies']:
+                        pkey = "objects_%s" % key
+                        stats.setdefault(pkey, {})
+                        stats[pkey][url] = response['policies'][key]['objects']
+        stats.update({"objects": objq, "containers": conq, "accounts": acctq})
         for item in stats:
             if len(stats[item]) > 0:
                 computed = self._gen_stats(stats[item].values(),

swift/common/middleware/recon.py

@@ -266,15 +266,28 @@ class ReconMiddleware(object):
 
     def get_quarantine_count(self):
         """get obj/container/account quarantine counts"""
-        qcounts = {"objects": 0, "containers": 0, "accounts": 0}
+        qcounts = {"objects": 0, "containers": 0, "accounts": 0,
+                   "policies": {}}
         qdir = "quarantined"
         for device in os.listdir(self.devices):
-            for qtype in qcounts:
-                qtgt = os.path.join(self.devices, device, qdir, qtype)
-                if os.path.exists(qtgt):
+            qpath = os.path.join(self.devices, device, qdir)
+            if os.path.exists(qpath):
+                for qtype in os.listdir(qpath):
+                    qtgt = os.path.join(qpath, qtype)
                     linkcount = os.lstat(qtgt).st_nlink
                     if linkcount > 2:
-                        qcounts[qtype] += linkcount - 2
+                        if qtype.startswith('objects'):
+                            if '-' in qtype:
+                                pkey = qtype.split('-', 1)[1]
+                            else:
+                                pkey = '0'
+                            qcounts['policies'].setdefault(pkey,
+                                                           {'objects': 0})
+                            qcounts['policies'][pkey]['objects'] \
+                                += linkcount - 2
+                            qcounts['objects'] += linkcount - 2
+                        else:
+                            qcounts[qtype] += linkcount - 2
         return qcounts
 
     def get_socket_info(self, openr=open):

test/unit/cli/test_recon.py

@@ -18,6 +18,7 @@ import json
 import mock
 import os
 import random
+import re
 import string
 from StringIO import StringIO
 import tempfile
@@ -211,6 +212,55 @@ class TestRecon(unittest.TestCase):
         for ring in ('account', 'container', 'object', 'object-1'):
             os.remove(os.path.join(self.swift_dir, "%s.ring.gz" % ring))
 
+    def test_quarantine_check(self):
+        hosts = [('127.0.0.1', 6010), ('127.0.0.1', 6020),
+                 ('127.0.0.1', 6030), ('127.0.0.1', 6040)]
+        # sample json response from http://<host>:<port>/recon/quarantined
+        responses = {6010: {'accounts': 0, 'containers': 0, 'objects': 1,
+                            'policies': {'0': {'objects': 0},
+                                         '1': {'objects': 1}}},
+                     6020: {'accounts': 1, 'containers': 1, 'objects': 3,
+                            'policies': {'0': {'objects': 1},
+                                         '1': {'objects': 2}}},
+                     6030: {'accounts': 2, 'containers': 2, 'objects': 5,
+                            'policies': {'0': {'objects': 2},
+                                         '1': {'objects': 3}}},
+                     6040: {'accounts': 3, 'containers': 3, 'objects': 7,
+                            'policies': {'0': {'objects': 3},
+                                         '1': {'objects': 4}}}}
+        # <low> <high> <avg> <total> <Failed> <no_result> <reported>
+        expected = {'objects_0': (0, 3, 1.5, 6, 0.0, 0, 4),
+                    'objects_1': (1, 4, 2.5, 10, 0.0, 0, 4),
+                    'objects': (1, 7, 4.0, 16, 0.0, 0, 4),
+                    'accounts': (0, 3, 1.5, 6, 0.0, 0, 4),
+                    'containers': (0, 3, 1.5, 6, 0.0, 0, 4)}
+
+        def mock_scout_quarantine(app, host):
+            url = 'http://%s:%s/recon/quarantined' % host
+            response = responses[host[1]]
+            status = 200
+            return url, response, status
+
+        stdout = StringIO()
+        patches = [
+            mock.patch('swift.cli.recon.Scout.scout', mock_scout_quarantine),
+            mock.patch('sys.stdout', new=stdout),
+        ]
+        with nested(*patches):
+            self.recon_instance.quarantine_check(hosts)
+
+        output = stdout.getvalue()
+        r = re.compile("\[quarantined_(.*)\](.*)")
+        for line in output.splitlines():
+            m = r.match(line)
+            if m:
+                ex = expected.pop(m.group(1))
+                self.assertEquals(m.group(2),
+                                  " low: %s, high: %s, avg: %s, total: %s,"
+                                  " Failed: %s%%, no_result: %s, reported: %s"
+                                  % ex)
+        self.assertFalse(expected)
+
 
 class TestReconCommands(unittest.TestCase):
     def setUp(self):

test/unit/common/middleware/test_recon.py

@@ -794,7 +794,7 @@ class TestReconSuccess(TestCase):
         self.assertEquals(rv, du_resp)
 
     def test_get_quarantine_count(self):
-        self.mockos.ls_output = ['sda']
+        dirs = [['sda'], ['accounts', 'containers', 'objects', 'objects-1']]
         self.mockos.ismount_output = True
 
         def fake_lstat(*args, **kwargs):
@@ -806,10 +806,16 @@ class TestReconSuccess(TestCase):
         def fake_exists(*args, **kwargs):
             return True
 
+        def fake_listdir(*args, **kwargs):
+            return dirs.pop(0)
+
         with mock.patch("os.lstat", fake_lstat):
             with mock.patch("os.path.exists", fake_exists):
-                rv = self.app.get_quarantine_count()
-        self.assertEquals(rv, {'objects': 2, 'accounts': 2, 'containers': 2})
+                with mock.patch("os.listdir", fake_listdir):
+                    rv = self.app.get_quarantine_count()
+        self.assertEquals(rv, {'objects': 4, 'accounts': 2, 'policies':
+                               {'1': {'objects': 2}, '0': {'objects': 2}},
+                               'containers': 2})
 
     def test_get_socket_info(self):
         sockstat_content = ['sockets: used 271',