openstack/swift
Code Issues Proposed changes

Show each policy's information on quarantined files in recon

Browse Source 
After the release of Swift ver. 2.0.0, some recon responses do not
show each policy's information yet. To make things worse, some recon
results only count on policy-0's score, therefore the total is not
shown in the recon results.

This patch makes the count of quarantined files policy-aware for recon
requests. Suppose a number of quarantined objects for policy-0 is 2
and a number for policy-1 is 3, recon sums up every policy's amount
and shows information for each policy as follows.

$ curl http://<host>:<port>/recon/quarantined
{"accounts": 0, "containers": 0, "objects": 5, "policies": {"0":
{"objects": 2}, "1": {"objects": 3}}}

Moreover, this patch adds stats for each policy in CLI output.

Change-Id: I07217c635f6fc4ea809ddbc3d859c4e81c4fde37
Related-Bug: 1375327
Related-Bug: 1375332
This commit is contained in:
Daisuke Morita 2014-11-28 20:19:02 +09:00
parent c3dadd94e0
commit f8fa1a9234
4 changed files with 84 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
swift/cli/recon.py
View File

@ -692,6 +692,7 @@ class SwiftRecon(object):
objq = {}
conq = {}
acctq = {}
stats = {}
recon = Scout("quarantined", self.verbose, self.suppress_errors,
self.timeout)
print("[%s] Checking quarantine" % self._ptime())
@ -700,7 +701,12 @@ class SwiftRecon(object):
objq[url] = response['objects']
conq[url] = response['containers']
acctq[url] = response['accounts']
stats = {"objects": objq, "containers": conq, "accounts": acctq}
if response['policies']:
for key in response['policies']:
pkey = "objects_%s" % key
stats.setdefault(pkey, {})
stats[pkey][url] = response['policies'][key]['objects']
stats.update({"objects": objq, "containers": conq, "accounts": acctq})
for item in stats:
if len(stats[item]) > 0:
computed = self._gen_stats(stats[item].values(),

23
swift/common/middleware/recon.py
View File

@ -266,15 +266,28 @@ class ReconMiddleware(object):
def get_quarantine_count(self):
"""get obj/container/account quarantine counts"""
qcounts = {"objects": 0, "containers": 0, "accounts": 0}
qcounts = {"objects": 0, "containers": 0, "accounts": 0,
"policies": {}}
qdir = "quarantined"
for device in os.listdir(self.devices):
for qtype in qcounts:
qtgt = os.path.join(self.devices, device, qdir, qtype)
if os.path.exists(qtgt):
qpath = os.path.join(self.devices, device, qdir)
if os.path.exists(qpath):
for qtype in os.listdir(qpath):
qtgt = os.path.join(qpath, qtype)
linkcount = os.lstat(qtgt).st_nlink
if linkcount > 2:
qcounts[qtype] += linkcount - 2
if qtype.startswith('objects'):
if '-' in qtype:
pkey = qtype.split('-', 1)[1]
else:
pkey = '0'
qcounts['policies'].setdefault(pkey,
{'objects': 0})
qcounts['policies'][pkey]['objects'] \
+= linkcount - 2
qcounts['objects'] += linkcount - 2
else:
qcounts[qtype] += linkcount - 2
return qcounts
def get_socket_info(self, openr=open):

50
test/unit/cli/test_recon.py
View File

@ -18,6 +18,7 @@ import json
import mock
import os
import random
import re
import string
from StringIO import StringIO
import tempfile
@ -211,6 +212,55 @@ class TestRecon(unittest.TestCase):
for ring in ('account', 'container', 'object', 'object-1'):
os.remove(os.path.join(self.swift_dir, "%s.ring.gz" % ring))
def test_quarantine_check(self):
hosts = [('127.0.0.1', 6010), ('127.0.0.1', 6020),
('127.0.0.1', 6030), ('127.0.0.1', 6040)]
# sample json response from http://<host>:<port>/recon/quarantined
responses = {6010: {'accounts': 0, 'containers': 0, 'objects': 1,
'policies': {'0': {'objects': 0},
'1': {'objects': 1}}},
6020: {'accounts': 1, 'containers': 1, 'objects': 3,
'policies': {'0': {'objects': 1},
'1': {'objects': 2}}},
6030: {'accounts': 2, 'containers': 2, 'objects': 5,
'policies': {'0': {'objects': 2},
'1': {'objects': 3}}},
6040: {'accounts': 3, 'containers': 3, 'objects': 7,
'policies': {'0': {'objects': 3},
'1': {'objects': 4}}}}
# <low> <high> <avg> <total> <Failed> <no_result> <reported>
expected = {'objects_0': (0, 3, 1.5, 6, 0.0, 0, 4),
'objects_1': (1, 4, 2.5, 10, 0.0, 0, 4),
'objects': (1, 7, 4.0, 16, 0.0, 0, 4),
'accounts': (0, 3, 1.5, 6, 0.0, 0, 4),
'containers': (0, 3, 1.5, 6, 0.0, 0, 4)}
def mock_scout_quarantine(app, host):
url = 'http://%s:%s/recon/quarantined' % host
response = responses[host[1]]
status = 200
return url, response, status
stdout = StringIO()
patches = [
mock.patch('swift.cli.recon.Scout.scout', mock_scout_quarantine),
mock.patch('sys.stdout', new=stdout),
]
with nested(*patches):
self.recon_instance.quarantine_check(hosts)
output = stdout.getvalue()
r = re.compile("\[quarantined_(.*)\](.*)")
for line in output.splitlines():
m = r.match(line)
if m:
ex = expected.pop(m.group(1))
self.assertEquals(m.group(2),
" low: %s, high: %s, avg: %s, total: %s,"
" Failed: %s%%, no_result: %s, reported: %s"
% ex)
self.assertFalse(expected)
class TestReconCommands(unittest.TestCase):
def setUp(self):

12
test/unit/common/middleware/test_recon.py
View File

@ -794,7 +794,7 @@ class TestReconSuccess(TestCase):
self.assertEquals(rv, du_resp)
def test_get_quarantine_count(self):
self.mockos.ls_output = ['sda']
dirs = [['sda'], ['accounts', 'containers', 'objects', 'objects-1']]
self.mockos.ismount_output = True
def fake_lstat(*args, **kwargs):
@ -806,10 +806,16 @@ class TestReconSuccess(TestCase):
def fake_exists(*args, **kwargs):
return True
def fake_listdir(*args, **kwargs):
return dirs.pop(0)
with mock.patch("os.lstat", fake_lstat):
with mock.patch("os.path.exists", fake_exists):
rv = self.app.get_quarantine_count()
self.assertEquals(rv, {'objects': 2, 'accounts': 2, 'containers': 2})
with mock.patch("os.listdir", fake_listdir):
rv = self.app.get_quarantine_count()
self.assertEquals(rv, {'objects': 4, 'accounts': 2, 'policies':
{'1': {'objects': 2}, '0': {'objects': 2}},
'containers': 2})
def test_get_socket_info(self):
sockstat_content = ['sockets: used 271',