openstack/swift
Code Issues Proposed changes

Make scripts in bin/ PEP8-compliant.

Browse Source 
Also made tox's PEP8 check look at the scripts in bin/ to keep them
PEP8-compliant.

Change-Id: I710365ea929d7fc15578d5f742a236bad47ef28e
This commit is contained in:
Samuel Merritt 2012-05-03 18:33:05 -07:00
parent 96476846b6
commit 2ccf219ec1
7 changed files with 56 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
bin/swift-account-audit
View File

@ -56,7 +56,8 @@ class Auditor(object):
error_file=None): error_file=None):
self.pool = GreenPool(concurrency) self.pool = GreenPool(concurrency)
self.object_ring = Ring(os.path.join(swift_dir, ring_name='object')) self.object_ring = Ring(os.path.join(swift_dir, ring_name='object'))
self.container_ring = Ring(os.path.join(swift_dir, ring_name='container')) self.container_ring = \
Ring(os.path.join(swift_dir, ring_name='container'))
self.account_ring = Ring(os.path.join(swift_dir, ring_name='account')) self.account_ring = Ring(os.path.join(swift_dir, ring_name='account'))
self.deep = deep self.deep = deep
self.error_file = error_file self.error_file = error_file
@ -73,7 +74,8 @@ class Auditor(object):
def audit_object(self, account, container, name): def audit_object(self, account, container, name):
path = '/%s/%s/%s' % (account, container, name) path = '/%s/%s/%s' % (account, container, name)
part, nodes = self.object_ring.get_nodes(account, container.encode('utf-8'), name.encode('utf-8')) part, nodes = self.object_ring.get_nodes(
account, container.encode('utf-8'), name.encode('utf-8'))
container_listing = self.audit_container(account, container) container_listing = self.audit_container(account, container)
consistent = True consistent = True
if name not in container_listing: if name not in container_listing:
@ -109,7 +111,8 @@ class Auditor(object):
etags.append(resp.getheader('ETag')) etags.append(resp.getheader('ETag'))
else: else:
conn = http_connect(node['ip'], node['port'], conn = http_connect(node['ip'], node['port'],
node['device'], part, 'HEAD', path.encode('utf-8'), {}) node['device'], part, 'HEAD',
path.encode('utf-8'), {})
resp = conn.getresponse() resp = conn.getresponse()
if resp.status // 100 != 2: if resp.status // 100 != 2:
self.object_not_found += 1 self.object_not_found += 1
@ -151,7 +154,8 @@ class Auditor(object):
if name not in account_listing: if name not in account_listing:
consistent = False consistent = False
print " Container %s not in account listing!" % path print " Container %s not in account listing!" % path
part, nodes = self.container_ring.get_nodes(account, name.encode('utf-8')) part, nodes = \
self.container_ring.get_nodes(account, name.encode('utf-8'))
rec_d = {} rec_d = {}
responses = {} responses = {}
for node in nodes: for node in nodes:
@ -160,15 +164,17 @@ class Auditor(object):
while results: while results:
node_id = node['id'] node_id = node['id']
try: try:
conn = http_connect(node['ip'], node['port'], node['device'], conn = http_connect(node['ip'], node['port'],
part, 'GET', path.encode('utf-8'), {}, node['device'], part, 'GET',
'format=json&marker=%s' % quote(marker.encode('utf-8'))) path.encode('utf-8'), {},
'format=json&marker=%s' %
quote(marker.encode('utf-8')))
resp = conn.getresponse() resp = conn.getresponse()
if resp.status // 100 != 2: if resp.status // 100 != 2:
self.container_not_found += 1 self.container_not_found += 1
consistent = False consistent = False
print ' Bad status GETting container "%s" on %s/%s' % \ print(' Bad status GETting container "%s" on %s/%s' %
(path, node['ip'], node['device']) (path, node['ip'], node['device']))
break break
if node['id'] not in responses: if node['id'] not in responses:
responses[node['id']] = dict(resp.getheaders()) responses[node['id']] = dict(resp.getheaders())
@ -185,12 +191,15 @@ class Auditor(object):
obj_name = obj['name'] obj_name = obj['name']
if obj_name not in rec_d: if obj_name not in rec_d:
rec_d[obj_name] = obj rec_d[obj_name] = obj
if obj['last_modified'] != rec_d[obj_name]['last_modified']: if (obj['last_modified'] !=
rec_d[obj_name]['last_modified']):
self.container_obj_mismatch += 1 self.container_obj_mismatch += 1
consistent = False consistent = False
print " Different versions of %s/%s in container dbs." % \ print(" Different versions of %s/%s "
(name, obj['name']) "in container dbs." % \
if obj['last_modified'] > rec_d[obj_name]['last_modified']: (name, obj['name']))
if (obj['last_modified'] >
rec_d[obj_name]['last_modified']):
rec_d[obj_name] = obj rec_d[obj_name] = obj
obj_counts = [int(header['x-container-object-count']) obj_counts = [int(header['x-container-object-count'])
for header in responses.values()] for header in responses.values()]
@ -233,18 +242,22 @@ class Auditor(object):
try: try:
conn = http_connect(node['ip'], node['port'], conn = http_connect(node['ip'], node['port'],
node['device'], part, 'GET', path, {}, node['device'], part, 'GET', path, {},
'format=json&marker=%s' % quote(marker.encode('utf-8'))) 'format=json&marker=%s' %
quote(marker.encode('utf-8')))
resp = conn.getresponse() resp = conn.getresponse()
if resp.status // 100 != 2: if resp.status // 100 != 2:
self.account_not_found += 1 self.account_not_found += 1
consistent = False consistent = False
print " Bad status GETting account '%s' from %ss:%ss" % (account, node['ip'], node['device']) print(" Bad status GETting account '%s' "
" from %ss:%ss" %
(account, node['ip'], node['device']))
break break
results = simplejson.loads(resp.read()) results = simplejson.loads(resp.read())
except Exception: except Exception:
self.account_exceptions += 1 self.account_exceptions += 1
consistent = False consistent = False
print " Exception GETting account '%s' on %ss:%ss" % (account, node['ip'], node['device']) print(" Exception GETting account '%s' on %ss:%ss" %
(account, node['ip'], node['device']))
break break
if node_id not in responses: if node_id not in responses:
responses[node_id] = [dict(resp.getheaders()), []] responses[node_id] = [dict(resp.getheaders()), []]
@ -257,17 +270,21 @@ class Auditor(object):
if len(set(cont_counts)) != 1: if len(set(cont_counts)) != 1:
self.account_container_mismatch += 1 self.account_container_mismatch += 1
consistent = False consistent = False
print " Account databases for '%s' don't agree on number of containers." % account print(" Account databases for '%s' don't agree on"
" number of containers." % account)
if cont_counts: if cont_counts:
print " Max: %s, Min: %s" % (max(cont_counts), min(cont_counts)) print " Max: %s, Min: %s" % (max(cont_counts),
min(cont_counts))
obj_counts = [int(header['x-account-object-count']) obj_counts = [int(header['x-account-object-count'])
for header in headers] for header in headers]
if len(set(obj_counts)) != 1: if len(set(obj_counts)) != 1:
self.account_object_mismatch += 1 self.account_object_mismatch += 1
consistent = False consistent = False
print " Account databases for '%s' don't agree on number of objects." % account print(" Account databases for '%s' don't agree on"
" number of objects." % account)
if obj_counts: if obj_counts:
print " Max: %s, Min: %s" % (max(obj_counts), min(obj_counts)) print " Max: %s, Min: %s" % (max(obj_counts),
min(obj_counts))
containers = set() containers = set()
for resp in responses.values(): for resp in responses.values():
containers.update(container['name'] for container in resp[1]) containers.update(container['name'] for container in resp[1])
@ -277,7 +294,8 @@ class Auditor(object):
self.accounts_checked += 1 self.accounts_checked += 1
if recurse: if recurse:
for container in containers: for container in containers:
self.pool.spawn_n(self.audit_container, account, container, True) self.pool.spawn_n(self.audit_container, account,
container, True)
if not consistent and self.error_file: if not consistent and self.error_file:
print >>open(self.error_file, 'a'), path print >>open(self.error_file, 'a'), path
return containers return containers
@ -349,4 +367,3 @@ if __name__ == '__main__':
auditor.audit(*split_path(path, 1, 3, True)) auditor.audit(*split_path(path, 1, 3, True))
auditor.wait() auditor.wait()
auditor.print_stats() auditor.print_stats()

3
bin/swift-dispersion-populate
View File

@ -96,7 +96,8 @@ if __name__ == '__main__':
retries_done = 0 retries_done = 0
url, token = get_auth(conf['auth_url'], conf['auth_user'], url, token = get_auth(conf['auth_url'], conf['auth_user'],
conf['auth_key'], auth_version=conf.get('auth_version', '1.0')) conf['auth_key'],
auth_version=conf.get('auth_version', '1.0'))
account = url.rsplit('/', 1)[1] account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency) connpool = Pool(max_size=concurrency)
connpool.create = lambda: Connection(conf['auth_url'], connpool.create = lambda: Connection(conf['auth_url'],

3
bin/swift-dispersion-report
View File

@ -281,7 +281,8 @@ Usage: %prog [options] [conf_file]
coropool = GreenPool(size=concurrency) coropool = GreenPool(size=concurrency)
url, token = get_auth(conf['auth_url'], conf['auth_user'], url, token = get_auth(conf['auth_url'], conf['auth_user'],
conf['auth_key'], auth_version=conf.get('auth_version', '1.0')) conf['auth_key'],
auth_version=conf.get('auth_version', '1.0'))
account = url.rsplit('/', 1)[1] account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency) connpool = Pool(max_size=concurrency)
connpool.create = lambda: Connection(conf['auth_url'], connpool.create = lambda: Connection(conf['auth_url'],

3
bin/swift-drive-audit
View File

@ -116,7 +116,8 @@ if __name__ == '__main__':
unmounts = 0 unmounts = 0
for kernel_device, count in errors.items(): for kernel_device, count in errors.items():
if count >= error_limit: if count >= error_limit:
device = [d for d in devices if d['kernel_device'] == kernel_device] device = \
[d for d in devices if d['kernel_device'] == kernel_device]
if device: if device:
mount_point = device[0]['mount_point'] mount_point = device[0]['mount_point']
if mount_point.startswith(device_dir): if mount_point.startswith(device_dir):

14
bin/swift-object-info
View File

@ -34,14 +34,14 @@ if __name__ == '__main__':
datafile = sys.argv[1] datafile = sys.argv[1]
fp = open(datafile, 'rb') fp = open(datafile, 'rb')
metadata = read_metadata(fp) metadata = read_metadata(fp)
path = metadata.pop('name','') path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type','') content_type = metadata.pop('Content-Type', '')
ts = metadata.pop('X-Timestamp','') ts = metadata.pop('X-Timestamp', '')
etag = metadata.pop('ETag','') etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length','') length = metadata.pop('Content-Length', '')
if path: if path:
print 'Path: %s' % path print 'Path: %s' % path
account, container, obj = path.split('/',3)[1:] account, container, obj = path.split('/', 3)[1:]
print ' Account: %s' % account print ' Account: %s' % account
print ' Container: %s' % container print ' Container: %s' % container
print ' Object: %s' % obj print ' Object: %s' % obj
@ -67,7 +67,7 @@ if __name__ == '__main__':
h = md5() h = md5()
file_len = 0 file_len = 0
while True: while True:
data = fp.read(64*1024) data = fp.read(64 * 1024)
if not data: if not data:
break break
h.update(data) h.update(data)

1
bin/swift-recon
View File

@ -427,7 +427,6 @@ class SwiftRecon(object):
help="Default = /etc/swift") help="Default = /etc/swift")
options, arguments = args.parse_args() options, arguments = args.parse_args()
if len(sys.argv) <= 1: if len(sys.argv) <= 1:
args.print_help() args.print_help()
sys.exit(0) sys.exit(0)

4
tox.ini
View File

@ -9,7 +9,9 @@ commands = nosetests test/unit []
[testenv:pep8] [testenv:pep8]
deps = pep8==0.6.1 deps = pep8==0.6.1
commands = pep8 --repeat --show-pep8 --show-source swift tools setup.py commands =
pep8 --repeat --show-pep8 --show-source swift tools setup.py
pep8 --repeat --show-pep8 --show-source --filename=swift* bin
[testenv:cover] [testenv:cover]
commands = commands =