From 6cc10d17de04a7ec19727880cd112898ca25d238 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 27 May 2014 17:25:15 -0700 Subject: [PATCH] Update bin scripts to be storage policy aware swift-container-info: Print policy container info swift-object-info: Allow to specify storage policy name when looking for object info Notify if there is missmatch between ring location and the actual object path in filesystem swift-get-nodes: Allow to specify storage policy name when looking for account/ container/object ring location Notify if there is missmatch between ring and the policy Lookup policy name in swift.conf; 'Legacy' container will use policy-0's name; 'Unknown' is shown if policy not found in swift.conf DocImpact Implements: blueprint storage-policies Change-Id: I450d40dc6e2d8f759187dff36d658e52737ae2a5 --- bin/swift-get-nodes | 171 ++++++----------- bin/swift-object-info | 107 ++--------- swift/cli/info.py | 376 ++++++++++++++++++++++++++++++++++--- test/unit/cli/test_info.py | 322 ++++++++++++++++++++++++++----- 4 files changed, 700 insertions(+), 276 deletions(-) diff --git a/bin/swift-get-nodes b/bin/swift-get-nodes index d36cbeb623..435cf0bda6 100755 --- a/bin/swift-get-nodes +++ b/bin/swift-get-nodes @@ -14,129 +14,70 @@ # See the License for the specific language governing permissions and # limitations under the License. -import optparse import sys -import urllib +import os +from optparse import OptionParser from swift.common.ring import Ring -from swift.common.utils import hash_path, storage_directory +from swift.cli.info import print_item_locations, InfoSystemExit -parser = optparse.OptionParser() -parser.add_option('-a', '--all', action='store_true', - help='Show all handoff nodes') -parser.add_option('-p', '--partition', metavar='PARTITION', - help='Show nodes for a given partition') -(options, args) = parser.parse_args() +if __name__ == '__main__': -if (len(args) < 2 or len(args) > 4) and \ - (options.partition is None or not args): - print 'Usage: %s [-a] [] []' \ - % sys.argv[0] - print ' Or: %s [-a] -p partition' % sys.argv[0] - print ' Note: account, container, object can also be a single arg ' \ - 'separated by /' - print 'Shows the nodes responsible for the item specified.' - print 'Example:' - print ' $ %s /etc/swift/account.ring.gz MyAccount' % sys.argv[0] - print ' Partition 5743883' - print ' Hash 96ae332a60b58910784e4417a03e1ad0' - print ' 10.1.1.7:8000 sdd1' - print ' 10.1.9.2:8000 sdb1' - print ' 10.1.5.5:8000 sdf1' - print ' 10.1.5.9:8000 sdt1 # [Handoff]' - sys.exit(1) + usage = ''' + Shows the nodes responsible for the item specified. + Usage: %prog [-a] [] [] + Or: %prog [-a] -p partition + Or: %prog [-a] -P policy_name + Note: account, container, object can also be a single arg separated by / + Example: + $ %prog -a /etc/swift/account.ring.gz MyAccount + Partition 5743883 + Hash 96ae332a60b58910784e4417a03e1ad0 + 10.1.1.7:8000 sdd1 + 10.1.9.2:8000 sdb1 + 10.1.5.5:8000 sdf1 + 10.1.5.9:8000 sdt1 # [Handoff] + ''' + parser = OptionParser(usage) + parser.add_option('-a', '--all', action='store_true', + help='Show all handoff nodes') + parser.add_option('-p', '--partition', metavar='PARTITION', + help='Show nodes for a given partition') + parser.add_option('-P', '--policy-name', dest='policy_name', + help='Specify which policy to use') + parser.add_option('-d', '--swift-dir', default='/etc/swift', + dest='swift_dir', help='Path to swift directory') + options, args = parser.parse_args() + + # swift-get-nodes -P nada -p 1 + if len(args) == 0: + if not options.policy_name or not options.partition: + sys.exit(parser.print_help()) + elif len(args) > 4 or len(args) < 1: + sys.exit(parser.print_help()) -if len(args) == 2 and '/' in args[1]: # Parse single path arg, as noted in above help text. - path = args[1].lstrip('/') - args = [args[0]] + [p for p in path.split('/', 2) if p] + # if len(args) == 1 and options.policy_name and '/' in args[0]: + if len(args) == 1 and not args[0].endswith('ring.gz'): + path = args[0].lstrip('/') + args = [p for p in path.split('/', 2) if p] + if len(args) == 2 and '/' in args[1]: + path = args[1].lstrip('/') + args = [args[0]] + [p for p in path.split('/', 2) if p] -ringloc = None -account = None -container = None -obj = None + ring = None + ring_name = None -if len(args) == 4: - # Account, Container and Object - ring_file, account, container, obj = args - ring = Ring(ring_file) - hash_str = hash_path(account, container, obj) - part, nodes = ring.get_nodes(account, container, obj) - target = "%s/%s/%s" % (account, container, obj) - loc = 'objects' -elif len(args) == 3: - # Account, Container - ring_file, account, container = args - ring = Ring(ring_file) - hash_str = hash_path(account, container) - part, nodes = ring.get_nodes(account, container) - target = "%s/%s" % (account, container) - loc = 'containers' -elif len(args) == 2: - # Account - ring_file, account = args - ring = Ring(ring_file) - hash_str = hash_path(account) - part, nodes = ring.get_nodes(account) - target = "%s" % (account) - loc = 'accounts' -elif len(args) == 1: - # Partition - ring_file = args[0] - ring = Ring(ring_file) - hash_str = None - part = int(options.partition) - nodes = ring.get_part_nodes(part) - target = '' - loc = ring_file.rsplit('/', 1)[-1].split('.', 1)[0] - if loc in ('account', 'container', 'object'): - loc += 's' - else: - loc = '' + if len(args) >= 1 and args[0].endswith('ring.gz'): + if os.path.exists(args[0]): + ring_name = args[0].rsplit('/', 1)[-1].split('.', 1)[0] + ring = Ring(args[0]) + else: + print 'Ring file does not exist' + args.pop(0) -more_nodes = [] -for more_node in ring.get_more_nodes(part): - more_nodes.append(more_node) - if not options.all and len(more_nodes) >= len(nodes): - break - -print '\nAccount \t%s' % account -print 'Container\t%s' % container -print 'Object \t%s\n' % obj -print '\nPartition\t%s' % part -print 'Hash \t%s\n' % hash_str - -for node in nodes: - print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'], - node['device']) -for mnode in more_nodes: - print 'Server:Port Device\t%s:%s %s\t [Handoff]' \ - % (mnode['ip'], mnode['port'], mnode['device']) -print "\n" -for node in nodes: - print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \ - % (node['ip'], node['port'], node['device'], part, - urllib.quote(target)) -for mnode in more_nodes: - print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s" # [Handoff]' \ - % (mnode['ip'], mnode['port'], mnode['device'], part, - urllib.quote(target)) -print "\n" -print 'Use your own device location of servers:' -print 'such as "export DEVICE=/srv/node"' -for node in nodes: - if hash_str: - print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/"' % ( - node['ip'], node['device'], storage_directory(loc, part, hash_str)) - else: - print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/"' % ( - node['ip'], node['device'], loc, part) -for mnode in more_nodes: - if hash_str: - print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/" '\ - '# [Handoff]' % (mnode['ip'], mnode['device'], - storage_directory(loc, part, hash_str)) - else: - print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/" # [Handoff]' % ( - mnode['ip'], mnode['device'], loc, part) + try: + print_item_locations(ring, ring_name, *args, **vars(options)) + except InfoSystemExit: + sys.exit(1) diff --git a/bin/swift-object-info b/bin/swift-object-info index 28ec3fc8dd..c625b47bb8 100755 --- a/bin/swift-object-info +++ b/bin/swift-object-info @@ -14,112 +14,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import sys -from datetime import datetime -from hashlib import md5 from optparse import OptionParser -from swift.common.ring import Ring -from swift.obj.diskfile import read_metadata -from swift.common.utils import hash_path, storage_directory - - -def print_object_info(datafile, check_etag=True, swift_dir='/etc/swift'): - if not os.path.exists(datafile) or not datafile.endswith('.data'): - print "Data file doesn't exist" - sys.exit(1) - try: - ring = Ring(swift_dir, ring_name='object') - except Exception: - ring = None - fp = open(datafile, 'rb') - metadata = read_metadata(fp) - path = metadata.pop('name', '') - content_type = metadata.pop('Content-Type', '') - ts = metadata.pop('X-Timestamp', '') - etag = metadata.pop('ETag', '') - length = metadata.pop('Content-Length', '') - if path: - print 'Path: %s' % path - account, container, obj = path.split('/', 3)[1:] - print ' Account: %s' % account - print ' Container: %s' % container - print ' Object: %s' % obj - obj_hash = hash_path(account, container, obj) - print ' Object hash: %s' % obj_hash - else: - print 'Path: Not found in metadata' - if content_type: - print 'Content-Type: %s' % content_type - else: - print 'Content-Type: Not found in metadata' - if ts: - print 'Timestamp: %s (%s)' % (datetime.fromtimestamp(float(ts)), ts) - else: - print 'Timestamp: Not found in metadata' - - file_len = None - if check_etag: - h = md5() - file_len = 0 - while True: - data = fp.read(64 * 1024) - if not data: - break - h.update(data) - file_len += len(data) - h = h.hexdigest() - if etag: - if h == etag: - print 'ETag: %s (valid)' % etag - else: - print "Etag: %s doesn't match file hash of %s!" % (etag, h) - else: - print 'ETag: Not found in metadata' - else: - print 'ETag: %s (not checked)' % etag - file_len = os.fstat(fp.fileno()).st_size - - if length: - if file_len == int(length): - print 'Content-Length: %s (valid)' % length - else: - print "Content-Length: %s doesn't match file length of %s" % ( - length, file_len) - else: - print 'Content-Length: Not found in metadata' - print 'User Metadata: %s' % metadata - if ring is not None: - print 'Ring locations:' - part, nodes = ring.get_nodes(account, container, obj) - for node in nodes: - print (' %s:%s - /srv/node/%s/%s/%s.data' % - (node['ip'], node['port'], node['device'], - storage_directory('objects', part, obj_hash), ts)) - print - print 'note: /srv/node is used as default value of `devices`, '\ - 'the real value is set in object-server.conf '\ - 'on each storage node.' - fp.close() +from swift.cli.info import print_obj, InfoSystemExit if __name__ == '__main__': - parser = OptionParser() - parser.set_defaults(check_etag=True, swift_dir='/etc/swift') + parser = OptionParser('%prog [options] OBJECT_FILE') parser.add_option( - '-n', '--no-check-etag', + '-n', '--no-check-etag', default=True, action="store_false", dest="check_etag", help="Don't verify file contents against stored etag") parser.add_option( - '-d', '--swift-dir', + '-d', '--swift-dir', default='/etc/swift', dest='swift_dir', help="Pass location of swift directory") + parser.add_option( + '-P', '--policy-name', dest='policy_name', + help="Specify storage policy name") options, args = parser.parse_args() - if len(args) < 1: - print "Usage: %s [-n] [-d] OBJECT_FILE" % sys.argv[0] - sys.exit(1) + if len(args) != 1: + sys.exit(parser.print_help()) - print_object_info(args[0], check_etag=options.check_etag, - swift_dir=options.swift_dir) + try: + print_obj(*args, **vars(options)) + except InfoSystemExit: + sys.exit(1) diff --git a/swift/cli/info.py b/swift/cli/info.py index 597303793d..fd0fa53206 100644 --- a/swift/cli/info.py +++ b/swift/cli/info.py @@ -10,9 +10,12 @@ # License for the specific language governing permissions and limitations # under the License. +import itertools import os import sqlite3 +import urllib from datetime import datetime +from hashlib import md5 from swift.common.utils import hash_path, storage_directory from swift.common.ring import Ring @@ -20,6 +23,9 @@ from swift.common.request_helpers import is_sys_meta, is_user_meta, \ strip_sys_meta_prefix, strip_user_meta_prefix from swift.account.backend import AccountBroker, DATADIR as ABDATADIR from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR +from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \ + extract_policy_index +from swift.common.storage_policy import POLICIES, POLICY_INDEX class InfoSystemExit(Exception): @@ -29,35 +35,105 @@ class InfoSystemExit(Exception): pass -def print_ring_locations(ring, datadir, account, container=None): +def print_ring_locations(ring, datadir, account, container=None, obj=None, + tpart=None, all_nodes=False, policy_index=None): """ print out ring locations of specified type :param ring: ring instance - :param datadir: high level directory to store account/container/objects + :param datadir: name of directory where things are stored. Usually one of + "accounts", "containers", "objects", or "objects-N". :param account: account name :param container: container name + :param obj: object name + :param tpart: target partition in ring + :param all_nodes: include all handoff nodes. If false, only the N primary + nodes and first N handoffs will be printed. + :param policy_index: include policy_index in curl headers """ - if ring is None or datadir is None or account is None: - raise ValueError('None type') - storage_type = 'account' - if container: - storage_type = 'container' - try: - part, nodes = ring.get_nodes(account, container, None) - except (ValueError, AttributeError): - raise ValueError('Ring error') + if not ring: + raise ValueError("No ring specified") + if not datadir: + raise ValueError("No datadir specified") + if tpart is None and not account: + raise ValueError("No partition or account/container/object specified") + if not account and (container or obj): + raise ValueError("Container/object specified without account") + if obj and not container: + raise ValueError('Object specified without container') + + if obj: + target = '%s/%s/%s' % (account, container, obj) + elif container: + target = '%s/%s' % (account, container) else: - path_hash = hash_path(account, container, None) - print '\nRing locations:' - for node in nodes: - print (' %s:%s - /srv/node/%s/%s/%s.db' % - (node['ip'], node['port'], node['device'], - storage_directory(datadir, part, path_hash), - path_hash)) - print '\nnote: /srv/node is used as default value of `devices`, the ' \ - 'real value is set in the %s config file on each storage node.' % \ - storage_type + target = '%s' % (account) + + if tpart: + part = int(tpart) + else: + part = ring.get_part(account, container, obj) + + primary_nodes = ring.get_part_nodes(part) + handoff_nodes = ring.get_more_nodes(part) + if not all_nodes: + handoff_nodes = itertools.islice(handoff_nodes, len(primary_nodes)) + handoff_nodes = list(handoff_nodes) + + if account and not tpart: + path_hash = hash_path(account, container, obj) + else: + path_hash = None + print 'Partition\t%s' % part + print 'Hash \t%s\n' % path_hash + + for node in primary_nodes: + print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'], + node['device']) + for node in handoff_nodes: + print 'Server:Port Device\t%s:%s %s\t [Handoff]' % ( + node['ip'], node['port'], node['device']) + + print "\n" + + for node in primary_nodes: + cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \ + % (node['ip'], node['port'], node['device'], part, + urllib.quote(target)) + if policy_index is not None: + cmd += ' -H "%s: %s"' % (POLICY_INDEX, policy_index) + print cmd + for node in handoff_nodes: + cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \ + % (node['ip'], node['port'], node['device'], part, + urllib.quote(target)) + if policy_index is not None: + cmd += ' -H "%s: %s"' % (POLICY_INDEX, policy_index) + cmd += ' # [Handoff]' + print cmd + + print "\n\nUse your own device location of servers:" + print "such as \"export DEVICE=/srv/node\"" + if path_hash: + for node in primary_nodes: + print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s"' % + (node['ip'], node['device'], + storage_directory(datadir, part, path_hash))) + for node in handoff_nodes: + print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s" # [Handoff]' % + (node['ip'], node['device'], + storage_directory(datadir, part, path_hash))) + else: + for node in primary_nodes: + print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"' % + (node['ip'], node['device'], datadir, part)) + for node in handoff_nodes: + print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"' + ' # [Handoff]' % + (node['ip'], node['device'], datadir, part)) + + print '\nnote: `/srv/node*` is used as default value of `devices`, the ' \ + 'real value is set in the config file on each storage node.' def print_db_info_metadata(db_type, info, metadata): @@ -106,11 +182,20 @@ def print_db_info_metadata(db_type, info, metadata): print (' Delete Timestamp: %s (%s)' % (datetime.utcfromtimestamp(float(info['delete_timestamp'])), info['delete_timestamp'])) + print (' Status Timestamp: %s (%s)' % + (datetime.utcfromtimestamp(float(info['status_changed_at'])), + info['status_changed_at'])) if db_type == 'account': print ' Container Count: %s' % info['container_count'] print ' Object Count: %s' % info['object_count'] print ' Bytes Used: %s' % info['bytes_used'] if db_type == 'container': + try: + policy_name = POLICIES[info['storage_policy_index']].name + except KeyError: + policy_name = 'Unknown' + print (' Storage Policy: %s (%s)' % ( + policy_name, info['storage_policy_index'])) print (' Reported Put Timestamp: %s (%s)' % (datetime.utcfromtimestamp( float(info['reported_put_timestamp'])), @@ -123,8 +208,8 @@ def print_db_info_metadata(db_type, info, metadata): print ' Reported Bytes Used: %s' % info['reported_bytes_used'] print ' Chexor: %s' % info['hash'] print ' UUID: %s' % info['id'] - except KeyError: - raise ValueError('Info is incomplete') + except KeyError as e: + raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.iteritems(): @@ -152,6 +237,53 @@ def print_db_info_metadata(db_type, info, metadata): print 'No user metadata found in db file' +def print_obj_metadata(metadata): + """ + Print out basic info and metadata from object, as returned from + :func:`swift.obj.diskfile.read_metadata`. + + Metadata should include the keys: name, Content-Type, and + X-Timestamp. + + Additional metadata is displayed unmodified. + + :param metadata: dict of object metadata + + :raises: ValueError + """ + if not metadata: + raise ValueError('Metadata is None') + path = metadata.pop('name', '') + content_type = metadata.pop('Content-Type', '') + ts = metadata.pop('X-Timestamp', 0) + account = container = obj = obj_hash = None + if path: + try: + account, container, obj = path.split('/', 3)[1:] + except ValueError: + raise ValueError('Path is invalid for object %r' % path) + else: + obj_hash = hash_path(account, container, obj) + print 'Path: %s' % path + print ' Account: %s' % account + print ' Container: %s' % container + print ' Object: %s' % obj + print ' Object hash: %s' % obj_hash + else: + print 'Path: Not found in metadata' + if content_type: + print 'Content-Type: %s' % content_type + else: + print 'Content-Type: Not found in metadata' + if ts: + print ('Timestamp: %s (%s)' % + (datetime.utcfromtimestamp(float(ts)), ts)) + else: + print 'Timestamp: Not found in metadata' + + print 'User Metadata: %s' % metadata + + def print_info(db_type, db_file, swift_dir='/etc/swift'): if db_type not in ('account', 'container'): print "Unrecognized DB type: internal error" @@ -184,3 +316,201 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'): ring = None else: print_ring_locations(ring, datadir, account, container) + + +def print_obj(datafile, check_etag=True, swift_dir='/etc/swift', + policy_name=''): + """ + Display information about an object read from the datafile. + Optionally verify the datafile content matches the ETag metadata. + + :param datafile: path on disk to object file + :param check_etag: boolean, will read datafile content and verify + computed checksum matches value stored in + metadata. + :param swift_dir: the path on disk to rings + :param policy_name: optionally the name to use when finding the ring + """ + if not os.path.exists(datafile) or not datafile.endswith('.data'): + print "Data file doesn't exist" + raise InfoSystemExit() + if not datafile.startswith(('/', './')): + datafile = './' + datafile + + policy_index = None + ring = None + datadir = DATADIR_BASE + + # try to extract policy index from datafile disk path + try: + policy_index = extract_policy_index(datafile) + except ValueError: + pass + + try: + if policy_index: + datadir += '-' + str(policy_index) + ring = Ring(swift_dir, ring_name='object-' + str(policy_index)) + elif policy_index == 0: + ring = Ring(swift_dir, ring_name='object') + except IOError: + # no such ring + pass + + if policy_name: + policy = POLICIES.get_by_name(policy_name) + if policy: + policy_index_for_name = policy.idx + if (policy_index is not None and + policy_index_for_name is not None and + policy_index != policy_index_for_name): + print 'Attention: Ring does not match policy!' + print 'Double check your policy name!' + if not ring and policy_index_for_name: + ring = POLICIES.get_object_ring(policy_index_for_name, + swift_dir) + datadir = get_data_dir(policy_index_for_name) + + with open(datafile, 'rb') as fp: + try: + metadata = read_metadata(fp) + except EOFError: + print "Invalid metadata" + raise InfoSystemExit() + + etag = metadata.pop('ETag', '') + length = metadata.pop('Content-Length', '') + path = metadata.get('name', '') + print_obj_metadata(metadata) + + # Optional integrity check; it's useful, but slow. + file_len = None + if check_etag: + h = md5() + file_len = 0 + while True: + data = fp.read(64 * 1024) + if not data: + break + h.update(data) + file_len += len(data) + h = h.hexdigest() + if etag: + if h == etag: + print 'ETag: %s (valid)' % etag + else: + print ("ETag: %s doesn't match file hash of %s!" % + (etag, h)) + else: + print 'ETag: Not found in metadata' + else: + print 'ETag: %s (not checked)' % etag + file_len = os.fstat(fp.fileno()).st_size + + if length: + if file_len == int(length): + print 'Content-Length: %s (valid)' % length + else: + print ("Content-Length: %s doesn't match file length of %s" + % (length, file_len)) + else: + print 'Content-Length: Not found in metadata' + + account, container, obj = path.split('/', 3)[1:] + if ring: + print_ring_locations(ring, datadir, account, container, obj, + policy_index=policy_index) + + +def print_item_locations(ring, ring_name=None, account=None, container=None, + obj=None, **kwargs): + """ + Display placement information for an item based on ring lookup. + + If a ring is provided it always takes precedence, but warnings will be + emitted if it doesn't match other optional arguments like the policy_name + or ring_name. + + If no ring is provided the ring_name and/or policy_name will be used to + lookup the ring. + + :param ring: a ring instance + :param ring_name: server type, or storage policy ring name if object ring + :param account: account name + :param container: container name + :param obj: object name + :param partition: part number for non path lookups + :param policy_name: name of storage policy to use to lookup the ring + :param all_nodes: include all handoff nodes. If false, only the N primary + nodes and first N handoffs will be printed. + """ + + policy_name = kwargs.get('policy_name', None) + part = kwargs.get('partition', None) + all_nodes = kwargs.get('all', False) + swift_dir = kwargs.get('swift_dir', '/etc/swift') + + if ring and policy_name: + policy = POLICIES.get_by_name(policy_name) + if policy: + if ring_name != policy.ring_name: + print 'Attention! mismatch between ring and policy detected!' + else: + print 'Attention! Policy %s is not valid' % policy_name + + policy_index = None + if ring is None and (obj or part): + if not policy_name: + print 'Need a ring or policy' + raise InfoSystemExit() + policy = POLICIES.get_by_name(policy_name) + if not policy: + print 'No policy named %r' % policy_name + raise InfoSystemExit() + policy_index = int(policy) + ring = POLICIES.get_object_ring(policy_index, swift_dir) + ring_name = (POLICIES.get_by_name(policy_name)).ring_name + + if account is None and (container is not None or obj is not None): + print 'No account specified' + raise InfoSystemExit() + + if container is None and obj is not None: + print 'No container specified' + raise InfoSystemExit() + + if account is None and part is None: + print 'No target specified' + raise InfoSystemExit() + + loc = '' + if part and ring_name: + if '-' in ring_name and ring_name.startswith('object'): + loc = 'objects-' + ring_name.split('-', 1)[1] + else: + loc = ring_name + 's' + if account and container and obj: + loc = 'objects' + if '-' in ring_name and ring_name.startswith('object'): + policy_index = int(ring_name.rsplit('-', 1)[1]) + loc = 'objects-%d' % policy_index + if account and container and not obj: + loc = 'containers' + if not any([ring, ring_name]): + ring = Ring(swift_dir, ring_name='container') + else: + if ring_name != 'container': + print 'Attention! mismatch between ring and item detected!' + if account and not container and not obj: + loc = 'accounts' + if not any([ring, ring_name]): + ring = Ring(swift_dir, ring_name='account') + else: + if ring_name != 'account': + print 'Attention! mismatch between ring and item detected!' + + print '\nAccount \t%s' % account + print 'Container\t%s' % container + print 'Object \t%s\n\n' % obj + print_ring_locations(ring, loc, account, container, obj, part, all_nodes, + policy_index=policy_index) diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py index 793d8b401a..d86c70354c 100644 --- a/test/unit/cli/test_info.py +++ b/test/unit/cli/test_info.py @@ -19,17 +19,22 @@ from cStringIO import StringIO from shutil import rmtree from tempfile import mkdtemp -from test.unit import write_fake_ring +from test.unit import patch_policies, write_fake_ring + from swift.common import ring, utils from swift.common.swob import Request +from swift.common.storage_policy import StoragePolicy, POLICIES from swift.cli.info import print_db_info_metadata, print_ring_locations, \ - print_info, InfoSystemExit + print_info, print_obj_metadata, print_obj, InfoSystemExit from swift.account.server import AccountController from swift.container.server import ContainerController +from swift.obj.diskfile import write_metadata -class TestCliInfo(unittest.TestCase): - +@patch_policies([StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'one', False), + StoragePolicy(2, 'two', False)]) +class TestCliInfoBase(unittest.TestCase): def setUp(self): self.orig_hp = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX utils.HASH_PATH_PREFIX = 'info' @@ -42,14 +47,30 @@ class TestCliInfo(unittest.TestCase): utils.mkdirs(os.path.join(self.testdir, 'sdb1')) utils.mkdirs(os.path.join(self.testdir, 'sdb1', 'tmp')) self.account_ring_path = os.path.join(self.testdir, 'account.ring.gz') - account_devs = [{'ip': '127.0.0.1', 'port': 42}, - {'ip': '127.0.0.2', 'port': 43}] + account_devs = [ + {'ip': '127.0.0.1', 'port': 42}, + {'ip': '127.0.0.2', 'port': 43}, + ] write_fake_ring(self.account_ring_path, *account_devs) self.container_ring_path = os.path.join(self.testdir, 'container.ring.gz') - container_devs = [{'ip': '127.0.0.3', 'port': 42}, - {'ip': '127.0.0.4', 'port': 43}] + container_devs = [ + {'ip': '127.0.0.3', 'port': 42}, + {'ip': '127.0.0.4', 'port': 43}, + ] write_fake_ring(self.container_ring_path, *container_devs) + self.object_ring_path = os.path.join(self.testdir, 'object.ring.gz') + object_devs = [ + {'ip': '127.0.0.3', 'port': 42}, + {'ip': '127.0.0.4', 'port': 43}, + ] + write_fake_ring(self.object_ring_path, *object_devs) + # another ring for policy 1 + self.one_ring_path = os.path.join(self.testdir, 'object-1.ring.gz') + write_fake_ring(self.one_ring_path, *object_devs) + # ... and another for policy 2 + self.two_ring_path = os.path.join(self.testdir, 'object-2.ring.gz') + write_fake_ring(self.two_ring_path, *object_devs) def tearDown(self): utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = self.orig_hp @@ -59,10 +80,13 @@ class TestCliInfo(unittest.TestCase): try: func(*args, **kwargs) except Exception, e: - self.assertEqual(msg, str(e)) + self.assertTrue(msg in str(e), + "Expected %r in %r" % (msg, str(e))) self.assertTrue(isinstance(e, exc), "Expected %s, got %s" % (exc, type(e))) + +class TestCliInfo(TestCliInfoBase): def test_print_db_info_metadata(self): self.assertRaisesMessage(ValueError, 'Wrong DB type', print_db_info_metadata, 't', {}, {}) @@ -76,6 +100,7 @@ class TestCliInfo(unittest.TestCase): created_at=100.1, put_timestamp=106.3, delete_timestamp=107.9, + status_changed_at=108.3, container_count='3', object_count='20', bytes_used='42') @@ -93,6 +118,7 @@ Metadata: Created at: 1970-01-01 00:01:40.100000 (100.1) Put Timestamp: 1970-01-01 00:01:46.300000 (106.3) Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9) + Status Timestamp: 1970-01-01 00:01:48.300000 (108.3) Container Count: 3 Object Count: 20 Bytes Used: 42 @@ -108,9 +134,11 @@ No system metadata found in db file info = dict( account='acct', container='cont', + storage_policy_index=0, created_at='0000000100.10000', put_timestamp='0000000106.30000', delete_timestamp='0000000107.90000', + status_changed_at='0000000108.30000', object_count='20', bytes_used='42', reported_put_timestamp='0000010106.30000', @@ -133,8 +161,10 @@ Metadata: Created at: 1970-01-01 00:01:40.100000 (0000000100.10000) Put Timestamp: 1970-01-01 00:01:46.300000 (0000000106.30000) Delete Timestamp: 1970-01-01 00:01:47.900000 (0000000107.90000) + Status Timestamp: 1970-01-01 00:01:48.300000 (0000000108.30000) Object Count: 20 Bytes Used: 42 + Storage Policy: %s (0) Reported Put Timestamp: 1970-01-01 02:48:26.300000 (0000010106.30000) Reported Delete Timestamp: 1970-01-01 02:48:27.900000 (0000010107.90000) Reported Object Count: 20 @@ -144,54 +174,62 @@ Metadata: X-Container-Bar: goo X-Container-Foo: bar System Metadata: {'mydata': 'swift'} -No user metadata found in db file''' +No user metadata found in db file''' % POLICIES[0].name self.assertEquals(sorted(out.getvalue().strip().split('\n')), sorted(exp_out.split('\n'))) - def test_print_ring_locations(self): - self.assertRaisesMessage(ValueError, 'None type', print_ring_locations, - None, 'dir', 'acct') - self.assertRaisesMessage(ValueError, 'None type', print_ring_locations, - [], None, 'acct') - self.assertRaisesMessage(ValueError, 'None type', print_ring_locations, - [], 'dir', None) - self.assertRaisesMessage(ValueError, 'Ring error', - print_ring_locations, - [], 'dir', 'acct', 'con') + def test_print_ring_locations_invalid_args(self): + self.assertRaises(ValueError, print_ring_locations, + None, 'dir', 'acct') + self.assertRaises(ValueError, print_ring_locations, + [], None, 'acct') + self.assertRaises(ValueError, print_ring_locations, + [], 'dir', None) + self.assertRaises(ValueError, print_ring_locations, + [], 'dir', 'acct', 'con') + self.assertRaises(ValueError, print_ring_locations, + [], 'dir', 'acct', obj='o') + def test_print_ring_locations_account(self): out = StringIO() with mock.patch('sys.stdout', out): acctring = ring.Ring(self.testdir, ring_name='account') print_ring_locations(acctring, 'dir', 'acct') - exp_db2 = os.path.join('/srv', 'node', 'sdb1', 'dir', '3', 'b47', - 'dc5be2aa4347a22a0fee6bc7de505b47', - 'dc5be2aa4347a22a0fee6bc7de505b47.db') - exp_db1 = os.path.join('/srv', 'node', 'sda1', 'dir', '3', 'b47', - 'dc5be2aa4347a22a0fee6bc7de505b47', - 'dc5be2aa4347a22a0fee6bc7de505b47.db') - exp_out = ('Ring locations:\n 127.0.0.2:43 - %s\n' - ' 127.0.0.1:42 - %s\n' - '\nnote: /srv/node is used as default value of `devices`,' - ' the real value is set in the account config file on' - ' each storage node.' % (exp_db2, exp_db1)) - self.assertEquals(out.getvalue().strip(), exp_out) + exp_db = os.path.join('${DEVICE:-/srv/node*}', 'sdb1', 'dir', '3', + 'b47', 'dc5be2aa4347a22a0fee6bc7de505b47') + self.assertTrue(exp_db in out.getvalue()) + self.assertTrue('127.0.0.1' in out.getvalue()) + self.assertTrue('127.0.0.2' in out.getvalue()) + def test_print_ring_locations_container(self): out = StringIO() with mock.patch('sys.stdout', out): contring = ring.Ring(self.testdir, ring_name='container') print_ring_locations(contring, 'dir', 'acct', 'con') - exp_db4 = os.path.join('/srv', 'node', 'sdb1', 'dir', '1', 'fe6', - '63e70955d78dfc62821edc07d6ec1fe6', - '63e70955d78dfc62821edc07d6ec1fe6.db') - exp_db3 = os.path.join('/srv', 'node', 'sda1', 'dir', '1', 'fe6', - '63e70955d78dfc62821edc07d6ec1fe6', - '63e70955d78dfc62821edc07d6ec1fe6.db') - exp_out = ('Ring locations:\n 127.0.0.4:43 - %s\n' - ' 127.0.0.3:42 - %s\n' - '\nnote: /srv/node is used as default value of `devices`,' - ' the real value is set in the container config file on' - ' each storage node.' % (exp_db4, exp_db3)) - self.assertEquals(out.getvalue().strip(), exp_out) + exp_db = os.path.join('${DEVICE:-/srv/node*}', 'sdb1', 'dir', '1', + 'fe6', '63e70955d78dfc62821edc07d6ec1fe6') + self.assertTrue(exp_db in out.getvalue()) + + def test_print_ring_locations_obj(self): + out = StringIO() + with mock.patch('sys.stdout', out): + objring = ring.Ring(self.testdir, ring_name='object') + print_ring_locations(objring, 'dir', 'acct', 'con', 'obj') + exp_obj = os.path.join('${DEVICE:-/srv/node*}', 'sda1', 'dir', '1', + '117', '4a16154fc15c75e26ba6afadf5b1c117') + self.assertTrue(exp_obj in out.getvalue()) + + def test_print_ring_locations_partition_number(self): + out = StringIO() + with mock.patch('sys.stdout', out): + objring = ring.Ring(self.testdir, ring_name='object') + print_ring_locations(objring, 'objects', None, tpart='1') + exp_obj1 = os.path.join('${DEVICE:-/srv/node*}', 'sda1', + 'objects', '1') + exp_obj2 = os.path.join('${DEVICE:-/srv/node*}', 'sdb1', + 'objects', '1') + self.assertTrue(exp_obj1 in out.getvalue()) + self.assertTrue(exp_obj2 in out.getvalue()) def test_print_info(self): db_file = 'foo' @@ -271,3 +309,199 @@ No user metadata found in db file''' self.assertEquals(out.getvalue().strip(), exp_out) else: self.fail("Expected an InfoSystemExit exception to be raised") + + +class TestPrintObj(TestCliInfoBase): + + def setUp(self): + super(TestPrintObj, self).setUp() + self.datafile = os.path.join(self.testdir, + '1402017432.46642.data') + with open(self.datafile, 'wb') as fp: + md = {'name': '/AUTH_admin/c/obj', + 'Content-Type': 'application/octet-stream'} + write_metadata(fp, md) + + def test_print_obj_invalid(self): + datafile = '1402017324.68634.data' + self.assertRaises(InfoSystemExit, print_obj, datafile) + datafile = os.path.join(self.testdir, './1234.data') + self.assertRaises(InfoSystemExit, print_obj, datafile) + + with open(datafile, 'wb') as fp: + fp.write('1234') + + out = StringIO() + with mock.patch('sys.stdout', out): + self.assertRaises(InfoSystemExit, print_obj, datafile) + self.assertEquals(out.getvalue().strip(), + 'Invalid metadata') + + def test_print_obj_valid(self): + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile, swift_dir=self.testdir) + etag_msg = 'ETag: Not found in metadata' + length_msg = 'Content-Length: Not found in metadata' + self.assertTrue(etag_msg in out.getvalue()) + self.assertTrue(length_msg in out.getvalue()) + + def test_print_obj_with_policy(self): + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile, swift_dir=self.testdir, policy_name='one') + etag_msg = 'ETag: Not found in metadata' + length_msg = 'Content-Length: Not found in metadata' + ring_loc_msg = 'ls -lah' + self.assertTrue(etag_msg in out.getvalue()) + self.assertTrue(length_msg in out.getvalue()) + self.assertTrue(ring_loc_msg in out.getvalue()) + + def test_missing_etag(self): + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile) + self.assertTrue('ETag: Not found in metadata' in out.getvalue()) + + +class TestPrintObjFullMeta(TestCliInfoBase): + def setUp(self): + super(TestPrintObjFullMeta, self).setUp() + self.datafile = os.path.join(self.testdir, + 'sda', 'objects-1', + '1', 'ea8', + 'db4449e025aca992307c7c804a67eea8', + '1402017884.18202.data') + utils.mkdirs(os.path.dirname(self.datafile)) + with open(self.datafile, 'wb') as fp: + md = {'name': '/AUTH_admin/c/obj', + 'Content-Type': 'application/octet-stream', + 'ETag': 'd41d8cd98f00b204e9800998ecf8427e', + 'Content-Length': 0} + write_metadata(fp, md) + + def test_print_obj(self): + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile, swift_dir=self.testdir) + self.assertTrue('/objects-1/' in out.getvalue()) + + def test_print_obj_no_ring(self): + no_rings_dir = os.path.join(self.testdir, 'no_rings_here') + os.mkdir(no_rings_dir) + + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile, swift_dir=no_rings_dir) + self.assertTrue('d41d8cd98f00b204e9800998ecf8427e' in out.getvalue()) + self.assertTrue('Partition' not in out.getvalue()) + + def test_print_obj_policy_name_mismatch(self): + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile, policy_name='two', swift_dir=self.testdir) + ring_alert_msg = 'Attention: Ring does not match policy' + self.assertTrue(ring_alert_msg in out.getvalue()) + + def test_valid_etag(self): + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile) + self.assertTrue('ETag: d41d8cd98f00b204e9800998ecf8427e (valid)' + in out.getvalue()) + + def test_invalid_etag(self): + with open(self.datafile, 'wb') as fp: + md = {'name': '/AUTH_admin/c/obj', + 'Content-Type': 'application/octet-stream', + 'ETag': 'badetag', + 'Content-Length': 0} + write_metadata(fp, md) + + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile) + self.assertTrue('ETag: badetag doesn\'t match file hash' + in out.getvalue()) + + def test_unchecked_etag(self): + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj(self.datafile, check_etag=False) + self.assertTrue('ETag: d41d8cd98f00b204e9800998ecf8427e (not checked)' + in out.getvalue()) + + def test_print_obj_metadata(self): + self.assertRaisesMessage(ValueError, 'Metadata is None', + print_obj_metadata, []) + + def reset_metadata(): + md = dict(name='/AUTH_admin/c/dummy') + md['Content-Type'] = 'application/octet-stream' + md['X-Timestamp'] = 106.3 + md['X-Object-Meta-Mtime'] = '107.3' + return md + + metadata = reset_metadata() + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj_metadata(metadata) + exp_out = '''Path: /AUTH_admin/c/dummy + Account: AUTH_admin + Container: c + Object: dummy + Object hash: 128fdf98bddd1b1e8695f4340e67a67a +Content-Type: application/octet-stream +Timestamp: 1970-01-01 00:01:46.300000 (106.3) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' + + self.assertEquals(out.getvalue().strip(), exp_out) + + metadata = reset_metadata() + metadata['name'] = '/a-s' + self.assertRaisesMessage(ValueError, 'Path is invalid', + print_obj_metadata, metadata) + + metadata = reset_metadata() + del metadata['name'] + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj_metadata(metadata) + exp_out = '''Path: Not found in metadata +Content-Type: application/octet-stream +Timestamp: 1970-01-01 00:01:46.300000 (106.3) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' + + self.assertEquals(out.getvalue().strip(), exp_out) + + metadata = reset_metadata() + del metadata['Content-Type'] + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj_metadata(metadata) + exp_out = '''Path: /AUTH_admin/c/dummy + Account: AUTH_admin + Container: c + Object: dummy + Object hash: 128fdf98bddd1b1e8695f4340e67a67a +Content-Type: Not found in metadata +Timestamp: 1970-01-01 00:01:46.300000 (106.3) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' + + self.assertEquals(out.getvalue().strip(), exp_out) + + metadata = reset_metadata() + del metadata['X-Timestamp'] + out = StringIO() + with mock.patch('sys.stdout', out): + print_obj_metadata(metadata) + exp_out = '''Path: /AUTH_admin/c/dummy + Account: AUTH_admin + Container: c + Object: dummy + Object hash: 128fdf98bddd1b1e8695f4340e67a67a +Content-Type: application/octet-stream +Timestamp: Not found in metadata +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' + + self.assertEquals(out.getvalue().strip(), exp_out)