openstack/swift
Code Issues Proposed changes

Cleaned up st command line parsing; always use included client.py as well

Browse Source
This commit is contained in:
gholt 2010-11-18 22:52:45 +00:00 committed by Tarmac
commit a91879f957
2 changed files with 981 additions and 927 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

312
bin/st
View File

@ -14,20 +14,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
try:
# Try to use installed swift.common.client...
from swift.common.client import get_auth, ClientException, Connection
except:
# But if not installed, use an included copy.
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Inclusion of swift.common.client
from errno import EEXIST, ENOENT
from hashlib import md5
from optparse import OptionParser
from os import environ, listdir, makedirs, utime
from os.path import basename, dirname, getmtime, getsize, isdir, join
from Queue import Empty, Queue
from sys import argv, exit, stderr, stdout
from threading import enumerate as threading_enumerate, Thread
from time import sleep
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Inclusion of swift.common.client for convenience of single file distribution
"""
Cloud Files client library used internally
"""
import socket
from cStringIO import StringIO
from httplib import HTTPConnection, HTTPException, HTTPSConnection
from httplib import HTTPException, HTTPSConnection
from re import compile, DOTALL
from tokenize import generate_tokens, STRING, NAME, OP
from urllib import quote as _quote, unquote
@ -38,6 +41,12 @@ except:
except:
from time import sleep
try:
from swift.common.bufferedhttp \
import BufferedHTTPConnection as HTTPConnection
except:
from httplib import HTTPConnection
def quote(value, safe='/'):
"""
@ -812,18 +821,7 @@ except:
return self._retry(delete_object, container, obj)
# End inclusion of swift.common.client
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
from errno import EEXIST, ENOENT
from hashlib import md5
from optparse import OptionParser
from os import environ, listdir, makedirs, utime
from os.path import basename, dirname, getmtime, getsize, isdir, join
from Queue import Empty, Queue
from sys import argv, exit, stderr, stdout
from threading import enumerate as threading_enumerate, Thread
from time import sleep
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
def mkdirs(path):
@ -865,12 +863,21 @@ st_delete_help = '''
delete --all OR delete container [object] [object] ...
Deletes everything in the account (with --all), or everything in a
container, or a list of objects depending on the args given.'''.strip('\n')
def st_delete(options, args):
def st_delete(parser, args, print_queue, error_queue):
parser.add_option('-a', '--all', action='store_true', dest='yes_all',
default=False, help='Indicates that you really want to delete '
'everything in the account')
(options, args) = parse_args(parser, args)
args = args[1:]
if (not args and not options.yes_all) or (args and options.yes_all):
options.error_queue.put('Usage: %s [options] %s' %
error_queue.put('Usage: %s [options] %s' %
(basename(argv[0]), st_delete_help))
return
object_queue = Queue(10000)
def _delete_object((container, obj), conn):
try:
conn.delete_object(container, obj)
@ -878,13 +885,14 @@ def st_delete(options, args):
path = options.yes_all and join(container, obj) or obj
if path[:1] in ('/', '\\'):
path = path[1:]
options.print_queue.put(path)
print_queue.put(path)
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Object %s not found' %
error_queue.put('Object %s not found' %
repr('%s/%s' % (container, obj)))
container_queue = Queue(10000)
def _delete_container(container, conn):
try:
marker = ''
@ -913,11 +921,12 @@ def st_delete(options, args):
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Container %s not found' % repr(container))
url, token = get_auth(options.auth, options.user, options.key, snet=options.snet)
error_queue.put('Container %s not found' % repr(container))
url, token = get_auth(options.auth, options.user, options.key,
snet=options.snet)
create_connection = lambda: Connection(options.auth, options.user,
options.key, preauthurl=url,
preauthtoken=token, snet=options.snet)
options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
object_threads = [QueueFunctionThread(object_queue, _delete_object,
create_connection()) for _ in xrange(10)]
for thread in object_threads:
@ -945,7 +954,7 @@ def st_delete(options, args):
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
error_queue.put('Account not found')
elif len(args) == 1:
conn = create_connection()
_delete_container(args[0], conn)
@ -969,15 +978,31 @@ def st_delete(options, args):
st_download_help = '''
download --all OR download container [object] [object] ...
Downloads everything in the account (with --all), or everything in a
container, or a list of objects depending on the args given. Use
the -o [--output] <filename> option to redirect the output to a file
or if "-" then the just redirect to stdout. '''.strip('\n')
def st_download(options, args):
container, or a list of objects depending on the args given. For a single
object download, you may use the -o [--output] <filename> option to
redirect the output to a specific file or if "-" then just redirect to
stdout.'''.strip('\n')
def st_download(options, args, print_queue, error_queue):
parser.add_option('-a', '--all', action='store_true', dest='yes_all',
default=False, help='Indicates that you really want to download '
'everything in the account')
parser.add_option('-o', '--output', dest='out_file', help='For a single '
'file download, stream the output to an alternate location ')
(options, args) = parse_args(parser, args)
args = args[1:]
if options.out_file == '-':
options.verbose = 0
if options.out_file and len(args) != 2:
exit('-o option only allowed for single file downloads')
if (not args and not options.yes_all) or (args and options.yes_all):
options.error_queue.put('Usage: %s [options] %s' %
error_queue.put('Usage: %s [options] %s' %
(basename(argv[0]), st_download_help))
return
object_queue = Queue(10000)
def _download_object(queue_arg, conn):
if len(queue_arg) == 2:
container, obj = queue_arg
@ -1021,23 +1046,24 @@ def st_download(options, args):
md5sum.update(chunk)
fp.close()
if md5sum.hexdigest() != etag:
options.error_queue.put('%s: md5sum != etag, %s != %s' %
error_queue.put('%s: md5sum != etag, %s != %s' %
(path, md5sum.hexdigest(), etag))
if read_length != content_length:
options.error_queue.put(
'%s: read_length != content_length, %d != %d' %
error_queue.put('%s: read_length != content_length, %d != %d' %
(path, read_length, content_length))
if 'x-object-meta-mtime' in headers and not options.out_file:
mtime = float(headers['x-object-meta-mtime'])
utime(path, (mtime, mtime))
if options.verbose:
options.print_queue.put(path)
print_queue.put(path)
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Object %s not found' %
error_queue.put('Object %s not found' %
repr('%s/%s' % (container, obj)))
container_queue = Queue(10000)
def _download_container(container, conn):
try:
marker = ''
@ -1052,11 +1078,12 @@ def st_download(options, args):
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Container %s not found' % repr(container))
url, token = get_auth(options.auth, options.user, options.key, snet=options.snet)
error_queue.put('Container %s not found' % repr(container))
url, token = get_auth(options.auth, options.user, options.key,
snet=options.snet)
create_connection = lambda: Connection(options.auth, options.user,
options.key, preauthurl=url,
preauthtoken=token, snet=options.snet)
options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
object_threads = [QueueFunctionThread(object_queue, _download_object,
create_connection()) for _ in xrange(10)]
for thread in object_threads:
@ -1080,7 +1107,7 @@ def st_download(options, args):
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
error_queue.put('Account not found')
elif len(args) == 1:
_download_container(args[0], create_connection())
else:
@ -1112,12 +1139,24 @@ list [options] [container]
items with the given delimiter (see Cloud Files general documentation for
what this means).
'''.strip('\n')
def st_list(options, args):
def st_list(options, args, print_queue, error_queue):
parser.add_option('-p', '--prefix', dest='prefix', help='Will only list '
'items beginning with the prefix')
parser.add_option('-d', '--delimiter', dest='delimiter', help='Will roll '
'up items with the given delimiter (see Cloud Files general '
'documentation for what this means)')
(options, args) = parse_args(parser, args)
args = args[1:]
if options.delimiter and not args:
exit('-d option only allowed for container listings')
if len(args) > 1:
options.error_queue.put('Usage: %s [options] %s' %
error_queue.put('Usage: %s [options] %s' %
(basename(argv[0]), st_list_help))
return
conn = Connection(options.auth, options.user, options.key, snet=options.snet)
conn = Connection(options.auth, options.user, options.key,
snet=options.snet)
try:
marker = ''
while True:
@ -1130,35 +1169,39 @@ def st_list(options, args):
if not items:
break
for item in items:
options.print_queue.put(item.get('name', item.get('subdir')))
print_queue.put(item.get('name', item.get('subdir')))
marker = items[-1].get('name', items[-1].get('subdir'))
except ClientException, err:
if err.http_status != 404:
raise
if not args:
options.error_queue.put('Account not found')
error_queue.put('Account not found')
else:
options.error_queue.put('Container %s not found' % repr(args[0]))
error_queue.put('Container %s not found' % repr(args[0]))
st_stat_help = '''
stat [container] [object]
Displays information for the account, container, or object depending on the
args given (if any).'''.strip('\n')
def st_stat(options, args):
def st_stat(options, args, print_queue, error_queue):
(options, args) = parse_args(parser, args)
args = args[1:]
conn = Connection(options.auth, options.user, options.key)
if not args:
try:
headers = conn.head_account()
if options.verbose > 1:
options.print_queue.put('''
print_queue.put('''
StorageURL: %s
Auth Token: %s
'''.strip('\n') % (conn.url, conn.token))
container_count = int(headers.get('x-account-container-count', 0))
object_count = int(headers.get('x-account-object-count', 0))
bytes_used = int(headers.get('x-account-bytes-used', 0))
options.print_queue.put('''
print_queue.put('''
Account: %s
Containers: %d
Objects: %d
@ -1166,24 +1209,24 @@ Containers: %d
object_count, bytes_used))
for key, value in headers.items():
if key.startswith('x-account-meta-'):
options.print_queue.put('%10s: %s' % ('Meta %s' %
print_queue.put('%10s: %s' % ('Meta %s' %
key[len('x-account-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-account-meta-') and key not in (
'content-length', 'date', 'x-account-container-count',
'x-account-object-count', 'x-account-bytes-used'):
options.print_queue.put(
print_queue.put(
'%10s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
error_queue.put('Account not found')
elif len(args) == 1:
try:
headers = conn.head_container(args[0])
object_count = int(headers.get('x-container-object-count', 0))
bytes_used = int(headers.get('x-container-bytes-used', 0))
options.print_queue.put('''
print_queue.put('''
Account: %s
Container: %s
Objects: %d
@ -1195,23 +1238,23 @@ Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
headers.get('x-container-write', '')))
for key, value in headers.items():
if key.startswith('x-container-meta-'):
options.print_queue.put('%9s: %s' % ('Meta %s' %
print_queue.put('%9s: %s' % ('Meta %s' %
key[len('x-container-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-container-meta-') and key not in (
'content-length', 'date', 'x-container-object-count',
'x-container-bytes-used', 'x-container-read',
'x-container-write'):
options.print_queue.put(
print_queue.put(
'%9s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Container %s not found' % repr(args[0]))
error_queue.put('Container %s not found' % repr(args[0]))
elif len(args) == 2:
try:
headers = conn.head_object(args[0], args[1])
options.print_queue.put('''
print_queue.put('''
Account: %s
Container: %s
Object: %s
@ -1225,21 +1268,21 @@ Content Length: %s
headers.get('etag')))
for key, value in headers.items():
if key.startswith('x-object-meta-'):
options.print_queue.put('%14s: %s' % ('Meta %s' %
print_queue.put('%14s: %s' % ('Meta %s' %
key[len('x-object-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-object-meta-') and key not in (
'content-type', 'content-length', 'last-modified',
'etag', 'date'):
options.print_queue.put(
print_queue.put(
'%14s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Object %s not found' %
error_queue.put('Object %s not found' %
repr('%s/%s' % (args[0], args[1])))
else:
options.error_queue.put('Usage: %s [options] %s' %
error_queue.put('Usage: %s [options] %s' %
(basename(argv[0]), st_stat_help))
@ -1252,7 +1295,22 @@ post [options] [container] [object]
or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
def st_post(options, args):
def st_post(options, args, print_queue, error_queue):
parser.add_option('-r', '--read-acl', dest='read_acl', help='Sets the '
'Read ACL for containers. Quick summary of ACL syntax: .r:*, '
'.r:-.example.com, .r:www.example.com, account1, account2:user2')
parser.add_option('-w', '--write-acl', dest='write_acl', help='Sets the '
'Write ACL for containers. Quick summary of ACL syntax: account1, '
'account2:user2')
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl) and not args:
exit('-r and -w options only allowed for containers')
conn = Connection(options.auth, options.user, options.key)
if not args:
headers = {}
@ -1265,7 +1323,7 @@ def st_post(options, args):
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
error_queue.put('Account not found')
elif len(args) == 1:
headers = {}
for item in options.meta:
@ -1293,10 +1351,10 @@ def st_post(options, args):
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Object %s not found' %
error_queue.put('Object %s not found' %
repr('%s/%s' % (args[0], args[1])))
else:
options.error_queue.put('Usage: %s [options] %s' %
error_queue.put('Usage: %s [options] %s' %
(basename(argv[0]), st_post_help))
@ -1305,12 +1363,21 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload.'''.strip('\n')
def st_upload(options, args):
def st_upload(options, args, print_queue, error_queue):
parser.add_option('-c', '--changed', action='store_true', dest='changed',
default=False, help='Will only upload files that have changed since '
'the last upload')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
options.error_queue.put('Usage: %s [options] %s' %
error_queue.put('Usage: %s [options] %s' %
(basename(argv[0]), st_upload_help))
return
file_queue = Queue(10000)
def _upload_file((path, dir_marker), conn):
try:
obj = path
@ -1352,11 +1419,12 @@ def st_upload(options, args):
content_length=getsize(path),
headers=put_headers)
if options.verbose:
options.print_queue.put(obj)
print_queue.put(obj)
except OSError, err:
if err.errno != ENOENT:
raise
options.error_queue.put('Local file %s not found' % repr(path))
error_queue.put('Local file %s not found' % repr(path))
def _upload_dir(path):
names = listdir(path)
if not names:
@ -1368,10 +1436,11 @@ def st_upload(options, args):
_upload_dir(subpath)
else:
file_queue.put((subpath, False)) # dir_marker = False
url, token = get_auth(options.auth, options.user, options.key, snet=options.snet)
url, token = get_auth(options.auth, options.user, options.key,
snet=options.snet)
create_connection = lambda: Connection(options.auth, options.user,
options.key, preauthurl=url,
preauthtoken=token, snet=options.snet)
options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
file_threads = [QueueFunctionThread(file_queue, _upload_file,
create_connection()) for _ in xrange(10)]
for thread in file_threads:
@ -1400,12 +1469,24 @@ def st_upload(options, args):
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
error_queue.put('Account not found')
def parse_args(parser, args, enforce_requires=True):
if not args:
args = ['-h']
(options, args) = parser.parse_args(args)
if enforce_requires and \
not (options.auth and options.user and options.key):
exit('''
Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or
overridden with -A, -U, or -K.'''.strip('\n'))
return options, args
if __name__ == '__main__':
parser = OptionParser(version='%prog 1.0', usage='''
Usage: %%prog [options] <command> [args]
Usage: %%prog <command> [options] [args]
Commands:
%(st_stat_help)s
@ -1424,55 +1505,18 @@ Example:
default=1, help='Print more info')
parser.add_option('-q', '--quiet', action='store_const', dest='verbose',
const=0, default=1, help='Suppress status output')
parser.add_option('-a', '--all', action='store_true', dest='yes_all',
default=False, help='Indicate that you really want the '
'whole account for commands that require --all in such '
'a case')
parser.add_option('-c', '--changed', action='store_true', dest='changed',
default=False, help='For the upload command: will '
'only upload files that have changed since the last '
'upload')
parser.add_option('-p', '--prefix', dest='prefix',
help='For the list command: will only list items '
'beginning with the prefix')
parser.add_option('-d', '--delimiter', dest='delimiter',
help='For the list command on containers: will roll up '
'items with the given delimiter (see Cloud Files '
'general documentation for what this means).')
parser.add_option('-r', '--read-acl', dest='read_acl',
help='Sets the Read ACL with post container commands. '
'Quick summary of ACL syntax: .r:*, .r:-.example.com, '
'.r:www.example.com, account1, account2:user2')
parser.add_option('-w', '--write-acl', dest='write_acl',
help='Sets the Write ACL with post container commands. '
'Quick summary of ACL syntax: account1, account2:user2')
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item of the syntax name:value '
'for use with post commands. This option may be '
'repeated. Example: -m Color:Blue -m Size:Large')
parser.add_option('-A', '--auth', dest='auth',
default=environ.get('ST_AUTH'),
help='URL for obtaining an auth token')
parser.add_option('-U', '--user', dest='user',
default=environ.get('ST_USER'),
help='User name for obtaining an auth token')
parser.add_option('-K', '--key', dest='key',
default=environ.get('ST_KEY'),
help='Key for obtaining an auth token')
parser.add_option('-o', '--output', dest='out_file',
help='For a single file download stream the output other location ')
args = argv[1:]
if not args:
args.append('-h')
(options, args) = parser.parse_args(args)
if options.out_file == '-':
options.verbose = 0
required_help = '''
Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or
overridden with -A, -U, or -K.'''.strip('\n')
for attr in ('auth', 'user', 'key'):
if not getattr(options, attr, None):
setattr(options, attr, environ.get('ST_%s' % attr.upper()))
if not getattr(options, attr, None):
exit(required_help)
parser.disable_interspersed_args()
(options, args) = parse_args(parser, argv[1:], enforce_requires=False)
parser.enable_interspersed_args()
commands = ('delete', 'download', 'list', 'post', 'stat', 'upload')
if not args or args[0] not in commands:
@ -1481,30 +1525,36 @@ overridden with -A, -U, or -K.'''.strip('\n')
exit('no such command: %s' % args[0])
exit()
options.print_queue = Queue(10000)
print_queue = Queue(10000)
def _print(item):
if isinstance(item, unicode):
item = item.encode('utf8')
print item
print_thread = QueueFunctionThread(options.print_queue, _print)
print_thread = QueueFunctionThread(print_queue, _print)
print_thread.start()
options.error_queue = Queue(10000)
error_queue = Queue(10000)
def _error(item):
if isinstance(item, unicode):
item = item.encode('utf8')
print >> stderr, item
error_thread = QueueFunctionThread(options.error_queue, _error)
error_thread = QueueFunctionThread(error_queue, _error)
error_thread.start()
try:
globals()['st_%s' % args[0]](options, args[1:])
while not options.print_queue.empty():
parser.usage = globals()['st_%s_help' % args[0]]
globals()['st_%s' % args[0]](parser, argv[1:], print_queue,
error_queue)
while not print_queue.empty():
sleep(0.01)
print_thread.abort = True
while print_thread.isAlive():
print_thread.join(0.01)
while not options.error_queue.empty():
while not error_queue.empty():
sleep(0.01)
error_thread.abort = True
while error_thread.isAlive():

4
swift/common/client.py
View File

@ -29,8 +29,12 @@ try:
except:
from time import sleep
try:
from swift.common.bufferedhttp \
import BufferedHTTPConnection as HTTPConnection
except:
from httplib import HTTPConnection
def quote(value, safe='/'):
"""