Merge "Bulk upload: treat user xattrs as object metadata"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
51565ab2fd
@ -75,6 +75,23 @@ def get_response_body(data_format, data_dict, error_list):
|
||||
return output
|
||||
|
||||
|
||||
def pax_key_to_swift_header(pax_key):
|
||||
if (pax_key == u"SCHILY.xattr.user.mime_type" or
|
||||
pax_key == u"LIBARCHIVE.xattr.user.mime_type"):
|
||||
return "Content-Type"
|
||||
elif pax_key.startswith(u"SCHILY.xattr.user.meta."):
|
||||
useful_part = pax_key[len(u"SCHILY.xattr.user.meta."):]
|
||||
return "X-Object-Meta-" + useful_part.encode("utf-8")
|
||||
elif pax_key.startswith(u"LIBARCHIVE.xattr.user.meta."):
|
||||
useful_part = pax_key[len(u"LIBARCHIVE.xattr.user.meta."):]
|
||||
return "X-Object-Meta-" + useful_part.encode("utf-8")
|
||||
else:
|
||||
# You can get things like atime/mtime/ctime or filesystem ACLs in
|
||||
# pax headers; those aren't really user metadata. The same goes for
|
||||
# other, non-user metadata.
|
||||
return None
|
||||
|
||||
|
||||
class Bulk(object):
|
||||
"""
|
||||
Middleware that will do many operations on a single request.
|
||||
@ -464,6 +481,16 @@ class Bulk(object):
|
||||
new_env['HTTP_USER_AGENT'] = \
|
||||
'%s BulkExpand' % req.environ.get('HTTP_USER_AGENT')
|
||||
create_obj_req = Request.blank(destination, new_env)
|
||||
|
||||
for pax_key, pax_value in tar_info.pax_headers.items():
|
||||
header_name = pax_key_to_swift_header(pax_key)
|
||||
if header_name:
|
||||
# Both pax_key and pax_value are unicode
|
||||
# strings; the key is already UTF-8 encoded, but
|
||||
# we still have to encode the value.
|
||||
create_obj_req.headers[header_name] = \
|
||||
pax_value.encode("utf-8")
|
||||
|
||||
resp = create_obj_req.get_response(self.app)
|
||||
containers_accessed.add(container)
|
||||
if resp.is_success:
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@ -25,9 +26,11 @@ from tempfile import mkdtemp
|
||||
from StringIO import StringIO
|
||||
from eventlet import sleep
|
||||
from mock import patch, call
|
||||
from test.unit.common.middleware.helpers import FakeSwift
|
||||
from swift.common import utils, constraints
|
||||
from swift.common.middleware import bulk
|
||||
from swift.common.swob import Request, Response, HTTPException
|
||||
from swift.common.swob import Request, Response, HTTPException, \
|
||||
HTTPNoContent, HTTPCreated, HeaderKeyDict
|
||||
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED
|
||||
|
||||
|
||||
@ -126,6 +129,104 @@ def build_tar_tree(tar, start_path, tree_obj, base_path=''):
|
||||
tar.addfile(tar_info)
|
||||
|
||||
|
||||
class TestUntarMetadata(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.app = FakeSwift()
|
||||
self.bulk = bulk.filter_factory({})(self.app)
|
||||
self.testdir = mkdtemp(suffix='tmp_test_bulk')
|
||||
|
||||
def tearDown(self):
|
||||
rmtree(self.testdir, ignore_errors=1)
|
||||
|
||||
def test_extract_metadata(self):
|
||||
self.app.register('HEAD', '/v1/a/c?extract-archive=tar',
|
||||
HTTPNoContent, {}, None)
|
||||
self.app.register('PUT', '/v1/a/c/obj1?extract-archive=tar',
|
||||
HTTPCreated, {}, None)
|
||||
self.app.register('PUT', '/v1/a/c/obj2?extract-archive=tar',
|
||||
HTTPCreated, {}, None)
|
||||
|
||||
# It's a real pain to instantiate TarInfo objects directly; they
|
||||
# really want to come from a file on disk or a tarball. So, we write
|
||||
# out some files and add pax headers to them as they get placed into
|
||||
# the tarball.
|
||||
with open(os.path.join(self.testdir, "obj1"), "w") as fh1:
|
||||
fh1.write("obj1 contents\n")
|
||||
with open(os.path.join(self.testdir, "obj2"), "w") as fh2:
|
||||
fh2.write("obj2 contents\n")
|
||||
|
||||
tar_ball = StringIO()
|
||||
tar_file = tarfile.TarFile.open(fileobj=tar_ball, mode="w",
|
||||
format=tarfile.PAX_FORMAT)
|
||||
|
||||
# With GNU tar 1.27.1 or later (possibly 1.27 as well), a file with
|
||||
# extended attribute user.thingy = dingy gets put into the tarfile
|
||||
# with pax_headers containing key/value pair
|
||||
# (SCHILY.xattr.user.thingy, dingy), both unicode strings (py2: type
|
||||
# unicode, not type str).
|
||||
#
|
||||
# With BSD tar (libarchive), you get key/value pair
|
||||
# (LIBARCHIVE.xattr.user.thingy, dingy), which strikes me as
|
||||
# gratuitous incompatibility.
|
||||
#
|
||||
# Still, we'll support uploads with both. Just heap more code on the
|
||||
# problem until you can forget it's under there.
|
||||
with open(os.path.join(self.testdir, "obj1")) as fh1:
|
||||
tar_info1 = tar_file.gettarinfo(fileobj=fh1,
|
||||
arcname="obj1")
|
||||
tar_info1.pax_headers[u'SCHILY.xattr.user.mime_type'] = \
|
||||
u'application/food-diary'
|
||||
tar_info1.pax_headers[u'SCHILY.xattr.user.meta.lunch'] = \
|
||||
u'sopa de albóndigas'
|
||||
tar_info1.pax_headers[
|
||||
u'SCHILY.xattr.user.meta.afternoon-snack'] = \
|
||||
u'gigantic bucket of coffee'
|
||||
tar_file.addfile(tar_info1, fh1)
|
||||
|
||||
with open(os.path.join(self.testdir, "obj2")) as fh2:
|
||||
tar_info2 = tar_file.gettarinfo(fileobj=fh2,
|
||||
arcname="obj2")
|
||||
tar_info2.pax_headers[
|
||||
u'LIBARCHIVE.xattr.user.meta.muppet'] = u'bert'
|
||||
tar_info2.pax_headers[
|
||||
u'LIBARCHIVE.xattr.user.meta.cat'] = u'fluffy'
|
||||
tar_info2.pax_headers[
|
||||
u'LIBARCHIVE.xattr.user.notmeta'] = u'skipped'
|
||||
tar_file.addfile(tar_info2, fh2)
|
||||
|
||||
tar_ball.seek(0)
|
||||
|
||||
req = Request.blank('/v1/a/c?extract-archive=tar')
|
||||
req.environ['REQUEST_METHOD'] = 'PUT'
|
||||
req.environ['wsgi.input'] = tar_ball
|
||||
req.headers['transfer-encoding'] = 'chunked'
|
||||
req.headers['accept'] = 'application/json;q=1.0'
|
||||
|
||||
resp = req.get_response(self.bulk)
|
||||
self.assertEqual(resp.status_int, 200)
|
||||
|
||||
# sanity check to make sure the upload worked
|
||||
upload_status = utils.json.loads(resp.body)
|
||||
self.assertEqual(upload_status['Number Files Created'], 2)
|
||||
|
||||
put1_headers = HeaderKeyDict(self.app.calls_with_headers[1][2])
|
||||
self.assertEqual(
|
||||
put1_headers.get('Content-Type'),
|
||||
'application/food-diary')
|
||||
self.assertEqual(
|
||||
put1_headers.get('X-Object-Meta-Lunch'),
|
||||
'sopa de alb\xc3\xb3ndigas')
|
||||
self.assertEqual(
|
||||
put1_headers.get('X-Object-Meta-Afternoon-Snack'),
|
||||
'gigantic bucket of coffee')
|
||||
|
||||
put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2])
|
||||
self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert')
|
||||
self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy')
|
||||
self.assertEqual(put2_headers.get('Content-Type'), None)
|
||||
self.assertEqual(put2_headers.get('X-Object-Meta-Blah'), None)
|
||||
|
||||
|
||||
class TestUntar(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user