openstack/swift
Code Issues Proposed changes
dc061dd95e
swift/test/functional/test_symlink.py
Ade Lee 5320ecbaf2 replace md5 with swift utils version 
md5 is not an approved algorithm in FIPS mode, and trying to
instantiate a hashlib.md5() will fail when the system is running in
FIPS mode.

md5 is allowed when in a non-security context.  There is a plan to
add a keyword parameter (usedforsecurity) to hashlib.md5() to annotate
whether or not the instance is being used in a security context.

In the case where it is not, the instantiation of md5 will be allowed.
See https://bugs.python.org/issue9216 for more details.

Some downstream python versions already support this parameter.  To
support these versions, a new encapsulation of md5() is added to
swift/common/utils.py.  This encapsulation is identical to the one being
added to oslo.utils, but is recreated here to avoid adding a dependency.

This patch is to replace the instances of hashlib.md5() with this new
encapsulation, adding an annotation indicating whether the usage is
a security context or not.

While this patch seems large, it is really just the same change over and
again.  Reviewers need to pay particular attention as to whether the
keyword parameter (usedforsecurity) is set correctly.   Right now, all
of them appear to be not used in a security context.

Now that all the instances have been converted, we can update the bandit
run to look for these instances and ensure that new invocations do not
creep in.

With this latest patch, the functional and unit tests all pass
on a FIPS enabled system.

Co-Authored-By: Pete Zaitcev
Change-Id: Ibb4917da4c083e1e094156d748708b87387f2d87
2020-12-15 09:52:55 -05:00

2479 lines
101 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python
# Copyright (c) 2010-2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hmac
import unittest
import itertools
import hashlib
import six
import time
from six.moves import urllib
from uuid import uuid4
from swift.common.http import is_success
from swift.common.swob import normalize_etag
from swift.common.utils import json, MD5_OF_EMPTY_STRING, md5
from swift.common.middleware.slo import SloGetContext
from test.functional import check_response, retry, requires_acls, \
cluster_info, SkipTest
from test.functional.tests import Base, TestFileComparisonEnv, Utils, BaseEnv
from test.functional.test_slo import TestSloEnv
from test.functional.test_dlo import TestDloEnv
from test.functional.test_tempurl import TestContainerTempurlEnv, \
TestTempurlEnv
from test.functional.swift_test_client import ResponseError
import test.functional as tf
from test.unit import group_by_byte
TARGET_BODY = b'target body'
def setUpModule():
tf.setup_package()
if 'symlink' not in cluster_info:
raise SkipTest("Symlinks not enabled")
def tearDownModule():
tf.teardown_package()
class TestSymlinkEnv(BaseEnv):
link_cont = uuid4().hex
tgt_cont = uuid4().hex
tgt_obj = uuid4().hex
@classmethod
def setUp(cls):
if tf.skip or tf.skip2:
raise SkipTest
cls._create_container(cls.tgt_cont) # use_account=1
cls._create_container(cls.link_cont) # use_account=1
# container in account 2
cls._create_container(cls.link_cont, use_account=2)
cls._create_tgt_object()
@classmethod
def containers(cls):
return (cls.link_cont, cls.tgt_cont)
@classmethod
def target_content_location(cls, override_obj=None, override_account=None):
account = override_account or tf.parsed[0].path.split('/', 2)[2]
return '/v1/%s/%s/%s' % (account, cls.tgt_cont,
override_obj or cls.tgt_obj)
@classmethod
def _make_request(cls, url, token, parsed, conn, method,
container, obj='', headers=None, body=b'',
query_args=None):
headers = headers or {}
headers.update({'X-Auth-Token': token})
path = '%s/%s/%s' % (parsed.path, container, obj) if obj \
else '%s/%s' % (parsed.path, container)
if query_args:
path += '?%s' % query_args
conn.request(method, path, body, headers)
resp = check_response(conn)
# to read the buffer and keep it in the attribute, call resp.content
resp.content
return resp
@classmethod
def _create_container(cls, name, headers=None, use_account=1):
headers = headers or {}
resp = retry(cls._make_request, method='PUT', container=name,
headers=headers, use_account=use_account)
if resp.status not in (201, 202):
raise ResponseError(resp)
return name
@classmethod
def _create_tgt_object(cls, body=TARGET_BODY):
resp = retry(cls._make_request, method='PUT',
headers={'Content-Type': 'application/target'},
container=cls.tgt_cont, obj=cls.tgt_obj,
body=body)
if resp.status != 201:
raise ResponseError(resp)
# sanity: successful put response has content-length 0
cls.tgt_length = str(len(body))
cls.tgt_etag = resp.getheader('etag')
resp = retry(cls._make_request, method='GET',
container=cls.tgt_cont, obj=cls.tgt_obj)
if resp.status != 200 and resp.content != body:
raise ResponseError(resp)
@classmethod
def tearDown(cls):
delete_containers = [
(use_account, containers) for use_account, containers in
enumerate([cls.containers(), [cls.link_cont]], 1)]
# delete objects inside container
for use_account, containers in delete_containers:
if use_account == 2 and tf.skip2:
continue
for container in containers:
while True:
cont = container
resp = retry(cls._make_request, method='GET',
container=cont, query_args='format=json',
use_account=use_account)
if resp.status == 404:
break
if not is_success(resp.status):
raise ResponseError(resp)
objs = json.loads(resp.content)
if not objs:
break
for obj in objs:
resp = retry(cls._make_request, method='DELETE',
container=container, obj=obj['name'],
use_account=use_account)
if resp.status not in (204, 404):
raise ResponseError(resp)
# delete the containers
for use_account, containers in delete_containers:
if use_account == 2 and tf.skip2:
continue
for container in containers:
resp = retry(cls._make_request, method='DELETE',
container=container,
use_account=use_account)
if resp.status not in (204, 404):
raise ResponseError(resp)
class TestSymlink(Base):
env = TestSymlinkEnv
@classmethod
def setUpClass(cls):
# To skip env setup for class setup, instead setUp the env for each
# test method
pass
def setUp(self):
self.env.setUp()
def object_name_generator():
while True:
yield uuid4().hex
self.obj_name_gen = object_name_generator()
self._account_name = None
def tearDown(self):
self.env.tearDown()
@property
def account_name(self):
if not self._account_name:
self._account_name = tf.parsed[0].path.split('/', 2)[2]
return self._account_name
def _make_request(self, url, token, parsed, conn, method,
container, obj='', headers=None, body=b'',
query_args=None, allow_redirects=True):
headers = headers or {}
headers.update({'X-Auth-Token': token})
path = '%s/%s/%s' % (parsed.path, container, obj) if obj \
else '%s/%s' % (parsed.path, container)
if query_args:
path += '?%s' % query_args
conn.requests_args['allow_redirects'] = allow_redirects
conn.request(method, path, body, headers)
resp = check_response(conn)
# to read the buffer and keep it in the attribute, call resp.content
resp.content
return resp
def _make_request_with_symlink_get(self, url, token, parsed, conn, method,
container, obj, headers=None, body=b''):
resp = self._make_request(
url, token, parsed, conn, method, container, obj, headers, body,
query_args='symlink=get')
return resp
def _test_put_symlink(self, link_cont, link_obj, tgt_cont, tgt_obj):
headers = {'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj)}
resp = retry(self._make_request, method='PUT',
container=link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 201)
def _test_put_symlink_with_etag(self, link_cont, link_obj, tgt_cont,
tgt_obj, etag, headers=None):
headers = headers or {}
headers.update({'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj),
'X-Symlink-Target-Etag': etag})
resp = retry(self._make_request, method='PUT',
container=link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 201, resp.content)
def _test_get_as_target_object(
self, link_cont, link_obj, expected_content_location,
use_account=1):
resp = retry(
self._make_request, method='GET',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200, resp.content)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'),
str(self.env.tgt_length))
self.assertEqual(resp.getheader('etag'), self.env.tgt_etag)
self.assertIn('Content-Location', resp.headers)
self.assertEqual(expected_content_location,
resp.getheader('content-location'))
return resp
def _test_head_as_target_object(self, link_cont, link_obj, use_account=1):
resp = retry(
self._make_request, method='HEAD',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200)
def _assertLinkObject(self, link_cont, link_obj, use_account=1):
# HEAD on link_obj itself
resp = retry(
self._make_request_with_symlink_get, method='HEAD',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200)
self.assertTrue(resp.getheader('x-symlink-target'))
# GET on link_obj itself
resp = retry(
self._make_request_with_symlink_get, method='GET',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, b'')
self.assertEqual(resp.getheader('content-length'), str(0))
self.assertTrue(resp.getheader('x-symlink-target'))
def _assertSymlink(self, link_cont, link_obj,
expected_content_location=None, use_account=1):
expected_content_location = \
expected_content_location or self.env.target_content_location()
# sanity: HEAD/GET on link_obj
self._assertLinkObject(link_cont, link_obj, use_account)
# HEAD target object via symlink
self._test_head_as_target_object(
link_cont=link_cont, link_obj=link_obj, use_account=use_account)
# GET target object via symlink
self._test_get_as_target_object(
link_cont=link_cont, link_obj=link_obj, use_account=use_account,
expected_content_location=expected_content_location)
def test_symlink_with_encoded_target_name(self):
# makes sure to test encoded characters as symlink target
target_obj = 'dealde%2Fl04 011e%204c8df/flash.png'
link_obj = uuid4().hex
# create target using unnormalized path
resp = retry(
self._make_request, method='PUT', container=self.env.tgt_cont,
obj=target_obj, body=TARGET_BODY)
self.assertEqual(resp.status, 201)
# you can get it using either name
resp = retry(
self._make_request, method='GET', container=self.env.tgt_cont,
obj=target_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
normalized_quoted_obj = 'dealde/l04%20011e%204c8df/flash.png'
self.assertEqual(normalized_quoted_obj, urllib.parse.quote(
urllib.parse.unquote(target_obj)))
resp = retry(
self._make_request, method='GET', container=self.env.tgt_cont,
obj=normalized_quoted_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
# create a symlink using the un-normalized target path
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=target_obj)
# and it's normalized
self._assertSymlink(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(
normalized_quoted_obj))
# create a symlink using the normalized target path
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=normalized_quoted_obj)
# and it's ALSO normalized
self._assertSymlink(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(
normalized_quoted_obj))
def test_symlink_put_head_get(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
self._assertSymlink(self.env.link_cont, link_obj)
def test_symlink_with_etag_put_head_get(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
self._assertSymlink(self.env.link_cont, link_obj)
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': self.env.tgt_etag})
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': 'not-the-etag'})
self.assertEqual(resp.status, 412)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_static_symlink_with_bad_etag_put_head_get(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# overwrite tgt object
self.env._create_tgt_object(body='updated target body')
resp = retry(
self._make_request, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
# but we still know where it points
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# uses a mechanism entirely divorced from if-match
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': self.env.tgt_etag})
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': 'not-the-etag'})
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='DELETE',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
# not-found-ness trumps if-match-ness
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 404)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_dynamic_link_to_static_link(self):
static_link_obj = uuid4().hex
# PUT static_link to tgt_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
symlink_obj = uuid4().hex
# PUT symlink to static_link
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.link_cont,
tgt_obj=static_link_obj)
self._test_get_as_target_object(
link_cont=self.env.link_cont, link_obj=symlink_obj,
expected_content_location=self.env.target_content_location())
def test_static_link_to_dynamic_link(self):
symlink_obj = uuid4().hex
# PUT symlink to tgt_obj
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
static_link_obj = uuid4().hex
# PUT a static_link to the symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
self._test_get_as_target_object(
link_cont=self.env.link_cont, link_obj=static_link_obj,
expected_content_location=self.env.target_content_location())
def test_static_link_to_nowhere(self):
missing_obj = uuid4().hex
static_link_obj = uuid4().hex
# PUT a static_link to the missing name
headers = {
'X-Symlink-Target': '%s/%s' % (self.env.link_cont, missing_obj),
'X-Symlink-Target-Etag': MD5_OF_EMPTY_STRING}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=static_link_obj,
headers=headers)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content, b'X-Symlink-Target does not exist')
def test_static_link_to_broken_symlink(self):
symlink_obj = uuid4().hex
# PUT symlink to tgt_obj
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
static_link_obj = uuid4().hex
# PUT a static_link to the symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
# break the symlink
resp = retry(
self._make_request, method='DELETE',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
self.assertEqual(resp.status // 100, 2)
# sanity
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=symlink_obj)
self.assertEqual(resp.status, 404)
# static_link is broken too!
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=static_link_obj)
self.assertEqual(resp.status, 404)
# interestingly you may create a static_link to a broken symlink
broken_static_link_obj = uuid4().hex
# PUT a static_link to the broken symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=broken_static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
def test_symlink_get_ranged(self):
link_obj = uuid4().hex
# PUT symlink
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
headers = {'Range': 'bytes=7-10'}
resp = retry(self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 206)
self.assertEqual(resp.content, b'body')
def test_create_symlink_before_target(self):
link_obj = uuid4().hex
target_obj = uuid4().hex
# PUT link_obj before target object is written
# PUT, GET, HEAD (on symlink) should all work ok without target object
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont, tgt_obj=target_obj)
# Try to GET target via symlink.
# 404 will be returned with Content-Location of target path.
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=1)
self.assertEqual(resp.status, 404)
self.assertIn('Content-Location', resp.headers)
self.assertEqual(self.env.target_content_location(target_obj),
resp.getheader('content-location'))
# HEAD on target object via symlink should return a 404 since target
# object has not yet been written
resp = retry(
self._make_request, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 404)
# GET on target object directly
resp = retry(
self._make_request, method='GET',
container=self.env.tgt_cont, obj=target_obj)
self.assertEqual(resp.status, 404)
# Now let's write target object and symlink will be able to return
# object
resp = retry(
self._make_request, method='PUT', container=self.env.tgt_cont,
obj=target_obj, body=TARGET_BODY)
self.assertEqual(resp.status, 201)
# successful put response has content-length 0
target_length = str(len(TARGET_BODY))
target_etag = resp.getheader('etag')
# sanity: HEAD/GET on link_obj itself
self._assertLinkObject(self.env.link_cont, link_obj)
# HEAD target object via symlink
self._test_head_as_target_object(
link_cont=self.env.link_cont, link_obj=link_obj)
# GET target object via symlink
resp = retry(self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'), str(target_length))
self.assertEqual(resp.getheader('etag'), target_etag)
self.assertIn('Content-Location', resp.headers)
self.assertEqual(self.env.target_content_location(target_obj),
resp.getheader('content-location'))
def test_symlink_chain(self):
# Testing to symlink chain like symlink -> symlink -> target.
symloop_max = cluster_info['symlink']['symloop_max']
# create symlink chain in a container. To simplify,
# use target container for all objects (symlinks and target) here
previous = self.env.tgt_obj
container = self.env.tgt_cont
for link_obj in itertools.islice(self.obj_name_gen, symloop_max):
# PUT link_obj point to tgt_obj
self._test_put_symlink(
link_cont=container, link_obj=link_obj,
tgt_cont=container, tgt_obj=previous)
# set corrent link_obj to previous
previous = link_obj
# the last link is valid for symloop_max constraint
max_chain_link = link_obj
self._assertSymlink(link_cont=container, link_obj=max_chain_link)
# PUT a new link_obj points to the max_chain_link
# that will result in 409 error on the HEAD/GET.
too_many_chain_link = next(self.obj_name_gen)
self._test_put_symlink(
link_cont=container, link_obj=too_many_chain_link,
tgt_cont=container, tgt_obj=max_chain_link)
# try to HEAD to target object via too_many_chain_link
resp = retry(self._make_request, method='HEAD',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content, b'')
# try to GET to target object via too_many_chain_link
resp = retry(self._make_request, method='GET',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(
resp.content,
b'Too many levels of symbolic links, maximum allowed is %d' %
symloop_max)
# However, HEAD/GET to the (just) link is still ok
self._assertLinkObject(container, too_many_chain_link)
def test_symlink_chain_with_etag(self):
# Testing to symlink chain like symlink -> symlink -> target.
symloop_max = cluster_info['symlink']['symloop_max']
# create symlink chain in a container. To simplify,
# use target container for all objects (symlinks and target) here
previous = self.env.tgt_obj
container = self.env.tgt_cont
for link_obj in itertools.islice(self.obj_name_gen, symloop_max):
# PUT link_obj point to tgt_obj
self._test_put_symlink_with_etag(link_cont=container,
link_obj=link_obj,
tgt_cont=container,
tgt_obj=previous,
etag=self.env.tgt_etag)
# set current link_obj to previous
previous = link_obj
# the last link is valid for symloop_max constraint
max_chain_link = link_obj
self._assertSymlink(link_cont=container, link_obj=max_chain_link)
# chained etag validation works as long as the target symlink works
headers = {'X-Symlink-Target': '%s/%s' % (container, max_chain_link),
'X-Symlink-Target-Etag': 'not-the-real-etag'}
resp = retry(self._make_request, method='PUT',
container=container, obj=uuid4().hex,
headers=headers)
self.assertEqual(resp.status, 409)
# PUT a new link_obj pointing to the max_chain_link can validate the
# ETag but will result in 409 error on the HEAD/GET.
too_many_chain_link = next(self.obj_name_gen)
self._test_put_symlink_with_etag(
link_cont=container, link_obj=too_many_chain_link,
tgt_cont=container, tgt_obj=max_chain_link,
etag=self.env.tgt_etag)
# try to HEAD to target object via too_many_chain_link
resp = retry(self._make_request, method='HEAD',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content, b'')
# try to GET to target object via too_many_chain_link
resp = retry(self._make_request, method='GET',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(
resp.content,
b'Too many levels of symbolic links, maximum allowed is %d' %
symloop_max)
# However, HEAD/GET to the (just) link is still ok
self._assertLinkObject(container, too_many_chain_link)
def test_symlink_and_slo_manifest_chain(self):
if 'slo' not in cluster_info:
raise SkipTest
symloop_max = cluster_info['symlink']['symloop_max']
# create symlink chain in a container. To simplify,
# use target container for all objects (symlinks and target) here
previous = self.env.tgt_obj
container = self.env.tgt_cont
# make symlink and slo manifest chain
# e.g. slo -> symlink -> symlink -> slo -> symlink -> symlink -> target
for _ in range(SloGetContext.max_slo_recursion_depth or 1):
for link_obj in itertools.islice(self.obj_name_gen, symloop_max):
# PUT link_obj point to previous object
self._test_put_symlink(
link_cont=container, link_obj=link_obj,
tgt_cont=container, tgt_obj=previous)
# next link will point to this link
previous = link_obj
else:
# PUT a manifest with single segment to the symlink
manifest_obj = next(self.obj_name_gen)
manifest = json.dumps(
[{'path': '/%s/%s' % (container, link_obj)}])
resp = retry(self._make_request, method='PUT',
container=container, obj=manifest_obj,
body=manifest,
query_args='multipart-manifest=put')
self.assertEqual(resp.status, 201) # sanity
previous = manifest_obj
# From the last manifest to the final target obj length is
# symloop_max * max_slo_recursion_depth
max_recursion_manifest = previous
# Check GET to max_recursion_manifest returns valid target object
resp = retry(
self._make_request, method='GET', container=container,
obj=max_recursion_manifest)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'),
str(self.env.tgt_length))
# N.B. since the last manifest is slo so it will remove
# content-location info from the response header
self.assertNotIn('Content-Location', resp.headers)
# sanity: one more link to the slo can work still
one_more_link = next(self.obj_name_gen)
self._test_put_symlink(
link_cont=container, link_obj=one_more_link,
tgt_cont=container, tgt_obj=max_recursion_manifest)
resp = retry(
self._make_request, method='GET', container=container,
obj=one_more_link)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'),
str(self.env.tgt_length))
self.assertIn('Content-Location', resp.headers)
self.assertIn('%s/%s' % (container, max_recursion_manifest),
resp.getheader('content-location'))
# PUT a new slo manifest point to the max_recursion_manifest
# Symlink and slo manifest chain from the new manifest to the final
# target has (max_slo_recursion_depth + 1) manifests.
too_many_recursion_manifest = next(self.obj_name_gen)
manifest = json.dumps(
[{'path': '/%s/%s' % (container, max_recursion_manifest)}])
resp = retry(self._make_request, method='PUT',
container=container, obj=too_many_recursion_manifest,
body=manifest.encode('ascii'),
query_args='multipart-manifest=put')
self.assertEqual(resp.status, 201) # sanity
# Check GET to too_many_recursion_mani returns 409 error
resp = retry(self._make_request, method='GET',
container=container, obj=too_many_recursion_manifest)
self.assertEqual(resp.status, 409)
# N.B. This error message is from slo middleware that uses default.
self.assertEqual(
resp.content,
b'<html><h1>Conflict</h1><p>There was a conflict when trying to'
b' complete your request.</p></html>')
def test_symlink_put_missing_target_container(self):
link_obj = uuid4().hex
# set only object, no container in the prefix
headers = {'X-Symlink-Target': self.env.tgt_obj}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 412)
self.assertEqual(resp.content,
b'X-Symlink-Target header must be of the form'
b' <container name>/<object name>')
def test_symlink_put_non_zero_length(self):
link_obj = uuid4().hex
headers = {'X-Symlink-Target':
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj)}
resp = retry(
self._make_request, method='PUT', container=self.env.link_cont,
obj=link_obj, body=b'non-zero-length', headers=headers)
self.assertEqual(resp.status, 400)
self.assertEqual(resp.content,
b'Symlink requests require a zero byte body')
def test_symlink_target_itself(self):
link_obj = uuid4().hex
headers = {
'X-Symlink-Target': '%s/%s' % (self.env.link_cont, link_obj)}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 400)
self.assertEqual(resp.content, b'Symlink cannot target itself')
def test_symlink_target_each_other(self):
symloop_max = cluster_info['symlink']['symloop_max']
link_obj1 = uuid4().hex
link_obj2 = uuid4().hex
# PUT two links which targets each other
self._test_put_symlink(
link_cont=self.env.link_cont, link_obj=link_obj1,
tgt_cont=self.env.link_cont, tgt_obj=link_obj2)
self._test_put_symlink(
link_cont=self.env.link_cont, link_obj=link_obj2,
tgt_cont=self.env.link_cont, tgt_obj=link_obj1)
for obj in (link_obj1, link_obj2):
# sanity: HEAD/GET on the link itself is ok
self._assertLinkObject(self.env.link_cont, obj)
for obj in (link_obj1, link_obj2):
resp = retry(self._make_request, method='HEAD',
container=self.env.link_cont, obj=obj)
self.assertEqual(resp.status, 409)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont, obj=obj)
self.assertEqual(resp.status, 409)
self.assertEqual(
resp.content,
b'Too many levels of symbolic links, maximum allowed is %d' %
symloop_max)
def test_symlink_put_copy_from(self):
link_obj1 = uuid4().hex
link_obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_src = '%s/%s' % (self.env.link_cont, link_obj1)
# copy symlink
headers = {'X-Copy-From': copy_src}
resp = retry(self._make_request_with_symlink_get,
method='PUT',
container=self.env.link_cont, obj=link_obj2,
headers=headers)
self.assertEqual(resp.status, 201)
self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2)
@requires_acls
def test_symlink_put_copy_from_cross_account(self):
link_obj1 = uuid4().hex
link_obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_src = '%s/%s' % (self.env.link_cont, link_obj1)
perm_two = tf.swift_test_perm[1]
# add X-Content-Read to account 1 link_cont and tgt_cont
# permit account 2 to read account 1 link_cont to perform copy_src
# and tgt_cont so that link_obj2 can refer to tgt_object
# this ACL allows the copy to succeed
headers = {'X-Container-Read': perm_two}
resp = retry(
self._make_request, method='POST',
container=self.env.link_cont, headers=headers)
self.assertEqual(resp.status, 204)
# this ACL allows link_obj in account 2 to target object in account 1
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=headers)
self.assertEqual(resp.status, 204)
# copy symlink itself to a different account w/o
# X-Symlink-Target-Account. This operation will result in copying
# symlink to the account 2 container that points to the
# container/object in the account 2.
# (the container/object is not prepared)
headers = {'X-Copy-From-Account': self.account_name,
'X-Copy-From': copy_src}
resp = retry(self._make_request_with_symlink_get, method='PUT',
container=self.env.link_cont, obj=link_obj2,
headers=headers, use_account=2)
self.assertEqual(resp.status, 201)
# sanity: HEAD/GET on link_obj itself
self._assertLinkObject(self.env.link_cont, link_obj2, use_account=2)
account_two = tf.parsed[1].path.split('/', 2)[2]
# no target object in the account 2
for method in ('HEAD', 'GET'):
resp = retry(
self._make_request, method=method,
container=self.env.link_cont, obj=link_obj2, use_account=2)
self.assertEqual(resp.status, 404)
self.assertIn('content-location', resp.headers)
self.assertEqual(
self.env.target_content_location(override_account=account_two),
resp.getheader('content-location'))
# copy symlink itself to a different account with target account
# the target path will be in account 1
# the target path will have an object
headers = {'X-Symlink-target-Account': self.account_name,
'X-Copy-From-Account': self.account_name,
'X-Copy-From': copy_src}
resp = retry(
self._make_request_with_symlink_get, method='PUT',
container=self.env.link_cont, obj=link_obj2,
headers=headers, use_account=2)
self.assertEqual(resp.status, 201)
self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2,
use_account=2)
def test_symlink_copy_from_target(self):
link_obj1 = uuid4().hex
obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_src = '%s/%s' % (self.env.link_cont, link_obj1)
# issuing a COPY request to a symlink w/o symlink=get, should copy
# the target object, not the symlink itself
headers = {'X-Copy-From': copy_src}
resp = retry(self._make_request, method='PUT',
container=self.env.tgt_cont, obj=obj2,
headers=headers)
self.assertEqual(resp.status, 201)
# HEAD to the copied object
resp = retry(self._make_request, method='HEAD',
container=self.env.tgt_cont, obj=obj2)
self.assertEqual(200, resp.status)
self.assertNotIn('Content-Location', resp.headers)
# GET to the copied object
resp = retry(self._make_request, method='GET',
container=self.env.tgt_cont, obj=obj2)
# But... this is a raw object (not a symlink)
self.assertEqual(200, resp.status)
self.assertNotIn('Content-Location', resp.headers)
self.assertEqual(TARGET_BODY, resp.content)
def test_symlink_copy(self):
link_obj1 = uuid4().hex
link_obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_dst = '%s/%s' % (self.env.link_cont, link_obj2)
# copy symlink
headers = {'Destination': copy_dst}
resp = retry(
self._make_request_with_symlink_get, method='COPY',
container=self.env.link_cont, obj=link_obj1, headers=headers)
self.assertEqual(resp.status, 201)
self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2)
def test_symlink_copy_target(self):
link_obj1 = uuid4().hex
obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_dst = '%s/%s' % (self.env.tgt_cont, obj2)
# copy target object
headers = {'Destination': copy_dst}
resp = retry(self._make_request, method='COPY',
container=self.env.link_cont, obj=link_obj1,
headers=headers)
self.assertEqual(resp.status, 201)
# HEAD to target object via symlink
resp = retry(self._make_request, method='HEAD',
container=self.env.tgt_cont, obj=obj2)
self.assertEqual(resp.status, 200)
self.assertNotIn('Content-Location', resp.headers)
# GET to the copied object that should be a raw object (not symlink)
resp = retry(self._make_request, method='GET',
container=self.env.tgt_cont, obj=obj2)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
self.assertNotIn('Content-Location', resp.headers)
def test_post_symlink(self):
link_obj = uuid4().hex
value1 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
# POSTing to a symlink is not allowed and should return a 307
headers = {'X-Object-Meta-Alpha': 'apple'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
# we are using account 0 in this test
expected_location_hdr = "%s/%s/%s" % (
tf.parsed[0].path, self.env.tgt_cont, self.env.tgt_obj)
self.assertEqual(resp.getheader('Location'), expected_location_hdr)
# Read header from symlink itself. The metadata is applied to symlink
resp = retry(self._make_request_with_symlink_get, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('X-Object-Meta-Alpha'), 'apple')
# Post the target object directly
headers = {'x-object-meta-test': value1}
resp = retry(
self._make_request, method='POST', container=self.env.tgt_cont,
obj=self.env.tgt_obj, headers=headers)
self.assertEqual(resp.status, 202)
resp = retry(self._make_request, method='GET',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('X-Object-Meta-Test'), value1)
# Read header from target object via symlink, should exist now.
resp = retry(
self._make_request, method='GET', container=self.env.link_cont,
obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('X-Object-Meta-Test'), value1)
# sanity: no X-Object-Meta-Alpha exists in the response header
self.assertNotIn('X-Object-Meta-Alpha', resp.headers)
def test_post_to_broken_dynamic_symlink(self):
# create a symlink to nowhere
link_obj = '%s-the-link' % uuid4().hex
tgt_obj = '%s-no-where' % uuid4().hex
headers = {'X-Symlink-Target': '%s/%s' % (self.env.tgt_cont, tgt_obj)}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 201)
# it's a real link!
self._assertLinkObject(self.env.link_cont, link_obj)
# ... it's just broken
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 404)
target_path = '/v1/%s/%s/%s' % (
self.account_name, self.env.tgt_cont, tgt_obj)
self.assertEqual(target_path, resp.headers['Content-Location'])
# we'll redirect with the Location header to the (invalid) target
headers = {'X-Object-Meta-Alpha': 'apple'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
self.assertEqual(target_path, resp.headers['Location'])
# and of course metadata *is* applied to the link
resp = retry(
self._make_request_with_symlink_get, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertTrue(resp.getheader('X-Object-Meta-Alpha'), 'apple')
def test_post_to_broken_static_symlink(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# overwrite tgt object
old_tgt_etag = normalize_etag(self.env.tgt_etag)
self.env._create_tgt_object(body='updated target body')
# sanity
resp = retry(
self._make_request, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
# but POST will still 307
headers = {'X-Object-Meta-Alpha': 'apple'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
target_path = '/v1/%s/%s/%s' % (
self.account_name, self.env.tgt_cont, self.env.tgt_obj)
self.assertEqual(target_path, resp.headers['Location'])
# but we give you the Etag just like... FYI?
self.assertEqual(old_tgt_etag, resp.headers['X-Symlink-Target-Etag'])
def test_post_with_symlink_header(self):
# POSTing to a symlink is not allowed and should return a 307
# updating the symlink target with a POST should always fail
headers = {'X-Symlink-Target': 'container/new_target'}
resp = retry(
self._make_request, method='POST', container=self.env.tgt_cont,
obj=self.env.tgt_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 400)
self.assertEqual(resp.content,
b'A PUT request is required to set a symlink target')
def test_overwrite_symlink(self):
link_obj = uuid4().hex
new_tgt_obj = "new_target_object_name"
new_tgt = '%s/%s' % (self.env.tgt_cont, new_tgt_obj)
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
# sanity
self._assertSymlink(self.env.link_cont, link_obj)
# Overwrite symlink with PUT
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=new_tgt_obj)
# head symlink to check X-Symlink-Target header
resp = retry(self._make_request_with_symlink_get, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
# target should remain with old target
self.assertEqual(resp.getheader('X-Symlink-Target'), new_tgt)
def test_delete_symlink(self):
link_obj = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
resp = retry(self._make_request, method='DELETE',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 204)
# make sure target object was not deleted and is still reachable
resp = retry(self._make_request, method='GET',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
@requires_acls
def test_symlink_put_target_account(self):
if tf.skip or tf.skip2:
raise SkipTest
link_obj = uuid4().hex
# create symlink in account 2
# pointing to account 1
headers = {'X-Symlink-Target-Account': self.account_name,
'X-Symlink-Target':
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj)}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers, use_account=2)
self.assertEqual(resp.status, 201)
perm_two = tf.swift_test_perm[1]
# sanity test:
# it should be ok to get the symlink itself, but not the target object
# because the read acl has not been configured yet
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 403)
# still know where it's pointing
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# add X-Content-Read to account 1 tgt_cont
# permit account 2 to read account 1 tgt_cont
# add acl to allow reading from source
headers = {'X-Container-Read': perm_two}
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=headers)
self.assertEqual(resp.status, 204)
# GET on link_obj itself
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
# GET to target object via symlink
resp = self._test_get_as_target_object(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(),
use_account=2)
@requires_acls
def test_symlink_with_etag_put_target_account(self):
if tf.skip or tf.skip2:
raise SkipTest
link_obj = uuid4().hex
# try to create a symlink in account 2 pointing to account 1
symlink_headers = {
'X-Symlink-Target-Account': self.account_name,
'X-Symlink-Target':
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj),
'X-Symlink-Target-Etag': self.env.tgt_etag}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=symlink_headers, use_account=2)
# since we don't have read access to verify the object we get the
# permissions error
self.assertEqual(resp.status, 403)
perm_two = tf.swift_test_perm[1]
# add X-Content-Read to account 1 tgt_cont
# permit account 2 to read account 1 tgt_cont
# add acl to allow reading from source
acl_headers = {'X-Container-Read': perm_two}
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=acl_headers)
self.assertEqual(resp.status, 204)
# now we can create the symlink
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=symlink_headers, use_account=2)
self.assertEqual(resp.status, 201)
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
# GET to target object via symlink
resp = self._test_get_as_target_object(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(),
use_account=2)
# Overwrite target
resp = retry(self._make_request, method='PUT',
container=self.env.tgt_cont, obj=self.env.tgt_obj,
body='some other content')
self.assertEqual(resp.status, 201)
# link is now broken
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 409)
# but we still know where it points
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# sanity test, remove permissions
headers = {'X-Remove-Container-Read': 'remove'}
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=headers)
self.assertEqual(resp.status, 204)
# it should be ok to get the symlink itself, but not the target object
# because the read acl has been revoked
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 403)
# Still know where it is, though
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_symlink_invalid_etag(self):
link_obj = uuid4().hex
headers = {'X-Symlink-Target': '%s/%s' % (self.env.tgt_cont,
self.env.tgt_obj),
'X-Symlink-Target-Etag': 'not-the-real-etag'}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content,
b"Object Etag 'ab706c400731332bffa67ed4bc15dcac' "
b"does not match X-Symlink-Target-Etag header "
b"'not-the-real-etag'")
def test_symlink_object_listing(self):
link_obj = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
# sanity
self._assertSymlink(self.env.link_cont, link_obj)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont,
query_args='format=json')
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
obj_info = object_list[0]
self.assertIn('symlink_path', obj_info)
self.assertEqual(self.env.target_content_location(),
obj_info['symlink_path'])
self.assertNotIn('symlink_etag', obj_info)
def test_static_link_object_listing(self):
link_obj = uuid4().hex
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# sanity
self._assertSymlink(self.env.link_cont, link_obj)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont,
query_args='format=json')
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
self.assertIn('symlink_path', object_list[0])
self.assertEqual(self.env.target_content_location(),
object_list[0]['symlink_path'])
obj_info = object_list[0]
self.assertIn('symlink_etag', obj_info)
self.assertEqual(normalize_etag(self.env.tgt_etag),
obj_info['symlink_etag'])
self.assertEqual(int(self.env.tgt_length),
obj_info['symlink_bytes'])
self.assertEqual(obj_info['content_type'], 'application/target')
# POSTing to a static_link can change the listing Content-Type
headers = {'Content-Type': 'application/foo'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont,
query_args='format=json')
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
obj_info = object_list[0]
self.assertEqual(obj_info['content_type'], 'application/foo')
class TestCrossPolicySymlinkEnv(TestSymlinkEnv):
multiple_policies_enabled = None
@classmethod
def setUp(cls):
if tf.skip or tf.skip2:
raise SkipTest
if cls.multiple_policies_enabled is None:
try:
cls.policies = tf.FunctionalStoragePolicyCollection.from_info()
except AssertionError:
pass
if cls.policies and len(cls.policies) > 1:
cls.multiple_policies_enabled = True
else:
cls.multiple_policies_enabled = False
return
link_policy = cls.policies.select()
tgt_policy = cls.policies.exclude(name=link_policy['name']).select()
link_header = {'X-Storage-Policy': link_policy['name']}
tgt_header = {'X-Storage-Policy': tgt_policy['name']}
cls._create_container(cls.link_cont, headers=link_header)
cls._create_container(cls.tgt_cont, headers=tgt_header)
# container in account 2
cls._create_container(cls.link_cont, headers=link_header,
use_account=2)
cls._create_tgt_object()
class TestCrossPolicySymlink(TestSymlink):
env = TestCrossPolicySymlinkEnv
def setUp(self):
super(TestCrossPolicySymlink, self).setUp()
if self.env.multiple_policies_enabled is False:
raise SkipTest('Cross policy test requires multiple policies')
elif self.env.multiple_policies_enabled is not True:
# just some sanity checking
raise Exception("Expected multiple_policies_enabled "
"to be True/False, got %r" % (
self.env.multiple_policies_enabled,))
def tearDown(self):
self.env.tearDown()
class TestSymlinkSlo(Base):
"""
Just some sanity testing of SLO + symlinks.
It is basically a copy of SLO tests in test_slo, but the tested object is
a symlink to the manifest (instead of the manifest itself)
"""
env = TestSloEnv
def setUp(self):
super(TestSymlinkSlo, self).setUp()
if self.env.slo_enabled is False:
raise SkipTest("SLO not enabled")
elif self.env.slo_enabled is not True:
# just some sanity checking
raise Exception(
"Expected slo_enabled to be True/False, got %r" %
(self.env.slo_enabled,))
self.file_symlink = self.env.container.file(uuid4().hex)
self.account_name = self.env.container.conn.storage_path.rsplit(
'/', 1)[-1]
def test_symlink_target_slo_manifest(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde')})
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(self.file_symlink.read()))
manifest_body = self.file_symlink.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == self.file_symlink.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.assertEqual(obj_info, {
'name': self.file_symlink.name,
'content_type': 'application/octet-stream',
'hash': 'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'symlink_path': '/v1/%s/%s/manifest-abcde' % (
self.account_name, self.env.container.name),
})
def test_static_link_target_slo_manifest(self):
manifest_info = self.env.container2.file(
"manifest-abcde").info(parms={
'multipart-manifest': 'get'})
manifest_etag = manifest_info['etag']
self.file_symlink.write(hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': manifest_etag,
})
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(self.file_symlink.read()))
manifest_body = self.file_symlink.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
# check listing
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == self.file_symlink.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.maxDiff = None
slo_info = self.env.container2.file("manifest-abcde").info()
self.assertEqual(obj_info, {
'name': self.file_symlink.name,
'content_type': 'application/octet-stream',
'hash': u'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'slo_etag': slo_info['etag'],
'symlink_path': '/v1/%s/%s/manifest-abcde' % (
self.account_name, self.env.container2.name),
'symlink_bytes': 4 * 2 ** 20 + 1,
'symlink_etag': normalize_etag(manifest_etag),
})
def test_static_link_target_slo_manifest_wrong_etag(self):
# try the slo "etag"
slo_etag = self.env.container2.file(
"manifest-abcde").info()['etag']
self.assertRaises(ResponseError, self.file_symlink.write, hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': slo_etag,
})
self.assert_status(409) # quotes OK, but doesn't match
# try the slo etag w/o the quotes
slo_etag = slo_etag.strip('"')
self.assertRaises(ResponseError, self.file_symlink.write, hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': slo_etag,
})
self.assert_status(409) # that still doesn't match
def test_static_link_target_symlink_to_slo_manifest(self):
# write symlink
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde')})
# write static_link
file_static_link = self.env.container.file(uuid4().hex)
file_static_link.write(hdrs={
'X-Symlink-Target': '%s/%s' % (
self.file_symlink.container, self.file_symlink.name),
'X-Symlink-Target-Etag': MD5_OF_EMPTY_STRING,
})
# validate reads
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(file_static_link.read()))
manifest_body = file_static_link.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
# check listing
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == file_static_link.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.maxDiff = None
self.assertEqual(obj_info, {
'name': file_static_link.name,
'content_type': 'application/octet-stream',
'hash': 'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'symlink_path': u'/v1/%s/%s/%s' % (
self.account_name, self.file_symlink.container,
self.file_symlink.name),
# the only time bytes/etag aren't the target object are when they
# validate through another static_link
'symlink_bytes': 0,
'symlink_etag': MD5_OF_EMPTY_STRING,
})
def test_symlink_target_slo_nested_manifest(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde-submanifest')})
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(self.file_symlink.read()))
def test_slo_get_ranged_manifest(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'ranged-manifest')})
self.assertEqual([
(b'c', 1),
(b'd', 1024 * 1024),
(b'e', 1),
(b'a', 512 * 1024),
(b'b', 512 * 1024),
(b'c', 1),
(b'd', 1),
], group_by_byte(self.file_symlink.read()))
def test_slo_ranged_get(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde')})
file_contents = self.file_symlink.read(size=1024 * 1024 + 2,
offset=1024 * 1024 - 1)
self.assertEqual([
(b'a', 1),
(b'b', 1024 * 1024),
(b'c', 1),
], group_by_byte(file_contents))
class TestSymlinkSloEnv(TestSloEnv):
@classmethod
def create_links_to_segments(cls, container):
seg_info = {}
for letter in ('a', 'b'):
seg_name = "linkto_seg_%s" % letter
file_item = container.file(seg_name)
sym_hdr = {'X-Symlink-Target': '%s/seg_%s' % (container.name,
letter)}
file_item.write(hdrs=sym_hdr)
seg_info[seg_name] = {
'path': '/%s/%s' % (container.name, seg_name)}
return seg_info
@classmethod
def setUp(cls):
super(TestSymlinkSloEnv, cls).setUp()
cls.link_seg_info = cls.create_links_to_segments(cls.container)
file_item = cls.container.file("manifest-linkto-ab")
file_item.write(
json.dumps([cls.link_seg_info['linkto_seg_a'],
cls.link_seg_info['linkto_seg_b']]).encode('ascii'),
parms={'multipart-manifest': 'put'})
class TestSymlinkToSloSegments(Base):
"""
This test class will contain various tests where the segments of the SLO
manifest are symlinks to the actual segments. Again the tests are basicaly
a copy/paste of the tests in test_slo, only the manifest has been modified
to contain symlinks as the segments.
"""
env = TestSymlinkSloEnv
def setUp(self):
super(TestSymlinkToSloSegments, self).setUp()
if self.env.slo_enabled is False:
raise SkipTest("SLO not enabled")
elif self.env.slo_enabled is not True:
# just some sanity checking
raise Exception(
"Expected slo_enabled to be True/False, got %r" %
(self.env.slo_enabled,))
def test_slo_get_simple_manifest_with_links(self):
file_item = self.env.container.file("manifest-linkto-ab")
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
], group_by_byte(file_item.read()))
def test_slo_container_listing(self):
# the listing object size should equal the sum of the size of the
# segments, not the size of the manifest body
file_item = self.env.container.file(Utils.create_name())
file_item.write(
json.dumps([
self.env.link_seg_info['linkto_seg_a']]).encode('ascii'),
parms={'multipart-manifest': 'put'})
# The container listing has the etag of the actual manifest object
# contents which we get using multipart-manifest=get. New enough swift
# also exposes the etag that we get when NOT using
# multipart-manifest=get. Verify that both remain consistent when the
# object is updated with a POST.
file_item.initialize()
slo_etag = file_item.etag
file_item.initialize(parms={'multipart-manifest': 'get'})
manifest_etag = file_item.etag
listing = self.env.container.files(parms={'format': 'json'})
for f_dict in listing:
if f_dict['name'] == file_item.name:
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual('application/octet-stream',
f_dict['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
self.fail('Failed to find manifest file in container listing')
# now POST updated content-type file
file_item.content_type = 'image/jpeg'
file_item.sync_metadata({'X-Object-Meta-Test': 'blah'})
file_item.initialize()
self.assertEqual('image/jpeg', file_item.content_type) # sanity
# verify that the container listing is consistent with the file
listing = self.env.container.files(parms={'format': 'json'})
for f_dict in listing:
if f_dict['name'] == file_item.name:
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual(file_item.content_type,
f_dict['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
self.fail('Failed to find manifest file in container listing')
# now POST with no change to content-type
file_item.sync_metadata({'X-Object-Meta-Test': 'blah'},
cfg={'no_content_type': True})
file_item.initialize()
self.assertEqual('image/jpeg', file_item.content_type) # sanity
# verify that the container listing is consistent with the file
listing = self.env.container.files(parms={'format': 'json'})
for f_dict in listing:
if f_dict['name'] == file_item.name:
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual(file_item.content_type,
f_dict['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
self.fail('Failed to find manifest file in container listing')
def test_slo_etag_is_hash_of_etags(self):
expected_hash = md5(usedforsecurity=False)
expected_hash.update((
md5(b'a' * 1024 * 1024, usedforsecurity=False)
.hexdigest().encode('ascii')))
expected_hash.update((
md5(b'b' * 1024 * 1024, usedforsecurity=False)
.hexdigest().encode('ascii')))
expected_etag = expected_hash.hexdigest()
file_item = self.env.container.file('manifest-linkto-ab')
self.assertEqual('"%s"' % expected_etag, file_item.info()['etag'])
def test_slo_copy(self):
file_item = self.env.container.file("manifest-linkto-ab")
file_item.copy(self.env.container.name, "copied-abcde")
copied = self.env.container.file("copied-abcde")
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
], group_by_byte(copied.read(parms={'multipart-manifest': 'get'})))
def test_slo_copy_the_manifest(self):
# first just perform some tests of the contents of the manifest itself
source = self.env.container.file("manifest-linkto-ab")
source_contents = source.read(parms={'multipart-manifest': 'get'})
source_json = json.loads(source_contents)
manifest_etag = md5(source_contents, usedforsecurity=False).hexdigest()
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
manifest_etag = '"%s"' % manifest_etag
source.initialize()
slo_etag = source.etag
self.assertEqual('application/octet-stream', source.content_type)
source.initialize(parms={'multipart-manifest': 'get'})
self.assertEqual(manifest_etag, source.etag)
self.assertEqual('application/json; charset=utf-8',
source.content_type)
# now, copy the manifest
self.assertTrue(source.copy(self.env.container.name,
"copied-ab-manifest-only",
parms={'multipart-manifest': 'get'}))
copied = self.env.container.file("copied-ab-manifest-only")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
try:
copied_json = json.loads(copied_contents)
except ValueError:
self.fail("COPY didn't copy the manifest (invalid json on GET)")
# make sure content of copied manifest is the same as original man.
self.assertEqual(source_json, copied_json)
copied.initialize()
self.assertEqual(copied.etag, slo_etag)
self.assertEqual('application/octet-stream', copied.content_type)
copied.initialize(parms={'multipart-manifest': 'get'})
self.assertEqual(source_contents, copied_contents)
self.assertEqual(copied.etag, manifest_etag)
self.assertEqual('application/json; charset=utf-8',
copied.content_type)
# verify the listing metadata
listing = self.env.container.files(parms={'format': 'json'})
names = {}
for f_dict in listing:
if f_dict['name'] in ('manifest-linkto-ab',
'copied-ab-manifest-only'):
names[f_dict['name']] = f_dict
self.assertIn('manifest-linkto-ab', names)
actual = names['manifest-linkto-ab']
self.assertEqual(2 * 1024 * 1024, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % actual['hash'])
else:
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
self.assertIn('copied-ab-manifest-only', names)
actual = names['copied-ab-manifest-only']
self.assertEqual(2 * 1024 * 1024, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % actual['hash'])
else:
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
class TestSymlinkDlo(Base):
env = TestDloEnv
def test_get_manifest(self):
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man1')})
self.assertEqual([
(b'a', 10),
(b'b', 10),
(b'c', 10),
(b'd', 10),
(b'e', 10),
], group_by_byte(file_symlink.read()))
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man2')})
self.assertEqual([
(b'A', 10),
(b'B', 10),
(b'C', 10),
(b'D', 10),
(b'E', 10),
], group_by_byte(file_symlink.read()))
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manall')})
self.assertEqual([
(b'a', 10),
(b'b', 10),
(b'c', 10),
(b'd', 10),
(b'e', 10),
(b'A', 10),
(b'B', 10),
(b'C', 10),
(b'D', 10),
(b'E', 10),
], group_by_byte(file_symlink.read()))
def test_get_manifest_document_itself(self):
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man1')})
file_contents = file_symlink.read(parms={'multipart-manifest': 'get'})
self.assertEqual(file_contents, b"man1-contents")
self.assertEqual(file_symlink.info()['x_object_manifest'],
"%s/%s/seg_lower" %
(self.env.container.name, self.env.segment_prefix))
def test_get_range(self):
link_obj = uuid4().hex + "_symlink"
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man1')})
self.assertEqual([
(b'a', 2),
(b'b', 10),
(b'c', 10),
(b'd', 3),
], group_by_byte(file_symlink.read(size=25, offset=8)))
file_contents = file_symlink.read(size=1, offset=47)
self.assertEqual(file_contents, b"e")
def test_get_range_out_of_range(self):
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man1')})
self.assertRaises(ResponseError, file_symlink.read, size=7, offset=50)
self.assert_status(416)
class TestSymlinkTargetObjectComparisonEnv(TestFileComparisonEnv):
@classmethod
def setUp(cls):
super(TestSymlinkTargetObjectComparisonEnv, cls).setUp()
cls.parms = None
cls.expect_empty_etag = False
cls.expect_body = True
class TestSymlinkComparisonEnv(TestFileComparisonEnv):
@classmethod
def setUp(cls):
super(TestSymlinkComparisonEnv, cls).setUp()
cls.parms = {'symlink': 'get'}
cls.expect_empty_etag = True
cls.expect_body = False
class TestSymlinkTargetObjectComparison(Base):
env = TestSymlinkTargetObjectComparisonEnv
def setUp(self):
super(TestSymlinkTargetObjectComparison, self).setUp()
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
file_item.name)})
def testIfMatch(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Match': md5}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-Match': 'bogus'}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
def testIfMatchMultipleEtags(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Match': '"bogus1", "%s", "bogus2"' % md5}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-Match': '"bogus1", "bogus2", "bogus3"'}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
def testIfNoneMatch(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-None-Match': 'bogus'}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-None-Match': md5}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfNoneMatchMultipleEtags(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-None-Match': '"bogus1", "bogus2", "bogus3"'}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-None-Match':
'"bogus1", "bogus2", "%s"' % md5}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfModifiedSince(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Modified-Since': self.env.time_old_f1}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms))
hdrs = {'If-Modified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfUnmodifiedSince(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Unmodified-Since': self.env.time_new}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms))
hdrs = {'If-Unmodified-Since': self.env.time_old_f2}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
def testIfMatchAndUnmodified(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Match': md5,
'If-Unmodified-Since': self.env.time_new}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-Match': 'bogus',
'If-Unmodified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
hdrs = {'If-Match': md5,
'If-Unmodified-Since': self.env.time_old_f3}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
def testLastModified(self):
file_item = self.env.container.file(Utils.create_name())
file_item.content_type = Utils.create_name()
resp = file_item.write_random_return_resp(self.env.file_size)
put_last_modified = resp.getheader('last-modified')
md5 = file_item.md5
# create symlink
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
file_item.name)})
info = file_symlink.info()
self.assertIn('last_modified', info)
last_modified = info['last_modified']
self.assertEqual(put_last_modified, info['last_modified'])
hdrs = {'If-Modified-Since': last_modified}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
hdrs = {'If-Unmodified-Since': last_modified}
self.assertTrue(file_symlink.read(hdrs=hdrs))
class TestSymlinkComparison(TestSymlinkTargetObjectComparison):
env = TestSymlinkComparisonEnv
def setUp(self):
super(TestSymlinkComparison, self).setUp()
def testLastModified(self):
file_item = self.env.container.file(Utils.create_name())
file_item.content_type = Utils.create_name()
resp = file_item.write_random_return_resp(self.env.file_size)
put_target_last_modified = resp.getheader('last-modified')
md5 = MD5_OF_EMPTY_STRING
# get different last-modified between file and symlink
time.sleep(1)
# create symlink
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
resp = file_symlink.write(return_resp=True,
hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
file_item.name)})
put_sym_last_modified = resp.getheader('last-modified')
info = file_symlink.info(parms=self.env.parms)
self.assertIn('last_modified', info)
last_modified = info['last_modified']
self.assertEqual(put_sym_last_modified, info['last_modified'])
hdrs = {'If-Modified-Since': put_target_last_modified}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-Modified-Since': last_modified}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
hdrs = {'If-Unmodified-Since': last_modified}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
class TestSymlinkAccountTempurl(Base):
env = TestTempurlEnv
digest_name = 'sha1'
def setUp(self):
super(TestSymlinkAccountTempurl, self).setUp()
if self.env.tempurl_enabled is False:
raise SkipTest("TempURL not enabled")
elif self.env.tempurl_enabled is not True:
# just some sanity checking
raise Exception(
"Expected tempurl_enabled to be True/False, got %r" %
(self.env.tempurl_enabled,))
if self.digest_name not in cluster_info['tempurl'].get(
'allowed_digests', ['sha1']):
raise SkipTest("tempurl does not support %s signatures" %
self.digest_name)
self.digest = getattr(hashlib, self.digest_name)
self.expires = int(time.time()) + 86400
self.obj_tempurl_parms = self.tempurl_parms(
'GET', self.expires, self.env.conn.make_path(self.env.obj.path),
self.env.tempurl_key)
def tempurl_parms(self, method, expires, path, key):
path = urllib.parse.unquote(path)
if not six.PY2:
method = method.encode('utf8')
path = path.encode('utf8')
key = key.encode('utf8')
sig = hmac.new(
key,
b'%s\n%d\n%s' % (method, expires, path),
self.digest).hexdigest()
return {'temp_url_sig': sig, 'temp_url_expires': str(expires)}
def test_PUT_symlink(self):
new_sym = self.env.container.file(Utils.create_name())
# give out a signature which allows a PUT to new_obj
expires = int(time.time()) + 86400
put_parms = self.tempurl_parms(
'PUT', expires, self.env.conn.make_path(new_sym.path),
self.env.tempurl_key)
# try to create symlink object
try:
new_sym.write(
b'', {'x-symlink-target': 'cont/foo'}, parms=put_parms,
cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 400)
else:
self.fail('request did not error')
def test_GET_symlink_inside_container(self):
tgt_obj = self.env.container.file(Utils.create_name())
sym = self.env.container.file(Utils.create_name())
tgt_obj.write(b"target object body")
sym.write(
b'',
{'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)})
expires = int(time.time()) + 86400
get_parms = self.tempurl_parms(
'GET', expires, self.env.conn.make_path(sym.path),
self.env.tempurl_key)
contents = sym.read(parms=get_parms, cfg={'no_auth_token': True})
self.assert_status([200])
self.assertEqual(contents, b"target object body")
def test_GET_symlink_outside_container(self):
tgt_obj = self.env.container.file(Utils.create_name())
tgt_obj.write(b"target object body")
container2 = self.env.account.container(Utils.create_name())
container2.create()
sym = container2.file(Utils.create_name())
sym.write(
b'',
{'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)})
expires = int(time.time()) + 86400
get_parms = self.tempurl_parms(
'GET', expires, self.env.conn.make_path(sym.path),
self.env.tempurl_key)
# cross container tempurl works fine for account tempurl key
contents = sym.read(parms=get_parms, cfg={'no_auth_token': True})
self.assert_status([200])
self.assertEqual(contents, b"target object body")
class TestSymlinkContainerTempurl(Base):
env = TestContainerTempurlEnv
digest_name = 'sha1'
def setUp(self):
super(TestSymlinkContainerTempurl, self).setUp()
if self.env.tempurl_enabled is False:
raise SkipTest("TempURL not enabled")
elif self.env.tempurl_enabled is not True:
# just some sanity checking
raise Exception(
"Expected tempurl_enabled to be True/False, got %r" %
(self.env.tempurl_enabled,))
if self.digest_name not in cluster_info['tempurl'].get(
'allowed_digests', ['sha1']):
raise SkipTest("tempurl does not support %s signatures" %
self.digest_name)
self.digest = getattr(hashlib, self.digest_name)
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'GET', expires, self.env.conn.make_path(self.env.obj.path),
self.env.tempurl_key)
self.obj_tempurl_parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
def tempurl_sig(self, method, expires, path, key):
path = urllib.parse.unquote(path)
if not six.PY2:
method = method.encode('utf8')
path = path.encode('utf8')
key = key.encode('utf8')
return hmac.new(
key,
b'%s\n%d\n%s' % (method, expires, path),
self.digest).hexdigest()
def test_PUT_symlink(self):
new_sym = self.env.container.file(Utils.create_name())
# give out a signature which allows a PUT to new_obj
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'PUT', expires, self.env.conn.make_path(new_sym.path),
self.env.tempurl_key)
put_parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
# try to create symlink object, should fail
try:
new_sym.write(
b'', {'x-symlink-target': 'cont/foo'}, parms=put_parms,
cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 400)
else:
self.fail('request did not error')
def test_GET_symlink_inside_container(self):
tgt_obj = self.env.container.file(Utils.create_name())
sym = self.env.container.file(Utils.create_name())
tgt_obj.write(b"target object body")
sym.write(
b'',
{'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)})
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'GET', expires, self.env.conn.make_path(sym.path),
self.env.tempurl_key)
parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
contents = sym.read(parms=parms, cfg={'no_auth_token': True})
self.assert_status([200])
self.assertEqual(contents, b"target object body")
def test_GET_symlink_outside_container(self):
tgt_obj = self.env.container.file(Utils.create_name())
tgt_obj.write(b"target object body")
container2 = self.env.account.container(Utils.create_name())
container2.create()
sym = container2.file(Utils.create_name())
sym.write(
b'',
{'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)})
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'GET', expires, self.env.conn.make_path(sym.path),
self.env.tempurl_key)
parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
# cross container tempurl does not work for container tempurl key
try:
sym.read(parms=parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 401)
else:
self.fail('request did not error')
try:
sym.info(parms=parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 401)
else:
self.fail('request did not error')
if __name__ == '__main__':
unittest.main()
View Git Blame Copy Permalink