Add Storage Policy Support
The basic idea here is to replace the use of a single object ring in
the Application class with a collection of object rings. The
collection includes not only the Ring object itself but the policy
name associated with it, the filename for the .gz and any other
metadata associated with the policy that may be needed. When
containers are created, a policy (thus a specific obj ring) is
selected allowing apps to specify policy at container creation time
and leverage policies simply by using different containers for object
operations.
The policy collection is based off of info in the swift.conf file.
The format of the sections in the .conf file is as follows:
swift.conf format:
[storage-policy:0]
name = chicken
[storage-policy:1]
name = turkey
default = yes
With the above format:
- Policy 0 will always be used for access to existing containers
without the policy specified. The ring name for policy 0 is always
'object', assuring backwards compatiblity. The parser will always
create a policy 0 even if not specified
- The policy with 'default=yes' is the one used for new container
creation. This allows the admin to specify which policy is used without
forcing the application to add the metadata.
This commit simply introduces storage policies and the loading
thereof; nobody's using it yet. That will follow in subsequent
commits.
Expose storage policies in /info
DocImpact
Implements: blueprint storage-policies
Change-Id: Ica05f41ecf3adb3648cc9182f11f1c8c5c678985
This commit is contained in:
Paul Luse
Clay Gerrard
parent
e00da6cabc
commit
46c68aebd1
@ -8,6 +8,37 @@
|
||||
swift_hash_path_suffix = changeme
|
||||
swift_hash_path_prefix = changeme
|
||||
|
||||
# storage policies are defined here and determine various characteristics
|
||||
# about how objects are stored and treated. Policies are specified by name on
|
||||
# a per container basis. Names are case-insensitive. The policy index is
|
||||
# specified in the section header and is used internally. The policy with
|
||||
# index 0 is always used for legacy containers and can be given a name for use
|
||||
# in metadata however the ring file name will always be 'object.ring.gz' for
|
||||
# backwards compatibility. If no policies are defined a policy with index 0
|
||||
# will be automatically created for backwards compatibility and given the name
|
||||
# Policy-0. A default policy is used when creating new containers when no
|
||||
# policy is specified in the request. If no other policies are defined the
|
||||
# policy with index 0 will be declared the default. If multiple policies are
|
||||
# defined you must define a policy with index 0 and you must specify a
|
||||
# default. It is recommended you always define a section for
|
||||
# storage-policy:0.
|
||||
[storage-policy:0]
|
||||
name = Policy-0
|
||||
default = yes
|
||||
|
||||
# the following section would declare a policy called 'silver', the number of
|
||||
# replicas will be determined by how the ring is built. In this example the
|
||||
# 'silver' policy could have a lower or higher # of replicas than the
|
||||
# 'Policy-0' policy above. The ring filename will be 'object-1.ring.gz'. You
|
||||
# may only specify one storage policy section as the default. If you changed
|
||||
# this section to specify 'silver' as the default, when a client created a new
|
||||
# container w/o a policy specified, it will get the 'silver' policy because
|
||||
# this config has specified it as the default. However if a legacy container
|
||||
# (one created with a pre-policy version of swift) is accessed, it is known
|
||||
# implicitly to be assigned to the policy with index 0 as opposed to the
|
||||
# current default.
|
||||
#[storage-policy:1]
|
||||
#name = silver
|
||||
|
||||
# The swift-constraints section sets the basic constraints on data
|
||||
# saved in the swift cluster. These constraints are automatically
|
||||
|
||||
354
swift/common/storage_policy.py
Normal file
354
swift/common/storage_policy.py
Normal file
@ -0,0 +1,354 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from ConfigParser import ConfigParser
|
||||
import textwrap
|
||||
import string
|
||||
|
||||
from swift.common.utils import config_true_value, SWIFT_CONF_FILE
|
||||
from swift.common.ring import Ring
|
||||
|
||||
POLICY = 'X-Storage-Policy'
|
||||
POLICY_INDEX = 'X-Backend-Storage-Policy-Index'
|
||||
LEGACY_POLICY_NAME = 'Policy-0'
|
||||
VALID_CHARS = '-' + string.letters + string.digits
|
||||
|
||||
|
||||
class PolicyError(ValueError):
|
||||
|
||||
def __init__(self, msg, index=None):
|
||||
if index is not None:
|
||||
msg += ', for index %r' % index
|
||||
super(PolicyError, self).__init__(msg)
|
||||
|
||||
|
||||
def _get_policy_string(base, policy_index):
|
||||
if policy_index == 0 or policy_index is None:
|
||||
return_string = base
|
||||
else:
|
||||
return_string = base + "-%d" % int(policy_index)
|
||||
return return_string
|
||||
|
||||
|
||||
def get_policy_string(base, policy_index):
|
||||
"""
|
||||
Helper function to construct a string from a base and the policy
|
||||
index. Used to encode the policy index into either a file name
|
||||
or a directory name by various modules.
|
||||
|
||||
:param base: the base string
|
||||
:param policy_index: the storage policy index
|
||||
|
||||
:returns: base name with policy index added
|
||||
"""
|
||||
if POLICIES.get_by_index(policy_index) is None:
|
||||
raise PolicyError("No policy with index %r" % policy_index)
|
||||
return _get_policy_string(base, policy_index)
|
||||
|
||||
|
||||
class StoragePolicy(object):
|
||||
"""
|
||||
Represents a storage policy.
|
||||
Not meant to be instantiated directly; use
|
||||
:func:`~swift.common.storage_policy.reload_storage_policies` to load
|
||||
POLICIES from ``swift.conf``.
|
||||
|
||||
The object_ring property is lazy loaded once the service's ``swift_dir``
|
||||
is known via :meth:`~StoragePolicyCollection.get_object_ring`, but it may
|
||||
be over-ridden via object_ring kwarg at create time for testing or
|
||||
actively loaded with :meth:`~StoragePolicy.load_ring`.
|
||||
"""
|
||||
def __init__(self, idx, name='', is_default=False, is_deprecated=False,
|
||||
object_ring=None):
|
||||
try:
|
||||
self.idx = int(idx)
|
||||
except ValueError:
|
||||
raise PolicyError('Invalid index', idx)
|
||||
if self.idx < 0:
|
||||
raise PolicyError('Invalid index', idx)
|
||||
if not name:
|
||||
raise PolicyError('Invalid name %r' % name, idx)
|
||||
# this is defensively restrictive, but could be expanded in the future
|
||||
if not all(c in VALID_CHARS for c in name):
|
||||
raise PolicyError('Names are used as HTTP headers, and can not '
|
||||
'reliably contain any characters not in %r. '
|
||||
'Invalid name %r' % (VALID_CHARS, name))
|
||||
if name.upper() == LEGACY_POLICY_NAME.upper() and self.idx != 0:
|
||||
msg = 'The name %s is reserved for policy index 0. ' \
|
||||
'Invalid name %r' % (LEGACY_POLICY_NAME, name)
|
||||
raise PolicyError(msg, idx)
|
||||
self.name = name
|
||||
self.is_deprecated = config_true_value(is_deprecated)
|
||||
self.is_default = config_true_value(is_default)
|
||||
if self.is_deprecated and self.is_default:
|
||||
raise PolicyError('Deprecated policy can not be default. '
|
||||
'Invalid config', self.idx)
|
||||
self.ring_name = _get_policy_string('object', self.idx)
|
||||
self.object_ring = object_ring
|
||||
|
||||
def __int__(self):
|
||||
return self.idx
|
||||
|
||||
def __cmp__(self, other):
|
||||
return cmp(self.idx, int(other))
|
||||
|
||||
def __repr__(self):
|
||||
return ("StoragePolicy(%d, %r, is_default=%s, is_deprecated=%s)") % (
|
||||
self.idx, self.name, self.is_default, self.is_deprecated)
|
||||
|
||||
def load_ring(self, swift_dir):
|
||||
"""
|
||||
Load the ring for this policy immediately.
|
||||
|
||||
:param swift_dir: path to rings
|
||||
"""
|
||||
if self.object_ring:
|
||||
return
|
||||
self.object_ring = Ring(swift_dir, ring_name=self.ring_name)
|
||||
|
||||
|
||||
class StoragePolicyCollection(object):
|
||||
"""
|
||||
This class represents the collection of valid storage policies for the
|
||||
cluster and is instantiated as :class:`StoragePolicy` objects are added to
|
||||
the collection when ``swift.conf`` is parsed by
|
||||
:func:`parse_storage_policies`.
|
||||
|
||||
When a StoragePolicyCollection is created, the following validation
|
||||
is enforced:
|
||||
|
||||
* If a policy with index 0 is not declared and no other policies defined,
|
||||
Swift will create one
|
||||
* The policy index must be a non-negative integer
|
||||
* If no policy is declared as the default and no other policies are
|
||||
defined, the policy with index 0 is set as the default
|
||||
* Policy indexes must be unique
|
||||
* Policy names are required
|
||||
* Policy names are case insensitive
|
||||
* Policy names must contain only letters, digits or a dash
|
||||
* Policy names must be unique
|
||||
* The policy name 'Policy-0' can only be used for the policy with index 0
|
||||
* If any policies are defined, exactly one policy must be declared default
|
||||
* Deprecated policies can not be declared the default
|
||||
|
||||
"""
|
||||
def __init__(self, pols):
|
||||
self.default = []
|
||||
self.by_name = {}
|
||||
self.by_index = {}
|
||||
self._validate_policies(pols)
|
||||
|
||||
def _add_policy(self, policy):
|
||||
"""
|
||||
Add pre-validated policies to internal indexes.
|
||||
"""
|
||||
self.by_name[policy.name.upper()] = policy
|
||||
self.by_index[int(policy)] = policy
|
||||
|
||||
def __repr__(self):
|
||||
return (textwrap.dedent("""
|
||||
StoragePolicyCollection([
|
||||
%s
|
||||
])
|
||||
""") % ',\n '.join(repr(p) for p in self)).strip()
|
||||
|
||||
def __len__(self):
|
||||
return len(self.by_index)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.by_index[key]
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.by_index.values())
|
||||
|
||||
def _validate_policies(self, policies):
|
||||
"""
|
||||
:param policies: list of policies
|
||||
"""
|
||||
|
||||
for policy in policies:
|
||||
if int(policy) in self.by_index:
|
||||
raise PolicyError('Duplicate index %s conflicts with %s' % (
|
||||
policy, self.get_by_index(int(policy))))
|
||||
if policy.name.upper() in self.by_name:
|
||||
raise PolicyError('Duplicate name %s conflicts with %s' % (
|
||||
policy, self.get_by_name(policy.name)))
|
||||
if policy.is_default:
|
||||
if not self.default:
|
||||
self.default = policy
|
||||
else:
|
||||
raise PolicyError(
|
||||
'Duplicate default %s conflicts with %s' % (
|
||||
policy, self.default))
|
||||
self._add_policy(policy)
|
||||
|
||||
# If a 0 policy wasn't explicitly given, or nothing was
|
||||
# provided, create the 0 policy now
|
||||
if 0 not in self.by_index:
|
||||
if len(self) != 0:
|
||||
raise PolicyError('You must specify a storage policy '
|
||||
'section for policy index 0 in order '
|
||||
'to define multiple policies')
|
||||
self._add_policy(StoragePolicy(0, name=LEGACY_POLICY_NAME))
|
||||
|
||||
# at least one policy must be enabled
|
||||
enabled_policies = [p for p in self if not p.is_deprecated]
|
||||
if not enabled_policies:
|
||||
raise PolicyError("Unable to find policy that's not deprecated!")
|
||||
|
||||
# if needed, specify default
|
||||
if not self.default:
|
||||
if len(self) > 1:
|
||||
raise PolicyError("Unable to find default policy")
|
||||
self.default = self[0]
|
||||
self.default.is_default = True
|
||||
|
||||
def get_by_name(self, name):
|
||||
"""
|
||||
Find a storage policy by its name.
|
||||
|
||||
:param name: name of the policy
|
||||
:returns: storage policy, or None
|
||||
"""
|
||||
return self.by_name.get(name.upper())
|
||||
|
||||
def get_by_index(self, index):
|
||||
"""
|
||||
Find a storage policy by its index.
|
||||
|
||||
An index of None will be treated as 0.
|
||||
|
||||
:param index: numeric index of the storage policy
|
||||
:returns: storage policy, or None if no such policy
|
||||
"""
|
||||
# makes it easier for callers to just pass in a header value
|
||||
index = int(index) if index else 0
|
||||
return self.by_index.get(index)
|
||||
|
||||
def get_object_ring(self, policy_idx, swift_dir):
|
||||
"""
|
||||
Get the ring object to use to handle a request based on its policy.
|
||||
|
||||
An index of None will be treated as 0.
|
||||
|
||||
:param policy_idx: policy index as defined in swift.conf
|
||||
:param swift_dir: swift_dir used by the caller
|
||||
:returns: appropriate ring object
|
||||
"""
|
||||
policy = self.get_by_index(policy_idx)
|
||||
if not policy:
|
||||
raise PolicyError("No policy with index %s" % policy_idx)
|
||||
if not policy.object_ring:
|
||||
policy.load_ring(swift_dir)
|
||||
return policy.object_ring
|
||||
|
||||
def get_policy_info(self):
|
||||
"""
|
||||
Build info about policies for the /info endpoint
|
||||
|
||||
:returns: list of dicts containing relevant policy information
|
||||
"""
|
||||
policy_info = []
|
||||
for pol in self:
|
||||
# delete from /info if deprecated
|
||||
if pol.is_deprecated:
|
||||
continue
|
||||
policy_entry = {}
|
||||
policy_entry['name'] = pol.name
|
||||
if pol.is_default:
|
||||
policy_entry['default'] = pol.is_default
|
||||
policy_info.append(policy_entry)
|
||||
return policy_info
|
||||
|
||||
|
||||
def parse_storage_policies(conf):
|
||||
"""
|
||||
Parse storage policies in ``swift.conf`` - note that validation
|
||||
is done when the :class:`StoragePolicyCollection` is instantiated.
|
||||
|
||||
:param conf: ConfigParser parser object for swift.conf
|
||||
"""
|
||||
policies = []
|
||||
for section in conf.sections():
|
||||
if not section.startswith('storage-policy:'):
|
||||
continue
|
||||
policy_index = section.split(':', 1)[1]
|
||||
# map config option name to StoragePolicy paramater name
|
||||
config_to_policy_option_map = {
|
||||
'name': 'name',
|
||||
'default': 'is_default',
|
||||
'deprecated': 'is_deprecated',
|
||||
}
|
||||
policy_options = {}
|
||||
for config_option, value in conf.items(section):
|
||||
try:
|
||||
policy_option = config_to_policy_option_map[config_option]
|
||||
except KeyError:
|
||||
raise PolicyError('Invalid option %r in '
|
||||
'storage-policy section %r' % (
|
||||
config_option, section))
|
||||
policy_options[policy_option] = value
|
||||
policy = StoragePolicy(policy_index, **policy_options)
|
||||
policies.append(policy)
|
||||
|
||||
return StoragePolicyCollection(policies)
|
||||
|
||||
|
||||
class StoragePolicySingleton(object):
|
||||
"""
|
||||
An instance of this class is the primary interface to storage policies
|
||||
exposed as a module level global named ``POLICIES``. This global
|
||||
reference wraps ``_POLICIES`` which is normally instantiated by parsing
|
||||
``swift.conf`` and will result in an instance of
|
||||
:class:`StoragePolicyCollection`.
|
||||
|
||||
You should never patch this instance directly, instead patch the module
|
||||
level ``_POLICIES`` instance so that swift code which imported
|
||||
``POLICIES`` directly will reference the patched
|
||||
:class:`StoragePolicyCollection`.
|
||||
"""
|
||||
|
||||
def __iter__(self):
|
||||
return iter(_POLICIES)
|
||||
|
||||
def __len__(self):
|
||||
return len(_POLICIES)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return _POLICIES[key]
|
||||
|
||||
def __getattribute__(self, name):
|
||||
return getattr(_POLICIES, name)
|
||||
|
||||
def __repr__(self):
|
||||
return repr(_POLICIES)
|
||||
|
||||
|
||||
def reload_storage_policies():
|
||||
"""
|
||||
Reload POLICIES from ``swift.conf``.
|
||||
"""
|
||||
global _POLICIES
|
||||
policy_conf = ConfigParser()
|
||||
policy_conf.read(SWIFT_CONF_FILE)
|
||||
try:
|
||||
_POLICIES = parse_storage_policies(policy_conf)
|
||||
except PolicyError as e:
|
||||
raise SystemExit('ERROR: Invalid Storage Policy Configuration '
|
||||
'in %s (%s)' % (SWIFT_CONF_FILE, e))
|
||||
|
||||
|
||||
# parse configuration and setup singleton
|
||||
_POLICIES = None
|
||||
reload_storage_policies()
|
||||
POLICIES = StoragePolicySingleton()
|
||||
@ -31,6 +31,7 @@ from swift.common.utils import cache_from_env, get_logger, \
|
||||
affinity_key_function, affinity_locality_predicate, list_from_csv, \
|
||||
register_swift_info
|
||||
from swift.common.constraints import check_utf8
|
||||
from swift.common.storage_policy import POLICIES
|
||||
from swift.proxy.controllers import AccountController, ObjectController, \
|
||||
ContainerController, InfoController
|
||||
from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
|
||||
@ -207,6 +208,7 @@ class Application(object):
|
||||
register_swift_info(
|
||||
version=swift_version,
|
||||
strict_cors_mode=self.strict_cors_mode,
|
||||
policies=POLICIES.get_policy_info(),
|
||||
**constraints.EFFECTIVE_CONSTRAINTS)
|
||||
|
||||
def check_config(self):
|
||||
|
||||
@ -21,7 +21,7 @@ import logging
|
||||
import errno
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from collections import defaultdict
|
||||
from collections import defaultdict, Iterable
|
||||
from tempfile import NamedTemporaryFile
|
||||
import time
|
||||
from eventlet.green import socket
|
||||
@ -34,6 +34,86 @@ from eventlet import sleep, Timeout
|
||||
import logging.handlers
|
||||
from httplib import HTTPException
|
||||
from numbers import Number
|
||||
from swift.common import storage_policy
|
||||
import functools
|
||||
|
||||
DEFAULT_PATCH_POLICIES = [storage_policy.StoragePolicy(0, 'nulo', True),
|
||||
storage_policy.StoragePolicy(1, 'unu')]
|
||||
LEGACY_PATCH_POLICIES = [storage_policy.StoragePolicy(0, 'legacy', True)]
|
||||
|
||||
|
||||
def patch_policies(thing_or_policies=None, legacy_only=False):
|
||||
if legacy_only:
|
||||
default_policies = LEGACY_PATCH_POLICIES
|
||||
else:
|
||||
default_policies = DEFAULT_PATCH_POLICIES
|
||||
|
||||
thing_or_policies = thing_or_policies or default_policies
|
||||
|
||||
if isinstance(thing_or_policies, (
|
||||
Iterable, storage_policy.StoragePolicyCollection)):
|
||||
return PatchPolicies(thing_or_policies)
|
||||
else:
|
||||
# it's a thing!
|
||||
return PatchPolicies(default_policies)(thing_or_policies)
|
||||
|
||||
|
||||
class PatchPolicies(object):
|
||||
"""
|
||||
Why not mock.patch? In my case, when used as a decorator on the class it
|
||||
seemed to patch setUp at the wrong time (i.e. in setup the global wasn't
|
||||
patched yet)
|
||||
"""
|
||||
|
||||
def __init__(self, policies):
|
||||
if isinstance(policies, storage_policy.StoragePolicyCollection):
|
||||
self.policies = policies
|
||||
else:
|
||||
self.policies = storage_policy.StoragePolicyCollection(policies)
|
||||
|
||||
def __call__(self, thing):
|
||||
if isinstance(thing, type):
|
||||
return self._patch_class(thing)
|
||||
else:
|
||||
return self._patch_method(thing)
|
||||
|
||||
def _patch_class(self, cls):
|
||||
|
||||
class NewClass(cls):
|
||||
|
||||
already_patched = False
|
||||
|
||||
def setUp(cls_self):
|
||||
self._orig_POLICIES = storage_policy._POLICIES
|
||||
if not cls_self.already_patched:
|
||||
storage_policy._POLICIES = self.policies
|
||||
cls_self.already_patched = True
|
||||
super(NewClass, cls_self).setUp()
|
||||
|
||||
def tearDown(cls_self):
|
||||
super(NewClass, cls_self).tearDown()
|
||||
storage_policy._POLICIES = self._orig_POLICIES
|
||||
|
||||
NewClass.__name__ = cls.__name__
|
||||
return NewClass
|
||||
|
||||
def _patch_method(self, f):
|
||||
@functools.wraps(f)
|
||||
def mywrapper(*args, **kwargs):
|
||||
self._orig_POLICIES = storage_policy._POLICIES
|
||||
try:
|
||||
storage_policy._POLICIES = self.policies
|
||||
return f(*args, **kwargs)
|
||||
finally:
|
||||
storage_policy._POLICIES = self._orig_POLICIES
|
||||
return mywrapper
|
||||
|
||||
def __enter__(self):
|
||||
self._orig_POLICIES = storage_policy._POLICIES
|
||||
storage_policy._POLICIES = self.policies
|
||||
|
||||
def __exit__(self, *args):
|
||||
storage_policy._POLICIES = self._orig_POLICIES
|
||||
|
||||
|
||||
class FakeRing(object):
|
||||
|
||||
517
test/unit/common/test_storage_policy.py
Normal file
517
test/unit/common/test_storage_policy.py
Normal file
@ -0,0 +1,517 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
""" Tests for swift.common.storage_policies """
|
||||
import unittest
|
||||
import StringIO
|
||||
from ConfigParser import ConfigParser
|
||||
import mock
|
||||
from tempfile import NamedTemporaryFile
|
||||
from test.unit import patch_policies, FakeRing
|
||||
from swift.common.storage_policy import (
|
||||
StoragePolicy, StoragePolicyCollection, POLICIES, PolicyError,
|
||||
parse_storage_policies, reload_storage_policies, get_policy_string)
|
||||
|
||||
|
||||
class TestStoragePolicies(unittest.TestCase):
|
||||
|
||||
def _conf(self, conf_str):
|
||||
conf_str = "\n".join(line.strip() for line in conf_str.split("\n"))
|
||||
conf = ConfigParser()
|
||||
conf.readfp(StringIO.StringIO(conf_str))
|
||||
return conf
|
||||
|
||||
@patch_policies([StoragePolicy(0, 'zero', True),
|
||||
StoragePolicy(1, 'one', False),
|
||||
StoragePolicy(2, 'two', False),
|
||||
StoragePolicy(3, 'three', False, is_deprecated=True)])
|
||||
def test_swift_info(self):
|
||||
# the deprecated 'three' should not exist in expect
|
||||
expect = [{'default': True, 'name': 'zero'},
|
||||
{'name': 'two'},
|
||||
{'name': 'one'}]
|
||||
swift_info = POLICIES.get_policy_info()
|
||||
self.assertEquals(sorted(expect, key=lambda k: k['name']),
|
||||
sorted(swift_info, key=lambda k: k['name']))
|
||||
|
||||
@patch_policies
|
||||
def test_get_policy_string(self):
|
||||
self.assertEquals(get_policy_string('something', 0), 'something')
|
||||
self.assertEquals(get_policy_string('something', None), 'something')
|
||||
self.assertEquals(get_policy_string('something', 1),
|
||||
'something' + '-1')
|
||||
self.assertRaises(PolicyError, get_policy_string, 'something', 99)
|
||||
|
||||
def test_defaults(self):
|
||||
self.assertTrue(len(POLICIES) > 0)
|
||||
|
||||
# test class functions
|
||||
default_policy = POLICIES.default
|
||||
self.assert_(default_policy.is_default)
|
||||
zero_policy = POLICIES.get_by_index(0)
|
||||
self.assert_(zero_policy.idx == 0)
|
||||
zero_policy_by_name = POLICIES.get_by_name(zero_policy.name)
|
||||
self.assert_(zero_policy_by_name.idx == 0)
|
||||
|
||||
def test_storage_policy_repr(self):
|
||||
test_policies = [StoragePolicy(0, 'aay', True),
|
||||
StoragePolicy(1, 'bee', False),
|
||||
StoragePolicy(2, 'cee', False)]
|
||||
policies = StoragePolicyCollection(test_policies)
|
||||
for policy in policies:
|
||||
policy_repr = repr(policy)
|
||||
self.assert_(policy.__class__.__name__ in policy_repr)
|
||||
self.assert_('is_default=%s' % policy.is_default in policy_repr)
|
||||
self.assert_('is_deprecated=%s' % policy.is_deprecated in
|
||||
policy_repr)
|
||||
self.assert_(policy.name in policy_repr)
|
||||
collection_repr = repr(policies)
|
||||
collection_repr_lines = collection_repr.splitlines()
|
||||
self.assert_(policies.__class__.__name__ in collection_repr_lines[0])
|
||||
self.assertEqual(len(policies), len(collection_repr_lines[1:-1]))
|
||||
for policy, line in zip(policies, collection_repr_lines[1:-1]):
|
||||
self.assert_(repr(policy) in line)
|
||||
with patch_policies(policies):
|
||||
self.assertEqual(repr(POLICIES), collection_repr)
|
||||
|
||||
def test_validate_policies_defaults(self):
|
||||
# 0 explicit default
|
||||
test_policies = [StoragePolicy(0, 'zero', True),
|
||||
StoragePolicy(1, 'one', False),
|
||||
StoragePolicy(2, 'two', False)]
|
||||
policies = StoragePolicyCollection(test_policies)
|
||||
self.assertEquals(policies.default, test_policies[0])
|
||||
self.assertEquals(policies.default.name, 'zero')
|
||||
|
||||
# non-zero explicit default
|
||||
test_policies = [StoragePolicy(0, 'zero', False),
|
||||
StoragePolicy(1, 'one', False),
|
||||
StoragePolicy(2, 'two', True)]
|
||||
policies = StoragePolicyCollection(test_policies)
|
||||
self.assertEquals(policies.default, test_policies[2])
|
||||
self.assertEquals(policies.default.name, 'two')
|
||||
|
||||
# multiple defaults
|
||||
test_policies = [StoragePolicy(0, 'zero', False),
|
||||
StoragePolicy(1, 'one', True),
|
||||
StoragePolicy(2, 'two', True)]
|
||||
self.assertRaisesWithMessage(
|
||||
PolicyError, 'Duplicate default', StoragePolicyCollection,
|
||||
test_policies)
|
||||
|
||||
# nothing specified
|
||||
test_policies = []
|
||||
policies = StoragePolicyCollection(test_policies)
|
||||
self.assertEquals(policies.default, policies[0])
|
||||
self.assertEquals(policies.default.name, 'Policy-0')
|
||||
|
||||
# no default specified with only policy index 0
|
||||
test_policies = [StoragePolicy(0, 'zero')]
|
||||
policies = StoragePolicyCollection(test_policies)
|
||||
self.assertEqual(policies.default, policies[0])
|
||||
|
||||
# no default specified with multiple policies
|
||||
test_policies = [StoragePolicy(0, 'zero', False),
|
||||
StoragePolicy(1, 'one', False),
|
||||
StoragePolicy(2, 'two', False)]
|
||||
self.assertRaisesWithMessage(
|
||||
PolicyError, 'Unable to find default policy',
|
||||
StoragePolicyCollection, test_policies)
|
||||
|
||||
def test_deprecate_policies(self):
|
||||
# deprecation specified
|
||||
test_policies = [StoragePolicy(0, 'zero', True),
|
||||
StoragePolicy(1, 'one', False),
|
||||
StoragePolicy(2, 'two', False, is_deprecated=True)]
|
||||
policies = StoragePolicyCollection(test_policies)
|
||||
self.assertEquals(policies.default, test_policies[0])
|
||||
self.assertEquals(policies.default.name, 'zero')
|
||||
self.assertEquals(len(policies), 3)
|
||||
|
||||
# multiple policies requires default
|
||||
test_policies = [StoragePolicy(0, 'zero', False),
|
||||
StoragePolicy(1, 'one', False, is_deprecated=True),
|
||||
StoragePolicy(2, 'two', False)]
|
||||
self.assertRaisesWithMessage(
|
||||
PolicyError, 'Unable to find default policy',
|
||||
StoragePolicyCollection, test_policies)
|
||||
|
||||
def test_validate_policies_indexes(self):
|
||||
# duplicate indexes
|
||||
test_policies = [StoragePolicy(0, 'zero', True),
|
||||
StoragePolicy(1, 'one', False),
|
||||
StoragePolicy(1, 'two', False)]
|
||||
self.assertRaises(PolicyError, StoragePolicyCollection,
|
||||
test_policies)
|
||||
|
||||
def test_validate_policy_params(self):
|
||||
StoragePolicy(0, 'name') # sanity
|
||||
# bogus indexes
|
||||
self.assertRaises(PolicyError, StoragePolicy, 'x', 'name')
|
||||
self.assertRaises(PolicyError, StoragePolicy, -1, 'name')
|
||||
# non-zero Policy-0
|
||||
self.assertRaisesWithMessage(PolicyError, 'reserved', StoragePolicy,
|
||||
1, 'policy-0')
|
||||
# deprecate default
|
||||
self.assertRaisesWithMessage(
|
||||
PolicyError, 'Deprecated policy can not be default',
|
||||
StoragePolicy, 1, 'Policy-1', is_default=True,
|
||||
is_deprecated=True)
|
||||
# weird names
|
||||
names = (
|
||||
'',
|
||||
'name_foo',
|
||||
'name\nfoo',
|
||||
'name foo',
|
||||
u'name \u062a',
|
||||
'name \xd8\xaa',
|
||||
)
|
||||
for name in names:
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid name',
|
||||
StoragePolicy, 1, name)
|
||||
|
||||
def test_validate_policies_names(self):
|
||||
# duplicate names
|
||||
test_policies = [StoragePolicy(0, 'zero', True),
|
||||
StoragePolicy(1, 'zero', False),
|
||||
StoragePolicy(2, 'two', False)]
|
||||
self.assertRaises(PolicyError, StoragePolicyCollection,
|
||||
test_policies)
|
||||
|
||||
def test_names_are_normalized(self):
|
||||
test_policies = [StoragePolicy(0, 'zero', True),
|
||||
StoragePolicy(1, 'ZERO', False)]
|
||||
self.assertRaises(PolicyError, StoragePolicyCollection,
|
||||
test_policies)
|
||||
|
||||
policies = StoragePolicyCollection([StoragePolicy(0, 'zEro', True),
|
||||
StoragePolicy(1, 'One', False)])
|
||||
|
||||
pol0 = policies[0]
|
||||
pol1 = policies[1]
|
||||
|
||||
for name in ('zero', 'ZERO', 'zErO', 'ZeRo'):
|
||||
self.assertEqual(pol0, policies.get_by_name(name))
|
||||
self.assertEqual(policies.get_by_name(name).name, 'zEro')
|
||||
for name in ('one', 'ONE', 'oNe', 'OnE'):
|
||||
self.assertEqual(pol1, policies.get_by_name(name))
|
||||
self.assertEqual(policies.get_by_name(name).name, 'One')
|
||||
|
||||
def assertRaisesWithMessage(self, exc_class, message, f, *args, **kwargs):
|
||||
try:
|
||||
f(*args, **kwargs)
|
||||
except exc_class as err:
|
||||
err_msg = str(err)
|
||||
self.assert_(message in err_msg, 'Error message %r did not '
|
||||
'have expected substring %r' % (err_msg, message))
|
||||
else:
|
||||
self.fail('%r did not raise %s' % (message, exc_class.__name__))
|
||||
|
||||
def test_deprecated_default(self):
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:1]
|
||||
name = one
|
||||
deprecated = yes
|
||||
default = yes
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(
|
||||
PolicyError, "Deprecated policy can not be default",
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
def test_multiple_policies_with_no_policy_index_zero(self):
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:1]
|
||||
name = one
|
||||
default = yes
|
||||
""")
|
||||
|
||||
# Policy-0 will not be implicitly added if other policies are defined
|
||||
self.assertRaisesWithMessage(
|
||||
PolicyError, "must specify a storage policy section "
|
||||
"for policy index 0", parse_storage_policies, bad_conf)
|
||||
|
||||
def test_no_default(self):
|
||||
orig_conf = self._conf("""
|
||||
[storage-policy:0]
|
||||
name = zero
|
||||
[storage-policy:1]
|
||||
name = one
|
||||
default = yes
|
||||
""")
|
||||
|
||||
policies = parse_storage_policies(orig_conf)
|
||||
self.assertEqual(policies.default, policies[1])
|
||||
self.assert_(policies[0].name, 'Policy-0')
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:0]
|
||||
name = zero
|
||||
[storage-policy:1]
|
||||
name = one
|
||||
deprecated = yes
|
||||
""")
|
||||
|
||||
# multiple polices and no explicit default
|
||||
self.assertRaisesWithMessage(
|
||||
PolicyError, "Unable to find default",
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
good_conf = self._conf("""
|
||||
[storage-policy:0]
|
||||
name = Policy-0
|
||||
default = yes
|
||||
[storage-policy:1]
|
||||
name = one
|
||||
deprecated = yes
|
||||
""")
|
||||
|
||||
policies = parse_storage_policies(good_conf)
|
||||
self.assertEqual(policies.default, policies[0])
|
||||
self.assert_(policies[1].is_deprecated, True)
|
||||
|
||||
def test_parse_storage_policies(self):
|
||||
# ValueError when deprecating policy 0
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:0]
|
||||
name = zero
|
||||
deprecated = yes
|
||||
|
||||
[storage-policy:1]
|
||||
name = one
|
||||
deprecated = yes
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(
|
||||
PolicyError, "Unable to find policy that's not deprecated",
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:]
|
||||
name = zero
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid index',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:-1]
|
||||
name = zero
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid index',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:x]
|
||||
name = zero
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid index',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:x-1]
|
||||
name = zero
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid index',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:x]
|
||||
name = zero
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid index',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:x:1]
|
||||
name = zero
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid index',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:1]
|
||||
name = zero
|
||||
boo = berries
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid option',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:0]
|
||||
name =
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid name',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:3]
|
||||
name = Policy-0
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid name',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:1]
|
||||
name = policY-0
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid name',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:0]
|
||||
name = one
|
||||
[storage-policy:1]
|
||||
name = ONE
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Duplicate name',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
bad_conf = self._conf("""
|
||||
[storage-policy:0]
|
||||
name = good_stuff
|
||||
""")
|
||||
|
||||
self.assertRaisesWithMessage(PolicyError, 'Invalid name',
|
||||
parse_storage_policies, bad_conf)
|
||||
|
||||
# Additional section added to ensure parser ignores other sections
|
||||
conf = self._conf("""
|
||||
[some-other-section]
|
||||
foo = bar
|
||||
[storage-policy:0]
|
||||
name = zero
|
||||
[storage-policy:5]
|
||||
name = one
|
||||
default = yes
|
||||
[storage-policy:6]
|
||||
name = duplicate-sections-are-ignored
|
||||
[storage-policy:6]
|
||||
name = apple
|
||||
""")
|
||||
policies = parse_storage_policies(conf)
|
||||
|
||||
self.assertEquals(True, policies.get_by_index(5).is_default)
|
||||
self.assertEquals(False, policies.get_by_index(0).is_default)
|
||||
self.assertEquals(False, policies.get_by_index(6).is_default)
|
||||
|
||||
self.assertEquals("object", policies.get_by_name("zero").ring_name)
|
||||
self.assertEquals("object-5", policies.get_by_name("one").ring_name)
|
||||
self.assertEquals("object-6", policies.get_by_name("apple").ring_name)
|
||||
|
||||
self.assertEqual(0, int(policies.get_by_name('zero')))
|
||||
self.assertEqual(5, int(policies.get_by_name('one')))
|
||||
self.assertEqual(6, int(policies.get_by_name('apple')))
|
||||
|
||||
self.assertEquals("zero", policies.get_by_index(0).name)
|
||||
self.assertEquals("zero", policies.get_by_index("0").name)
|
||||
self.assertEquals("one", policies.get_by_index(5).name)
|
||||
self.assertEquals("apple", policies.get_by_index(6).name)
|
||||
self.assertEquals("zero", policies.get_by_index(None).name)
|
||||
self.assertEquals("zero", policies.get_by_index('').name)
|
||||
|
||||
def test_reload_invalid_storage_policies(self):
|
||||
conf = self._conf("""
|
||||
[storage-policy:0]
|
||||
name = zero
|
||||
[storage-policy:00]
|
||||
name = double-zero
|
||||
""")
|
||||
with NamedTemporaryFile() as f:
|
||||
conf.write(f)
|
||||
f.flush()
|
||||
with mock.patch('swift.common.storage_policy.SWIFT_CONF_FILE',
|
||||
new=f.name):
|
||||
try:
|
||||
reload_storage_policies()
|
||||
except SystemExit as e:
|
||||
err_msg = str(e)
|
||||
else:
|
||||
self.fail('SystemExit not raised')
|
||||
parts = [
|
||||
'Invalid Storage Policy Configuration',
|
||||
'Duplicate index',
|
||||
]
|
||||
for expected in parts:
|
||||
self.assert_(expected in err_msg, '%s was not in %s' % (expected,
|
||||
err_msg))
|
||||
|
||||
def test_storage_policy_ordering(self):
|
||||
test_policies = StoragePolicyCollection([
|
||||
StoragePolicy(0, 'zero', is_default=True),
|
||||
StoragePolicy(503, 'error'),
|
||||
StoragePolicy(204, 'empty'),
|
||||
StoragePolicy(404, 'missing'),
|
||||
])
|
||||
self.assertEqual([0, 204, 404, 503], [int(p) for p in
|
||||
sorted(list(test_policies))])
|
||||
|
||||
p503 = test_policies[503]
|
||||
self.assertTrue(501 < p503 < 507)
|
||||
|
||||
def test_get_object_ring(self):
|
||||
test_policies = [StoragePolicy(0, 'aay', True),
|
||||
StoragePolicy(1, 'bee', False),
|
||||
StoragePolicy(2, 'cee', False)]
|
||||
policies = StoragePolicyCollection(test_policies)
|
||||
|
||||
class NamedFakeRing(FakeRing):
|
||||
|
||||
def __init__(self, swift_dir, ring_name=None):
|
||||
self.ring_name = ring_name
|
||||
super(NamedFakeRing, self).__init__()
|
||||
|
||||
with mock.patch('swift.common.storage_policy.Ring',
|
||||
new=NamedFakeRing):
|
||||
for policy in policies:
|
||||
self.assertFalse(policy.object_ring)
|
||||
ring = policies.get_object_ring(int(policy), '/path/not/used')
|
||||
self.assertEqual(ring.ring_name, policy.ring_name)
|
||||
self.assertTrue(policy.object_ring)
|
||||
self.assert_(isinstance(policy.object_ring, NamedFakeRing))
|
||||
|
||||
def blow_up(*args, **kwargs):
|
||||
raise Exception('kaboom!')
|
||||
|
||||
with mock.patch('swift.common.storage_policy.Ring', new=blow_up):
|
||||
for policy in policies:
|
||||
policy.load_ring('/path/not/used')
|
||||
expected = policies.get_object_ring(int(policy),
|
||||
'/path/not/used')
|
||||
self.assertEqual(policy.object_ring, expected)
|
||||
|
||||
# bad policy index
|
||||
self.assertRaises(PolicyError, policies.get_object_ring, 99,
|
||||
'/path/not/used')
|
||||
|
||||
def test_singleton_passthrough(self):
|
||||
test_policies = [StoragePolicy(0, 'aay', True),
|
||||
StoragePolicy(1, 'bee', False),
|
||||
StoragePolicy(2, 'cee', False)]
|
||||
with patch_policies(test_policies):
|
||||
for policy in POLICIES:
|
||||
self.assertEqual(POLICIES[int(policy)], policy)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -27,6 +27,7 @@ from urllib import quote
|
||||
from hashlib import md5
|
||||
from tempfile import mkdtemp
|
||||
import weakref
|
||||
import operator
|
||||
import re
|
||||
|
||||
import mock
|
||||
@ -34,7 +35,8 @@ from eventlet import sleep, spawn, wsgi, listen
|
||||
import simplejson
|
||||
|
||||
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \
|
||||
fake_http_connect, FakeRing, FakeMemcache, debug_logger
|
||||
fake_http_connect, FakeRing, FakeMemcache, debug_logger, \
|
||||
patch_policies
|
||||
from swift.proxy import server as proxy_server
|
||||
from swift.account import server as account_server
|
||||
from swift.container import server as container_server
|
||||
@ -51,6 +53,7 @@ from swift.proxy.controllers.base import get_container_memcache_key, \
|
||||
get_account_memcache_key, cors_validation
|
||||
import swift.proxy.controllers
|
||||
from swift.common.request_helpers import get_sys_meta_prefix
|
||||
from swift.common.storage_policy import StoragePolicy
|
||||
from swift.common.swob import Request, Response, HTTPUnauthorized, \
|
||||
HTTPException
|
||||
|
||||
@ -5928,6 +5931,9 @@ class TestProxyObjectPerformance(unittest.TestCase):
|
||||
print "Run %02d took %07.03f" % (i, end - start)
|
||||
|
||||
|
||||
@patch_policies([StoragePolicy(0, 'migrated'),
|
||||
StoragePolicy(1, 'ernie', True),
|
||||
StoragePolicy(3, 'bert')])
|
||||
class TestSwiftInfo(unittest.TestCase):
|
||||
def setUp(self):
|
||||
utils._swift_info = {}
|
||||
@ -5963,7 +5969,14 @@ class TestSwiftInfo(unittest.TestCase):
|
||||
self.assertTrue('strict_cors_mode' in si)
|
||||
# this next test is deliberately brittle in order to alert if
|
||||
# other items are added to swift info
|
||||
self.assertEqual(len(si), 13)
|
||||
self.assertEqual(len(si), 14)
|
||||
|
||||
self.assertTrue('policies' in si)
|
||||
sorted_pols = sorted(si['policies'], key=operator.itemgetter('name'))
|
||||
self.assertEqual(len(sorted_pols), 3)
|
||||
self.assertEqual(sorted_pols[0]['name'], 'bert')
|
||||
self.assertEqual(sorted_pols[1]['name'], 'ernie')
|
||||
self.assertEqual(sorted_pols[2]['name'], 'migrated')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Loading…
Reference in New Issue
Block a user