diff --git a/etc/swift.conf-sample b/etc/swift.conf-sample index 8bb8c4ae5e..fac17676cf 100644 --- a/etc/swift.conf-sample +++ b/etc/swift.conf-sample @@ -8,6 +8,37 @@ swift_hash_path_suffix = changeme swift_hash_path_prefix = changeme +# storage policies are defined here and determine various characteristics +# about how objects are stored and treated. Policies are specified by name on +# a per container basis. Names are case-insensitive. The policy index is +# specified in the section header and is used internally. The policy with +# index 0 is always used for legacy containers and can be given a name for use +# in metadata however the ring file name will always be 'object.ring.gz' for +# backwards compatibility. If no policies are defined a policy with index 0 +# will be automatically created for backwards compatibility and given the name +# Policy-0. A default policy is used when creating new containers when no +# policy is specified in the request. If no other policies are defined the +# policy with index 0 will be declared the default. If multiple policies are +# defined you must define a policy with index 0 and you must specify a +# default. It is recommended you always define a section for +# storage-policy:0. +[storage-policy:0] +name = Policy-0 +default = yes + +# the following section would declare a policy called 'silver', the number of +# replicas will be determined by how the ring is built. In this example the +# 'silver' policy could have a lower or higher # of replicas than the +# 'Policy-0' policy above. The ring filename will be 'object-1.ring.gz'. You +# may only specify one storage policy section as the default. If you changed +# this section to specify 'silver' as the default, when a client created a new +# container w/o a policy specified, it will get the 'silver' policy because +# this config has specified it as the default. However if a legacy container +# (one created with a pre-policy version of swift) is accessed, it is known +# implicitly to be assigned to the policy with index 0 as opposed to the +# current default. +#[storage-policy:1] +#name = silver # The swift-constraints section sets the basic constraints on data # saved in the swift cluster. These constraints are automatically diff --git a/swift/common/storage_policy.py b/swift/common/storage_policy.py new file mode 100644 index 0000000000..0c0c0eee2c --- /dev/null +++ b/swift/common/storage_policy.py @@ -0,0 +1,354 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +import textwrap +import string + +from swift.common.utils import config_true_value, SWIFT_CONF_FILE +from swift.common.ring import Ring + +POLICY = 'X-Storage-Policy' +POLICY_INDEX = 'X-Backend-Storage-Policy-Index' +LEGACY_POLICY_NAME = 'Policy-0' +VALID_CHARS = '-' + string.letters + string.digits + + +class PolicyError(ValueError): + + def __init__(self, msg, index=None): + if index is not None: + msg += ', for index %r' % index + super(PolicyError, self).__init__(msg) + + +def _get_policy_string(base, policy_index): + if policy_index == 0 or policy_index is None: + return_string = base + else: + return_string = base + "-%d" % int(policy_index) + return return_string + + +def get_policy_string(base, policy_index): + """ + Helper function to construct a string from a base and the policy + index. Used to encode the policy index into either a file name + or a directory name by various modules. + + :param base: the base string + :param policy_index: the storage policy index + + :returns: base name with policy index added + """ + if POLICIES.get_by_index(policy_index) is None: + raise PolicyError("No policy with index %r" % policy_index) + return _get_policy_string(base, policy_index) + + +class StoragePolicy(object): + """ + Represents a storage policy. + Not meant to be instantiated directly; use + :func:`~swift.common.storage_policy.reload_storage_policies` to load + POLICIES from ``swift.conf``. + + The object_ring property is lazy loaded once the service's ``swift_dir`` + is known via :meth:`~StoragePolicyCollection.get_object_ring`, but it may + be over-ridden via object_ring kwarg at create time for testing or + actively loaded with :meth:`~StoragePolicy.load_ring`. + """ + def __init__(self, idx, name='', is_default=False, is_deprecated=False, + object_ring=None): + try: + self.idx = int(idx) + except ValueError: + raise PolicyError('Invalid index', idx) + if self.idx < 0: + raise PolicyError('Invalid index', idx) + if not name: + raise PolicyError('Invalid name %r' % name, idx) + # this is defensively restrictive, but could be expanded in the future + if not all(c in VALID_CHARS for c in name): + raise PolicyError('Names are used as HTTP headers, and can not ' + 'reliably contain any characters not in %r. ' + 'Invalid name %r' % (VALID_CHARS, name)) + if name.upper() == LEGACY_POLICY_NAME.upper() and self.idx != 0: + msg = 'The name %s is reserved for policy index 0. ' \ + 'Invalid name %r' % (LEGACY_POLICY_NAME, name) + raise PolicyError(msg, idx) + self.name = name + self.is_deprecated = config_true_value(is_deprecated) + self.is_default = config_true_value(is_default) + if self.is_deprecated and self.is_default: + raise PolicyError('Deprecated policy can not be default. ' + 'Invalid config', self.idx) + self.ring_name = _get_policy_string('object', self.idx) + self.object_ring = object_ring + + def __int__(self): + return self.idx + + def __cmp__(self, other): + return cmp(self.idx, int(other)) + + def __repr__(self): + return ("StoragePolicy(%d, %r, is_default=%s, is_deprecated=%s)") % ( + self.idx, self.name, self.is_default, self.is_deprecated) + + def load_ring(self, swift_dir): + """ + Load the ring for this policy immediately. + + :param swift_dir: path to rings + """ + if self.object_ring: + return + self.object_ring = Ring(swift_dir, ring_name=self.ring_name) + + +class StoragePolicyCollection(object): + """ + This class represents the collection of valid storage policies for the + cluster and is instantiated as :class:`StoragePolicy` objects are added to + the collection when ``swift.conf`` is parsed by + :func:`parse_storage_policies`. + + When a StoragePolicyCollection is created, the following validation + is enforced: + + * If a policy with index 0 is not declared and no other policies defined, + Swift will create one + * The policy index must be a non-negative integer + * If no policy is declared as the default and no other policies are + defined, the policy with index 0 is set as the default + * Policy indexes must be unique + * Policy names are required + * Policy names are case insensitive + * Policy names must contain only letters, digits or a dash + * Policy names must be unique + * The policy name 'Policy-0' can only be used for the policy with index 0 + * If any policies are defined, exactly one policy must be declared default + * Deprecated policies can not be declared the default + + """ + def __init__(self, pols): + self.default = [] + self.by_name = {} + self.by_index = {} + self._validate_policies(pols) + + def _add_policy(self, policy): + """ + Add pre-validated policies to internal indexes. + """ + self.by_name[policy.name.upper()] = policy + self.by_index[int(policy)] = policy + + def __repr__(self): + return (textwrap.dedent(""" + StoragePolicyCollection([ + %s + ]) + """) % ',\n '.join(repr(p) for p in self)).strip() + + def __len__(self): + return len(self.by_index) + + def __getitem__(self, key): + return self.by_index[key] + + def __iter__(self): + return iter(self.by_index.values()) + + def _validate_policies(self, policies): + """ + :param policies: list of policies + """ + + for policy in policies: + if int(policy) in self.by_index: + raise PolicyError('Duplicate index %s conflicts with %s' % ( + policy, self.get_by_index(int(policy)))) + if policy.name.upper() in self.by_name: + raise PolicyError('Duplicate name %s conflicts with %s' % ( + policy, self.get_by_name(policy.name))) + if policy.is_default: + if not self.default: + self.default = policy + else: + raise PolicyError( + 'Duplicate default %s conflicts with %s' % ( + policy, self.default)) + self._add_policy(policy) + + # If a 0 policy wasn't explicitly given, or nothing was + # provided, create the 0 policy now + if 0 not in self.by_index: + if len(self) != 0: + raise PolicyError('You must specify a storage policy ' + 'section for policy index 0 in order ' + 'to define multiple policies') + self._add_policy(StoragePolicy(0, name=LEGACY_POLICY_NAME)) + + # at least one policy must be enabled + enabled_policies = [p for p in self if not p.is_deprecated] + if not enabled_policies: + raise PolicyError("Unable to find policy that's not deprecated!") + + # if needed, specify default + if not self.default: + if len(self) > 1: + raise PolicyError("Unable to find default policy") + self.default = self[0] + self.default.is_default = True + + def get_by_name(self, name): + """ + Find a storage policy by its name. + + :param name: name of the policy + :returns: storage policy, or None + """ + return self.by_name.get(name.upper()) + + def get_by_index(self, index): + """ + Find a storage policy by its index. + + An index of None will be treated as 0. + + :param index: numeric index of the storage policy + :returns: storage policy, or None if no such policy + """ + # makes it easier for callers to just pass in a header value + index = int(index) if index else 0 + return self.by_index.get(index) + + def get_object_ring(self, policy_idx, swift_dir): + """ + Get the ring object to use to handle a request based on its policy. + + An index of None will be treated as 0. + + :param policy_idx: policy index as defined in swift.conf + :param swift_dir: swift_dir used by the caller + :returns: appropriate ring object + """ + policy = self.get_by_index(policy_idx) + if not policy: + raise PolicyError("No policy with index %s" % policy_idx) + if not policy.object_ring: + policy.load_ring(swift_dir) + return policy.object_ring + + def get_policy_info(self): + """ + Build info about policies for the /info endpoint + + :returns: list of dicts containing relevant policy information + """ + policy_info = [] + for pol in self: + # delete from /info if deprecated + if pol.is_deprecated: + continue + policy_entry = {} + policy_entry['name'] = pol.name + if pol.is_default: + policy_entry['default'] = pol.is_default + policy_info.append(policy_entry) + return policy_info + + +def parse_storage_policies(conf): + """ + Parse storage policies in ``swift.conf`` - note that validation + is done when the :class:`StoragePolicyCollection` is instantiated. + + :param conf: ConfigParser parser object for swift.conf + """ + policies = [] + for section in conf.sections(): + if not section.startswith('storage-policy:'): + continue + policy_index = section.split(':', 1)[1] + # map config option name to StoragePolicy paramater name + config_to_policy_option_map = { + 'name': 'name', + 'default': 'is_default', + 'deprecated': 'is_deprecated', + } + policy_options = {} + for config_option, value in conf.items(section): + try: + policy_option = config_to_policy_option_map[config_option] + except KeyError: + raise PolicyError('Invalid option %r in ' + 'storage-policy section %r' % ( + config_option, section)) + policy_options[policy_option] = value + policy = StoragePolicy(policy_index, **policy_options) + policies.append(policy) + + return StoragePolicyCollection(policies) + + +class StoragePolicySingleton(object): + """ + An instance of this class is the primary interface to storage policies + exposed as a module level global named ``POLICIES``. This global + reference wraps ``_POLICIES`` which is normally instantiated by parsing + ``swift.conf`` and will result in an instance of + :class:`StoragePolicyCollection`. + + You should never patch this instance directly, instead patch the module + level ``_POLICIES`` instance so that swift code which imported + ``POLICIES`` directly will reference the patched + :class:`StoragePolicyCollection`. + """ + + def __iter__(self): + return iter(_POLICIES) + + def __len__(self): + return len(_POLICIES) + + def __getitem__(self, key): + return _POLICIES[key] + + def __getattribute__(self, name): + return getattr(_POLICIES, name) + + def __repr__(self): + return repr(_POLICIES) + + +def reload_storage_policies(): + """ + Reload POLICIES from ``swift.conf``. + """ + global _POLICIES + policy_conf = ConfigParser() + policy_conf.read(SWIFT_CONF_FILE) + try: + _POLICIES = parse_storage_policies(policy_conf) + except PolicyError as e: + raise SystemExit('ERROR: Invalid Storage Policy Configuration ' + 'in %s (%s)' % (SWIFT_CONF_FILE, e)) + + +# parse configuration and setup singleton +_POLICIES = None +reload_storage_policies() +POLICIES = StoragePolicySingleton() diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 59123ae8f2..5107a06ac2 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -31,6 +31,7 @@ from swift.common.utils import cache_from_env, get_logger, \ affinity_key_function, affinity_locality_predicate, list_from_csv, \ register_swift_info from swift.common.constraints import check_utf8 +from swift.common.storage_policy import POLICIES from swift.proxy.controllers import AccountController, ObjectController, \ ContainerController, InfoController from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ @@ -207,6 +208,7 @@ class Application(object): register_swift_info( version=swift_version, strict_cors_mode=self.strict_cors_mode, + policies=POLICIES.get_policy_info(), **constraints.EFFECTIVE_CONSTRAINTS) def check_config(self): diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 5ccc5906cd..084a2523c4 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -21,7 +21,7 @@ import logging import errno import sys from contextlib import contextmanager -from collections import defaultdict +from collections import defaultdict, Iterable from tempfile import NamedTemporaryFile import time from eventlet.green import socket @@ -34,6 +34,86 @@ from eventlet import sleep, Timeout import logging.handlers from httplib import HTTPException from numbers import Number +from swift.common import storage_policy +import functools + +DEFAULT_PATCH_POLICIES = [storage_policy.StoragePolicy(0, 'nulo', True), + storage_policy.StoragePolicy(1, 'unu')] +LEGACY_PATCH_POLICIES = [storage_policy.StoragePolicy(0, 'legacy', True)] + + +def patch_policies(thing_or_policies=None, legacy_only=False): + if legacy_only: + default_policies = LEGACY_PATCH_POLICIES + else: + default_policies = DEFAULT_PATCH_POLICIES + + thing_or_policies = thing_or_policies or default_policies + + if isinstance(thing_or_policies, ( + Iterable, storage_policy.StoragePolicyCollection)): + return PatchPolicies(thing_or_policies) + else: + # it's a thing! + return PatchPolicies(default_policies)(thing_or_policies) + + +class PatchPolicies(object): + """ + Why not mock.patch? In my case, when used as a decorator on the class it + seemed to patch setUp at the wrong time (i.e. in setup the global wasn't + patched yet) + """ + + def __init__(self, policies): + if isinstance(policies, storage_policy.StoragePolicyCollection): + self.policies = policies + else: + self.policies = storage_policy.StoragePolicyCollection(policies) + + def __call__(self, thing): + if isinstance(thing, type): + return self._patch_class(thing) + else: + return self._patch_method(thing) + + def _patch_class(self, cls): + + class NewClass(cls): + + already_patched = False + + def setUp(cls_self): + self._orig_POLICIES = storage_policy._POLICIES + if not cls_self.already_patched: + storage_policy._POLICIES = self.policies + cls_self.already_patched = True + super(NewClass, cls_self).setUp() + + def tearDown(cls_self): + super(NewClass, cls_self).tearDown() + storage_policy._POLICIES = self._orig_POLICIES + + NewClass.__name__ = cls.__name__ + return NewClass + + def _patch_method(self, f): + @functools.wraps(f) + def mywrapper(*args, **kwargs): + self._orig_POLICIES = storage_policy._POLICIES + try: + storage_policy._POLICIES = self.policies + return f(*args, **kwargs) + finally: + storage_policy._POLICIES = self._orig_POLICIES + return mywrapper + + def __enter__(self): + self._orig_POLICIES = storage_policy._POLICIES + storage_policy._POLICIES = self.policies + + def __exit__(self, *args): + storage_policy._POLICIES = self._orig_POLICIES class FakeRing(object): diff --git a/test/unit/common/test_storage_policy.py b/test/unit/common/test_storage_policy.py new file mode 100644 index 0000000000..e154a11b63 --- /dev/null +++ b/test/unit/common/test_storage_policy.py @@ -0,0 +1,517 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for swift.common.storage_policies """ +import unittest +import StringIO +from ConfigParser import ConfigParser +import mock +from tempfile import NamedTemporaryFile +from test.unit import patch_policies, FakeRing +from swift.common.storage_policy import ( + StoragePolicy, StoragePolicyCollection, POLICIES, PolicyError, + parse_storage_policies, reload_storage_policies, get_policy_string) + + +class TestStoragePolicies(unittest.TestCase): + + def _conf(self, conf_str): + conf_str = "\n".join(line.strip() for line in conf_str.split("\n")) + conf = ConfigParser() + conf.readfp(StringIO.StringIO(conf_str)) + return conf + + @patch_policies([StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'one', False), + StoragePolicy(2, 'two', False), + StoragePolicy(3, 'three', False, is_deprecated=True)]) + def test_swift_info(self): + # the deprecated 'three' should not exist in expect + expect = [{'default': True, 'name': 'zero'}, + {'name': 'two'}, + {'name': 'one'}] + swift_info = POLICIES.get_policy_info() + self.assertEquals(sorted(expect, key=lambda k: k['name']), + sorted(swift_info, key=lambda k: k['name'])) + + @patch_policies + def test_get_policy_string(self): + self.assertEquals(get_policy_string('something', 0), 'something') + self.assertEquals(get_policy_string('something', None), 'something') + self.assertEquals(get_policy_string('something', 1), + 'something' + '-1') + self.assertRaises(PolicyError, get_policy_string, 'something', 99) + + def test_defaults(self): + self.assertTrue(len(POLICIES) > 0) + + # test class functions + default_policy = POLICIES.default + self.assert_(default_policy.is_default) + zero_policy = POLICIES.get_by_index(0) + self.assert_(zero_policy.idx == 0) + zero_policy_by_name = POLICIES.get_by_name(zero_policy.name) + self.assert_(zero_policy_by_name.idx == 0) + + def test_storage_policy_repr(self): + test_policies = [StoragePolicy(0, 'aay', True), + StoragePolicy(1, 'bee', False), + StoragePolicy(2, 'cee', False)] + policies = StoragePolicyCollection(test_policies) + for policy in policies: + policy_repr = repr(policy) + self.assert_(policy.__class__.__name__ in policy_repr) + self.assert_('is_default=%s' % policy.is_default in policy_repr) + self.assert_('is_deprecated=%s' % policy.is_deprecated in + policy_repr) + self.assert_(policy.name in policy_repr) + collection_repr = repr(policies) + collection_repr_lines = collection_repr.splitlines() + self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) + self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) + for policy, line in zip(policies, collection_repr_lines[1:-1]): + self.assert_(repr(policy) in line) + with patch_policies(policies): + self.assertEqual(repr(POLICIES), collection_repr) + + def test_validate_policies_defaults(self): + # 0 explicit default + test_policies = [StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'one', False), + StoragePolicy(2, 'two', False)] + policies = StoragePolicyCollection(test_policies) + self.assertEquals(policies.default, test_policies[0]) + self.assertEquals(policies.default.name, 'zero') + + # non-zero explicit default + test_policies = [StoragePolicy(0, 'zero', False), + StoragePolicy(1, 'one', False), + StoragePolicy(2, 'two', True)] + policies = StoragePolicyCollection(test_policies) + self.assertEquals(policies.default, test_policies[2]) + self.assertEquals(policies.default.name, 'two') + + # multiple defaults + test_policies = [StoragePolicy(0, 'zero', False), + StoragePolicy(1, 'one', True), + StoragePolicy(2, 'two', True)] + self.assertRaisesWithMessage( + PolicyError, 'Duplicate default', StoragePolicyCollection, + test_policies) + + # nothing specified + test_policies = [] + policies = StoragePolicyCollection(test_policies) + self.assertEquals(policies.default, policies[0]) + self.assertEquals(policies.default.name, 'Policy-0') + + # no default specified with only policy index 0 + test_policies = [StoragePolicy(0, 'zero')] + policies = StoragePolicyCollection(test_policies) + self.assertEqual(policies.default, policies[0]) + + # no default specified with multiple policies + test_policies = [StoragePolicy(0, 'zero', False), + StoragePolicy(1, 'one', False), + StoragePolicy(2, 'two', False)] + self.assertRaisesWithMessage( + PolicyError, 'Unable to find default policy', + StoragePolicyCollection, test_policies) + + def test_deprecate_policies(self): + # deprecation specified + test_policies = [StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'one', False), + StoragePolicy(2, 'two', False, is_deprecated=True)] + policies = StoragePolicyCollection(test_policies) + self.assertEquals(policies.default, test_policies[0]) + self.assertEquals(policies.default.name, 'zero') + self.assertEquals(len(policies), 3) + + # multiple policies requires default + test_policies = [StoragePolicy(0, 'zero', False), + StoragePolicy(1, 'one', False, is_deprecated=True), + StoragePolicy(2, 'two', False)] + self.assertRaisesWithMessage( + PolicyError, 'Unable to find default policy', + StoragePolicyCollection, test_policies) + + def test_validate_policies_indexes(self): + # duplicate indexes + test_policies = [StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'one', False), + StoragePolicy(1, 'two', False)] + self.assertRaises(PolicyError, StoragePolicyCollection, + test_policies) + + def test_validate_policy_params(self): + StoragePolicy(0, 'name') # sanity + # bogus indexes + self.assertRaises(PolicyError, StoragePolicy, 'x', 'name') + self.assertRaises(PolicyError, StoragePolicy, -1, 'name') + # non-zero Policy-0 + self.assertRaisesWithMessage(PolicyError, 'reserved', StoragePolicy, + 1, 'policy-0') + # deprecate default + self.assertRaisesWithMessage( + PolicyError, 'Deprecated policy can not be default', + StoragePolicy, 1, 'Policy-1', is_default=True, + is_deprecated=True) + # weird names + names = ( + '', + 'name_foo', + 'name\nfoo', + 'name foo', + u'name \u062a', + 'name \xd8\xaa', + ) + for name in names: + self.assertRaisesWithMessage(PolicyError, 'Invalid name', + StoragePolicy, 1, name) + + def test_validate_policies_names(self): + # duplicate names + test_policies = [StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'zero', False), + StoragePolicy(2, 'two', False)] + self.assertRaises(PolicyError, StoragePolicyCollection, + test_policies) + + def test_names_are_normalized(self): + test_policies = [StoragePolicy(0, 'zero', True), + StoragePolicy(1, 'ZERO', False)] + self.assertRaises(PolicyError, StoragePolicyCollection, + test_policies) + + policies = StoragePolicyCollection([StoragePolicy(0, 'zEro', True), + StoragePolicy(1, 'One', False)]) + + pol0 = policies[0] + pol1 = policies[1] + + for name in ('zero', 'ZERO', 'zErO', 'ZeRo'): + self.assertEqual(pol0, policies.get_by_name(name)) + self.assertEqual(policies.get_by_name(name).name, 'zEro') + for name in ('one', 'ONE', 'oNe', 'OnE'): + self.assertEqual(pol1, policies.get_by_name(name)) + self.assertEqual(policies.get_by_name(name).name, 'One') + + def assertRaisesWithMessage(self, exc_class, message, f, *args, **kwargs): + try: + f(*args, **kwargs) + except exc_class as err: + err_msg = str(err) + self.assert_(message in err_msg, 'Error message %r did not ' + 'have expected substring %r' % (err_msg, message)) + else: + self.fail('%r did not raise %s' % (message, exc_class.__name__)) + + def test_deprecated_default(self): + bad_conf = self._conf(""" + [storage-policy:1] + name = one + deprecated = yes + default = yes + """) + + self.assertRaisesWithMessage( + PolicyError, "Deprecated policy can not be default", + parse_storage_policies, bad_conf) + + def test_multiple_policies_with_no_policy_index_zero(self): + bad_conf = self._conf(""" + [storage-policy:1] + name = one + default = yes + """) + + # Policy-0 will not be implicitly added if other policies are defined + self.assertRaisesWithMessage( + PolicyError, "must specify a storage policy section " + "for policy index 0", parse_storage_policies, bad_conf) + + def test_no_default(self): + orig_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = one + default = yes + """) + + policies = parse_storage_policies(orig_conf) + self.assertEqual(policies.default, policies[1]) + self.assert_(policies[0].name, 'Policy-0') + + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:1] + name = one + deprecated = yes + """) + + # multiple polices and no explicit default + self.assertRaisesWithMessage( + PolicyError, "Unable to find default", + parse_storage_policies, bad_conf) + + good_conf = self._conf(""" + [storage-policy:0] + name = Policy-0 + default = yes + [storage-policy:1] + name = one + deprecated = yes + """) + + policies = parse_storage_policies(good_conf) + self.assertEqual(policies.default, policies[0]) + self.assert_(policies[1].is_deprecated, True) + + def test_parse_storage_policies(self): + # ValueError when deprecating policy 0 + bad_conf = self._conf(""" + [storage-policy:0] + name = zero + deprecated = yes + + [storage-policy:1] + name = one + deprecated = yes + """) + + self.assertRaisesWithMessage( + PolicyError, "Unable to find policy that's not deprecated", + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:] + name = zero + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid index', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:-1] + name = zero + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid index', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:x] + name = zero + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid index', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:x-1] + name = zero + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid index', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:x] + name = zero + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid index', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:x:1] + name = zero + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid index', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:1] + name = zero + boo = berries + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid option', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:0] + name = + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid name', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:3] + name = Policy-0 + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid name', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:1] + name = policY-0 + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid name', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:0] + name = one + [storage-policy:1] + name = ONE + """) + + self.assertRaisesWithMessage(PolicyError, 'Duplicate name', + parse_storage_policies, bad_conf) + + bad_conf = self._conf(""" + [storage-policy:0] + name = good_stuff + """) + + self.assertRaisesWithMessage(PolicyError, 'Invalid name', + parse_storage_policies, bad_conf) + + # Additional section added to ensure parser ignores other sections + conf = self._conf(""" + [some-other-section] + foo = bar + [storage-policy:0] + name = zero + [storage-policy:5] + name = one + default = yes + [storage-policy:6] + name = duplicate-sections-are-ignored + [storage-policy:6] + name = apple + """) + policies = parse_storage_policies(conf) + + self.assertEquals(True, policies.get_by_index(5).is_default) + self.assertEquals(False, policies.get_by_index(0).is_default) + self.assertEquals(False, policies.get_by_index(6).is_default) + + self.assertEquals("object", policies.get_by_name("zero").ring_name) + self.assertEquals("object-5", policies.get_by_name("one").ring_name) + self.assertEquals("object-6", policies.get_by_name("apple").ring_name) + + self.assertEqual(0, int(policies.get_by_name('zero'))) + self.assertEqual(5, int(policies.get_by_name('one'))) + self.assertEqual(6, int(policies.get_by_name('apple'))) + + self.assertEquals("zero", policies.get_by_index(0).name) + self.assertEquals("zero", policies.get_by_index("0").name) + self.assertEquals("one", policies.get_by_index(5).name) + self.assertEquals("apple", policies.get_by_index(6).name) + self.assertEquals("zero", policies.get_by_index(None).name) + self.assertEquals("zero", policies.get_by_index('').name) + + def test_reload_invalid_storage_policies(self): + conf = self._conf(""" + [storage-policy:0] + name = zero + [storage-policy:00] + name = double-zero + """) + with NamedTemporaryFile() as f: + conf.write(f) + f.flush() + with mock.patch('swift.common.storage_policy.SWIFT_CONF_FILE', + new=f.name): + try: + reload_storage_policies() + except SystemExit as e: + err_msg = str(e) + else: + self.fail('SystemExit not raised') + parts = [ + 'Invalid Storage Policy Configuration', + 'Duplicate index', + ] + for expected in parts: + self.assert_(expected in err_msg, '%s was not in %s' % (expected, + err_msg)) + + def test_storage_policy_ordering(self): + test_policies = StoragePolicyCollection([ + StoragePolicy(0, 'zero', is_default=True), + StoragePolicy(503, 'error'), + StoragePolicy(204, 'empty'), + StoragePolicy(404, 'missing'), + ]) + self.assertEqual([0, 204, 404, 503], [int(p) for p in + sorted(list(test_policies))]) + + p503 = test_policies[503] + self.assertTrue(501 < p503 < 507) + + def test_get_object_ring(self): + test_policies = [StoragePolicy(0, 'aay', True), + StoragePolicy(1, 'bee', False), + StoragePolicy(2, 'cee', False)] + policies = StoragePolicyCollection(test_policies) + + class NamedFakeRing(FakeRing): + + def __init__(self, swift_dir, ring_name=None): + self.ring_name = ring_name + super(NamedFakeRing, self).__init__() + + with mock.patch('swift.common.storage_policy.Ring', + new=NamedFakeRing): + for policy in policies: + self.assertFalse(policy.object_ring) + ring = policies.get_object_ring(int(policy), '/path/not/used') + self.assertEqual(ring.ring_name, policy.ring_name) + self.assertTrue(policy.object_ring) + self.assert_(isinstance(policy.object_ring, NamedFakeRing)) + + def blow_up(*args, **kwargs): + raise Exception('kaboom!') + + with mock.patch('swift.common.storage_policy.Ring', new=blow_up): + for policy in policies: + policy.load_ring('/path/not/used') + expected = policies.get_object_ring(int(policy), + '/path/not/used') + self.assertEqual(policy.object_ring, expected) + + # bad policy index + self.assertRaises(PolicyError, policies.get_object_ring, 99, + '/path/not/used') + + def test_singleton_passthrough(self): + test_policies = [StoragePolicy(0, 'aay', True), + StoragePolicy(1, 'bee', False), + StoragePolicy(2, 'cee', False)] + with patch_policies(test_policies): + for policy in POLICIES: + self.assertEqual(POLICIES[int(policy)], policy) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 1372e822d5..6d91e3b5e5 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -27,6 +27,7 @@ from urllib import quote from hashlib import md5 from tempfile import mkdtemp import weakref +import operator import re import mock @@ -34,7 +35,8 @@ from eventlet import sleep, spawn, wsgi, listen import simplejson from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \ - fake_http_connect, FakeRing, FakeMemcache, debug_logger + fake_http_connect, FakeRing, FakeMemcache, debug_logger, \ + patch_policies from swift.proxy import server as proxy_server from swift.account import server as account_server from swift.container import server as container_server @@ -51,6 +53,7 @@ from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_memcache_key, cors_validation import swift.proxy.controllers from swift.common.request_helpers import get_sys_meta_prefix +from swift.common.storage_policy import StoragePolicy from swift.common.swob import Request, Response, HTTPUnauthorized, \ HTTPException @@ -5928,6 +5931,9 @@ class TestProxyObjectPerformance(unittest.TestCase): print "Run %02d took %07.03f" % (i, end - start) +@patch_policies([StoragePolicy(0, 'migrated'), + StoragePolicy(1, 'ernie', True), + StoragePolicy(3, 'bert')]) class TestSwiftInfo(unittest.TestCase): def setUp(self): utils._swift_info = {} @@ -5963,7 +5969,14 @@ class TestSwiftInfo(unittest.TestCase): self.assertTrue('strict_cors_mode' in si) # this next test is deliberately brittle in order to alert if # other items are added to swift info - self.assertEqual(len(si), 13) + self.assertEqual(len(si), 14) + + self.assertTrue('policies' in si) + sorted_pols = sorted(si['policies'], key=operator.itemgetter('name')) + self.assertEqual(len(sorted_pols), 3) + self.assertEqual(sorted_pols[0]['name'], 'bert') + self.assertEqual(sorted_pols[1]['name'], 'ernie') + self.assertEqual(sorted_pols[2]['name'], 'migrated') if __name__ == '__main__':