From 7463a46a6262eb95572f90d1bf93abc461576700 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 11 Mar 2014 14:18:55 -0700 Subject: [PATCH] DRY out common.constraints a bit more Change-Id: If8a71692b79e381036b34245e09697ac1009beca --- swift/common/constraints.py | 100 +++++++++++++++------------ test/functional/tests.py | 10 +-- test/unit/common/test_constraints.py | 81 ++++++++++++++++++++++ 3 files changed, 140 insertions(+), 51 deletions(-) diff --git a/swift/common/constraints.py b/swift/common/constraints.py index 0c23132129..7a480eaec0 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -22,56 +22,64 @@ from swift.common.utils import ismount, split_path, SWIFT_CONF_FILE from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \ HTTPRequestEntityTooLarge, HTTPPreconditionFailed -constraints_conf = ConfigParser() -constraints_conf_exists = constraints_conf.read(SWIFT_CONF_FILE) +MAX_FILE_SIZE = 5368709122 +MAX_META_NAME_LENGTH = 128 +MAX_META_VALUE_LENGTH = 256 +MAX_META_COUNT = 90 +MAX_META_OVERALL_SIZE = 4096 +MAX_HEADER_SIZE = 8192 +MAX_OBJECT_NAME_LENGTH = 1024 +CONTAINER_LISTING_LIMIT = 10000 +ACCOUNT_LISTING_LIMIT = 10000 +MAX_ACCOUNT_NAME_LENGTH = 256 +MAX_CONTAINER_NAME_LENGTH = 256 + +DEFAULT_CONSTRAINTS = { + 'max_file_size': MAX_FILE_SIZE, + 'max_meta_name_length': MAX_META_NAME_LENGTH, + 'max_meta_value_length': MAX_META_VALUE_LENGTH, + 'max_meta_count': MAX_META_COUNT, + 'max_meta_overall_size': MAX_META_OVERALL_SIZE, + 'max_header_size': MAX_HEADER_SIZE, + 'max_object_name_length': MAX_OBJECT_NAME_LENGTH, + 'container_listing_limit': CONTAINER_LISTING_LIMIT, + 'account_listing_limit': ACCOUNT_LISTING_LIMIT, + 'max_account_name_length': MAX_ACCOUNT_NAME_LENGTH, + 'max_container_name_length': MAX_CONTAINER_NAME_LENGTH, +} + +SWIFT_CONSTRAINTS_LOADED = False +OVERRIDE_CONSTRAINTS = {} # any constraints overridden by SWIFT_CONF_FILE +EFFECTIVE_CONSTRAINTS = {} # populated by reload_constraints -def constraints_conf_int(name, default): - try: - return int(constraints_conf.get('swift-constraints', name)) - except (NoSectionError, NoOptionError): - return default +def reload_constraints(): + """ + Parse SWIFT_CONF_FILE and reset module level global contraint attrs, + populating OVERRIDE_CONSTRAINTS AND EFFECTIVE_CONSTRAINTS along the way. + """ + global SWIFT_CONSTRAINTS_LOADED, OVERRIDE_CONSTRAINTS + SWIFT_CONSTRAINTS_LOADED = False + OVERRIDE_CONSTRAINTS = {} + constraints_conf = ConfigParser() + if constraints_conf.read(SWIFT_CONF_FILE): + SWIFT_CONSTRAINTS_LOADED = True + for name in DEFAULT_CONSTRAINTS: + try: + value = int(constraints_conf.get('swift-constraints', name)) + except (NoSectionError, NoOptionError): + pass + else: + OVERRIDE_CONSTRAINTS[name] = value + for name, default in DEFAULT_CONSTRAINTS.items(): + value = OVERRIDE_CONSTRAINTS.get(name, default) + EFFECTIVE_CONSTRAINTS[name] = value + # "globals" in this context is module level globals, always. + globals()[name.upper()] = value -#: Max file size allowed for objects -MAX_FILE_SIZE = constraints_conf_int('max_file_size', - 5368709122) # 5 * 1024 * 1024 * 1024 + 2 -#: Max length of the name of a key for metadata -MAX_META_NAME_LENGTH = constraints_conf_int('max_meta_name_length', 128) -#: Max length of the value of a key for metadata -MAX_META_VALUE_LENGTH = constraints_conf_int('max_meta_value_length', 256) -#: Max number of metadata items -MAX_META_COUNT = constraints_conf_int('max_meta_count', 90) -#: Max overall size of metadata -MAX_META_OVERALL_SIZE = constraints_conf_int('max_meta_overall_size', 4096) -#: Max size of any header -MAX_HEADER_SIZE = constraints_conf_int('max_header_size', 8192) -#: Max object name length -MAX_OBJECT_NAME_LENGTH = constraints_conf_int('max_object_name_length', 1024) -#: Max object list length of a get request for a container -CONTAINER_LISTING_LIMIT = constraints_conf_int('container_listing_limit', - 10000) -#: Max container list length of a get request for an account -ACCOUNT_LISTING_LIMIT = constraints_conf_int('account_listing_limit', 10000) -#: Max account name length -MAX_ACCOUNT_NAME_LENGTH = constraints_conf_int('max_account_name_length', 256) -#: Max container name length -MAX_CONTAINER_NAME_LENGTH = constraints_conf_int('max_container_name_length', - 256) -# A simple dictionary of all the constraints that can be specified in the -# SWIFT_CONF_FILE. -default_constraints = dict(( - ('max_file_size', MAX_FILE_SIZE), - ('max_meta_name_length', MAX_META_NAME_LENGTH), - ('max_meta_value_length', MAX_META_VALUE_LENGTH), - ('max_meta_count', MAX_META_COUNT), - ('max_meta_overall_size', MAX_META_OVERALL_SIZE), - ('max_header_size', MAX_HEADER_SIZE), - ('max_object_name_length', MAX_OBJECT_NAME_LENGTH), - ('container_listing_limit', CONTAINER_LISTING_LIMIT), - ('account_listing_limit', ACCOUNT_LISTING_LIMIT), - ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH), - ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH))) +reload_constraints() + # Maximum slo segments in buffer MAX_BUFFERED_SLO_SEGMENTS = 10000 diff --git a/test/functional/tests.py b/test/functional/tests.py index 6a9f78d611..d65b69883a 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -29,19 +29,19 @@ from nose import SkipTest from test import get_config from test.functional.swift_test_client import Account, Connection, File, \ ResponseError -from swift.common.constraints import default_constraints, \ - constraints_conf_exists +from swift.common import constraints + config = get_config('func_test') -for k in default_constraints: +for k in constraints.DEFAULT_CONSTRAINTS: if k in config: # prefer what's in test.conf config[k] = int(config[k]) - elif constraints_conf_exists: + elif constraints.SWIFT_CONSTRAINTS_LOADED: # swift.conf exists, so use what's defined there (or swift defaults) # This normally happens when the test is running locally to the cluster # as in a SAIO. - config[k] = default_constraints[k] + config[k] = constraints.EFFECTIVE_CONSTRAINTS[k] else: # .functests don't know what the constraints of the tested cluster are, # so the tests can't reliably pass or fail. Therefore, skip those diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index aaf5269ac7..9abf153db6 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -15,6 +15,7 @@ import unittest import mock +import tempfile from test import safe_repr from test.unit import MockTrue @@ -245,5 +246,85 @@ class TestConstraints(unittest.TestCase): constraints.check_copy_from_header, req) +class TestConstraintsConfig(unittest.TestCase): + + def test_default_constraints(self): + for key in constraints.DEFAULT_CONSTRAINTS: + # if there is local over-rides in swift.conf we just continue on + if key in constraints.OVERRIDE_CONSTRAINTS: + continue + # module level attrs (that aren't in OVERRIDE) should have the + # same value as the DEFAULT map + module_level_value = getattr(constraints, key.upper()) + self.assertEquals(constraints.DEFAULT_CONSTRAINTS[key], + module_level_value) + + def test_effective_constraints(self): + for key in constraints.DEFAULT_CONSTRAINTS: + # module level attrs should always mirror the same value as the + # EFFECTIVE map + module_level_value = getattr(constraints, key.upper()) + self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], + module_level_value) + # if there are local over-rides in swift.conf those should be + # reflected in the EFFECTIVE, otherwise we expect the DEFAULTs + self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], + constraints.OVERRIDE_CONSTRAINTS.get( + key, constraints.DEFAULT_CONSTRAINTS[key])) + + def test_override_constraints(self): + try: + with tempfile.NamedTemporaryFile() as f: + f.write('[swift-constraints]\n') + # set everything to 1 + for key in constraints.DEFAULT_CONSTRAINTS: + f.write('%s = 1\n' % key) + f.flush() + with mock.patch.object(constraints, 'SWIFT_CONF_FILE', + f.name): + constraints.reload_constraints() + for key in constraints.DEFAULT_CONSTRAINTS: + # module level attrs should all be 1 + module_level_value = getattr(constraints, key.upper()) + self.assertEquals(module_level_value, 1) + # all keys should be in OVERRIDE + self.assertEquals(constraints.OVERRIDE_CONSTRAINTS[key], + module_level_value) + # module level attrs should always mirror the same value as + # the EFFECTIVE map + self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], + module_level_value) + finally: + constraints.reload_constraints() + + def test_reload_reset(self): + try: + with tempfile.NamedTemporaryFile() as f: + f.write('[swift-constraints]\n') + # set everything to 1 + for key in constraints.DEFAULT_CONSTRAINTS: + f.write('%s = 1\n' % key) + f.flush() + with mock.patch.object(constraints, 'SWIFT_CONF_FILE', + f.name): + constraints.reload_constraints() + self.assertTrue(constraints.SWIFT_CONSTRAINTS_LOADED) + self.assertEquals(sorted(constraints.DEFAULT_CONSTRAINTS.keys()), + sorted(constraints.OVERRIDE_CONSTRAINTS.keys())) + # file is now deleted... + with mock.patch.object(constraints, 'SWIFT_CONF_FILE', + f.name): + constraints.reload_constraints() + # no constraints have been loaded from non-existant swift.conf + self.assertFalse(constraints.SWIFT_CONSTRAINTS_LOADED) + # no constraints are in OVERRIDE + self.assertEquals([], constraints.OVERRIDE_CONSTRAINTS.keys()) + # the EFFECTIVE constraints mirror DEFAULT + self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS, + constraints.DEFAULT_CONSTRAINTS) + finally: + constraints.reload_constraints() + + if __name__ == '__main__': unittest.main()