Merge "DRY out common.constraints a bit more"

swift/common/constraints.py

@@ -22,56 +22,64 @@ from swift.common.utils import ismount, split_path, SWIFT_CONF_FILE
 from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \
     HTTPRequestEntityTooLarge, HTTPPreconditionFailed
 
-constraints_conf = ConfigParser()
+MAX_FILE_SIZE = 5368709122
-constraints_conf_exists = constraints_conf.read(SWIFT_CONF_FILE)
+MAX_META_NAME_LENGTH = 128
+MAX_META_VALUE_LENGTH = 256
+MAX_META_COUNT = 90
+MAX_META_OVERALL_SIZE = 4096
+MAX_HEADER_SIZE = 8192
+MAX_OBJECT_NAME_LENGTH = 1024
+CONTAINER_LISTING_LIMIT = 10000
+ACCOUNT_LISTING_LIMIT = 10000
+MAX_ACCOUNT_NAME_LENGTH = 256
+MAX_CONTAINER_NAME_LENGTH = 256
+
+DEFAULT_CONSTRAINTS = {
+    'max_file_size': MAX_FILE_SIZE,
+    'max_meta_name_length': MAX_META_NAME_LENGTH,
+    'max_meta_value_length': MAX_META_VALUE_LENGTH,
+    'max_meta_count': MAX_META_COUNT,
+    'max_meta_overall_size': MAX_META_OVERALL_SIZE,
+    'max_header_size': MAX_HEADER_SIZE,
+    'max_object_name_length': MAX_OBJECT_NAME_LENGTH,
+    'container_listing_limit': CONTAINER_LISTING_LIMIT,
+    'account_listing_limit': ACCOUNT_LISTING_LIMIT,
+    'max_account_name_length': MAX_ACCOUNT_NAME_LENGTH,
+    'max_container_name_length': MAX_CONTAINER_NAME_LENGTH,
+}
+
+SWIFT_CONSTRAINTS_LOADED = False
+OVERRIDE_CONSTRAINTS = {}  # any constraints overridden by SWIFT_CONF_FILE
+EFFECTIVE_CONSTRAINTS = {}  # populated by reload_constraints
 
 
-def constraints_conf_int(name, default):
+def reload_constraints():
-    try:
+    """
-        return int(constraints_conf.get('swift-constraints', name))
+    Parse SWIFT_CONF_FILE and reset module level global contraint attrs,
-    except (NoSectionError, NoOptionError):
+    populating OVERRIDE_CONSTRAINTS AND EFFECTIVE_CONSTRAINTS along the way.
-        return default
+    """
+    global SWIFT_CONSTRAINTS_LOADED, OVERRIDE_CONSTRAINTS
+    SWIFT_CONSTRAINTS_LOADED = False
+    OVERRIDE_CONSTRAINTS = {}
+    constraints_conf = ConfigParser()
+    if constraints_conf.read(SWIFT_CONF_FILE):
+        SWIFT_CONSTRAINTS_LOADED = True
+        for name in DEFAULT_CONSTRAINTS:
+            try:
+                value = int(constraints_conf.get('swift-constraints', name))
+            except (NoSectionError, NoOptionError):
+                pass
+            else:
+                OVERRIDE_CONSTRAINTS[name] = value
+    for name, default in DEFAULT_CONSTRAINTS.items():
+        value = OVERRIDE_CONSTRAINTS.get(name, default)
+        EFFECTIVE_CONSTRAINTS[name] = value
+        # "globals" in this context is module level globals, always.
+        globals()[name.upper()] = value
 
 
-#: Max file size allowed for objects
+reload_constraints()
-MAX_FILE_SIZE = constraints_conf_int('max_file_size',
+
-                                     5368709122)  # 5 * 1024 * 1024 * 1024 + 2
-#: Max length of the name of a key for metadata
-MAX_META_NAME_LENGTH = constraints_conf_int('max_meta_name_length', 128)
-#: Max length of the value of a key for metadata
-MAX_META_VALUE_LENGTH = constraints_conf_int('max_meta_value_length', 256)
-#: Max number of metadata items
-MAX_META_COUNT = constraints_conf_int('max_meta_count', 90)
-#: Max overall size of metadata
-MAX_META_OVERALL_SIZE = constraints_conf_int('max_meta_overall_size', 4096)
-#: Max size of any header
-MAX_HEADER_SIZE = constraints_conf_int('max_header_size', 8192)
-#: Max object name length
-MAX_OBJECT_NAME_LENGTH = constraints_conf_int('max_object_name_length', 1024)
-#: Max object list length of a get request for a container
-CONTAINER_LISTING_LIMIT = constraints_conf_int('container_listing_limit',
-                                               10000)
-#: Max container list length of a get request for an account
-ACCOUNT_LISTING_LIMIT = constraints_conf_int('account_listing_limit', 10000)
-#: Max account name length
-MAX_ACCOUNT_NAME_LENGTH = constraints_conf_int('max_account_name_length', 256)
-#: Max container name length
-MAX_CONTAINER_NAME_LENGTH = constraints_conf_int('max_container_name_length',
-                                                 256)
-# A simple dictionary of all the constraints that can be specified in the
-# SWIFT_CONF_FILE.
-default_constraints = dict((
-    ('max_file_size', MAX_FILE_SIZE),
-    ('max_meta_name_length', MAX_META_NAME_LENGTH),
-    ('max_meta_value_length', MAX_META_VALUE_LENGTH),
-    ('max_meta_count', MAX_META_COUNT),
-    ('max_meta_overall_size', MAX_META_OVERALL_SIZE),
-    ('max_header_size', MAX_HEADER_SIZE),
-    ('max_object_name_length', MAX_OBJECT_NAME_LENGTH),
-    ('container_listing_limit', CONTAINER_LISTING_LIMIT),
-    ('account_listing_limit', ACCOUNT_LISTING_LIMIT),
-    ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH),
-    ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH)))
 
 # Maximum slo segments in buffer
 MAX_BUFFERED_SLO_SEGMENTS = 10000

test/functional/tests.py

@@ -31,19 +31,19 @@ from nose import SkipTest
 from test import get_config
 from test.functional.swift_test_client import Account, Connection, File, \
     ResponseError
-from swift.common.constraints import default_constraints, \
+from swift.common import constraints
-    constraints_conf_exists
+
 
 config = get_config('func_test')
-for k in default_constraints:
+for k in constraints.DEFAULT_CONSTRAINTS:
     if k in config:
         # prefer what's in test.conf
         config[k] = int(config[k])
-    elif constraints_conf_exists:
+    elif constraints.SWIFT_CONSTRAINTS_LOADED:
         # swift.conf exists, so use what's defined there (or swift defaults)
         # This normally happens when the test is running locally to the cluster
         # as in a SAIO.
-        config[k] = default_constraints[k]
+        config[k] = constraints.EFFECTIVE_CONSTRAINTS[k]
     else:
         # .functests don't know what the constraints of the tested cluster are,
         # so the tests can't reliably pass or fail. Therefore, skip those

test/unit/common/test_constraints.py

@@ -15,6 +15,7 @@
 
 import unittest
 import mock
+import tempfile
 
 from test import safe_repr
 from test.unit import MockTrue
@@ -245,5 +246,85 @@ class TestConstraints(unittest.TestCase):
                           constraints.check_copy_from_header, req)
 
 
+class TestConstraintsConfig(unittest.TestCase):
+
+    def test_default_constraints(self):
+        for key in constraints.DEFAULT_CONSTRAINTS:
+            # if there is local over-rides in swift.conf we just continue on
+            if key in constraints.OVERRIDE_CONSTRAINTS:
+                continue
+            # module level attrs (that aren't in OVERRIDE) should have the
+            # same value as the DEFAULT map
+            module_level_value = getattr(constraints, key.upper())
+            self.assertEquals(constraints.DEFAULT_CONSTRAINTS[key],
+                              module_level_value)
+
+    def test_effective_constraints(self):
+        for key in constraints.DEFAULT_CONSTRAINTS:
+            # module level attrs should always mirror the same value as the
+            # EFFECTIVE map
+            module_level_value = getattr(constraints, key.upper())
+            self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key],
+                              module_level_value)
+            # if there are local over-rides in swift.conf those should be
+            # reflected in the EFFECTIVE, otherwise we expect the DEFAULTs
+            self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key],
+                              constraints.OVERRIDE_CONSTRAINTS.get(
+                                  key, constraints.DEFAULT_CONSTRAINTS[key]))
+
+    def test_override_constraints(self):
+        try:
+            with tempfile.NamedTemporaryFile() as f:
+                f.write('[swift-constraints]\n')
+                # set everything to 1
+                for key in constraints.DEFAULT_CONSTRAINTS:
+                    f.write('%s = 1\n' % key)
+                f.flush()
+                with mock.patch.object(constraints, 'SWIFT_CONF_FILE',
+                                       f.name):
+                    constraints.reload_constraints()
+            for key in constraints.DEFAULT_CONSTRAINTS:
+                # module level attrs should all be 1
+                module_level_value = getattr(constraints, key.upper())
+                self.assertEquals(module_level_value, 1)
+                # all keys should be in OVERRIDE
+                self.assertEquals(constraints.OVERRIDE_CONSTRAINTS[key],
+                                  module_level_value)
+                # module level attrs should always mirror the same value as
+                # the EFFECTIVE map
+                self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key],
+                                  module_level_value)
+        finally:
+            constraints.reload_constraints()
+
+    def test_reload_reset(self):
+        try:
+            with tempfile.NamedTemporaryFile() as f:
+                f.write('[swift-constraints]\n')
+                # set everything to 1
+                for key in constraints.DEFAULT_CONSTRAINTS:
+                    f.write('%s = 1\n' % key)
+                f.flush()
+                with mock.patch.object(constraints, 'SWIFT_CONF_FILE',
+                                       f.name):
+                    constraints.reload_constraints()
+            self.assertTrue(constraints.SWIFT_CONSTRAINTS_LOADED)
+            self.assertEquals(sorted(constraints.DEFAULT_CONSTRAINTS.keys()),
+                              sorted(constraints.OVERRIDE_CONSTRAINTS.keys()))
+            # file is now deleted...
+            with mock.patch.object(constraints, 'SWIFT_CONF_FILE',
+                                   f.name):
+                constraints.reload_constraints()
+            # no constraints have been loaded from non-existant swift.conf
+            self.assertFalse(constraints.SWIFT_CONSTRAINTS_LOADED)
+            # no constraints are in OVERRIDE
+            self.assertEquals([], constraints.OVERRIDE_CONSTRAINTS.keys())
+            # the EFFECTIVE constraints mirror DEFAULT
+            self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS,
+                              constraints.DEFAULT_CONSTRAINTS)
+        finally:
+            constraints.reload_constraints()
+
+
 if __name__ == '__main__':
     unittest.main()