Merge "Select policy when running functional test"

doc/source/development_guidelines.rst

@@ -71,6 +71,18 @@ The endpoint and authorization credentials to be used by functional tests
 should be configured in the ``test.conf`` file as described in the section
 :ref:`setup_scripts`.
 
+The environment variable ``SWIFT_TEST_POLICY`` may be set to specify a
+particular storage policy *name* that will be used for testing. When set, tests
+that would otherwise not specify a policy or choose a random policy from
+those available will instead use the policy specified. Tests that use more than
+one policy will include the specified policy in the set of policies used. The
+specified policy must be available on the cluster under test.
+
+For example, this command would run the functional tests using policy
+'silver'::
+
+  SWIFT_TEST_POLICY=silver tox -e func
+
 If the ``test.conf`` file is not found then the functional test framework will
 instantiate a set of Swift servers in the same process that executes the
 functional tests. This 'in-process test' mode may also be enabled (or disabled)
@@ -95,13 +107,14 @@ found in ``<custom_conf_source_dir>``, the search will then look in the
 the corresponding sample config file from ``etc/`` is used (e.g.
 ``proxy-server.conf-sample`` or ``swift.conf-sample``).
 
-The environment variable ``SWIFT_TEST_POLICY`` may be set to specify
-a particular storage policy *name* that will be used for testing. When set,
-this policy must exist in the ``swift.conf`` file and its corresponding ring
-file must exist in ``<custom_conf_source_dir>`` (if specified) or ``etc/``. The
-test setup will set the specified policy to be the default and use its ring
-file properties for constructing the test object ring. This allows in-process
-testing to be run against various policy types and ring files.
+When using the 'in-process test' mode ``SWIFT_TEST_POLICY`` may be set to
+specify a particular storage policy *name* that will be used for testing as
+described above. When set, this policy must exist in the ``swift.conf`` file
+and its corresponding ring file must exist in ``<custom_conf_source_dir>`` (if
+specified) or ``etc/``. The test setup will set the specified policy to be the
+default and use its ring file properties for constructing the test object ring.
+This allows in-process testing to be run against various policy types and ring
+files.
 
 For example, this command would run the in-process mode functional tests
 using config files found in ``$HOME/my_tests`` and policy 'silver'::

test/functional/__init__.py

@@ -37,7 +37,7 @@ from swift.common.middleware.memcache import MemcacheMiddleware
 from swift.common.storage_policy import parse_storage_policies, PolicyError
 
 from test import get_config
-from test.functional.swift_test_client import Account, Connection, \
+from test.functional.swift_test_client import Account, Connection, Container, \
     ResponseError
 # This has the side effect of mocking out the xattr module so that unit tests
 # (and in this case, when in-process functional tests are called for) can run
@@ -47,7 +47,7 @@ from test.unit import debug_logger, FakeMemcache
 from swift.common import constraints, utils, ring, storage_policy
 from swift.common.ring import Ring
 from swift.common.wsgi import monkey_patch_mimetools, loadapp
-from swift.common.utils import config_true_value
+from swift.common.utils import config_true_value, split_path
 from swift.account import server as account_server
 from swift.container import server as container_server
 from swift.obj import server as object_server, mem_server as mem_object_server
@@ -106,6 +106,7 @@ orig_swift_conf_name = None
 
 in_process = False
 _testdir = _test_servers = _test_coros = None
+policy_specified = None
 
 
 class FakeMemcacheMiddleware(MemcacheMiddleware):
@@ -210,7 +211,6 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
     for policy in policies:
         conf.remove_section(sp_prefix + str(policy.idx))
 
-    policy_specified = os.environ.get('SWIFT_TEST_POLICY')
     if policy_specified:
         policy_to_test = policies.get_by_name(policy_specified)
         if policy_to_test is None:
@@ -521,6 +521,9 @@ def get_cluster_info():
 
 
 def setup_package():
+
+    global policy_specified
+    policy_specified = os.environ.get('SWIFT_TEST_POLICY')
     in_process_env = os.environ.get('SWIFT_TEST_IN_PROCESS')
     if in_process_env is not None:
         use_in_process = utils.config_true_value(in_process_env)
@@ -700,6 +703,22 @@ def setup_package():
         print >>sys.stderr, \
             'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS'
 
+    if policy_specified:
+        policies = FunctionalStoragePolicyCollection.from_info()
+        for p in policies:
+            # policy names are case-insensitive
+            if policy_specified.lower() == p['name'].lower():
+                _info('Using specified policy %s' % policy_specified)
+                FunctionalStoragePolicyCollection.policy_specified = p
+                Container.policy_specified = policy_specified
+                break
+        else:
+            _info(
+                'SKIPPING FUNCTIONAL TESTS: Failed to find specified policy %s'
+                % policy_specified)
+            raise Exception('Failed to find specified policy %s'
+                            % policy_specified)
+
     get_cluster_info()
 
 
@@ -747,8 +766,24 @@ conn = [None, None, None, None, None]
 
 def connection(url):
     if has_insecure:
-        return http_connection(url, insecure=insecure)
-    return http_connection(url)
+        parsed_url, http_conn = http_connection(url, insecure=insecure)
+    else:
+        parsed_url, http_conn = http_connection(url)
+
+    orig_request = http_conn.request
+
+    # Add the policy header if policy_specified is set
+    def request_with_policy(method, url, body=None, headers={}):
+        version, account, container, obj = split_path(url, 1, 4, True)
+        if policy_specified and method == 'PUT' and container and not obj \
+                and 'X-Storage-Policy' not in headers:
+            headers['X-Storage-Policy'] = policy_specified
+
+        return orig_request(method, url, body, headers)
+
+    http_conn.request = request_with_policy
+
+    return parsed_url, http_conn
 
 
 def get_url_token(user_index, os_options):
@@ -899,6 +934,9 @@ def requires_acls(f):
 
 class FunctionalStoragePolicyCollection(object):
 
+    # policy_specified is set in __init__.py when tests are being set up.
+    policy_specified = None
+
     def __init__(self, policies):
         self._all = policies
         self.default = None
@@ -940,7 +978,12 @@ class FunctionalStoragePolicyCollection(object):
             p.get(k) != v for k, v in kwargs.items())])
 
     def select(self):
-        return random.choice(self)
+        # check that a policy was specified and that it is available
+        # in the current list (i.e., hasn't been excluded of the current list)
+        if self.policy_specified and self.policy_specified in self:
+            return self.policy_specified
+        else:
+            return random.choice(self)
 
 
 def requires_policies(f):

test/functional/swift_test_client.py

@@ -488,6 +488,9 @@ class Account(Base):
 
 
 class Container(Base):
+    # policy_specified is set in __init__.py when tests are being set up.
+    policy_specified = None
+
     def __init__(self, conn, account, name):
         self.conn = conn
         self.account = str(account)
@@ -500,6 +503,8 @@ class Container(Base):
             parms = {}
         if cfg is None:
             cfg = {}
+        if self.policy_specified and 'X-Storage-Policy' not in hdrs:
+            hdrs['X-Storage-Policy'] = self.policy_specified
         return self.conn.make_request('PUT', self.path, hdrs=hdrs,
                                       parms=parms, cfg=cfg) in (201, 202)
 

test/functional/test_container.py

@@ -1398,8 +1398,11 @@ class TestContainer(unittest.TestCase):
             raise SkipTest()
 
         def put(url, token, parsed, conn):
+            # using the empty storage policy header value here to ensure
+            # that the default policy is chosen in case policy_specified is set
+            # see __init__.py for details on policy_specified
             conn.request('PUT', parsed.path + '/' + self.container, '',
-                         {'X-Auth-Token': token})
+                         {'X-Auth-Token': token, 'X-Storage-Policy': ''})
             return check_response(conn)
         resp = retry(put)
         resp.read()