From 7bdb8266d4b926526acdea22e8c9c634855eeecc Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Wed, 25 Mar 2015 13:19:36 +0000 Subject: [PATCH] Select policy when running functional test Add ability to select a storage policy when running the functional tests by defining the environment variable SWIFT_TEST_POLICY. This patch was first introduced by Janie Richling in feature/ec: https://review.openstack.org/#/c/166097 Co-Authored-By: Janie Richling Co-Authored-By: Christian Schwede Co-Authored-By: Kota Tsuyuzaki Co-Authored-By: Alistair Coles Change-Id: Id96354546088df137def0ae7fd434baf64ed0c51 Signed-off-by: Thiago da Silva --- doc/source/development_guidelines.rst | 27 +++++++++---- test/functional/__init__.py | 55 ++++++++++++++++++++++++--- test/functional/swift_test_client.py | 5 +++ test/functional/test_container.py | 5 ++- 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/doc/source/development_guidelines.rst b/doc/source/development_guidelines.rst index 241eda6cf5..c2af0d6224 100644 --- a/doc/source/development_guidelines.rst +++ b/doc/source/development_guidelines.rst @@ -60,6 +60,18 @@ The endpoint and authorization credentials to be used by functional tests should be configured in the ``test.conf`` file as described in the section :ref:`setup_scripts`. +The environment variable ``SWIFT_TEST_POLICY`` may be set to specify a +particular storage policy *name* that will be used for testing. When set, tests +that would otherwise not specify a policy or choose a random policy from +those available will instead use the policy specified. Tests that use more than +one policy will include the specified policy in the set of policies used. The +specified policy must be available on the cluster under test. + +For example, this command would run the functional tests using policy +'silver':: + + SWIFT_TEST_POLICY=silver tox -e func + If the ``test.conf`` file is not found then the functional test framework will instantiate a set of Swift servers in the same process that executes the functional tests. This 'in-process test' mode may also be enabled (or disabled) @@ -84,13 +96,14 @@ found in ````, the search will then look in the the corresponding sample config file from ``etc/`` is used (e.g. ``proxy-server.conf-sample`` or ``swift.conf-sample``). -The environment variable ``SWIFT_TEST_POLICY`` may be set to specify -a particular storage policy *name* that will be used for testing. When set, -this policy must exist in the ``swift.conf`` file and its corresponding ring -file must exist in ```` (if specified) or ``etc/``. The -test setup will set the specified policy to be the default and use its ring -file properties for constructing the test object ring. This allows in-process -testing to be run against various policy types and ring files. +When using the 'in-process test' mode ``SWIFT_TEST_POLICY`` may be set to +specify a particular storage policy *name* that will be used for testing as +described above. When set, this policy must exist in the ``swift.conf`` file +and its corresponding ring file must exist in ```` (if +specified) or ``etc/``. The test setup will set the specified policy to be the +default and use its ring file properties for constructing the test object ring. +This allows in-process testing to be run against various policy types and ring +files. For example, this command would run the in-process mode functional tests using config files found in ``$HOME/my_tests`` and policy 'silver':: diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 4a8cb80bd9..5dedadb7fc 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -36,7 +36,7 @@ from swift.common.middleware.memcache import MemcacheMiddleware from swift.common.storage_policy import parse_storage_policies, PolicyError from test import get_config -from test.functional.swift_test_client import Account, Connection, \ +from test.functional.swift_test_client import Account, Connection, Container, \ ResponseError # This has the side effect of mocking out the xattr module so that unit tests # (and in this case, when in-process functional tests are called for) can run @@ -46,7 +46,7 @@ from test.unit import debug_logger, FakeMemcache from swift.common import constraints, utils, ring, storage_policy from swift.common.ring import Ring from swift.common.wsgi import monkey_patch_mimetools, loadapp -from swift.common.utils import config_true_value +from swift.common.utils import config_true_value, split_path from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server, mem_server as mem_object_server @@ -105,6 +105,7 @@ orig_swift_conf_name = None in_process = False _testdir = _test_servers = _test_coros = None +policy_specified = None class FakeMemcacheMiddleware(MemcacheMiddleware): @@ -209,7 +210,6 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir): for policy in policies: conf.remove_section(sp_prefix + str(policy.idx)) - policy_specified = os.environ.get('SWIFT_TEST_POLICY') if policy_specified: policy_to_test = policies.get_by_name(policy_specified) if policy_to_test is None: @@ -520,6 +520,9 @@ def get_cluster_info(): def setup_package(): + + global policy_specified + policy_specified = os.environ.get('SWIFT_TEST_POLICY') in_process_env = os.environ.get('SWIFT_TEST_IN_PROCESS') if in_process_env is not None: use_in_process = utils.config_true_value(in_process_env) @@ -699,6 +702,22 @@ def setup_package(): print >>sys.stderr, \ 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS' + if policy_specified: + policies = FunctionalStoragePolicyCollection.from_info() + for p in policies: + # policy names are case-insensitive + if policy_specified.lower() == p['name'].lower(): + _info('Using specified policy %s' % policy_specified) + FunctionalStoragePolicyCollection.policy_specified = p + Container.policy_specified = policy_specified + break + else: + _info( + 'SKIPPING FUNCTIONAL TESTS: Failed to find specified policy %s' + % policy_specified) + raise Exception('Failed to find specified policy %s' + % policy_specified) + get_cluster_info() @@ -746,8 +765,24 @@ conn = [None, None, None, None, None] def connection(url): if has_insecure: - return http_connection(url, insecure=insecure) - return http_connection(url) + parsed_url, http_conn = http_connection(url, insecure=insecure) + else: + parsed_url, http_conn = http_connection(url) + + orig_request = http_conn.request + + # Add the policy header if policy_specified is set + def request_with_policy(method, url, body=None, headers={}): + version, account, container, obj = split_path(url, 1, 4, True) + if policy_specified and method == 'PUT' and container and not obj \ + and 'X-Storage-Policy' not in headers: + headers['X-Storage-Policy'] = policy_specified + + return orig_request(method, url, body, headers) + + http_conn.request = request_with_policy + + return parsed_url, http_conn def get_url_token(user_index, os_options): @@ -898,6 +933,9 @@ def requires_acls(f): class FunctionalStoragePolicyCollection(object): + # policy_specified is set in __init__.py when tests are being set up. + policy_specified = None + def __init__(self, policies): self._all = policies self.default = None @@ -939,7 +977,12 @@ class FunctionalStoragePolicyCollection(object): p.get(k) != v for k, v in kwargs.items())]) def select(self): - return random.choice(self) + # check that a policy was specified and that it is available + # in the current list (i.e., hasn't been excluded of the current list) + if self.policy_specified and self.policy_specified in self: + return self.policy_specified + else: + return random.choice(self) def requires_policies(f): diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index f68dc035f0..6c8e083221 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -481,6 +481,9 @@ class Account(Base): class Container(Base): + # policy_specified is set in __init__.py when tests are being set up. + policy_specified = None + def __init__(self, conn, account, name): self.conn = conn self.account = str(account) @@ -493,6 +496,8 @@ class Container(Base): parms = {} if cfg is None: cfg = {} + if self.policy_specified and 'X-Storage-Policy' not in hdrs: + hdrs['X-Storage-Policy'] = self.policy_specified return self.conn.make_request('PUT', self.path, hdrs=hdrs, parms=parms, cfg=cfg) in (201, 202) diff --git a/test/functional/test_container.py b/test/functional/test_container.py index d7896a42e7..a8da114179 100755 --- a/test/functional/test_container.py +++ b/test/functional/test_container.py @@ -1381,8 +1381,11 @@ class TestContainer(unittest.TestCase): raise SkipTest() def put(url, token, parsed, conn): + # using the empty storage policy header value here to ensure + # that the default policy is chosen in case policy_specified is set + # see __init__.py for details on policy_specified conn.request('PUT', parsed.path + '/' + self.container, '', - {'X-Auth-Token': token}) + {'X-Auth-Token': token, 'X-Storage-Policy': ''}) return check_response(conn) resp = retry(put) resp.read()