diff --git a/doc/requirements.txt b/doc/requirements.txt index de9a2f53ad..13c167a85b 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -6,3 +6,4 @@ sphinx>=1.6.2 # BSD openstackdocstheme>=1.11.0 # Apache-2.0 reno>=1.8.0 # Apache-2.0 os-api-ref>=1.0.0 # Apache-2.0 +python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0 diff --git a/swift/common/middleware/s3api/s3token.py b/swift/common/middleware/s3api/s3token.py index e8990c0e14..db4aa77314 100644 --- a/swift/common/middleware/s3api/s3token.py +++ b/swift/common/middleware/s3api/s3token.py @@ -30,19 +30,25 @@ This middleware: access key. * Validates s3 token with Keystone. * Transforms the account name to AUTH_%(tenant_name). +* Optionally can retrieve and cache secret from keystone + to validate signature locally """ import base64 import json +from keystoneclient.v3 import client as keystone_client +from keystoneauth1 import session as keystone_session +from keystoneauth1 import loading as keystone_loading import requests import six from six.moves import urllib from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \ HTTPException -from swift.common.utils import config_true_value, split_path, get_logger +from swift.common.utils import config_true_value, split_path, get_logger, \ + cache_from_env from swift.common.wsgi import ConfigFileError @@ -155,6 +161,31 @@ class S3Token(object): else: self._verify = None + self._secret_cache_duration = int(conf.get('secret_cache_duration', 0)) + if self._secret_cache_duration > 0: + try: + auth_plugin = keystone_loading.get_plugin_loader( + conf.get('auth_type')) + available_auth_options = auth_plugin.get_options() + auth_options = {} + for option in available_auth_options: + name = option.name.replace('-', '_') + value = conf.get(name) + if value: + auth_options[name] = value + + auth = auth_plugin.load_from_options(**auth_options) + session = keystone_session.Session(auth=auth) + self.keystoneclient = keystone_client.Client(session=session) + self._logger.info("Caching s3tokens for %s seconds", + self._secret_cache_duration) + except Exception: + self._logger.warning("Unable to load keystone auth_plugin. " + "Secret caching will be unavailable.", + exc_info=True) + self.keystoneclient = None + self._secret_cache_duration = 0 + def _deny_request(self, code): error_cls, message = { 'AccessDenied': (HTTPUnauthorized, 'Access denied'), @@ -245,64 +276,99 @@ class S3Token(object): creds = {'credentials': {'access': access, 'token': token, 'signature': signature}} - creds_json = json.dumps(creds) - self._logger.debug('Connecting to Keystone sending this JSON: %s', - creds_json) - # NOTE(vish): We could save a call to keystone by having - # keystone return token, tenant, user, and roles - # from this call. - # - # NOTE(chmou): We still have the same problem we would need to - # change token_auth to detect if we already - # identified and not doing a second query and just - # pass it through to swiftauth in this case. - try: - # NB: requests.Response, not swob.Response - resp = self._json_request(creds_json) - except HTTPException as e_resp: - if self._delay_auth_decision: - msg = 'Received error, deferring rejection based on error: %s' - self._logger.debug(msg, e_resp.status) - return self._app(environ, start_response) - else: - msg = 'Received error, rejecting request with error: %s' - self._logger.debug(msg, e_resp.status) - # NB: swob.Response, not requests.Response - return e_resp(environ, start_response) - self._logger.debug('Keystone Reply: Status: %d, Output: %s', - resp.status_code, resp.content) + memcache_client = None + memcache_token_key = 's3secret/%s' % access + if self._secret_cache_duration > 0: + memcache_client = cache_from_env(environ) + cached_auth_data = None - try: - token = resp.json() - if 'access' in token: - headers, token_id, tenant = parse_v2_response(token) - elif 'token' in token: - headers, token_id, tenant = parse_v3_response(token) - else: - raise ValueError + if memcache_client: + cached_auth_data = memcache_client.get(memcache_token_key) + if cached_auth_data: + headers, token_id, tenant, secret = cached_auth_data + if s3_auth_details['check_signature'](secret): + self._logger.debug("Cached creds valid") + else: + self._logger.debug("Cached creds invalid") + cached_auth_data = None - # Populate the environment similar to auth_token, - # so we don't have to contact Keystone again. + if not cached_auth_data: + creds_json = json.dumps(creds) + self._logger.debug('Connecting to Keystone sending this JSON: %s', + creds_json) + # NOTE(vish): We could save a call to keystone by having + # keystone return token, tenant, user, and roles + # from this call. # - # Note that although the strings are unicode following json - # deserialization, Swift's HeaderEnvironProxy handles ensuring - # they're stored as native strings - req.headers.update(headers) - req.environ['keystone.token_info'] = token - except (ValueError, KeyError, TypeError): - if self._delay_auth_decision: - error = ('Error on keystone reply: %d %s - ' - 'deferring rejection downstream') - self._logger.debug(error, resp.status_code, resp.content) - return self._app(environ, start_response) - else: - error = ('Error on keystone reply: %d %s - ' - 'rejecting request') - self._logger.debug(error, resp.status_code, resp.content) - return self._deny_request('InvalidURI')( - environ, start_response) + # NOTE(chmou): We still have the same problem we would need to + # change token_auth to detect if we already + # identified and not doing a second query and just + # pass it through to swiftauth in this case. + try: + # NB: requests.Response, not swob.Response + resp = self._json_request(creds_json) + except HTTPException as e_resp: + if self._delay_auth_decision: + msg = ('Received error, deferring rejection based on ' + 'error: %s') + self._logger.debug(msg, e_resp.status) + return self._app(environ, start_response) + else: + msg = 'Received error, rejecting request with error: %s' + self._logger.debug(msg, e_resp.status) + # NB: swob.Response, not requests.Response + return e_resp(environ, start_response) + self._logger.debug('Keystone Reply: Status: %d, Output: %s', + resp.status_code, resp.content) + + try: + token = resp.json() + if 'access' in token: + headers, token_id, tenant = parse_v2_response(token) + elif 'token' in token: + headers, token_id, tenant = parse_v3_response(token) + else: + raise ValueError + if memcache_client: + user_id = headers.get('X-User-Id') + if not user_id: + raise ValueError + try: + cred_ref = self.keystoneclient.ec2.get( + user_id=user_id, + access=access) + memcache_client.set( + memcache_token_key, + (headers, token_id, tenant, cred_ref.secret), + time=self._secret_cache_duration) + self._logger.debug("Cached keystone credentials") + except Exception: + self._logger.warning("Unable to cache secret", + exc_info=True) + + # Populate the environment similar to auth_token, + # so we don't have to contact Keystone again. + # + # Note that although the strings are unicode following json + # deserialization, Swift's HeaderEnvironProxy handles ensuring + # they're stored as native strings + req.environ['keystone.token_info'] = token + except (ValueError, KeyError, TypeError): + if self._delay_auth_decision: + error = ('Error on keystone reply: %d %s - ' + 'deferring rejection downstream') + self._logger.debug(error, resp.status_code, resp.content) + return self._app(environ, start_response) + else: + error = ('Error on keystone reply: %d %s - ' + 'rejecting request') + self._logger.debug(error, resp.status_code, resp.content) + return self._deny_request('InvalidURI')( + environ, start_response) + + req.headers.update(headers) req.headers['X-Auth-Token'] = token_id tenant_to_connect = force_tenant or tenant['id'] if six.PY2 and isinstance(tenant_to_connect, six.text_type): diff --git a/test/unit/common/middleware/s3api/test_s3token.py b/test/unit/common/middleware/s3api/test_s3token.py index 4e9251c01f..e84099d83d 100644 --- a/test/unit/common/middleware/s3api/test_s3token.py +++ b/test/unit/common/middleware/s3api/test_s3token.py @@ -504,6 +504,98 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): self._assert_authorized(req, account_path='/v1/') self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o') + @mock.patch('swift.common.middleware.s3api.s3token.cache_from_env') + @mock.patch('keystoneclient.v3.client.Client') + @mock.patch.object(requests, 'post') + def test_secret_is_cached(self, MOCK_REQUEST, MOCK_KEYSTONE, + MOCK_CACHE_FROM_ENV): + self.middleware = s3token.filter_factory({ + 'auth_uri': 'http://example.com', + 'secret_cache_duration': '20', + 'auth_type': 'v3password', + 'auth_url': 'http://example.com:5000/v3', + 'username': 'swift', + 'password': 'secret', + 'project_name': 'service', + 'user_domain_name': 'default', + 'project_domain_name': 'default', + })(FakeApp()) + self.assertEqual(20, self.middleware._secret_cache_duration) + + cache = MOCK_CACHE_FROM_ENV.return_value + + fake_cache_response = ({}, 'token_id', {'id': 'tenant_id'}, 'secret') + cache.get.return_value = fake_cache_response + + MOCK_REQUEST.return_value = TestResponse({ + 'status_code': 201, + 'text': json.dumps(GOOD_RESPONSE_V2)}) + + req = Request.blank('/v1/AUTH_cfa/c/o') + req.environ['s3api.auth_details'] = { + 'access_key': u'access', + 'signature': u'signature', + 'string_to_sign': u'token', + 'check_signature': lambda x: True + } + req.get_response(self.middleware) + # Ensure we don't request auth from keystone + self.assertFalse(MOCK_REQUEST.called) + + @mock.patch('swift.common.middleware.s3api.s3token.cache_from_env') + @mock.patch('keystoneclient.v3.client.Client') + @mock.patch.object(requests, 'post') + def test_secret_sets_cache(self, MOCK_REQUEST, MOCK_KEYSTONE, + MOCK_CACHE_FROM_ENV): + self.middleware = s3token.filter_factory({ + 'auth_uri': 'http://example.com', + 'secret_cache_duration': '20', + 'auth_type': 'v3password', + 'auth_url': 'http://example.com:5000/v3', + 'username': 'swift', + 'password': 'secret', + 'project_name': 'service', + 'user_domain_name': 'default', + 'project_domain_name': 'default', + })(FakeApp()) + self.assertEqual(20, self.middleware._secret_cache_duration) + + cache = MOCK_CACHE_FROM_ENV.return_value + cache.get.return_value = None + + keystone_client = MOCK_KEYSTONE.return_value + keystone_client.ec2.get.return_value = mock.Mock(secret='secret') + + MOCK_REQUEST.return_value = TestResponse({ + 'status_code': 201, + 'text': json.dumps(GOOD_RESPONSE_V2)}) + + req = Request.blank('/v1/AUTH_cfa/c/o') + req.environ['s3api.auth_details'] = { + 'access_key': u'access', + 'signature': u'signature', + 'string_to_sign': u'token', + 'check_signature': lambda x: True + } + req.get_response(self.middleware) + expected_headers = { + 'X-Identity-Status': u'Confirmed', + 'X-Roles': u'swift-user,_member_', + 'X-User-Id': u'USER_ID', + 'X-User-Name': u'S3_USER', + 'X-Tenant-Id': u'TENANT_ID', + 'X-Tenant-Name': u'TENANT_NAME', + 'X-Project-Id': u'TENANT_ID', + 'X-Project-Name': u'TENANT_NAME', + } + + self.assertTrue(MOCK_REQUEST.called) + tenant = GOOD_RESPONSE_V2['access']['token']['tenant'] + token = GOOD_RESPONSE_V2['access']['token']['id'] + expected_cache = (expected_headers, token, tenant, 'secret') + cache.set.assert_called_once_with('s3secret/access', expected_cache, + time=20) + class S3TokenMiddlewareTestBad(S3TokenMiddlewareTestBase): def test_unauthorized_token(self):