Sync with OpenStack v1.11.0 Jan 10 2014

Updated tox.ini, functional tests, and proxy
unit tests.

BUG: https://bugs.launchpad.net/bugs/1268017

Change-Id: I5ff8359b8abdb8fe5ae82492c12f57c395992735
Signed-off-by: Luis Pabon <lpabon@redhat.com>
Reviewed-on: http://review.gluster.org/6682
Reviewed-by: Thiago da Silva <thiago@redhat.com>
Tested-by: Thiago da Silva <thiago@redhat.com>

gluster/swift/__init__.py

@@ -45,6 +45,6 @@ class PkgInfo(object):
 ###
 ### Change the Package version here
 ###
-_pkginfo = PkgInfo('1.10.2', '0', 'gluster_swift', False)
+_pkginfo = PkgInfo('1.11.0', '0', 'gluster_swift', False)
 __version__ = _pkginfo.pretty_version
 __canonical_version__ = _pkginfo.canonical_version

modules/swift

@@ -1 +1 @@
-Subproject commit 4bfe6748fd8e746460f5428f3dd161c82b1443a2
+Subproject commit f310006fae1af991097eee5929a1c73051eb1e00

test/functional/swift_test_client.py

@@ -65,7 +65,7 @@ class ResponseError(Exception):
 
 
 def listing_empty(method):
-    for i in xrange(0, 6):
+    for i in xrange(6):
         if len(method()) == 0:
             return True
 
@@ -208,7 +208,11 @@ class Connection(object):
 
     def make_request(self, method, path=[], data='', hdrs={}, parms={},
                      cfg={}):
-        path = self.make_path(path, cfg=cfg)
+        if not cfg.get('verbatim_path'):
+            # Set verbatim_path=True to make a request to exactly the given
+            # path, not storage path + given path. Useful for
+            # non-account/container/object requests.
+            path = self.make_path(path, cfg=cfg)
         headers = self.make_headers(hdrs, cfg=cfg)
         if isinstance(parms, dict) and parms:
             quote = urllib.quote
@@ -719,7 +723,8 @@ class File(Base):
         else:
             raise RuntimeError
 
-    def write(self, data='', hdrs={}, parms={}, callback=None, cfg={}):
+    def write(self, data='', hdrs={}, parms={}, callback=None, cfg={},
+              return_resp=False):
         block_size = 2 ** 20
 
         if isinstance(data, file):
@@ -763,6 +768,9 @@ class File(Base):
             pass
         self.md5 = self.compute_md5sum(data)
 
+        if return_resp:
+            return self.conn.response
+
         return True
 
     def write_random(self, size=None, hdrs={}, parms={}, cfg={}):
@@ -772,3 +780,12 @@ class File(Base):
                                 self.conn.make_path(self.path))
         self.md5 = self.compute_md5sum(StringIO.StringIO(data))
         return data
+
+    def write_random_return_resp(self, size=None, hdrs={}, parms={}, cfg={}):
+        data = self.random_data(size)
+        resp = self.write(data, hdrs=hdrs, parms=parms, cfg=cfg,
+                          return_resp=True)
+        if not resp:
+            raise ResponseError(self.conn.response)
+        self.md5 = self.compute_md5sum(StringIO.StringIO(data))
+        return resp

test/functional/tests.py

@@ -1704,8 +1704,14 @@ class TestFileComparisonEnv:
             file_item.write_random(cls.file_size)
             cls.files.append(file_item)
 
-        cls.time_old = time.asctime(time.localtime(time.time() - 86400))
+        cls.time_old_f1 = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
-        cls.time_new = time.asctime(time.localtime(time.time() + 86400))
+                                        time.gmtime(time.time() - 86400))
+        cls.time_old_f2 = time.strftime("%A, %d-%b-%y %H:%M:%S GMT",
+                                        time.gmtime(time.time() - 86400))
+        cls.time_old_f3 = time.strftime("%a %b %d %H:%M:%S %Y",
+                                        time.gmtime(time.time() - 86400))
+        cls.time_new = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
+                                     time.gmtime(time.time() + 86400))
 
 
 class TestFileComparison(Base):
@@ -1732,7 +1738,7 @@ class TestFileComparison(Base):
 
     def testIfModifiedSince(self):
         for file_item in self.env.files:
-            hdrs = {'If-Modified-Since': self.env.time_old}
+            hdrs = {'If-Modified-Since': self.env.time_old_f1}
             self.assert_(file_item.read(hdrs=hdrs))
 
             hdrs = {'If-Modified-Since': self.env.time_new}
@@ -1744,7 +1750,7 @@ class TestFileComparison(Base):
             hdrs = {'If-Unmodified-Since': self.env.time_new}
             self.assert_(file_item.read(hdrs=hdrs))
 
-            hdrs = {'If-Unmodified-Since': self.env.time_old}
+            hdrs = {'If-Unmodified-Since': self.env.time_old_f2}
             self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
             self.assert_status(412)
 
@@ -1760,10 +1766,32 @@ class TestFileComparison(Base):
             self.assert_status(412)
 
             hdrs = {'If-Match': file_item.md5,
-                    'If-Unmodified-Since': self.env.time_old}
+                    'If-Unmodified-Since': self.env.time_old_f3}
             self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
             self.assert_status(412)
 
+    def testLastModified(self):
+        file_name = Utils.create_name()
+        content_type = Utils.create_name()
+
+        file = self.env.container.file(file_name)
+        file.content_type = content_type
+        resp = file.write_random_return_resp(self.env.file_size)
+        put_last_modified = resp.getheader('last-modified')
+
+        file = self.env.container.file(file_name)
+        info = file.info()
+        self.assert_('last_modified' in info)
+        last_modified = info['last_modified']
+        self.assertEqual(put_last_modified, info['last_modified'])
+
+        hdrs = {'If-Modified-Since': last_modified}
+        self.assertRaises(ResponseError, file.read, hdrs=hdrs)
+        self.assert_status(304)
+
+        hdrs = {'If-Unmodified-Since': last_modified}
+        self.assert_(file.read(hdrs=hdrs))
+
 
 class TestFileComparisonUTF8(Base2, TestFileComparison):
     set_up = False
@@ -1776,6 +1804,24 @@ class TestSloEnv(object):
     def setUp(cls):
         cls.conn = Connection(config)
         cls.conn.authenticate()
+
+        if cls.slo_enabled is None:
+            status = cls.conn.make_request('GET', '/info',
+                                           cfg={'verbatim_path': True})
+            if not (200 <= status <= 299):
+                # Can't tell if SLO is enabled or not since we're running
+                # against an old cluster, so let's skip the tests instead of
+                # possibly having spurious failures.
+                cls.slo_enabled = False
+            else:
+                # Don't bother looking for ValueError here. If something is
+                # responding to a GET /info request with invalid JSON, then
+                # the cluster is broken and a test failure will let us know.
+                cluster_info = json.loads(cls.conn.response.read())
+                cls.slo_enabled = 'slo' in cluster_info
+            if not cls.slo_enabled:
+                return
+
         cls.account = Account(cls.conn, config.get('account',
                                                    config['username']))
         cls.account.delete_containers()
@@ -1785,28 +1831,6 @@ class TestSloEnv(object):
         if not cls.container.create():
             raise ResponseError(cls.conn.response)
 
-        # TODO(seriously, anyone can do this): make this use the /info API once
-        # it lands, both for detection of SLO and for minimum segment size
-        if cls.slo_enabled is None:
-            test_file = cls.container.file(".test-slo")
-            try:
-                # If SLO is enabled, this'll raise an error since
-                # X-Static-Large-Object is a reserved header.
-                #
-                # If SLO is not enabled, then this will get the usual 2xx
-                # response.
-                test_file.write(
-                    "some contents",
-                    hdrs={'X-Static-Large-Object': 'true'})
-            except ResponseError as err:
-                if err.status == 400:
-                    cls.slo_enabled = True
-                else:
-                    raise
-            else:
-                cls.slo_enabled = False
-                return
-
         seg_info = {}
         for letter, size in (('a', 1024 * 1024),
                              ('b', 1024 * 1024),
@@ -1865,6 +1889,7 @@ class TestSlo(Base):
     set_up = False
 
     def setUp(self):
+        raise SkipTest("SLO not enabled yet in gluster-swift")
         super(TestSlo, self).setUp()
         if self.env.slo_enabled is False:
             raise SkipTest("SLO not enabled")

test/unit/__init__.py

@@ -17,9 +17,9 @@
 
 import os
 import copy
-import errno
 import logging
-from sys import exc_info
+import errno
+import sys
 from contextlib import contextmanager
 from collections import defaultdict
 from tempfile import NamedTemporaryFile
@@ -28,7 +28,7 @@ from eventlet.green import socket
 from tempfile import mkdtemp
 from shutil import rmtree
 from test import get_config
-from swift.common.utils import config_true_value
+from swift.common.utils import config_true_value, LogAdapter
 from hashlib import md5
 from eventlet import sleep, Timeout
 import logging.handlers

test/unit/proxy/controllers/test_account.py

@@ -19,7 +19,9 @@ import unittest
 from swift.common.swob import Request
 from swift.proxy import server as proxy_server
 from swift.proxy.controllers.base import headers_to_account_info
+from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 class TestAccountController(unittest.TestCase):
@@ -32,8 +34,8 @@ class TestAccountController(unittest.TestCase):
     def test_account_info_in_response_env(self):
         controller = proxy_server.AccountController(self.app, 'AUTH_bob')
         with mock.patch('swift.proxy.controllers.base.http_connect',
-                        fake_http_connect(200, 200, body='')):
+                        fake_http_connect(200, body='')):
-            req = Request.blank('/AUTH_bob', {'PATH_INFO': '/AUTH_bob'})
+            req = Request.blank('/v1/AUTH_bob', {'PATH_INFO': '/v1/AUTH_bob'})
             resp = controller.HEAD(req)
         self.assertEqual(2, resp.status_int // 100)
         self.assertTrue('swift.account/AUTH_bob' in resp.environ)
@@ -46,22 +48,110 @@ class TestAccountController(unittest.TestCase):
             'x-account-meta-temp-url-key-2': 'value'}
         controller = proxy_server.AccountController(self.app, 'a')
 
-        req = Request.blank('/a')
+        req = Request.blank('/v1/a')
         with mock.patch('swift.proxy.controllers.base.http_connect',
-                        fake_http_connect(200, 200, headers=owner_headers)):
+                        fake_http_connect(200, headers=owner_headers)):
             resp = controller.HEAD(req)
         self.assertEquals(2, resp.status_int // 100)
         for key in owner_headers:
             self.assertTrue(key not in resp.headers)
 
-        req = Request.blank('/a', environ={'swift_owner': True})
+        req = Request.blank('/v1/a', environ={'swift_owner': True})
         with mock.patch('swift.proxy.controllers.base.http_connect',
-                        fake_http_connect(200, 200, headers=owner_headers)):
+                        fake_http_connect(200, headers=owner_headers)):
             resp = controller.HEAD(req)
         self.assertEquals(2, resp.status_int // 100)
         for key in owner_headers:
             self.assertTrue(key in resp.headers)
 
+    def test_get_deleted_account(self):
+        resp_headers = {
+            'x-account-status': 'deleted',
+        }
+        controller = proxy_server.AccountController(self.app, 'a')
+
+        req = Request.blank('/v1/a')
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(404, headers=resp_headers)):
+            resp = controller.HEAD(req)
+        self.assertEquals(410, resp.status_int)
+
+    def test_long_acct_names(self):
+        long_acct_name = '%sLongAccountName' % ('Very' * (MAX_ANAME_LEN // 4))
+        controller = proxy_server.AccountController(self.app, long_acct_name)
+
+        req = Request.blank('/v1/%s' % long_acct_name)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200)):
+            resp = controller.HEAD(req)
+        self.assertEquals(400, resp.status_int)
+
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200)):
+            resp = controller.GET(req)
+        self.assertEquals(400, resp.status_int)
+
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200)):
+            resp = controller.POST(req)
+        self.assertEquals(400, resp.status_int)
+
+    def _make_callback_func(self, context):
+        def callback(ipaddr, port, device, partition, method, path,
+                     headers=None, query_string=None, ssl=False):
+            context['method'] = method
+            context['path'] = path
+            context['headers'] = headers or {}
+        return callback
+
+    def test_sys_meta_headers_PUT(self):
+        # check that headers in sys meta namespace make it through
+        # the proxy controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('account')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Account-Meta-Test'
+        # allow PUTs to account...
+        self.app.allow_account_management = True
+        controller = proxy_server.AccountController(self.app, 'a')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.PUT(req)
+        self.assertEqual(context['method'], 'PUT')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
+    def test_sys_meta_headers_POST(self):
+        # check that headers in sys meta namespace make it through
+        # the proxy controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('account')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Account-Meta-Test'
+        controller = proxy_server.AccountController(self.app, 'a')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.POST(req)
+        self.assertEqual(context['method'], 'POST')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
 
 if __name__ == '__main__':
     unittest.main()

test/unit/proxy/controllers/test_base.py

@@ -18,11 +18,13 @@ from mock import patch
 from swift.proxy.controllers.base import headers_to_container_info, \
     headers_to_account_info, headers_to_object_info, get_container_info, \
     get_container_memcache_key, get_account_info, get_account_memcache_key, \
-    get_object_env_key, _get_cache_key, get_info, get_object_info, Controller
+    get_object_env_key, _get_cache_key, get_info, get_object_info, \
-from swift.common.swob import Request
+    Controller, GetOrHeadHandler
+from swift.common.swob import Request, HTTPException, HeaderKeyDict
 from swift.common.utils import split_path
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
 from swift.proxy import server as proxy_server
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 FakeResponse_status_int = 201
@@ -88,7 +90,7 @@ class TestFuncs(unittest.TestCase):
 
     def test_GETorHEAD_base(self):
         base = Controller(self.app)
-        req = Request.blank('/a/c/o/with/slashes')
+        req = Request.blank('/v1/a/c/o/with/slashes')
         with patch('swift.proxy.controllers.base.'
                    'http_connect', fake_http_connect(200)):
             resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part',
@@ -96,14 +98,14 @@ class TestFuncs(unittest.TestCase):
         self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ)
         self.assertEqual(
             resp.environ['swift.object/a/c/o/with/slashes']['status'], 200)
-        req = Request.blank('/a/c/o')
+        req = Request.blank('/v1/a/c/o')
         with patch('swift.proxy.controllers.base.'
                    'http_connect', fake_http_connect(200)):
             resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part',
                                        '/a/c/o')
         self.assertTrue('swift.object/a/c/o' in resp.environ)
         self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200)
-        req = Request.blank('/a/c')
+        req = Request.blank('/v1/a/c')
         with patch('swift.proxy.controllers.base.'
                    'http_connect', fake_http_connect(200)):
             resp = base.GETorHEAD_base(req, 'container', FakeRing(), 'part',
@@ -111,7 +113,7 @@ class TestFuncs(unittest.TestCase):
         self.assertTrue('swift.container/a/c' in resp.environ)
         self.assertEqual(resp.environ['swift.container/a/c']['status'], 200)
 
-        req = Request.blank('/a')
+        req = Request.blank('/v1/a')
         with patch('swift.proxy.controllers.base.'
                    'http_connect', fake_http_connect(200)):
             resp = base.GETorHEAD_base(req, 'account', FakeRing(), 'part',
@@ -250,7 +252,9 @@ class TestFuncs(unittest.TestCase):
     def test_get_container_info_cache(self):
         cached = {'status': 404,
                   'bytes': 3333,
-                  'object_count': 10}
+                  'object_count': 10,
+                  # simplejson sometimes hands back strings, sometimes unicodes
+                  'versions': u"\u1F4A9"}
         req = Request.blank("/v1/account/cont",
                             environ={'swift.cache': FakeCache(cached)})
         with patch('swift.proxy.controllers.base.'
@@ -259,6 +263,7 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(resp['bytes'], 3333)
         self.assertEquals(resp['object_count'], 10)
         self.assertEquals(resp['status'], 404)
+        self.assertEquals(resp['versions'], "\xe1\xbd\x8a\x39")
 
     def test_get_container_info_env(self):
         cache_key = get_container_memcache_key("account", "cont")
@@ -361,6 +366,15 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(resp['meta']['whatevs'], 14)
         self.assertEquals(resp['meta']['somethingelse'], 0)
 
+    def test_headers_to_container_info_sys_meta(self):
+        prefix = get_sys_meta_prefix('container')
+        headers = {'%sWhatevs' % prefix: 14,
+                   '%ssomethingelse' % prefix: 0}
+        resp = headers_to_container_info(headers.items(), 200)
+        self.assertEquals(len(resp['sysmeta']), 2)
+        self.assertEquals(resp['sysmeta']['whatevs'], 14)
+        self.assertEquals(resp['sysmeta']['somethingelse'], 0)
+
     def test_headers_to_container_info_values(self):
         headers = {
             'x-container-read': 'readvalue',
@@ -392,6 +406,15 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(resp['meta']['whatevs'], 14)
         self.assertEquals(resp['meta']['somethingelse'], 0)
 
+    def test_headers_to_account_info_sys_meta(self):
+        prefix = get_sys_meta_prefix('account')
+        headers = {'%sWhatevs' % prefix: 14,
+                   '%ssomethingelse' % prefix: 0}
+        resp = headers_to_account_info(headers.items(), 200)
+        self.assertEquals(len(resp['sysmeta']), 2)
+        self.assertEquals(resp['sysmeta']['whatevs'], 14)
+        self.assertEquals(resp['sysmeta']['somethingelse'], 0)
+
     def test_headers_to_account_info_values(self):
         headers = {
             'x-account-object-count': '10',
@@ -433,3 +456,79 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(
             resp,
             headers_to_object_info(headers.items(), 200))
+
+    def test_have_quorum(self):
+        base = Controller(self.app)
+        # just throw a bunch of test cases at it
+        self.assertEqual(base.have_quorum([201, 404], 3), False)
+        self.assertEqual(base.have_quorum([201, 201], 4), False)
+        self.assertEqual(base.have_quorum([201, 201, 404, 404], 4), False)
+        self.assertEqual(base.have_quorum([201, 503, 503, 201], 4), False)
+        self.assertEqual(base.have_quorum([201, 201], 3), True)
+        self.assertEqual(base.have_quorum([404, 404], 3), True)
+        self.assertEqual(base.have_quorum([201, 201], 2), True)
+        self.assertEqual(base.have_quorum([404, 404], 2), True)
+        self.assertEqual(base.have_quorum([201, 404, 201, 201], 4), True)
+
+    def test_range_fast_forward(self):
+        req = Request.blank('/')
+        handler = GetOrHeadHandler(None, req, None, None, None, None, {})
+        handler.fast_forward(50)
+        self.assertEquals(handler.backend_headers['Range'], 'bytes=50-')
+
+        handler = GetOrHeadHandler(None, req, None, None, None, None,
+                                   {'Range': 'bytes=23-50'})
+        handler.fast_forward(20)
+        self.assertEquals(handler.backend_headers['Range'], 'bytes=43-50')
+        self.assertRaises(HTTPException,
+                          handler.fast_forward, 80)
+
+        handler = GetOrHeadHandler(None, req, None, None, None, None,
+                                   {'Range': 'bytes=23-'})
+        handler.fast_forward(20)
+        self.assertEquals(handler.backend_headers['Range'], 'bytes=43-')
+
+        handler = GetOrHeadHandler(None, req, None, None, None, None,
+                                   {'Range': 'bytes=-100'})
+        handler.fast_forward(20)
+        self.assertEquals(handler.backend_headers['Range'], 'bytes=-80')
+
+    def test_transfer_headers_with_sysmeta(self):
+        base = Controller(self.app)
+        good_hdrs = {'x-base-sysmeta-foo': 'ok',
+                     'X-Base-sysmeta-Bar': 'also ok'}
+        bad_hdrs = {'x-base-sysmeta-': 'too short'}
+        hdrs = dict(good_hdrs)
+        hdrs.update(bad_hdrs)
+        dst_hdrs = HeaderKeyDict()
+        base.transfer_headers(hdrs, dst_hdrs)
+        self.assertEqual(HeaderKeyDict(good_hdrs), dst_hdrs)
+
+    def test_generate_request_headers(self):
+        base = Controller(self.app)
+        src_headers = {'x-remove-base-meta-owner': 'x',
+                       'x-base-meta-size': '151M',
+                       'new-owner': 'Kun'}
+        req = Request.blank('/v1/a/c/o', headers=src_headers)
+        dst_headers = base.generate_request_headers(req, transfer=True)
+        expected_headers = {'x-base-meta-owner': '',
+                            'x-base-meta-size': '151M'}
+        for k, v in expected_headers.iteritems():
+            self.assertTrue(k in dst_headers)
+            self.assertEqual(v, dst_headers[k])
+        self.assertFalse('new-owner' in dst_headers)
+
+    def test_generate_request_headers_with_sysmeta(self):
+        base = Controller(self.app)
+        good_hdrs = {'x-base-sysmeta-foo': 'ok',
+                     'X-Base-sysmeta-Bar': 'also ok'}
+        bad_hdrs = {'x-base-sysmeta-': 'too short'}
+        hdrs = dict(good_hdrs)
+        hdrs.update(bad_hdrs)
+        req = Request.blank('/v1/a/c/o', headers=hdrs)
+        dst_headers = base.generate_request_headers(req, transfer=True)
+        for k, v in good_hdrs.iteritems():
+            self.assertTrue(k.lower() in dst_headers)
+            self.assertEqual(v, dst_headers[k.lower()])
+        for k, v in bad_hdrs.iteritems():
+            self.assertFalse(k.lower() in dst_headers)

test/unit/proxy/controllers/test_container.py

@@ -20,6 +20,7 @@ from swift.common.swob import Request
 from swift.proxy import server as proxy_server
 from swift.proxy.controllers.base import headers_to_container_info
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 class TestContainerController(unittest.TestCase):
@@ -33,7 +34,7 @@ class TestContainerController(unittest.TestCase):
         controller = proxy_server.ContainerController(self.app, 'a', 'c')
         with mock.patch('swift.proxy.controllers.base.http_connect',
                         fake_http_connect(200, 200, body='')):
-            req = Request.blank('/a/c', {'PATH_INFO': '/a/c'})
+            req = Request.blank('/v1/a/c', {'PATH_INFO': '/v1/a/c'})
             resp = controller.HEAD(req)
         self.assertEqual(2, resp.status_int // 100)
         self.assertTrue("swift.container/a/c" in resp.environ)
@@ -46,7 +47,7 @@ class TestContainerController(unittest.TestCase):
             'x-container-sync-key': 'value', 'x-container-sync-to': 'value'}
         controller = proxy_server.ContainerController(self.app, 'a', 'c')
 
-        req = Request.blank('/a/c')
+        req = Request.blank('/v1/a/c')
         with mock.patch('swift.proxy.controllers.base.http_connect',
                         fake_http_connect(200, 200, headers=owner_headers)):
             resp = controller.HEAD(req)
@@ -54,7 +55,7 @@ class TestContainerController(unittest.TestCase):
         for key in owner_headers:
             self.assertTrue(key not in resp.headers)
 
-        req = Request.blank('/a/c', environ={'swift_owner': True})
+        req = Request.blank('/v1/a/c', environ={'swift_owner': True})
         with mock.patch('swift.proxy.controllers.base.http_connect',
                         fake_http_connect(200, 200, headers=owner_headers)):
             resp = controller.HEAD(req)
@@ -62,6 +63,61 @@ class TestContainerController(unittest.TestCase):
         for key in owner_headers:
             self.assertTrue(key in resp.headers)
 
+    def _make_callback_func(self, context):
+        def callback(ipaddr, port, device, partition, method, path,
+                     headers=None, query_string=None, ssl=False):
+            context['method'] = method
+            context['path'] = path
+            context['headers'] = headers or {}
+        return callback
+
+    def test_sys_meta_headers_PUT(self):
+        # check that headers in sys meta namespace make it through
+        # the container controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('container')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Container-Meta-Test'
+        controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a/c', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.PUT(req)
+        self.assertEqual(context['method'], 'PUT')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
+    def test_sys_meta_headers_POST(self):
+        # check that headers in sys meta namespace make it through
+        # the container controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('container')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Container-Meta-Test'
+        controller = proxy_server.ContainerController(self.app, 'a', 'c')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a/c', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.POST(req)
+        self.assertEqual(context['method'], 'POST')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
 
 if __name__ == '__main__':
     unittest.main()

test/unit/proxy/controllers/test_info.py

@@ -0,0 +1,293 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import time
+from mock import Mock
+
+from swift.proxy.controllers import InfoController
+from swift.proxy.server import Application as ProxyApp
+from swift.common import utils
+from swift.common.utils import json
+from swift.common.swob import Request, HTTPException
+
+
+class TestInfoController(unittest.TestCase):
+
+    def setUp(self):
+        utils._swift_info = {}
+        utils._swift_admin_info = {}
+
+    def get_controller(self, expose_info=None, disallowed_sections=None,
+                       admin_key=None):
+        disallowed_sections = disallowed_sections or []
+
+        app = Mock(spec=ProxyApp)
+        return InfoController(app, None, expose_info,
+                              disallowed_sections, admin_key)
+
+    def start_response(self, status, headers):
+        self.got_statuses.append(status)
+        for h in headers:
+            self.got_headers.append({h[0]: h[1]})
+
+    def test_disabled_info(self):
+        controller = self.get_controller(expose_info=False)
+
+        req = Request.blank(
+            '/info', environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('403 Forbidden', str(resp))
+
+    def test_get_info(self):
+        controller = self.get_controller(expose_info=True)
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        req = Request.blank(
+            '/info', environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+        info = json.loads(resp.body)
+        self.assertTrue('admin' not in info)
+        self.assertTrue('foo' in info)
+        self.assertTrue('bar' in info['foo'])
+        self.assertEqual(info['foo']['bar'], 'baz')
+
+    def test_options_info(self):
+        controller = self.get_controller(expose_info=True)
+
+        req = Request.blank(
+            '/info', environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.OPTIONS(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+        self.assertTrue('Allow' in resp.headers)
+
+    def test_get_info_cors(self):
+        controller = self.get_controller(expose_info=True)
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        req = Request.blank(
+            '/info', environ={'REQUEST_METHOD': 'GET'},
+            headers={'Origin': 'http://example.com'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+        info = json.loads(resp.body)
+        self.assertTrue('admin' not in info)
+        self.assertTrue('foo' in info)
+        self.assertTrue('bar' in info['foo'])
+        self.assertEqual(info['foo']['bar'], 'baz')
+        self.assertTrue('Access-Control-Allow-Origin' in resp.headers)
+        self.assertTrue('Access-Control-Expose-Headers' in resp.headers)
+
+    def test_head_info(self):
+        controller = self.get_controller(expose_info=True)
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        req = Request.blank(
+            '/info', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = controller.HEAD(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+
+    def test_disallow_info(self):
+        controller = self.get_controller(expose_info=True,
+                                         disallowed_sections=['foo2'])
+        utils._swift_info = {'foo': {'bar': 'baz'},
+                             'foo2': {'bar2': 'baz2'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        req = Request.blank(
+            '/info', environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+        info = json.loads(resp.body)
+        self.assertTrue('foo' in info)
+        self.assertTrue('bar' in info['foo'])
+        self.assertEqual(info['foo']['bar'], 'baz')
+        self.assertTrue('foo2' not in info)
+
+    def test_disabled_admin_info(self):
+        controller = self.get_controller(expose_info=True, admin_key='')
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        expires = int(time.time() + 86400)
+        sig = utils.get_hmac('GET', '/info', expires, '')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('403 Forbidden', str(resp))
+
+    def test_get_admin_info(self):
+        controller = self.get_controller(expose_info=True,
+                                         admin_key='secret-admin-key')
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        expires = int(time.time() + 86400)
+        sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+        info = json.loads(resp.body)
+        self.assertTrue('admin' in info)
+        self.assertTrue('qux' in info['admin'])
+        self.assertTrue('quux' in info['admin']['qux'])
+        self.assertEqual(info['admin']['qux']['quux'], 'corge')
+
+    def test_head_admin_info(self):
+        controller = self.get_controller(expose_info=True,
+                                         admin_key='secret-admin-key')
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        expires = int(time.time() + 86400)
+        sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'HEAD'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+
+        expires = int(time.time() + 86400)
+        sig = utils.get_hmac('HEAD', '/info', expires, 'secret-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'HEAD'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+
+    def test_get_admin_info_invalid_method(self):
+        controller = self.get_controller(expose_info=True,
+                                         admin_key='secret-admin-key')
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        expires = int(time.time() + 86400)
+        sig = utils.get_hmac('HEAD', '/info', expires, 'secret-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('401 Unauthorized', str(resp))
+
+    def test_get_admin_info_invalid_expires(self):
+        controller = self.get_controller(expose_info=True,
+                                         admin_key='secret-admin-key')
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        expires = 1
+        sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('401 Unauthorized', str(resp))
+
+        expires = 'abc'
+        sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('401 Unauthorized', str(resp))
+
+    def test_get_admin_info_invalid_path(self):
+        controller = self.get_controller(expose_info=True,
+                                         admin_key='secret-admin-key')
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        expires = int(time.time() + 86400)
+        sig = utils.get_hmac('GET', '/foo', expires, 'secret-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('401 Unauthorized', str(resp))
+
+    def test_get_admin_info_invalid_key(self):
+        controller = self.get_controller(expose_info=True,
+                                         admin_key='secret-admin-key')
+        utils._swift_info = {'foo': {'bar': 'baz'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        expires = int(time.time() + 86400)
+        sig = utils.get_hmac('GET', '/foo', expires, 'invalid-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('401 Unauthorized', str(resp))
+
+    def test_admin_disallow_info(self):
+        controller = self.get_controller(expose_info=True,
+                                         disallowed_sections=['foo2'],
+                                         admin_key='secret-admin-key')
+        utils._swift_info = {'foo': {'bar': 'baz'},
+                             'foo2': {'bar2': 'baz2'}}
+        utils._swift_admin_info = {'qux': {'quux': 'corge'}}
+
+        expires = int(time.time() + 86400)
+        sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key')
+        path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format(
+            sig=sig, expires=expires)
+        req = Request.blank(
+            path, environ={'REQUEST_METHOD': 'GET'})
+        resp = controller.GET(req)
+        self.assertTrue(isinstance(resp, HTTPException))
+        self.assertEqual('200 OK', str(resp))
+        info = json.loads(resp.body)
+        self.assertTrue('foo2' not in info)
+        self.assertTrue('admin' in info)
+        self.assertTrue('disallowed_sections' in info['admin'])
+        self.assertTrue('foo2' in info['admin']['disallowed_sections'])
+        self.assertTrue('qux' in info['admin'])
+        self.assertTrue('quux' in info['admin']['qux'])
+        self.assertEqual(info['admin']['qux']['quux'], 'corge')
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/requirements.txt

@@ -0,0 +1,8 @@
+dnspython>=1.9.4
+eventlet>=0.9.15
+greenlet>=0.3.1
+netifaces>=0.5
+pastedeploy>=1.3.3
+simplejson>=2.0.9
+xattr>=0.4
+python-swiftclient

tools/test-requires

@@ -8,7 +8,7 @@ nose
 nosexcover
 openstack.nose_plugin
 nosehtmloutput
-sphinx>=1.1.2
+sphinx>=1.1.2,<1.2
 mock>=0.8.0
 python-keystoneclient
 prettytable

tox.ini

@@ -1,7 +1,11 @@
 [tox]
 envlist = py26,py27,pep8,functest,ksfunctest
+minversion = 1.6
+skipsdist = True
 
 [testenv]
+usedevelop = True
+install_command = pip install --allow-external netifaces --allow-insecure netifaces -U {opts} {packages}
 whitelist_externals=bash
 setenv = VIRTUAL_ENV={envdir}
          NOSE_WITH_OPENSTACK=1
@@ -11,9 +15,10 @@ setenv = VIRTUAL_ENV={envdir}
          NOSE_OPENSTACK_SHOW_ELAPSED=1
          NOSE_OPENSTACK_STDOUT=1
 deps =
-  https://launchpad.net/gluster-swift/icehouse/1.10.2/+download/swift-1.10.0.172.g9fe7748.tar.gz
+  https://launchpad.net/gluster-swift/icehouse/1.11.0/+download/swift-1.11.0.71.gf310006.tar.gz
   --download-cache={homedir}/.pipcache
   -r{toxinidir}/tools/test-requires
+  -r{toxinidir}/tools/requirements.txt
 changedir = {toxinidir}/test/unit
 commands = nosetests -v --exe --with-xunit --with-coverage --cover-package gluster --cover-erase --cover-xml --cover-html --cover-branches --with-html-output {posargs}