Pass HTTP_REFERER down to subrequests

Currently a HTTP_REFERER (Referer) header isn't passed down to
subrequests. This means *LO subrequests to segment containers
return a 403 on a *LO GET when accessed by requests using referer
ACLs.
Currently the only way around referer access to *LO's is to make the
segments container world readable.

This change makes sure the referer header is passed into subrequests
allowing a segments container to only need to be locked down with
the same referer as the *LO container.

This is a 1 line change to code, but also adds a unit and 2 functional
functional tests (one for DLO and one for SLO).

Change-Id: I1fa5328979302d9c8133aa739787c8dae6084f54
Closes-Bug: #1526575

swift/common/wsgi.py

@@ -1095,7 +1095,8 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None,
                  'HTTP_ORIGIN', 'HTTP_ACCESS_CONTROL_REQUEST_METHOD',
                  'SERVER_PROTOCOL', 'swift.cache', 'swift.source',
                  'swift.trans_id', 'swift.authorize_override',
-                 'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID'):
+                 'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID',
+                 'HTTP_REFERER'):
         if name in env:
             newenv[name] = env[name]
     if method:

test/functional/tests.py

@@ -2178,14 +2178,23 @@ class TestDloEnv(object):
     def setUp(cls):
         cls.conn = Connection(tf.config)
         cls.conn.authenticate()
+
+        config2 = tf.config.copy()
+        config2['username'] = tf.config['username3']
+        config2['password'] = tf.config['password3']
+        cls.conn2 = Connection(config2)
+        cls.conn2.authenticate()
+
         cls.account = Account(cls.conn, tf.config.get('account',
                                                       tf.config['username']))
         cls.account.delete_containers()
 
         cls.container = cls.account.container(Utils.create_name())
+        cls.container2 = cls.account.container(Utils.create_name())
 
-        if not cls.container.create():
+        for cont in (cls.container, cls.container2):
-            raise ResponseError(cls.conn.response)
+            if not cont.create():
+                raise ResponseError(cls.conn.response)
 
         # avoid getting a prefix that stops halfway through an encoded
         # character
@@ -2199,13 +2208,18 @@ class TestDloEnv(object):
             file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
             file_item.write(letter.upper() * 10)
 
+        for letter in ('f', 'g', 'h', 'i', 'j'):
+            file_item = cls.container2.file("%s/seg_lower%s" %
+                                            (prefix, letter))
+            file_item.write(letter * 10)
+
         man1 = cls.container.file("man1")
         man1.write('man1-contents',
                    hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
                          (cls.container.name, prefix)})
 
-        man1 = cls.container.file("man2")
+        man2 = cls.container.file("man2")
-        man1.write('man2-contents',
+        man2.write('man2-contents',
                    hdrs={"X-Object-Manifest": "%s/%s/seg_upper" %
                          (cls.container.name, prefix)})
 
@@ -2214,6 +2228,12 @@ class TestDloEnv(object):
                      hdrs={"X-Object-Manifest": "%s/%s/seg" %
                            (cls.container.name, prefix)})
 
+        mancont2 = cls.container.file("mancont2")
+        mancont2.write(
+            'mancont2-contents',
+            hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
+                                       (cls.container2.name, prefix)})
+
 
 class TestDlo(Base):
     env = TestDloEnv
@@ -2375,6 +2395,31 @@ class TestDlo(Base):
         manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
         self.assert_status(200)
 
+    def test_dlo_referer_on_segment_container(self):
+        # First the account2 (test3) should fail
+        headers = {'X-Auth-Token': self.env.conn2.storage_token,
+                   'Referer': 'http://blah.example.com'}
+        dlo_file = self.env.container.file("mancont2")
+        self.assertRaises(ResponseError, dlo_file.read,
+                          hdrs=headers)
+        self.assert_status(403)
+
+        # Now set the referer on the dlo container only
+        referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'}
+        self.env.container.update_metadata(referer_metadata)
+
+        self.assertRaises(ResponseError, dlo_file.read,
+                          hdrs=headers)
+        self.assert_status(403)
+
+        # Finally set the referer on the segment container
+        self.env.container2.update_metadata(referer_metadata)
+
+        contents = dlo_file.read(hdrs=headers)
+        self.assertEqual(
+            contents,
+            "ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj")
+
 
 class TestDloUTF8(Base2, TestDlo):
     set_up = False
@@ -2516,6 +2561,11 @@ class TestSloEnv(object):
         cls.conn2.authenticate()
         cls.account2 = cls.conn2.get_account()
         cls.account2.delete_containers()
+        config3 = tf.config.copy()
+        config3['username'] = tf.config['username3']
+        config3['password'] = tf.config['password3']
+        cls.conn3 = Connection(config3)
+        cls.conn3.authenticate()
 
         if cls.slo_enabled is None:
             cls.slo_enabled = 'slo' in cluster_info
@@ -2527,9 +2577,11 @@ class TestSloEnv(object):
         cls.account.delete_containers()
 
         cls.container = cls.account.container(Utils.create_name())
+        cls.container2 = cls.account.container(Utils.create_name())
 
-        if not cls.container.create():
+        for cont in (cls.container, cls.container2):
-            raise ResponseError(cls.conn.response)
+            if not cont.create():
+                raise ResponseError(cls.conn.response)
 
         cls.seg_info = seg_info = {}
         for letter, size in (('a', 1024 * 1024),
@@ -2552,6 +2604,14 @@ class TestSloEnv(object):
                         seg_info['seg_e']]),
             parms={'multipart-manifest': 'put'})
 
+        # Put the same manifest in the container2
+        file_item = cls.container2.file("manifest-abcde")
+        file_item.write(
+            json.dumps([seg_info['seg_a'], seg_info['seg_b'],
+                        seg_info['seg_c'], seg_info['seg_d'],
+                        seg_info['seg_e']]),
+            parms={'multipart-manifest': 'put'})
+
         file_item = cls.container.file('manifest-cd')
         cd_json = json.dumps([seg_info['seg_c'], seg_info['seg_d']])
         file_item.write(cd_json, parms={'multipart-manifest': 'put'})
@@ -3083,6 +3143,33 @@ class TestSlo(Base):
         manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
         self.assert_status(200)
 
+    def test_slo_referer_on_segment_container(self):
+        # First the account2 (test3) should fail
+        headers = {'X-Auth-Token': self.env.conn3.storage_token,
+                   'Referer': 'http://blah.example.com'}
+        slo_file = self.env.container2.file('manifest-abcde')
+        self.assertRaises(ResponseError, slo_file.read,
+                          hdrs=headers)
+        self.assert_status(403)
+
+        # Now set the referer on the slo container only
+        referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'}
+        self.env.container2.update_metadata(referer_metadata)
+
+        self.assertRaises(ResponseError, slo_file.read,
+                          hdrs=headers)
+        self.assert_status(409)
+
+        # Finally set the referer on the segment container
+        self.env.container.update_metadata(referer_metadata)
+        contents = slo_file.read(hdrs=headers)
+        self.assertEqual(4 * 1024 * 1024 + 1, len(contents))
+        self.assertEqual('a', contents[0])
+        self.assertEqual('a', contents[1024 * 1024 - 1])
+        self.assertEqual('b', contents[1024 * 1024])
+        self.assertEqual('d', contents[-2])
+        self.assertEqual('e', contents[-1])
+
 
 class TestSloUTF8(Base2, TestSlo):
     set_up = False

test/unit/common/test_wsgi.py

@@ -825,6 +825,13 @@ class TestWSGI(unittest.TestCase):
         self.assertTrue('HTTP_X_PROJECT_ID' in newenv)
         self.assertEqual(newenv['HTTP_X_PROJECT_ID'], '5678')
 
+    def test_make_env_keeps_referer(self):
+        oldenv = {'HTTP_REFERER': 'http://blah.example.com'}
+        newenv = wsgi.make_env(oldenv)
+
+        self.assertTrue('HTTP_REFERER' in newenv)
+        self.assertEqual(newenv['HTTP_REFERER'], 'http://blah.example.com')
+
 
 class TestServersPerPortStrategy(unittest.TestCase):
     def setUp(self):