openstack/swift
Code Issues Proposed changes

Add support for Hash Prefix

Browse Source 
A new configuration parameter is added to /etc/swift/swift.conf
[swift-hash]
swift_hash_path_prefix = 'random unique string'

New installations are advised to set this parameter to a random secret,
which would not be disclosed ouside the organization.
The same secret needs to be used by all swift servers of the same cluster.

Existing installations should set this parameter to an empty string
(the default)

DocImpact

Fixes: Bug #1157454

Change-Id: I63b10d0b7d6dd3f74e0f10bb41b5f240fa03578a
This commit is contained in:
David Hadas 2013-03-20 01:35:41 +02:00
parent 37e96663ec
commit a979c8007b
12 changed files with 55 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/source/development_saio.rst
View File

@ -318,7 +318,8 @@ Sample configuration files are provided with all defaults in line-by-line commen
#. Create `/etc/swift/swift.conf`::
[swift-hash]
# random unique string that can never change (DO NOT LOSE)
# random unique strings that can never change (DO NOT LOSE)
swift_hash_path_prefix = changeme
swift_hash_path_suffix = changeme
#. Create `/etc/swift/account-server/1.conf`::

3
doc/source/howto_installmultinode.rst
View File

@ -73,7 +73,8 @@ General OS configuration and partitioning for each node
cat >/etc/swift/swift.conf <<EOF
[swift-hash]
# random unique string that can never change (DO NOT LOSE)
# random unique strings that can never change (DO NOT LOSE)
swift_hash_path_prefix = `od -t x8 -N 8 -A n </dev/random`
swift_hash_path_suffix = `od -t x8 -N 8 -A n </dev/random`
EOF

17
swift/common/utils.py
View File

@ -76,12 +76,18 @@ FALLOCATE_RESERVE = 0
# will end up with would also require knowing this suffix.
hash_conf = ConfigParser()
HASH_PATH_SUFFIX = ''
HASH_PATH_PREFIX = ''
if hash_conf.read('/etc/swift/swift.conf'):
try:
HASH_PATH_SUFFIX = hash_conf.get('swift-hash',
'swift_hash_path_suffix')
except (NoSectionError, NoOptionError):
pass
try:
HASH_PATH_PREFIX = hash_conf.get('swift-hash',
'swift_hash_path_prefix')
except (NoSectionError, NoOptionError):
pass
def backward(f, blocksize=4096):
@ -134,8 +140,9 @@ def noop_libc_function(*args):
def validate_configuration():
if HASH_PATH_SUFFIX == '':
sys.exit("Error: [swift-hash]: swift_hash_path_suffix missing "
if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX:
sys.exit("Error: [swift-hash]: both swift_hash_path_suffix "
"and swift_hash_path_prefix are missing "
"from /etc/swift/swift.conf")
@ -991,9 +998,11 @@ def hash_path(account, container=None, object=None, raw_digest=False):
if object:
paths.append(object)
if raw_digest:
return md5('/' + '/'.join(paths) + HASH_PATH_SUFFIX).digest()
return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths)
+ HASH_PATH_SUFFIX).digest()
else:
return md5('/' + '/'.join(paths) + HASH_PATH_SUFFIX).hexdigest()
return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths)
+ HASH_PATH_SUFFIX).hexdigest()
@contextmanager

1
test/unit/common/middleware/test_list_endpoints.py
View File

@ -36,6 +36,7 @@ def start_response(*args):
class TestListEndpoints(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = ''
self.testdir = os.path.join(os.path.dirname(__file__), 'ring')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)

4
test/unit/common/ring/test_ring.py
View File

@ -97,6 +97,7 @@ class TestRing(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = ''
self.testdir = os.path.join(os.path.dirname(__file__), 'ring')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
@ -133,11 +134,14 @@ class TestRing(unittest.TestCase):
self.assertEquals(self.ring.serialized_path, self.testgz)
# test invalid endcap
_orig_hash_path_suffix = utils.HASH_PATH_SUFFIX
_orig_hash_path_prefix = utils.HASH_PATH_PREFIX
try:
utils.HASH_PATH_SUFFIX = ''
utils.HASH_PATH_PREFIX = ''
self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever')
finally:
utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix
utils.HASH_PATH_PREFIX = _orig_hash_path_prefix
def test_has_changed(self):
self.assertEquals(self.ring.has_changed(), False)

1
test/unit/common/test_daemon.py
View File

@ -62,6 +62,7 @@ class TestRunDaemon(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = 'startcap'
utils.drop_privileges = lambda *args: None
utils.capture_stdio = lambda *args: None

33
test/unit/common/test_utils.py
View File

@ -127,6 +127,7 @@ class TestUtils(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = 'startcap'
def test_normalize_timestamp(self):
""" Test swift.common.utils.normalize_timestamp """
@ -626,20 +627,28 @@ class TestUtils(unittest.TestCase):
self.assert_('127.0.0.1' in myips)
def test_hash_path(self):
_prefix = utils.HASH_PATH_PREFIX
utils.HASH_PATH_PREFIX = ''
# Yes, these tests are deliberately very fragile. We want to make sure
# that if someones changes the results hash_path produces, they know it
self.assertEquals(utils.hash_path('a'),
'1c84525acb02107ea475dcd3d09c2c58')
self.assertEquals(utils.hash_path('a', 'c'),
'33379ecb053aa5c9e356c68997cbb59e')
self.assertEquals(utils.hash_path('a', 'c', 'o'),
'06fbf0b514e5199dfc4e00f42eb5ea83')
self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False),
'06fbf0b514e5199dfc4e00f42eb5ea83')
self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True),
'\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN'
'\x00\xf4.\xb5\xea\x83')
self.assertRaises(ValueError, utils.hash_path, 'a', object='o')
try:
self.assertEquals(utils.hash_path('a'),
'1c84525acb02107ea475dcd3d09c2c58')
self.assertEquals(utils.hash_path('a', 'c'),
'33379ecb053aa5c9e356c68997cbb59e')
self.assertEquals(utils.hash_path('a', 'c', 'o'),
'06fbf0b514e5199dfc4e00f42eb5ea83')
self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False),
'06fbf0b514e5199dfc4e00f42eb5ea83')
self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True),
'\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN'
'\x00\xf4.\xb5\xea\x83')
self.assertRaises(ValueError, utils.hash_path, 'a', object='o')
utils.HASH_PATH_PREFIX = 'abcdef'
self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False),
'363f9b535bfb7d17a43a46a358afca0e')
finally:
utils.HASH_PATH_PREFIX = _prefix
def test_load_libc_function(self):
self.assert_(callable(

1
test/unit/container/test_sync.py
View File

@ -23,6 +23,7 @@ from swiftclient import ClientException
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = 'endcap'
class FakeRing(object):

1
test/unit/container/test_updater.py
View File

@ -35,6 +35,7 @@ class TestContainerUpdater(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = 'startcap'
self.testdir = os.path.join(mkdtemp(), 'tmp_test_container_updater')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)

1
test/unit/obj/test_replicator.py
View File

@ -135,6 +135,7 @@ class TestObjectReplicator(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = ''
# Setup a test ring (stolen from common/test_ring.py)
self.testdir = tempfile.mkdtemp()
self.devices = os.path.join(self.testdir, 'node')

7
test/unit/obj/test_server.py
View File

@ -383,6 +383,7 @@ class TestObjectController(unittest.TestCase):
def setUp(self):
""" Set up for testing swift.object_server.ObjectController """
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = 'startcap'
self.testdir = \
os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController')
mkdirs(os.path.join(self.testdir, 'sda1', 'tmp'))
@ -1808,6 +1809,8 @@ class TestObjectController(unittest.TestCase):
'x-trans-id': '-'}})
def test_async_update_saves_on_exception(self):
_prefix = utils.HASH_PATH_PREFIX
utils.HASH_PATH_PREFIX = ''
def fake_http_connect(*args):
raise Exception('test')
@ -1820,6 +1823,7 @@ class TestObjectController(unittest.TestCase):
{'x-timestamp': '1', 'x-out': 'set'}, 'sda1')
finally:
object_server.http_connect = orig_http_connect
utils.HASH_PATH_PREFIX = _prefix
self.assertEquals(
pickle.load(open(os.path.join(self.testdir, 'sda1',
'async_pending', 'a83',
@ -1828,6 +1832,8 @@ class TestObjectController(unittest.TestCase):
'container': 'c', 'obj': 'o', 'op': 'PUT'})
def test_async_update_saves_on_non_2xx(self):
_prefix = utils.HASH_PATH_PREFIX
utils.HASH_PATH_PREFIX = ''
def fake_http_connect(status):
@ -1860,6 +1866,7 @@ class TestObjectController(unittest.TestCase):
'op': 'PUT'})
finally:
object_server.http_connect = orig_http_connect
utils.HASH_PATH_PREFIX = _prefix
def test_async_update_does_not_save_on_2xx(self):

1
test/unit/obj/test_updater.py
View File

@ -37,6 +37,7 @@ class TestObjectUpdater(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = ''
self.testdir = os.path.join(os.path.dirname(__file__),
'object_updater')
rmtree(self.testdir, ignore_errors=1)