Port backup test_storage to Python 3

* Swift: use byte strings
* Encode Unicode (to UTF-8) before hashing
* tox.ini: run backup/test_storage.py on Python 3.4

Partially implements: blueprint trove-python3
Change-Id: Ia5201d2f15db4a66ce9c51d3b9404e3c82fe7c13

tox.ini

@@ -37,6 +37,7 @@ commands =
         trove/tests/unittests/backup/test_backup_controller.py \
         trove/tests/unittests/backup/test_backup_models.py \
         trove/tests/unittests/backup/test_backupagent.py \
+        trove/tests/unittests/backup/test_storage.py \
         trove/tests/unittests/cluster/test_cassandra_cluster.py \
         trove/tests/unittests/cluster/test_cluster.py \
         trove/tests/unittests/cluster/test_cluster_controller.py \

trove/common/strategies/storage/swift.py

@@ -18,6 +18,7 @@ import hashlib
 import json
 
 from oslo_log import log as logging
+import six
 
 from trove.common import cfg
 from trove.common.i18n import _
@@ -156,7 +157,10 @@ class SwiftStorage(base.Storage):
                 'size_bytes': stream_reader.segment_length
             })
 
-            swift_checksum.update(segment_checksum)
+            if six.PY3:
+                swift_checksum.update(segment_checksum.encode())
+            else:
+                swift_checksum.update(segment_checksum)
 
         # All segments uploaded.
         num_segments = len(segment_results)
@@ -171,7 +175,7 @@ class SwiftStorage(base.Storage):
             metadata = {}
         metadata.update(stream.metadata())
         headers = {}
-        for key, value in metadata.iteritems():
+        for key, value in metadata.items():
             headers[self._set_attr(key)] = value
 
         LOG.debug('Metadata headers: %s' % str(headers))

trove/tests/fakes/swift.py

@@ -23,6 +23,7 @@ import swiftclient.client as swift_client
 import uuid
 
 from oslo_log import log as logging
+import six
 from six.moves import http_client
 from swiftclient import client as swift
 
@@ -108,7 +109,10 @@ class FakeSwiftConnection(object):
                 object_checksum = md5(self.container_objects[object_name])
                 # The manifest file etag for a HEAD or GET is the checksum of
                 # the concatenated checksums.
-                checksum.update(object_checksum.hexdigest())
+                if six.PY3:
+                    checksum.update(object_checksum.hexdigest().encode())
+                else:
+                    checksum.update(object_checksum.hexdigest())
             # this is included to test bad swift segment etags
             if name.startswith("bad_manifest_etag_"):
                 return {'etag': '"this_is_an_intentional_bad_manifest_etag"'}
@@ -172,7 +176,10 @@ class FakeSwiftConnection(object):
             # container is where the object segments are in and prefix is the
             # common prefix for all segments.
             self.manifest_name = name
-            object_checksum.update(contents)
+            if isinstance(contents, six.text_type):
+                object_checksum.update(contents.encode('utf-8'))
+            else:
+                object_checksum.update(contents)
         elif self.COPY_OBJECT_HEADER_KEY in headers:
             # this is a copy object operation
             source_path = headers.get(self.COPY_OBJECT_HEADER_KEY)
@@ -181,7 +188,7 @@ class FakeSwiftConnection(object):
         else:
             if hasattr(contents, 'read'):
                 chunk_size = 128
-                object_content = ""
+                object_content = b""
                 chunk = contents.read(chunk_size)
                 while chunk:
                     object_content += chunk

trove/tests/unittests/backup/test_storage.py

@@ -254,7 +254,7 @@ class SwiftStorageLoad(trove_testtools.TestCase):
 class MockBackupStream(MockBackupRunner):
 
     def read(self, chunk_size):
-        return 'X' * chunk_size
+        return b'X' * chunk_size
 
 
 class StreamReaderTests(trove_testtools.TestCase):
@@ -287,11 +287,11 @@ class StreamReaderTests(trove_testtools.TestCase):
     def test_segment_almost_complete(self):
         self.stream.segment_length = 98
         results = self.stream.read(2)
-        self.assertEqual('XX', results)
+        self.assertEqual(b'XX', results)
         self.assertEqual('123_00000000', self.stream.segment,
                          "The Segment should still be the same")
         self.assertEqual(100, self.stream.segment_length)
-        checksum = hashlib.md5('XX')
+        checksum = hashlib.md5(b'XX')
         checksum = checksum.hexdigest()
         segment_checksum = self.stream.segment_checksum.hexdigest()
         self.assertEqual(checksum, segment_checksum,