From 14c371e944eabafa213142bfe35a4932f1bf00e0 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 21 Jan 2017 12:15:22 +0100 Subject: [PATCH] Add ability to stream object directly to file For object downloads, allow the user to specify a file to write the content to, rather than returning it all in memory. Change-Id: Ic926fd63a9801049e8120d7597df9961a5f9e657 --- .../stream-to-file-91f48d6dcea399c6.yaml | 3 + shade/openstackcloud.py | 30 ++++++-- shade/tests/functional/test_object.py | 69 +++++++++++++++++++ 3 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/stream-to-file-91f48d6dcea399c6.yaml diff --git a/releasenotes/notes/stream-to-file-91f48d6dcea399c6.yaml b/releasenotes/notes/stream-to-file-91f48d6dcea399c6.yaml new file mode 100644 index 000000000..60e6d64c8 --- /dev/null +++ b/releasenotes/notes/stream-to-file-91f48d6dcea399c6.yaml @@ -0,0 +1,3 @@ +--- +features: + - get_object now supports streaming output directly to a file. diff --git a/shade/openstackcloud.py b/shade/openstackcloud.py index 361e08ad0..afc523211 100644 --- a/shade/openstackcloud.py +++ b/shade/openstackcloud.py @@ -6063,14 +6063,21 @@ class OpenStackCloud(_normalize.Normalizer): raise def get_object(self, container, obj, query_string=None, - resp_chunk_size=None): + resp_chunk_size=1024, outfile=None): """Get the headers and body of an object from swift :param string container: name of the container. :param string obj: name of the object. :param string query_string: query args for uri. (delimiter, prefix, etc.) - :param int resp_chunk_size: chunk size of data to read. + :param int resp_chunk_size: chunk size of data to read. Only used + if the results are being written to a + file. (optional, defaults to 1k) + :param outfile: Write the object to a file instead of + returning the contents. If this option is + given, body in the return tuple will be None. outfile + can either be a file path given as a string, or a + File like object. :returns: Tuple (headers, body) of the object, or None if the object is not found (404) @@ -6083,10 +6090,25 @@ class OpenStackCloud(_normalize.Normalizer): if query_string: endpoint = '{endpoint}?{query_string}'.format( endpoint=endpoint, query_string=query_string) - response = self._object_store_client.get(endpoint) + response = self._object_store_client.get( + endpoint, stream=True) response_headers = { k.lower(): v for k, v in response.headers.items()} - return (response_headers, response.text) + if outfile: + if isinstance(outfile, six.string_types): + outfile_handle = open(outfile, 'wb') + else: + outfile_handle = outfile + for chunk in response.iter_content( + resp_chunk_size, decode_unicode=False): + outfile_handle.write(chunk) + if isinstance(outfile, six.string_types): + outfile_handle.close() + else: + outfile_handle.flush() + return (response_headers, None) + else: + return (response_headers, response.text) except OpenStackCloudHTTPError as e: if e.response.status_code == 404: return None diff --git a/shade/tests/functional/test_object.py b/shade/tests/functional/test_object.py index f32802e74..5d624e765 100644 --- a/shade/tests/functional/test_object.py +++ b/shade/tests/functional/test_object.py @@ -97,3 +97,72 @@ class TestObject(base.BaseFunctionalTestCase): self.assertEqual(container_name, self.demo_cloud.list_containers()[0]['name']) self.demo_cloud.delete_container(container_name) + + def test_download_object_to_file(self): + '''Test uploading small and large files.''' + container_name = self.getUniqueString('container') + self.addDetail('container', content.text_content(container_name)) + self.addCleanup(self.demo_cloud.delete_container, container_name) + self.demo_cloud.create_container(container_name) + self.assertEqual(container_name, + self.demo_cloud.list_containers()[0]['name']) + sizes = ( + (64 * 1024, 1), # 64K, one segment + (64 * 1024, 5) # 64MB, 5 segments + ) + for size, nseg in sizes: + fake_content = '' + segment_size = int(round(size / nseg)) + with tempfile.NamedTemporaryFile() as fake_file: + fake_content = ''.join(random.SystemRandom().choice( + string.ascii_uppercase + string.digits) + for _ in range(size)).encode('latin-1') + + fake_file.write(fake_content) + fake_file.flush() + name = 'test-%d' % size + self.addCleanup( + self.demo_cloud.delete_object, container_name, name) + self.demo_cloud.create_object( + container_name, name, + fake_file.name, + segment_size=segment_size, + metadata={'foo': 'bar'}) + self.assertFalse(self.demo_cloud.is_object_stale( + container_name, name, + fake_file.name + ) + ) + self.assertEqual( + 'bar', self.demo_cloud.get_object_metadata( + container_name, name)['x-object-meta-foo'] + ) + self.demo_cloud.update_object(container=container_name, name=name, + metadata={'testk': 'testv'}) + self.assertEqual( + 'testv', self.demo_cloud.get_object_metadata( + container_name, name)['x-object-meta-testk'] + ) + try: + with tempfile.NamedTemporaryFile() as fake_file: + self.demo_cloud.get_object( + container_name, name, outfile=fake_file.name) + downloaded_content = open(fake_file.name, 'rb').read() + self.assertEqual(fake_content, downloaded_content) + except exc.OpenStackCloudException as e: + self.addDetail( + 'failed_response', + content.text_content(str(e.response.headers))) + self.addDetail( + 'failed_response', + content.text_content(e.response.text)) + raise + self.assertEqual( + name, + self.demo_cloud.list_objects(container_name)[0]['name']) + self.assertTrue( + self.demo_cloud.delete_object(container_name, name)) + self.assertEqual([], self.demo_cloud.list_objects(container_name)) + self.assertEqual(container_name, + self.demo_cloud.list_containers()[0]['name']) + self.demo_cloud.delete_container(container_name)