fix download stream iterators
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/1eb58838 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/1eb58838 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/1eb58838 Branch: refs/heads/trunk Commit: 1eb58838df91ff8d9156a341b1ef560f601f7277 Parents: 366310b Author: Anthony Shaw <anthonys...@apache.org> Authored: Thu Jan 12 12:12:26 2017 +1100 Committer: Anthony Shaw <anthonys...@apache.org> Committed: Thu Jan 12 12:12:26 2017 +1100 ---------------------------------------------------------------------- libcloud/storage/base.py | 20 +++----------------- libcloud/storage/drivers/s3.py | 2 +- 2 files changed, 4 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/1eb58838/libcloud/storage/base.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index 84d8904..233cc25 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -25,7 +25,6 @@ import hashlib from os.path import join as pjoin from libcloud.utils.py3 import httplib -from libcloud.utils.py3 import next from libcloud.utils.py3 import b import libcloud.utils.files @@ -557,25 +556,12 @@ class StorageDriver(BaseDriver): 'overwrite_existing=False', driver=self) - stream = response.iter_content(chunk_size) - - try: - data_read = next(stream) - except StopIteration: - # Empty response? - return False - bytes_transferred = 0 with open(file_path, 'wb') as file_handle: - while len(data_read) > 0: - file_handle.write(b(data_read)) - bytes_transferred += len(data_read) - - try: - data_read = next(stream) - except StopIteration: - data_read = '' + for chunk in response._response.iter_content(chunk_size): + file_handle.write(b(chunk)) + bytes_transferred += len(chunk) if int(obj.size) != int(bytes_transferred): # Transfer failed, support retry? http://git-wip-us.apache.org/repos/asf/libcloud/blob/1eb58838/libcloud/storage/drivers/s3.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 72ddcf6..a0b8ef0 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -419,7 +419,7 @@ class BaseS3StorageDriver(StorageDriver): return self._get_object( obj=obj, callback=read_in_chunks, response=response, - callback_kwargs={'iterator': response.iter_content, + callback_kwargs={'iterator': response.iter_content(), 'chunk_size': chunk_size}, success_status_code=httplib.OK)
