switch back to raw for _upload_object, encode strings for StringIO hashing, fix various tests
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/cd399d91 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/cd399d91 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/cd399d91 Branch: refs/heads/trunk Commit: cd399d9101a94956cf3e04ee5d5c488510b9103b Parents: 6e0040d Author: Anthony Shaw <anthonys...@apache.org> Authored: Sat Jan 7 12:24:18 2017 +1100 Committer: Anthony Shaw <anthonys...@apache.org> Committed: Sat Jan 7 12:24:18 2017 +1100 ---------------------------------------------------------------------- libcloud/common/base.py | 14 +++ libcloud/storage/base.py | 16 ++- libcloud/test/__init__.py | 188 ++++++++++++++++++------------- libcloud/test/storage/test_atmos.py | 3 +- libcloud/test/storage/test_base.py | 101 +++++------------ 5 files changed, 165 insertions(+), 157 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/common/base.py ---------------------------------------------------------------------- diff --git a/libcloud/common/base.py b/libcloud/common/base.py index 5266163..180cc11 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -284,6 +284,20 @@ class RawResponse(Response): self._reason = None self.connection = connection + def success(self): + """ + Determine if our request was successful. + + The meaning of this can be arbitrary; did we receive OK status? Did + the node get created? Were we authenticated? + + :rtype: ``bool`` + :return: ``True`` or ``False`` + """ + # pylint: disable=E1101 + return self.status in [requests.codes.ok, requests.codes.created, + httplib.OK, httplib.CREATED, httplib.ACCEPTED] + @property def response(self): if not self._response: http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/storage/base.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index c290ffd..29b84fb 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -629,7 +629,7 @@ class StorageDriver(BaseDriver): response = self.connection.request( request_path, method=request_method, data=stream, - headers=headers, raw=False) + headers=headers, raw=True) stream_hash, stream_length = self._hash_buffered_stream( stream, self._get_hash_function()) @@ -638,7 +638,7 @@ class StorageDriver(BaseDriver): response = self.connection.request( request_path, method=request_method, data=file_stream, - headers=headers, raw=False) + headers=headers, raw=True) with open(file_path, 'rb') as file_stream: stream_hash, stream_length = self._hash_buffered_stream( file_stream, @@ -656,12 +656,20 @@ class StorageDriver(BaseDriver): 'data_hash': stream_hash} def _hash_buffered_stream(self, stream, hasher, blocksize=65536): + total_len = 0 + if not hasattr(stream, '__exit__'): + for s in iter(stream): + hasher.update(s) + total_len = total_len + len(s) + return (hasher.hexdigest(), total_len) with stream: buf = stream.read(blocksize) - total_len = 0 while len(buf) > 0: total_len = total_len + len(buf) - hasher.update(buf) + if isinstance(buf, str): + hasher.update(buf.encode()) + else: + hasher.update(buf) buf = stream.read(blocksize) return (hasher.hexdigest(), total_len) http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/test/__init__.py ---------------------------------------------------------------------- diff --git a/libcloud/test/__init__.py b/libcloud/test/__init__.py index ba911a0..3909d2e 100644 --- a/libcloud/test/__init__.py +++ b/libcloud/test/__init__.py @@ -150,6 +150,93 @@ class BaseMockHttpObject(object): return meth_name +class MockRawResponse(BaseMockHttpObject): + """ + Mock RawResponse object suitable for testing. + """ + + type = None + responseCls = MockResponse + + def __init__(self, connection, path=None): + super(MockRawResponse, self).__init__() + self._data = [] + self._current_item = 0 + self._response = None + self._status = None + self._headers = None + self._reason = None + self._path = path + self.connection = connection + + def next(self): + if self._current_item == len(self._data): + raise StopIteration + + value = self._data[self._current_item] + self._current_item += 1 + return value + + def __next__(self): + return self.next() + + def _generate_random_data(self, size): + data = '' + current_size = 0 + while current_size < size: + value = str(random.randint(0, 9)) + value_size = len(value) + data += value + current_size += value_size + + return data + + @property + def response(self): + return self._get_response_if_not_available() + + @property + def status(self): + self._get_response_if_not_available() + return self._status + + @property + def status_code(self): + self._get_response_if_not_available() + return self._status + + def success(self): + self._get_response_if_not_available() + return self._status in [httplib.OK, httplib.CREATED, httplib.ACCEPTED] + + @property + def headers(self): + self._get_response_if_not_available() + return self._headers + + @property + def reason(self): + self._get_response_if_not_available() + return self._reason + + def _get_response_if_not_available(self): + if not self._response: + meth_name = self._get_method_name(type=self.type, + use_param=False, qs=None, + path=self._path if self._path else self.connection.action) + meth = getattr(self, meth_name.replace('%', '_')) + result = meth(self.connection.method, None, None, None) + self._status, self._body, self._headers, self._reason = result + self._response = self.responseCls(self._status, self._body, + self._headers, self._reason) + return self._response + + @property + def text(self): + self._get_response_if_not_available() + return self._body + + class MockHttp(BaseMockHttpObject): """ A mock HTTP client/server suitable for testing purposes. This replaces @@ -181,6 +268,7 @@ class MockHttp(BaseMockHttpObject): """ responseCls = MockResponse + rawResponseCls = MockRawResponse host = None port = None response = None @@ -284,10 +372,31 @@ class MockHttpTestCase(MockHttp, unittest.TestCase): self.assertEqual(params[key], value) +class MockConnection(object): + def __init__(self, action): + self.action = action + + class StorageMockHttp(MockHttp): def prepared_request(self, method, url, body=None, headers=None, raw=False, stream=False): - pass + # Find a method we can use for this request + parsed = urlparse.urlparse(url) + scheme, netloc, path, params, query, fragment = parsed + qs = parse_qs(query) + if path.endswith('/'): + path = path[:-1] + meth_name = self._get_method_name(type=self.type, + use_param=self.use_param, + qs=qs, path=path) + meth = getattr(self, meth_name.replace('%', '_')) + + if self.test and isinstance(self.test, LibcloudTestCase): + self.test._add_visited_url(url=url) + self.test._add_executed_mock_method(method_name=meth_name) + + status, body, headers, reason = meth(method, url, body, headers) + self.response = self.rawResponseCls(MockConnection(action=method), path=path) def putrequest(self, method, action, skip_host=0, skip_accept_encoding=0): pass @@ -302,83 +411,6 @@ class StorageMockHttp(MockHttp): pass -class MockRawResponse(BaseMockHttpObject): - """ - Mock RawResponse object suitable for testing. - """ - - type = None - responseCls = MockResponse - - def __init__(self, connection): - super(MockRawResponse, self).__init__() - self._data = [] - self._current_item = 0 - - self._status = None - self._response = None - self._headers = None - self._reason = None - self.connection = connection - - def next(self): - if self._current_item == len(self._data): - raise StopIteration - - value = self._data[self._current_item] - self._current_item += 1 - return value - - def __next__(self): - return self.next() - - def _generate_random_data(self, size): - data = '' - current_size = 0 - while current_size < size: - value = str(random.randint(0, 9)) - value_size = len(value) - data += value - current_size += value_size - - return data - - @property - def response(self): - return self._get_response_if_not_available() - - @property - def status(self): - self._get_response_if_not_available() - return self._status - - @property - def status_code(self): - self._get_response_if_not_availale() - return self._status - - @property - def headers(self): - self._get_response_if_not_available() - return self._headers - - @property - def reason(self): - self._get_response_if_not_available() - return self._reason - - def _get_response_if_not_available(self): - if not self._response: - meth_name = self._get_method_name(type=self.type, - use_param=False, qs=None, - path=self.connection.action) - meth = getattr(self, meth_name.replace('%', '_')) - result = meth(self.connection.method, None, None, None) - self._status, self._body, self._headers, self._reason = result - self._response = self.responseCls(self._status, self._body, - self._headers, self._reason) - return self._response - if __name__ == "__main__": import doctest doctest.testmod() http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/test/storage/test_atmos.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_atmos.py b/libcloud/test/storage/test_atmos.py index d847fc0..1371b5a 100644 --- a/libcloud/test/storage/test_atmos.py +++ b/libcloud/test/storage/test_atmos.py @@ -491,7 +491,8 @@ class AtmosMockHttp(StorageMockHttp, unittest.TestCase): def runTest(self): pass - def request(self, method, url, body=None, headers=None, raw=False): + def request(self, method, url, body=None, headers=None, raw=False, + stream=False): headers = headers or {} parsed = urlparse.urlparse(url) if parsed.query.startswith('metadata/'): http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/test/storage/test_base.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_base.py b/libcloud/test/storage/test_base.py index 91bbe2f..2a6316d 100644 --- a/libcloud/test/storage/test_base.py +++ b/libcloud/test/storage/test_base.py @@ -15,12 +15,14 @@ import sys import hashlib +from libcloud.utils.py3 import httplib +from io import BytesIO from mock import Mock from libcloud.utils.py3 import StringIO from libcloud.utils.py3 import PY3 -from libcloud.utils.py3 import b +from libcloud.utils.py3 import b, u if PY3: from io import FileIO as file @@ -30,16 +32,33 @@ from libcloud.storage.base import DEFAULT_CONTENT_TYPE from libcloud.test import unittest from libcloud.test import StorageMockHttp +from libcloud.test import MockRawResponse +class BaseMockHttp(StorageMockHttp): + def __init__(self, *args, **kwargs): + self.text = '' + self.status_code = 200 + + def root(self, method, url, body, headers): + body = '' + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _(self, method, url, body, headers): + body = '' + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + class BaseStorageTests(unittest.TestCase): def setUp(self): self.send_called = 0 - StorageDriver.connectionCls.conn_class = StorageMockHttp - + StorageDriver.connectionCls.conn_class = BaseMockHttp + BaseMockHttp.rawResponseCls = BaseMockHttp self.driver1 = StorageDriver('username', 'key', host='localhost') self.driver1.supports_chunked_encoding = True + + #self.driver1.connectionCls.rawResponseCls = BaseMockHttp + self.driver2 = StorageDriver('username', 'key', host='localhost') self.driver2.supports_chunked_encoding = False @@ -47,20 +66,8 @@ class BaseStorageTests(unittest.TestCase): self.driver1.strict_mode = False def test__upload_object_iterator_must_have_next_method(self): - class Iterator(object): - - def next(self): - pass - - class Iterator2(file): - - def __init__(self): - pass - - class SomeClass(object): - pass - valid_iterators = [Iterator(), Iterator2(), StringIO('bar')] + valid_iterators = [BytesIO(b('134')), StringIO('bar')] invalid_iterators = ['foobar', '', False, True, 1, object()] def upload_func(*args, **kwargs): @@ -71,11 +78,11 @@ class BaseStorageTests(unittest.TestCase): 'request_path': '/', 'headers': {}} for value in valid_iterators: - kwargs['iterator'] = value + kwargs['stream'] = value self.driver1._upload_object(**kwargs) for value in invalid_iterators: - kwargs['iterator'] = value + kwargs['stream'] = value try: self.driver1._upload_object(**kwargs) @@ -84,60 +91,6 @@ class BaseStorageTests(unittest.TestCase): else: self.fail('Exception was not thrown') - def test_upload_zero_bytes_long_object_via_stream(self): - iterator = Mock() - - if PY3: - iterator.__next__ = Mock() - iterator.__next__.side_effect = StopIteration() - else: - iterator.next.side_effect = StopIteration() - - def mock_send(data): - self.send_called += 1 - - response = Mock() - response.connection.connection.send = mock_send - - # Normal - success, data_hash, bytes_transferred = \ - self.driver1._stream_data(response=response, - iterator=iterator, - chunked=False, calculate_hash=True) - - self.assertTrue(success) - self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest()) - self.assertEqual(bytes_transferred, 0) - self.assertEqual(self.send_called, 1) - - # Chunked - success, data_hash, bytes_transferred = \ - self.driver1._stream_data(response=response, - iterator=iterator, - chunked=True, calculate_hash=True) - - self.assertTrue(success) - self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest()) - self.assertEqual(bytes_transferred, 0) - self.assertEqual(self.send_called, 5) - - def test__upload_data(self): - def mock_send(data): - self.send_called += 1 - - response = Mock() - response.connection.connection.send = mock_send - - data = '123456789901234567' - success, data_hash, bytes_transferred = \ - self.driver1._upload_data(response=response, data=data, - calculate_hash=True) - - self.assertTrue(success) - self.assertEqual(data_hash, hashlib.md5(b(data)).hexdigest()) - self.assertEqual(bytes_transferred, (len(data))) - self.assertEqual(self.send_called, 1) - def test__get_hash_function(self): self.driver1.hash_type = 'md5' func = self.driver1._get_hash_function() @@ -169,7 +122,7 @@ class BaseStorageTests(unittest.TestCase): upload_func=upload_func, upload_func_kwargs={}, request_path='/', - iterator=iterator) + stream=iterator) headers = self.driver1.connection.request.call_args[-1]['headers'] self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE) @@ -186,7 +139,7 @@ class BaseStorageTests(unittest.TestCase): upload_func=upload_func, upload_func_kwargs={}, request_path='/', - iterator=iterator) + stream=iterator) if __name__ == '__main__':
