From 1f8bbe563e1c843b35db56a2f01275d0524b5ceb Mon Sep 17 00:00:00 2001 From: kevinli7 Date: Fri, 8 Dec 2017 16:59:14 -0500 Subject: [PATCH] Revert change to transfer.Upload that converts streams into strings in StreamInChunks due to memory concerns. (#193) --- apitools/base/py/transfer.py | 6 ------ apitools/base/py/transfer_test.py | 33 ------------------------------- 2 files changed, 39 deletions(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 402281f6..e795a1a2 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -1032,12 +1032,6 @@ def __SendChunk(self, start, additional_headers=None): else: end = min(start + self.chunksize, self.total_size) body_stream = stream_slice.StreamSlice(self.stream, end - start) - # Change body_stream from a stream to a string object. This is - # because httpwrapper.MakeRequest doesn't handle the case where - # request.body is a stream. In the case that the body is a stream, - # if a request has to be retried, then the stream will be exhausted - # and the request will hang. - body_stream = body_stream.read() # TODO(craigcitro): Think about clearer errors on "no data in # stream". request.body = body_stream diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index e1080fa5..a944b057 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -17,7 +17,6 @@ """Tests for transfer.py.""" import string -import httplib2 import mock import six from six.moves import http_client @@ -499,35 +498,3 @@ def _side_effect(uri, **kwargs): # pylint: disable=unused-argument len(body)) return responses.pop(0) return _side_effect - - def testRetryRequestChunks(self): - """Test that StreamInChunks will retry correctly.""" - # Create and configure the upload object. - bytes_http = httplib2.Http() - upload = transfer.Upload( - stream=self.sample_stream, - mime_type='text/plain', - total_size=len(self.sample_data), - close_stream=False, - http=bytes_http) - - upload.strategy = transfer.RESUMABLE_UPLOAD - # Set the chunk size so the entire stream is uploaded. - upload.chunksize = len(self.sample_data) - # Mock the upload to return the sample response. - with mock.patch.object(bytes_http, - 'request') as make_request: - # This side effect also checks the request body. - responses = [ - self.response, # Initial request in InitializeUpload(). - self.fail_response, # 503 status code from server. - self.response # Successful request. - ] - make_request.side_effect = self.HttpRequestSideEffect(responses) - - # Initialization. - upload.InitializeUpload(self.request, bytes_http) - upload.StreamInChunks() - - # Ensure the mock was called the correct number of times. - self.assertEquals(make_request.call_count, len(responses))