Garrett Holmstrom
9 years ago
parent
38a34855c4
commit
33721f912b
@ -1,73 +0,0 @@
|
|||||||
From 53be601f35e7bccd699439b3356bd37f49ab1933 Mon Sep 17 00:00:00 2001
|
|
||||||
From: "Ryan S. Brown" <sb@ryansb.com>
|
|
||||||
Date: Mon, 30 Nov 2015 13:02:22 -0500
|
|
||||||
Subject: [PATCH 1/1] Fix Glacier test failure in python 3.5 due to MagicMock
|
|
||||||
|
|
||||||
In Python 3.5+, the ConcurrentUploader tests were failing due to an
|
|
||||||
equality check.
|
|
||||||
|
|
||||||
```
|
|
||||||
FAIL: test_correct_low_level_api_calls (tests.unit.glacier.test_concurrent.TestConcurrentUploader)
|
|
||||||
----------------------------------------------------------------------
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "/tmp/boto/tests/unit/glacier/test_concurrent.py", line 105, in test_correct_low_level_api_calls
|
|
||||||
'vault_name', mock.ANY, mock.ANY, 8 * 1024 * 1024)
|
|
||||||
File "/home/ryansb/.pyenv/versions/3.5.0/lib/python3.5/unittest/mock.py", line 792, in assert_called_with
|
|
||||||
raise AssertionError(_error_message()) from cause
|
|
||||||
AssertionError: Expected call: complete_multipart_upload('vault_name', <ANY>, <ANY>, 8388608)
|
|
||||||
Actual call: complete_multipart_upload('vault_name', <MagicMock name='mock.initiate_multipart_upload().__getitem__()' id='140364396570440'>, b'c5aac592460a9ac
|
|
||||||
7845e341090f6f9c81f201b63e5338ee8948a6fe6830c55dc', 8388608)
|
|
||||||
```
|
|
||||||
|
|
||||||
Note that the MagicMock in the second call position was failing to be equal to mock.ANY.
|
|
||||||
|
|
||||||
In a Python 3.5 shell, this can be demonstrated as:
|
|
||||||
|
|
||||||
```
|
|
||||||
>>> from unittest import mock
|
|
||||||
>>> m = mock.MagicMock()
|
|
||||||
>>> mock.ANY == m
|
|
||||||
True
|
|
||||||
>>> m == mock.ANY
|
|
||||||
False
|
|
||||||
```
|
|
||||||
|
|
||||||
The tests pass in all supported versions below 3.4, and to get around
|
|
||||||
this in 3.5 I added a return value for the `initiate_multipart_upload`
|
|
||||||
attribute of the mock that contains a request ID to be checked. This
|
|
||||||
prevents `complete_multipart_upload` from being called with the
|
|
||||||
MagicMock.
|
|
||||||
---
|
|
||||||
tests/unit/glacier/test_concurrent.py | 10 ++++++++--
|
|
||||||
1 file changed, 8 insertions(+), 2 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/tests/unit/glacier/test_concurrent.py b/tests/unit/glacier/test_concurrent.py
|
|
||||||
index dd33e17..9ed9a89 100644
|
|
||||||
--- a/tests/unit/glacier/test_concurrent.py
|
|
||||||
+++ b/tests/unit/glacier/test_concurrent.py
|
|
||||||
@@ -95,14 +95,20 @@ class TestConcurrentUploader(unittest.TestCase):
|
|
||||||
|
|
||||||
def test_correct_low_level_api_calls(self):
|
|
||||||
api_mock = mock.MagicMock()
|
|
||||||
+ upload_id = '0898d645-ea45-4548-9a67-578f507ead49'
|
|
||||||
+ initiate_upload_mock = mock.Mock(
|
|
||||||
+ return_value={'UploadId': upload_id})
|
|
||||||
+ # initiate_multipart_upload must return a body containing an `UploadId`
|
|
||||||
+ api_mock.attach_mock(initiate_upload_mock, 'initiate_multipart_upload')
|
|
||||||
+
|
|
||||||
uploader = FakeThreadedConcurrentUploader(api_mock, 'vault_name')
|
|
||||||
uploader.upload('foofile')
|
|
||||||
# The threads call the upload_part, so we're just verifying the
|
|
||||||
# initiate/complete multipart API calls.
|
|
||||||
- api_mock.initiate_multipart_upload.assert_called_with(
|
|
||||||
+ initiate_upload_mock.assert_called_with(
|
|
||||||
'vault_name', 4 * 1024 * 1024, None)
|
|
||||||
api_mock.complete_multipart_upload.assert_called_with(
|
|
||||||
- 'vault_name', mock.ANY, mock.ANY, 8 * 1024 * 1024)
|
|
||||||
+ 'vault_name', upload_id, mock.ANY, 8 * 1024 * 1024)
|
|
||||||
|
|
||||||
def test_downloader_work_queue_is_correctly_populated(self):
|
|
||||||
job = mock.MagicMock()
|
|
||||||
--
|
|
||||||
2.4.3
|
|
||||||
|
|
||||||
@ -1,44 +1,48 @@
|
|||||||
Index: boto/boto/compat.py
|
Index: boto-2.39.0/boto/compat.py
|
||||||
===================================================================
|
===================================================================
|
||||||
--- boto.orig/boto/compat.py
|
--- boto-2.39.0.orig/boto/compat.py
|
||||||
+++ boto/boto/compat.py
|
+++ boto-2.39.0/boto/compat.py
|
||||||
@@ -46,15 +46,23 @@ except (AttributeError, ImportError):
|
@@ -46,16 +46,27 @@ except (AttributeError, ImportError):
|
||||||
# This is probably running on App Engine.
|
# This is probably running on App Engine.
|
||||||
expanduser = (lambda x: x)
|
expanduser = (lambda x: x)
|
||||||
|
|
||||||
-from boto.vendored import six
|
-from boto.vendored import six
|
||||||
-
|
|
||||||
-from boto.vendored.six import BytesIO, StringIO
|
|
||||||
-from boto.vendored.six.moves import filter, http_client, map, _thread, \
|
|
||||||
- urllib, zip
|
|
||||||
-from boto.vendored.six.moves.queue import Queue
|
|
||||||
-from boto.vendored.six.moves.urllib.parse import parse_qs, quote, unquote, \
|
|
||||||
- urlparse, urlsplit
|
|
||||||
-from boto.vendored.six.moves.urllib.request import urlopen
|
|
||||||
+try:
|
+try:
|
||||||
+ import six
|
+ import six
|
||||||
+ from six import BytesIO, StringIO
|
+ from six import BytesIO, StringIO
|
||||||
+ from six.moves import filter, http_client, map, _thread, urllib, zip
|
+ from six.moves import filter, http_client, map, _thread, \
|
||||||
|
+ urllib, zip
|
||||||
+ from six.moves.queue import Queue
|
+ from six.moves.queue import Queue
|
||||||
+ from six.moves.urllib.parse import parse_qs, quote, unquote, urlparse, \
|
+ from six.moves.urllib.parse import parse_qs, quote, unquote, \
|
||||||
+ urlsplit
|
+ urlparse, urlsplit
|
||||||
|
+ from six.moves.urllib.parse import unquote_plus
|
||||||
+ from six.moves.urllib.request import urlopen
|
+ from six.moves.urllib.request import urlopen
|
||||||
+except ImportError:
|
+except ImportError:
|
||||||
+ from boto.vendored import six
|
+ from boto.vendored import six
|
||||||
|
|
||||||
|
-from boto.vendored.six import BytesIO, StringIO
|
||||||
|
-from boto.vendored.six.moves import filter, http_client, map, _thread, \
|
||||||
|
- urllib, zip
|
||||||
|
-from boto.vendored.six.moves.queue import Queue
|
||||||
|
-from boto.vendored.six.moves.urllib.parse import parse_qs, quote, unquote, \
|
||||||
|
- urlparse, urlsplit
|
||||||
|
-from boto.vendored.six.moves.urllib.parse import unquote_plus
|
||||||
|
-from boto.vendored.six.moves.urllib.request import urlopen
|
||||||
+ from boto.vendored.six import BytesIO, StringIO
|
+ from boto.vendored.six import BytesIO, StringIO
|
||||||
+ from boto.vendored.six.moves import filter, http_client, map, _thread, \
|
+ from boto.vendored.six.moves import filter, http_client, map, _thread, \
|
||||||
+ urllib, zip
|
+ urllib, zip
|
||||||
+ from boto.vendored.six.moves.queue import Queue
|
+ from boto.vendored.six.moves.queue import Queue
|
||||||
+ from boto.vendored.six.moves.urllib.parse import parse_qs, quote, unquote, \
|
+ from boto.vendored.six.moves.urllib.parse import parse_qs, quote, unquote, \
|
||||||
+ urlparse, urlsplit
|
+ urlparse, urlsplit
|
||||||
|
+ from boto.vendored.six.moves.urllib.parse import unquote_plus
|
||||||
+ from boto.vendored.six.moves.urllib.request import urlopen
|
+ from boto.vendored.six.moves.urllib.request import urlopen
|
||||||
|
|
||||||
if six.PY3:
|
if six.PY3:
|
||||||
# StandardError was removed, so use the base exception type instead
|
# StandardError was removed, so use the base exception type instead
|
||||||
Index: boto/setup.py
|
Index: boto-2.39.0/setup.py
|
||||||
===================================================================
|
===================================================================
|
||||||
--- boto.orig/setup.py
|
--- boto-2.39.0.orig/setup.py
|
||||||
+++ boto/setup.py
|
+++ boto-2.39.0/setup.py
|
||||||
@@ -76,7 +76,7 @@ setup(name = "boto",
|
@@ -76,7 +76,7 @@ setup(name = "boto",
|
||||||
"boto.elastictranscoder", "boto.opsworks", "boto.redshift",
|
"boto.elastictranscoder", "boto.opsworks", "boto.redshift",
|
||||||
"boto.dynamodb2", "boto.support", "boto.cloudtrail",
|
"boto.dynamodb2", "boto.support", "boto.cloudtrail",
|
||||||
@ -0,0 +1,46 @@
|
|||||||
|
commit c5047b3dc1800dda7a6a02766c9bafdaa139014e
|
||||||
|
Author: Garrett Holmstrom <gholms@devzero.com>
|
||||||
|
Date: Fri Jan 29 16:37:13 2016 -0800
|
||||||
|
|
||||||
|
Decode request bodies before passing them to json
|
||||||
|
|
||||||
|
AWSAuthConnection._mexe encodes request bodies as UTF-8, mutating the
|
||||||
|
original request object in the process. This breaks kinesis's unit
|
||||||
|
tests on at least python 3.4 and 3.5 on Fedora, because those unit tests
|
||||||
|
call json.loads, which expect str objects rather than the bytes objects
|
||||||
|
that _mexe converted it to.
|
||||||
|
|
||||||
|
This commit makes the test cases decode request bodies before feeding
|
||||||
|
them to json.loads.
|
||||||
|
|
||||||
|
Index: boto-2.39.0/tests/unit/kinesis/test_kinesis.py
|
||||||
|
===================================================================
|
||||||
|
--- boto-2.39.0.orig/tests/unit/kinesis/test_kinesis.py
|
||||||
|
+++ boto-2.39.0/tests/unit/kinesis/test_kinesis.py
|
||||||
|
@@ -36,7 +36,7 @@ class TestKinesis(AWSMockServiceTestCase
|
||||||
|
self.service_connection.put_record('stream-name',
|
||||||
|
b'\x00\x01\x02\x03\x04\x05', 'partition-key')
|
||||||
|
|
||||||
|
- body = json.loads(self.actual_request.body)
|
||||||
|
+ body = json.loads(self.actual_request.body.decode('utf-8'))
|
||||||
|
self.assertEqual(body['Data'], 'AAECAwQF')
|
||||||
|
|
||||||
|
target = self.actual_request.headers['X-Amz-Target']
|
||||||
|
@@ -47,7 +47,7 @@ class TestKinesis(AWSMockServiceTestCase
|
||||||
|
self.service_connection.put_record('stream-name',
|
||||||
|
'data', 'partition-key')
|
||||||
|
|
||||||
|
- body = json.loads(self.actual_request.body)
|
||||||
|
+ body = json.loads(self.actual_request.body.decode('utf-8'))
|
||||||
|
self.assertEqual(body['Data'], 'ZGF0YQ==')
|
||||||
|
|
||||||
|
target = self.actual_request.headers['X-Amz-Target']
|
||||||
|
@@ -66,7 +66,7 @@ class TestKinesis(AWSMockServiceTestCase
|
||||||
|
self.service_connection.put_records(stream_name='stream-name',
|
||||||
|
records=[record_binary, record_str])
|
||||||
|
|
||||||
|
- body = json.loads(self.actual_request.body)
|
||||||
|
+ body = json.loads(self.actual_request.body.decode('utf-8'))
|
||||||
|
self.assertEqual(body['Records'][0]['Data'], 'AAECAwQF')
|
||||||
|
self.assertEqual(body['Records'][1]['Data'], 'ZGF0YQ==')
|
||||||
|
|
||||||
Loading…
Reference in new issue