From af7729f64a97ab35e83a1a7971781e69d124d99e Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Thu, 16 Jun 2016 14:56:18 -0500 Subject: [PATCH 1/5] Use seek from end rather than getvalue --- requests/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index dfeb77d9..0982ca41 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -45,7 +45,7 @@ def dict_to_sequence(d): def super_len(o): - total_length = 0 + total_length = None current_position = 0 if hasattr(o, '__len__'): @@ -54,10 +54,6 @@ def super_len(o): elif hasattr(o, 'len'): total_length = o.len - elif hasattr(o, 'getvalue'): - # e.g. BytesIO, cStringIO.StringIO - total_length = len(o.getvalue()) - elif hasattr(o, 'fileno'): try: fileno = o.fileno() @@ -89,6 +85,13 @@ def super_len(o): # let requests chunk it instead. current_position = total_length + if hasattr(o, 'seek') and total_length is None: + # StringIO has a notimplemented fileno + # BytesIO has seek and not fileno + total_length = o.seek(0, 2) + # seek back to current position to support partially read file-like + o.seek(current_position) + return max(0, total_length - current_position) From d4965a4ec6fc205f6a35aa2ff33514e4903fd3c4 Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Fri, 17 Jun 2016 10:43:10 -0500 Subject: [PATCH 2/5] Python 2 compatibility --- requests/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 0982ca41..f1e40bde 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -89,8 +89,14 @@ def super_len(o): # StringIO has a notimplemented fileno # BytesIO has seek and not fileno total_length = o.seek(0, 2) + # StringIO objects in py2 returns None + if total_length is None and hasattr(o, 'tell'): + total_length = o.tell() + elif total_length is None: + total_length = 0 + # seek back to current position to support partially read file-like - o.seek(current_position) + o.seek(current_position or 0) return max(0, total_length - current_position) From 8873a090979ffb627b21d54c04e7ddd1943eea03 Mon Sep 17 00:00:00 2001 From: Skipper Seabold Date: Fri, 17 Jun 2016 13:27:02 -0500 Subject: [PATCH 3/5] Fall back to streaming --- requests/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index f1e40bde..c847e601 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -83,7 +83,8 @@ def super_len(o): # is actually a special file descriptor like stdin. In this # instance, we don't know what the length is, so set it to zero and # let requests chunk it instead. - current_position = total_length + if total_length is not None: + current_position = total_length if hasattr(o, 'seek') and total_length is None: # StringIO has a notimplemented fileno @@ -98,6 +99,9 @@ def super_len(o): # seek back to current position to support partially read file-like o.seek(current_position or 0) + if total_length is None: + total_length = 0 + return max(0, total_length - current_position) From 99364b6b0b8d1ce5ac69121040e641db99d562d1 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 24 Aug 2016 10:25:37 -0600 Subject: [PATCH 4/5] removing use of seek to set total_length --- requests/utils.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index c847e601..a225d09a 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -86,18 +86,16 @@ def super_len(o): if total_length is not None: current_position = total_length - if hasattr(o, 'seek') and total_length is None: - # StringIO has a notimplemented fileno - # BytesIO has seek and not fileno - total_length = o.seek(0, 2) - # StringIO objects in py2 returns None - if total_length is None and hasattr(o, 'tell'): - total_length = o.tell() - elif total_length is None: - total_length = 0 + if hasattr(o, 'seek') and total_length is None: + # StringIO and BytesIO have seek but no useable fileno - # seek back to current position to support partially read file-like - o.seek(current_position or 0) + # seek to end of file + o.seek(0, 2) + total_length = o.tell() + + # seek back to current position to support + # partially read file-like objects + o.seek(current_position or 0) if total_length is None: total_length = 0 From f5c59743e7a45b68eaa82c83baaaed5607348afe Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Thu, 25 Aug 2016 10:51:06 -0600 Subject: [PATCH 5/5] adding tests for super_len conditional flow --- tests/test_utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 2e28c068..0a73cc81 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -66,6 +66,34 @@ class TestSuperLen: assert super_len(fd) == 4 assert len(recwarn) == warnings_num + def test_super_len_with__len__(self): + foo = [1,2,3,4] + len_foo = super_len(foo) + assert len_foo == 4 + + def test_super_len_with_no__len__(self): + class LenFile(object): + def __init__(self): + self.len = 5 + + assert super_len(LenFile()) == 5 + + def test_super_len_with_tell(self): + foo = StringIO.StringIO('12345') + assert super_len(foo) == 5 + foo.read(2) + assert super_len(foo) == 3 + + def test_super_len_with_fileno(self): + with open(__file__, 'rb') as f: + length = super_len(f) + file_data = f.read() + assert length == len(file_data) + + def test_super_len_with_no_matches(self): + """Ensure that objects without any length methods default to 0""" + assert super_len(object()) == 0 + class TestToKeyValList: