From cfa627ae62cebff20edf0816ce327c61603ef6a8 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Fri, 27 Jul 2012 17:08:16 +0200 Subject: [PATCH 1/3] Fixed encoding of fields with the same name. * Properly handle repeated data fields for multipart/form-data requests (#737) * Allow a list of 2-tuples as the `files` agument. * Consistently serialize lists a of parameters (#729). --- AUTHORS.rst | 1 + requests/models.py | 47 ++++++++++++++++++++++++++++-------------- tests/test_requests.py | 38 ++++++++++++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 17 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 1ff53148..f21d1736 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -110,3 +110,4 @@ Patches and Suggestions - Victoria Mo - Leila Muhtasib - Matthias Rahlf +- Jakub Roztocil diff --git a/requests/models.py b/requests/models.py index 8f31c3a7..1a74dc2b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -344,16 +344,33 @@ class Request(object): return data def _encode_files(self, files): + """Build the body for a multipart/form-data request. + Will successfully encode files when passed as a dict or a list of + 2-tuples. Order is retained if data is a list of 2-tuples but abritrary + if parameters are supplied as a dict. + + """ if (not files) or isinstance(self.data, str): return None - try: - fields = self.data.copy() - except AttributeError: - fields = dict(self.data) + def tuples(obj): + """Ensure 2-tuples. A dict or a 2-tuples list can be supplied.""" + if isinstance(obj, dict): + return list(obj.items()) + elif hasattr(obj, '__iter__'): + try: + dict(obj) + except ValueError: + pass + else: + return obj + raise ValueError('A dict or a list of 2-tuples required.') - for (k, v) in list(files.items()): + # 2-tuples containing both file and data fields. + fields = [] + + for k, v in tuples(files): # support for explicit filename if isinstance(v, (tuple, list)): fn, fp = v @@ -362,18 +379,18 @@ class Request(object): fp = v if isinstance(fp, (bytes, str)): fp = StringIO(fp) - fields.update({k: (fn, fp.read())}) + fields.append((k, (fn, fp.read()))) - for field in fields: - if isinstance(fields[field], numeric_types): - fields[field] = str(fields[field]) - if isinstance(fields[field], list): - newvalue = ', '.join(fields[field]) - fields[field] = newvalue - - (body, content_type) = encode_multipart_formdata(fields) + for k, vs in tuples(self.data): + if isinstance(vs, list): + for v in vs: + fields.append((k, str(v))) + else: + fields.append((k, str(vs))) - return (body, content_type) + body, content_type = encode_multipart_formdata(fields) + + return body, content_type @property def full_url(self): diff --git a/tests/test_requests.py b/tests/test_requests.py index e8bfc881..abc57e12 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -981,10 +981,10 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): list for a value in the data argument.""" data = {'field': ['a', 'b']} - files = {'file': 'Garbled data'} + files = {'field': 'Garbled data'} r = post(httpbin('post'), data=data, files=files) t = json.loads(r.text) - self.assertEqual(t.get('form'), {'field': 'a, b'}) + self.assertEqual(t.get('form'), {'field': ['a', 'b']}) self.assertEqual(t.get('files'), files) def test_str_data_content_type(self): @@ -1028,5 +1028,39 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): r = get(URL()) self.assertEqual(r.status_code, 200) + def test_post_fields_with_multiple_values_and_files_as_tuples(self): + """Test that it is possible to POST multiple data and file fields + with the same name.""" + + data = [ + ('__field__', '__value__'), + ('__field__', '__value__'), + ] + files = [ + ('__field__', '__value__'), + ('__field__', '__value__'), + ] + + r = post(httpbin('post'), data=data, files=files) + t = json.loads(r.text) + + self.assertEqual(t.get('form'), { + '__field__': [ + '__value__', + '__value__', + ] + }) + + # It's not currently possible to test for multiple file fields with + # the same name against httpbin so we need to inspect the encoded + # body manually. + request = r.request + body, content_type = request._encode_files(request.files) + file_field = ('Content-Disposition: form-data;' + ' name="__field__"; filename="__field__"') + self.assertEqual(body.count('__value__'), 4) + self.assertEqual(body.count(file_field), 2) + + if __name__ == '__main__': unittest.main() From dee3693ea004e2f859e4fba4cfedb376e4d0bb2b Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 30 Jul 2012 10:35:47 +0200 Subject: [PATCH 2/3] Use BytesIO for bytes. This fixes a TypeError on Python 3 that ocurred when passing bytes as the values for files. --- requests/models.py | 5 ++++- tests/test_requests.py | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 1a74dc2b..136427fe 100644 --- a/requests/models.py +++ b/requests/models.py @@ -10,6 +10,7 @@ This module contains the primary objects that power Requests. import os import socket from datetime import datetime +from io import BytesIO from .hooks import dispatch_hook, HOOKS from .structures import CaseInsensitiveDict @@ -377,8 +378,10 @@ class Request(object): else: fn = guess_filename(v) or k fp = v - if isinstance(fp, (bytes, str)): + if isinstance(fp, str): fp = StringIO(fp) + if isinstance(fp, bytes): + fp = BytesIO(fp) fields.append((k, (fn, fp.read()))) for k, vs in tuples(self.data): diff --git a/tests/test_requests.py b/tests/test_requests.py index abc57e12..e196af44 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -7,7 +7,6 @@ import sys import os sys.path.insert(0, os.path.abspath('..')) - import json import os import unittest @@ -1030,7 +1029,9 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): def test_post_fields_with_multiple_values_and_files_as_tuples(self): """Test that it is possible to POST multiple data and file fields - with the same name.""" + with the same name. + https://github.com/kennethreitz/requests/pull/746 + """ data = [ ('__field__', '__value__'), @@ -1061,6 +1062,10 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): self.assertEqual(body.count('__value__'), 4) self.assertEqual(body.count(file_field), 2) + def test_bytes_files(self): + """Test that `bytes` can be used as the values of `files`.""" + post(httpbin('post'), files={'test': b'test'}) + if __name__ == '__main__': unittest.main() From 88c762e256647a03f10a7a8c8707043a1b33aa5b Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Sat, 11 Aug 2012 08:19:35 +0200 Subject: [PATCH 3/3] Fixed tests for Python 3 (text vs. bytes). --- tests/test_requests.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index e196af44..f43ccac8 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1033,16 +1033,12 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): https://github.com/kennethreitz/requests/pull/746 """ - data = [ - ('__field__', '__value__'), - ('__field__', '__value__'), - ] - files = [ + fields = [ ('__field__', '__value__'), ('__field__', '__value__'), ] - r = post(httpbin('post'), data=data, files=files) + r = post(httpbin('post'), data=fields, files=fields) t = json.loads(r.text) self.assertEqual(t.get('form'), { @@ -1057,9 +1053,9 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): # body manually. request = r.request body, content_type = request._encode_files(request.files) - file_field = ('Content-Disposition: form-data;' - ' name="__field__"; filename="__field__"') - self.assertEqual(body.count('__value__'), 4) + file_field = (b'Content-Disposition: form-data;' + b' name="__field__"; filename="__field__"') + self.assertEqual(body.count(b'__value__'), 4) self.assertEqual(body.count(file_field), 2) def test_bytes_files(self):