diff --git a/requests/exceptions.py b/requests/exceptions.py index 82797664..cddffd95 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -107,6 +107,10 @@ class RetryError(RequestException): class UnrewindableBodyError(RequestException): """Requests encountered an error when trying to rewind a body""" +class ConflictingHeaderError(RequestException): + """Mutually exclusive request headers set""" + + # Warnings diff --git a/requests/models.py b/requests/models.py index df94f026..365c0ed8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -32,12 +32,14 @@ from .packages.urllib3.exceptions import ( LocationParseError, ConnectionError) from .exceptions import ( HTTPError, MissingScheme, InvalidURL, ChunkedEncodingError, - ContentDecodingError, ConnectionError, StreamConsumedError) + ContentDecodingError, ConnectionError, StreamConsumedError, + ConflictingHeaderError) from ._internal_utils import to_native_string, unicode_is_ascii from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, check_header_validity) + iter_slices, guess_json_utf, super_len, check_header_validity, + determine_if_stream) from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, is_py2, chardet, builtin_str, basestring) @@ -466,15 +468,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if not isinstance(body, bytes): body = body.encode('utf-8') - is_stream = all([ - hasattr(data, '__iter__'), - not isinstance(data, (basestring, list, tuple, collections.Mapping)) - ]) - - try: - length = super_len(data) - except (TypeError, AttributeError, UnsupportedOperation): - length = None + is_stream = determine_if_stream(data) if is_stream: body = data @@ -493,10 +487,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if files: raise NotImplementedError('Streamed bodies and files are mutually exclusive.') - if length: - self.headers['Content-Length'] = builtin_str(length) - else: - self.headers['Transfer-Encoding'] = 'chunked' else: # Multi-part file uploads. if files: @@ -509,27 +499,39 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): else: content_type = 'application/x-www-form-urlencoded' - self.prepare_content_length(body) - # Add content-type if it wasn't explicitly provided. if content_type and ('content-type' not in self.headers): self.headers['Content-Type'] = content_type + self.prepare_content_length(body) self.body = body def prepare_content_length(self, body): - """Prepare Content-Length header based on request method and body""" + """Prepares Content-Length header. + + If the length of the body of the request can be computed, Content-Length is set using + super_len. If user has manually set either a Transfer-Encoding or Content-Length header + when it should not be set (they should be mutually exclusive) an ConflictingHeaderError + error will be raised. + """ if body is not None: - length = super_len(body) + is_stream = determine_if_stream(body) + + try: + length = super_len(body) + except (TypeError, AttributeError, UnsupportedOperation): + length = None + if length: - # If length exists, set it. Otherwise, we fallback - # to Transfer-Encoding: chunked. self.headers['Content-Length'] = builtin_str(length) - elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None: - # Set Content-Length to 0 for methods that can have a body - # but don't provide one. (i.e. not GET or HEAD) + elif is_stream and not length: + self.headers['Transfer-Encoding'] = 'chunked' + elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None): self.headers['Content-Length'] = '0' + if 'Transfer-Encoding' in self.headers and 'Content-Length' in self.headers: + raise ConflictingHeaderError('Transfer-Encoding and Content-Length headers both set') + def prepare_auth(self, auth, url=''): """Prepares the given HTTP auth data.""" diff --git a/requests/utils.py b/requests/utils.py index e9460be4..e36d62af 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -865,6 +865,7 @@ def urldefragauth(url): return urlunparse((scheme, netloc, path, params, query, '')) + def rewind_body(prepared_request): """Move file pointer back to its recorded starting position so it can be read again on redirect. @@ -878,3 +879,11 @@ def rewind_body(prepared_request): "body for redirect.") else: raise UnrewindableBodyError("Unable to rewind request body for redirect.") + + +def determine_if_stream(data): + """Given data, determines if it should be sent as a stream. + """ + is_iterable = hasattr(data, '__iter__') + is_io_type = not isinstance(data, (basestring, list, tuple, dict)) + return is_iterable and is_io_type diff --git a/tests/test_requests.py b/tests/test_requests.py index 696cb2bd..a9de9c17 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -25,7 +25,7 @@ from requests.cookies import ( from requests.exceptions import ( ConnectionError, ConnectTimeout, InvalidScheme, InvalidURL, MissingScheme, ReadTimeout, Timeout, RetryError, TooManyRedirects, - ProxyError, InvalidHeader, UnrewindableBodyError) + ProxyError, InvalidHeader, UnrewindableBodyError, ConflictingHeaderError) from requests.models import PreparedRequest from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin @@ -1924,6 +1924,29 @@ class TestRequests: assert 'Transfer-Encoding' in prepared_request.headers assert 'Content-Length' not in prepared_request.headers + def test_chunked_upload_with_manually_set_content_length_header_raises_error(self, httpbin): + """Ensure that if a user manually sets a content length header when the data + is chunked that an ConflictingHeaderError is raised""" + data = (i for i in [b'a', b'b', b'c']) + url = httpbin('post') + with pytest.raises(ConflictingHeaderError): + r = requests.post(url, data=data, headers={'Content-Length': 'foo'}) + + def test_content_length_with_manually_set_transfer_encoding_raises_error(self, httpbin): + """Ensure that if a user manually sets a Transfer-Encoding header when data is not chunked + that an ConflictingHeaderError is raised""" + data = 'test data' + url = httpbin('post') + with pytest.raises(ConflictingHeaderError): + r = requests.post(url, data=data, headers={'Transfer-Encoding': 'chunked'}) + + def test_null_body_does_not_raise_error(self, httpbin): + url = httpbin('post') + try: + requests.post(url, data=None) + except ConflictingHeaderError: + pytest.fail('ConflictingHeaderError raised') + def test_custom_redirect_mixin(self, httpbin): """Tests a custom mixin to overwrite ``get_redirect_target``.