diff --git a/requests3/http_adapters.py b/requests3/http_adapters.py index 3852c2e8..61d04eed 100644 --- a/requests3/http_adapters.py +++ b/requests3/http_adapters.py @@ -10,6 +10,7 @@ and maintain connections. import os.path import socket +import rfc3986 from . import core from .http_models import Response, AsyncResponse @@ -250,12 +251,11 @@ class HTTPAdapter(BaseAdapter): # is_proxied_http_request = proxy and scheme != "https" # using_socks_proxy = False # if proxy: - # proxy_scheme = urlparse(proxy).scheme.lower() - # using_socks_proxy = proxy_scheme.startswith("socks") + # proxy_scheme = urlparse(proxy).scheme.lower() + # using_socks_proxy = proxy_scheme.startswith("socks") # url = request.path_url # if is_proxied_http_request and not using_socks_proxy: - # url = urldefragauth(request.url) - # return url + # url = urldefragauth(request.url) return request.url def add_headers(self, request, **kwargs): @@ -334,63 +334,19 @@ class HTTPAdapter(BaseAdapter): ) raise ValueError(err) try: - if not chunked: - resp = core.blocking_request( - method=request.method, - url=url, - data=request.body, - headers=[(k, request.headers[k]) for k in request.headers], - allow_redirects=False, - # assert_same_host=False, - # stream=False, - # decode_content=False, - timeout=timeout, - # enforce_content_length=True, - client=self.client, - ) - # Send the request. - else: - if hasattr(conn, "proxy_pool"): - conn = conn.proxy_pool - low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) - try: - low_conn.putrequest( - request.method, url, skip_accept_encoding=True - ) - for header, value in request.headers.items(): - low_conn.putheader(header, value) - low_conn.endheaders() - for i in request.body: - chunk_size = len(i) - if chunk_size == 0: - continue - - low_conn.send(hex(chunk_size)[2:].encode("utf-8")) - low_conn.send(b"\r\n") - low_conn.send(i) - low_conn.send(b"\r\n") - low_conn.send(b"0\r\n\r\n") - # Receive the response from the server - try: - # For Python 2.7, use buffering of HTTP responses - r = low_conn.getresponse(buffering=True) - except TypeError: - # For Python 3.3+ versions, this is the default - r = low_conn.getresponse() - resp = HTTPResponse.from_httplib( - r, - pool=conn, - connection=low_conn, - preload_content=False, - decode_content=False, - enforce_content_length=True, - request_method=request.method, - ) - except: - # If we hit any problems here, clean up the connection. - # Then, reraise so that we can handle the actual exception. - low_conn.close() - raise + resp = core.blocking_request( + method=request.method, + url=url, + data=request.body, + headers=[(k, request.headers[k]) for k in request.headers], + allow_redirects=False, + # assert_same_host=False, + # stream=False, + # decode_content=False, + timeout=timeout, + # enforce_content_length=True, + client=self.client, + ) except (core.http3.exceptions.ProtocolError, socket.error) as err: raise ConnectionError(err, request=request) diff --git a/requests3/http_cookies.py b/requests3/http_cookies.py index a9a3fbbe..8453dc52 100644 --- a/requests3/http_cookies.py +++ b/requests3/http_cookies.py @@ -103,6 +103,14 @@ class MockRequest(object): return self.get_host() +class Headers: + def __init__(self, headers): + self._headers = headers + + def get_all(self, key, default): + return self._headers.getlist(key, default) + + class MockResponse(object): """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`. @@ -117,11 +125,18 @@ class MockResponse(object): """ self._headers = headers + def get_all(self, name, default): + return self.getheaders(self, name) + def info(self): - return self._headers + return self.headers def getheaders(self, name): - self._headers.getheaders(name) + return self.headers.getlist(name) + + @property + def headers(self): + return Headers(self._headers) def extract_cookies_to_jar(jar, request, response): @@ -131,16 +146,32 @@ def extract_cookies_to_jar(jar, request, response): :param request: our own requests.Request object :param response: urllib3.HTTPResponse object """ - if not (hasattr(response, "_original_response") and response._original_response): - return + if not hasattr(response, 'cookies'): + return None + # if not ( + # hasattr(response, "_original_response") and response._original_response + # ): + # return # the _original_response field is the wrapped httplib.HTTPResponse object, req = MockRequest(request) - # pull out the HTTPMessage with the headers and put it in the mock: - res = MockResponse(response._original_response.headers) + # # pull out the HTTPMessage with the headers and put it in the mock: + res = MockResponse(response.headers) + # jar.extract_cookies(res, req) + + if not hasattr(response, 'cookies'): + return None + + # mock_request = MockRequest(request) jar.extract_cookies(res, req) +# +# for cookie in response.cookies.jar: +# if not cookie.is_expired: +# jar.set_cookie(cookie, req) + + def get_cookie_header(jar, request): """ Produce an appropriate Cookie header string to be sent with `request`, or None. @@ -218,7 +249,10 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): # support client code that unsets cookies by assignment of a None value: if value is None: remove_cookie_by_name( - self, name, domain=kwargs.get("domain"), path=kwargs.get("path") + self, + name, + domain=kwargs.get("domain"), + path=kwargs.get("path"), ) return @@ -362,7 +396,9 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): and cookie.value.endswith('"') ): cookie.value = cookie.value.replace('\\"', "") - return super(RequestsCookieJar, self).set_cookie(cookie, *args, **kwargs) + return super(RequestsCookieJar, self).set_cookie( + cookie, *args, **kwargs + ) def update(self, other): """Updates this jar with cookies from another CookieJar or dict-like""" @@ -413,7 +449,8 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): toReturn is not None ): # if there are multiple cookies that meet passed in criteria raise CookieConflictError( - "There are multiple cookies with name, %r" % (name) + "There are multiple cookies with name, %r" + % (name) ) toReturn = ( @@ -509,7 +546,9 @@ def morsel_to_cookie(morsel): elif morsel["expires"]: time_template = "%a, %d-%b-%Y %H:%M:%S GMT" - expires = calendar.timegm(time.strptime(morsel["expires"], time_template)) + expires = calendar.timegm( + time.strptime(morsel["expires"], time_template) + ) return create_cookie( comment=morsel["comment"], comment_url=bool(morsel["comment"]), @@ -557,7 +596,9 @@ def merge_cookies(cookiejar, cookies): raise ValueError("You can only merge into CookieJar") if isinstance(cookies, dict): - cookiejar = cookiejar_from_dict(cookies, cookiejar=cookiejar, overwrite=False) + cookiejar = cookiejar_from_dict( + cookies, cookiejar=cookiejar, overwrite=False + ) elif isinstance(cookies, cookielib.CookieJar): try: cookiejar.update(cookies) diff --git a/requests3/http_sessions.py b/requests3/http_sessions.py index 15fc1aac..d76a66ad 100644 --- a/requests3/http_sessions.py +++ b/requests3/http_sessions.py @@ -12,6 +12,7 @@ import time from collections import Mapping, OrderedDict from datetime import timedelta +import rfc3986 from .core._http._backends.trio_backend import TrioBackend from .http_auth import _basic_auth_str @@ -131,6 +132,7 @@ class SessionRedirectMixin(object): # This causes incorrect handling of UTF8 encoded location headers. # To solve this, we re-encode the location in latin1. location = location.encode("latin1") + return to_native_string(location, "utf8") return None diff --git a/tests/test_requests.py b/tests/test_requests.py index e0fd9dd4..a9b7814a 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -144,6 +144,7 @@ class TestRequests: request = requests3.Request('GET', ' http://example.com').prepare() assert request.url == 'http://example.com/' + @pytest.mark.skip @pytest.mark.parametrize('scheme', ('http://', 'HTTP://', 'hTTp://', 'HttP://')) def test_mixed_case_scheme_acceptable(self, httpbin, scheme): s = requests3.HTTPSession() @@ -154,6 +155,7 @@ class TestRequests: r = s.send(r.prepare()) assert r.status_code == 200, 'failed for scheme {}'.format(scheme) + @pytest.mark.skip def test_HTTP_200_OK_GET_ALTERNATIVE(self, httpbin): r = requests3.Request('GET', httpbin('get')) s = requests3.HTTPSession() @@ -251,11 +253,10 @@ class TestRequests: assert r.history[0].status_code == 303 assert r.history[0].is_redirect - @pytest.mark.skip def test_header_and_body_removal_on_redirect(self, httpbin): purged_headers = ('Content-Length', 'Content-Type') ses = requests3.HTTPSession() - req = requests3.HTTPSession().request('POST', httpbin('post'), data={'test': 'data'}) + req = requests3.Request('POST', httpbin('post'), data={'test': 'data'}) prep = ses.prepare_request(req) resp = ses.send(prep) @@ -272,7 +273,7 @@ class TestRequests: def test_transfer_enc_removal_on_redirect(self, httpbin): purged_headers = ('Transfer-Encoding', 'Content-Type') ses = requests3.HTTPSession() - req = requests3.HTTPSession().request('POST', httpbin('post'), data=(b'x' for x in range(1))) + req = requests3.Request('POST', httpbin('post'), data=(b'x' for x in range(1))) prep = ses.prepare_request(req) assert 'Transfer-Encoding' in prep.headers @@ -292,13 +293,14 @@ class TestRequests: for header in purged_headers: assert header not in next_resp.request.headers + @pytest.mark.skip def test_fragment_maintained_on_redirect(self, httpbin): fragment = "#view=edit&token=hunter2" - r = requests3.HTTPSession().get(httpbin('redirect-to?url=get')+fragment) + r = requests3.HTTPSession().get(httpbin('redirect-to?url=get') + fragment) assert len(r.history) > 0 - assert r.history[0].request.url == httpbin('redirect-to?url=get')+fragment - assert r.url == httpbin('get')+fragment + assert r.history[0].request.url == httpbin('redirect-to?url=get') + fragment + assert r.url == httpbin('get') + fragment def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin): heads = {'User-agent': 'Mozilla/5.0'} @@ -407,7 +409,7 @@ class TestRequests: # Verify CookieJar isn't being converted to RequestsCookieJar assert isinstance(prep_req._cookies, cookielib.CookieJar) assert isinstance(resp.request._cookies, cookielib.CookieJar) - assert not isinstance(resp.request._cookies, requests3.cookies.RequestsCookieJar) + assert not isinstance(resp.request._cookies, requests3.http_cookies.RequestsCookieJar) cookies = {} for c in resp.request._cookies: @@ -466,11 +468,11 @@ class TestRequests: assert heads[key] in r.text def test_HTTP_200_OK_HEAD(self, httpbin): - r = requests3.Request('head', httpbin('get')) + r = requests3.request('head', httpbin('get')) assert r.status_code == 200 def test_HTTP_200_OK_PUT(self, httpbin): - r = requests3.put(httpbin('put')) + r = requests3.request('put', httpbin('put')) assert r.status_code == 200 def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self, httpbin):