From 7c5151e8e71551f566edd10df943eb322a4a1dbc Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Tue, 1 May 2012 15:07:19 -0700 Subject: [PATCH 01/12] changes to .gitignore --- .gitignore | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index aaf729b7..6cd2d38d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,8 @@ MANIFEST coverage.xml nosetests.xml pylint.txt -*.pyc -docs/_build toy.py -.gitignore junit-report.xml -requests.egg-info/ \ No newline at end of file +requests.egg-info/ +*.pyc +*.swp From 2a27b123fd5bda2030e81995ff77e0e06f09ceca Mon Sep 17 00:00:00 2001 From: miguel Date: Fri, 3 Feb 2012 14:16:23 -0500 Subject: [PATCH 02/12] catch TooManyRedirects in safe_mode --- requests/models.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index c11858b8..be6a4edc 100644 --- a/requests/models.py +++ b/requests/models.py @@ -631,7 +631,15 @@ class Request(object): else: raise - self._build_response(r) + # build_response can throw TooManyRedirects + try: + self._build_response(r) + except RequestException as e: + if self.config.get('safe_mode', False): + # In safe mode, catch the exception + self.response.error = e + else: + raise # Response manipulation hook. self.response = dispatch_hook('response', self.hooks, self.response) From fed6cfbf4d282cc5056fd1e26b999523fdac73b1 Mon Sep 17 00:00:00 2001 From: miguel Date: Fri, 3 Feb 2012 14:40:20 -0500 Subject: [PATCH 03/12] mostly complete cookie support based on cookiejar --- requests/packages/oreos/cookiejar.py | 188 +++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 requests/packages/oreos/cookiejar.py diff --git a/requests/packages/oreos/cookiejar.py b/requests/packages/oreos/cookiejar.py new file mode 100644 index 00000000..0e21e443 --- /dev/null +++ b/requests/packages/oreos/cookiejar.py @@ -0,0 +1,188 @@ +import urlparse +import Cookie +from Cookie import Morsel +import cookielib +import collections + +class MockRequest: + """ + Wraps a *requests.Request* to mimic a *urllib2.Request*. + """ + def __init__(self, request): + self._r = request + + def get_type(self): + return urlparse.urlparse(self._r.full_url).scheme + + def get_host(self): + return urlparse.urlparse(self._r.full_url).netloc + + def get_origin_req_host(self): + if self._r.history: + r = self._r.history[0] + return urlparse.urlparse(r).netloc + else: + return self.get_host() + + def get_full_url(self): + return self._r.full_url + + def has_header(self, name): + return name in self._r.headers + + def get_header(self, name, default=None): + return self._r.headers.get(name, default) + + def add_unredirected_header(self, name, value): + # XXX: This is incorrect. + # To do this correctly the *requests.Request* class would need + # to keep track of headers that are only sent with the original + # request. + self._r.headers[name] = value + + def is_unverifiable(self): + # unverifiable == redirected + return bool(self.history) + +class MockResponse: + """ + Wraps a *requests.Response* to mimic a *urllib.addinfourl*. + """ + def __init__(self, response): + self._r = response + + def info(self): + return self._r.msg + + def getheaders(self, name): + self._r.msg.getheaders(name) + +class CookieJar(cookielib.LWPCookieJar, collections.MutableMapping): + def extract_cookies(self, response, request): + if response.raw._original_response: + req = MockRequest(request) + res = MockResponse(response.raw._original_response) + cookielib.CookieJar.extract_cookies(self, res, req) + + def get_header(self, request): + r = MockRequest(request) + self.add_cookie_header(r) + return r._r.headers['Cookie'] + + def get(self, name, domain=None, path=None, default=None): + try: + return self._find(name, domain, path) + except KeyError: + return default + + def set(self, name, value, **kwargs): + if isinstance(value, Morsel): + c = morsel_to_cookielib(value) + else: + c = create_cookie(name, value, **kwargs) + self.set_cookie(c) + return c + + def update(self, other): + if isinstance(other, cookielib.CookieJar): + for cookie in other: + self.set_cookie(cookie) + else: + collections.MutableMapping.update(self, other) + + def __getitem__(self, name): + return self._find(name) + + def __setitem__(self, name, value): + self.set(name, value) + + def __delitem__(self, name): + raise NotImplementedError('cookies can be removed using *clear*') + + def _find(self, name, domain=None, path=None): + try: + if domain is None: + dompaths = self._cookies.itervalues() + else: + dompaths = [self._cookies[domain]] + for paths in dompaths: + if path is None: + pathcookies = paths.itervalues() + else: + pathcookies = [paths[path]] + for cookies in pathcookies: + for cname, cookie in cookies.iteritems(): + if name == cname: + return cookielib_to_morsel(cookie) + except KeyError: + pass + raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) + +def create_cookie(name, value, **kwargs): + result = dict( + version=0, + name=name, + value=value, + port=None, + domain='', + path='/', + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False, + ) + + badargs = set(kwargs) - set(result) + if badargs: + err = 'create_cookie() got unexpected keyword arguments: %s' + raise TypeError(err % list(badargs)) + + result.update(kwargs) + result['port_specified'] = bool(result['port']) + result['domain_specified'] = bool(result['domain']) + result['domain_initial_dot'] = result['domain'].startswith('.') + result['path_specified'] = bool(result['path']) + + return cookielib.Cookie(**result) + +def cookielib_to_morsel(cookie): + m = Morsel() + m.key = cookie.name + m.value = cookie.value + m.coded_value = Cookie._quote(cookie.value) + m.update({ + 'expires': cookie.expires or '', + 'path': cookie.path or '', + 'comment': cookie.comment or '', + 'domain': cookie.domain or '', + 'max-age': cookie.expires or '', + 'secure': cookie.secure or '', + 'httponly': cookie._rest.get('HttpOnly', ''), + 'version': cookie.version or '', + }) + return m + +def morsel_to_cookielib(morsel): + c = create_cookie( + name=morsel.key, + value=morsel.value, + version=morsel['version'] or 0, + port=None, + port_specified=False, + domain=morsel['domain'], + domain_specified=bool(morsel['domain']), + domain_initial_dot=morsel['domain'].startswith('.'), + path=morsel['path'], + path_specified=bool(morsel['path']), + secure=bool(morsel['secure']), + expires=morsel['max-age'] or morsel['expires'], + discard=False, + comment=morsel['comment'], + comment_url=bool(morsel['comment']), + rest={'HttpOnly': morsel['httponly']}, + rfc2109=False, + ) + return c From c4ab96b63800f04511aada9b5899540a7937967a Mon Sep 17 00:00:00 2001 From: miguel Date: Fri, 3 Feb 2012 15:12:41 -0500 Subject: [PATCH 04/12] mostly complete cookie support based on cookiejar --- requests/models.py | 47 +++++++++++++--------------- requests/packages/oreos/cookiejar.py | 13 +++++--- requests/sessions.py | 17 +++++----- 3 files changed, 40 insertions(+), 37 deletions(-) diff --git a/requests/models.py b/requests/models.py index be6a4edc..4e6f26a6 100644 --- a/requests/models.py +++ b/requests/models.py @@ -15,6 +15,7 @@ from .structures import CaseInsensitiveDict from .status_codes import codes from .auth import HTTPBasicAuth, HTTPProxyAuth +from .packages.oreos.cookiejar import CookieJar from .packages.urllib3.response import HTTPResponse from .packages.urllib3.exceptions import MaxRetryError, LocationParseError from .packages.urllib3.exceptions import SSLError as _SSLError @@ -126,7 +127,14 @@ class Request(object): self.auth = auth #: CookieJar to attach to :class:`Request `. - self.cookies = dict(cookies or []) + if isinstance(cookies, CookieJar): + self.cookies = cookies + else: + self.cookies = CookieJar() + + # Add passed cookies in. + if cookies is not None: + self.cookies.update(cookies) #: True if Request has been sent. self.sent = False @@ -182,27 +190,26 @@ class Request(object): # Pass settings over. response.config = self.config + # Save original response for later. + response.raw = resp + response.url = self.full_url + if resp: # Fallback to None if there's no status_code, for whatever reason. response.status_code = getattr(resp, 'status', None) - + # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) - # Start off with our local cookies. - cookies = self.cookies or dict() - # Add new cookies from the server. - if 'set-cookie' in response.headers: - cookie_header = response.headers['set-cookie'] - cookies = dict_from_string(cookie_header) + self.cookies.extract_cookies(response, self) # Save cookies in Response. - response.cookies = cookies + response.cookies = self.cookies # No exceptions were harmed in the making of this request. response.error = getattr(resp, 'error', None) @@ -220,8 +227,6 @@ class Request(object): r = build(resp) - self.cookies.update(r.cookies) - if r.status_code in REDIRECT_STATI and not self.redirect: while (('location' in r.headers) and @@ -299,13 +304,11 @@ class Request(object): request.send() r = request.response - self.cookies.update(r.cookies) r.history = history self.response = r self.response.request = self - self.response.cookies.update(self.cookies) @staticmethod def _encode_params(data): @@ -573,19 +576,13 @@ class Request(object): if not self.sent or anyway: - if self.cookies: - - # Skip if 'cookie' header is explicitly set. - if 'cookie' not in self.headers: - - # Simple cookie with our dict. - c = SimpleCookie() - for (k, v) in list(self.cookies.items()): - c[k] = v - - # Turn it into a header. - cookie_header = c.output(header='', sep='; ').strip() + # Skip if 'cookie' header is explicitly set. + if 'cookie' not in self.headers: + # Turn it into a header. + cookie_header = self.cookies.get_header(self) + + if cookie_header: # Attach Cookie header to request. self.headers['Cookie'] = cookie_header diff --git a/requests/packages/oreos/cookiejar.py b/requests/packages/oreos/cookiejar.py index 0e21e443..9327e158 100644 --- a/requests/packages/oreos/cookiejar.py +++ b/requests/packages/oreos/cookiejar.py @@ -1,3 +1,8 @@ +""" +Wrapper for CookieJar allow dict-like access. +Ideally there would be wrappers for LWPCookieJar and MozillaCookieJar as well. +""" + import urlparse import Cookie from Cookie import Morsel @@ -18,8 +23,8 @@ class MockRequest: return urlparse.urlparse(self._r.full_url).netloc def get_origin_req_host(self): - if self._r.history: - r = self._r.history[0] + if self._r.response.history: + r = self._r.response.history[0] return urlparse.urlparse(r).netloc else: return self.get_host() @@ -42,7 +47,7 @@ class MockRequest: def is_unverifiable(self): # unverifiable == redirected - return bool(self.history) + return bool(self._r.response.history) class MockResponse: """ @@ -57,7 +62,7 @@ class MockResponse: def getheaders(self, name): self._r.msg.getheaders(name) -class CookieJar(cookielib.LWPCookieJar, collections.MutableMapping): +class CookieJar(cookielib.CookieJar, collections.MutableMapping): def extract_cookies(self, response, request): if response.raw._original_response: req = MockRequest(request) diff --git a/requests/sessions.py b/requests/sessions.py index f032cdcf..141bbf69 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -14,7 +14,7 @@ from .models import Request from .hooks import dispatch_hook from .utils import header_expand from .packages.urllib3.poolmanager import PoolManager - +from .packages.oreos.cookiejar import CookieJar def merge_kwargs(local_kwarg, default_kwarg): """Merges kwarg dictionaries. @@ -69,7 +69,6 @@ class Session(object): cert=None): self.headers = headers or {} - self.cookies = cookies or {} self.auth = auth self.timeout = timeout self.proxies = proxies or {} @@ -86,11 +85,14 @@ class Session(object): self.init_poolmanager() # Set up a CookieJar to be used by default - self.cookies = {} - - # Add passed cookies in. - if cookies is not None: - self.cookies.update(cookies) + if isinstance(cookies, CookieJar): + self.cookies = cookies + else: + self.cookies = CookieJar() + + # Add passed cookies in. + if cookies is not None: + self.cookies.update(cookies) def init_poolmanager(self): self.poolmanager = PoolManager( @@ -148,7 +150,6 @@ class Session(object): method = str(method).upper() # Default empty dicts for dict params. - cookies = {} if cookies is None else cookies data = {} if data is None else data files = {} if files is None else files headers = {} if headers is None else headers From 4d6871d9176c13affe625b1885278d396a39f21d Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Wed, 25 Apr 2012 00:50:34 -0700 Subject: [PATCH 05/12] Further changes to move cookies to CookieJar --- AUTHORS.rst | 1 + requests/compat.py | 4 +- requests/cookies.py | 258 +++++++++++++++++++++++++++ requests/models.py | 36 ++-- requests/packages/oreos/cookiejar.py | 193 -------------------- requests/sessions.py | 38 +++- requests/utils.py | 68 +------ tests/test_cookies.py | 149 ++++++++++++++++ tests/test_requests.py | 24 ++- 9 files changed, 473 insertions(+), 298 deletions(-) create mode 100644 requests/cookies.py delete mode 100644 requests/packages/oreos/cookiejar.py create mode 100755 tests/test_cookies.py diff --git a/AUTHORS.rst b/AUTHORS.rst index 50e7e1b5..6d4f9793 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -96,3 +96,4 @@ Patches and Suggestions - Michael Newman - Jonty Wareing - Shivaram Lingamneni +- Miguel (dhagrow) diff --git a/requests/compat.py b/requests/compat.py index fec7a01d..37063f58 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -83,7 +83,7 @@ if is_py2: from urlparse import urlparse, urlunparse, urljoin, urlsplit from urllib2 import parse_http_list import cookielib - from .packages.oreos.monkeys import SimpleCookie + from Cookie import Morsel from StringIO import StringIO bytes = str @@ -96,7 +96,7 @@ elif is_py3: from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote from urllib.request import parse_http_list from http import cookiejar as cookielib - from http.cookies import SimpleCookie + from http.cookies import Morsel from io import StringIO str = str diff --git a/requests/cookies.py b/requests/cookies.py new file mode 100644 index 00000000..1e795581 --- /dev/null +++ b/requests/cookies.py @@ -0,0 +1,258 @@ +""" +Compatibility code to be able to use `cookielib.CookieJar` with requests. + +requests.utils imports from here, so be careful with imports. +""" + +import collections +from .compat import cookielib, urlparse, Morsel + +try: + import threading + # grr, pyflakes: this fixes "redefinition of unused 'threading'" + threading +except ImportError: + import dummy_threading as threading + +class MockRequest(object): + """Wraps a `requests.Request` to mimic a `urllib2.Request`. + + The code in `cookielib.CookieJar` expects this interface in order to correctly + manage cookie policies, i.e., determine whether a cookie can be set, given the + domains of the request and the cookie. + + The original request object is read-only. The client is responsible for collecting + the new headers via `get_new_headers()` and interpreting them appropriately. You + probably want `get_cookie_header`, defined below. + """ + + def __init__(self, request): + self._r = request + self._new_headers = {} + + def get_type(self): + return urlparse.urlparse(self._r.full_url).scheme + + def get_host(self): + return urlparse.urlparse(self._r.full_url).netloc + + def get_origin_req_host(self): + if self._r.response.history: + r = self._r.response.history[0] + return urlparse.urlparse(r).netloc + else: + return self.get_host() + + def get_full_url(self): + return self._r.full_url + + def is_unverifiable(self): + # unverifiable == redirected + return bool(self._r.response.history) + + def has_header(self, name): + return name in self._r.headers or name in self._new_headers + + def get_header(self, name, default=None): + return self._r.headers.get(name, self._new_headers.get(name, default)) + + def add_header(self, key, val): + """cookielib has no legitimate use for this method; add it back if you find one.""" + raise NotImplementedError("Cookie headers should be added with add_unredirected_header()") + + def add_unredirected_header(self, name, value): + self._new_headers[name] = value + + def get_new_headers(self): + return self._new_headers + +class MockResponse(object): + """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`. + + ...what? Basically, expose the parsed HTTP headers from the server response + the way `cookielib` expects to see them. + """ + + def __init__(self, headers): + """Make a MockResponse for `cookielib` to read. + + :param headers: a httplib.HTTPMessage or analogous carrying the headers + """ + self._headers = headers + + def info(self): + return self._headers + + def getheaders(self, name): + self._headers.getheaders(name) + +def extract_cookies_to_jar(jar, request, response): + """Extract the cookies from the headers of `response`, into `jar`.""" + if response._original_response is None: + # TODO why would this happen? + return + req = MockRequest(request) + # pull out the HTTPMessage with the headers and put it in the mock: + res = MockResponse(response._original_response.msg) + jar.extract_cookies(res, req) + +def get_cookie_header(jar, request): + """Produce an appropriate Cookie header string to be sent with `request`, or None.""" + r = MockRequest(request) + jar.add_cookie_header(r) + return r.get_new_headers().get('Cookie') + +def remove_cookie_by_name(cookiejar, name, domain=None, path=None): + """Unsets a cookie by name, by default over all domains and paths. + + Wraps CookieJar.clear(), is O(n). + """ + clearables = [] + for cookie in cookiejar: + if cookie.name == name: + if domain is None or domain == cookie.domain: + if path is None or path == cookie.path: + clearables.append((cookie.domain, cookie.path, cookie.name)) + + for domain, path, name in clearables: + cookiejar.clear(domain, path, name) + +class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): + """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface. + + This is the CookieJar we create by default for requests and sessions that + don't specify one, since some clients may expect response.cookies and + session.cookies to support dict operations. + + Don't use the dict interface internally; it's just for compatibility with + with external client code. All `requests` code should work out of the box + with externally provided instances of CookieJar, e.g., LWPCookieJar and + FileCookieJar. + + Caution: dictionary operations that are normally O(1) may be O(n). + + Unlike a regular CookieJar, this class is pickleable. + """ + + def get(self, name, domain=None, path=None, default=None): + try: + return self._find(name, domain, path) + except KeyError: + return default + + def set(self, name, value, **kwargs): + # support client code that unsets cookies by assignment of a None value: + if value is None: + remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path')) + return + + if isinstance(value, Morsel): + c = morsel_to_cookie(value) + else: + c = create_cookie(name, value, **kwargs) + self.set_cookie(c) + return c + + def __getitem__(self, name): + return self._find(name) + + def __setitem__(self, name, value): + self.set(name, value) + + def __delitem__(self, name): + remove_cookie_by_name(self, name) + + def _find(self, name, domain=None, path=None): + for cookie in iter(self): + if cookie.name == name: + if domain is None or cookie.domain == domain: + if path is None or cookie.path == path: + return cookie.value + + raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) + + def __getstate__(self): + state = self.__dict__.copy() + # remove the unpickleable RLock object + state.pop('_cookies_lock') + return state + + def __setstate__(self, state): + self.__dict__.update(state) + if '_cookies_lock' not in self.__dict__: + self._cookies_lock = threading.RLock() + + def copy(self): + """We're probably better off forbidding this.""" + raise NotImplementedError + +def create_cookie(name, value, **kwargs): + """Make a cookie from underspecified parameters. + + By default, the pair of `name` and `value` will be set for the domain '' + and sent on every request (this is sometimes called a "supercookie"). + """ + result = dict( + version=0, + name=name, + value=value, + port=None, + domain='', + path='/', + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False, + ) + + badargs = set(kwargs) - set(result) + if badargs: + err = 'create_cookie() got unexpected keyword arguments: %s' + raise TypeError(err % list(badargs)) + + result.update(kwargs) + result['port_specified'] = bool(result['port']) + result['domain_specified'] = bool(result['domain']) + result['domain_initial_dot'] = result['domain'].startswith('.') + result['path_specified'] = bool(result['path']) + + return cookielib.Cookie(**result) + +def morsel_to_cookie(morsel): + """Convert a Morsel object into a Cookie containing the one k/v pair.""" + c = create_cookie( + name=morsel.key, + value=morsel.value, + version=morsel['version'] or 0, + port=None, + port_specified=False, + domain=morsel['domain'], + domain_specified=bool(morsel['domain']), + domain_initial_dot=morsel['domain'].startswith('.'), + path=morsel['path'], + path_specified=bool(morsel['path']), + secure=bool(morsel['secure']), + expires=morsel['max-age'] or morsel['expires'], + discard=False, + comment=morsel['comment'], + comment_url=bool(morsel['comment']), + rest={'HttpOnly': morsel['httponly']}, + rfc2109=False, + ) + return c + +def cookiejar_from_dict(cookie_dict, cookiejar=None): + """Returns a CookieJar from a key/value dictionary. + + :param cookie_dict: Dict of key/values to insert into CookieJar. + """ + if cookiejar is None: + cookiejar = RequestsCookieJar() + + if cookie_dict is not None: + for name, value in cookie_dict.iteritems(): + cookiejar.set_cookie(create_cookie(name, value)) + return cookiejar diff --git a/requests/models.py b/requests/models.py index 4e6f26a6..00d5cc40 100644 --- a/requests/models.py +++ b/requests/models.py @@ -15,7 +15,7 @@ from .structures import CaseInsensitiveDict from .status_codes import codes from .auth import HTTPBasicAuth, HTTPProxyAuth -from .packages.oreos.cookiejar import CookieJar +from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header from .packages.urllib3.response import HTTPResponse from .packages.urllib3.exceptions import MaxRetryError, LocationParseError from .packages.urllib3.exceptions import SSLError as _SSLError @@ -28,11 +28,11 @@ from .exceptions import ( URLRequired, SSLError, MissingSchema, InvalidSchema, InvalidURL) from .utils import ( get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri, - dict_from_string, stream_decode_response_unicode, get_netrc_auth, + stream_decode_response_unicode, get_netrc_auth, DEFAULT_CA_BUNDLE_PATH) from .compat import ( - urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, - SimpleCookie, is_py2) + cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, + is_py2) # Import chardet if it is available. try: @@ -127,14 +127,10 @@ class Request(object): self.auth = auth #: CookieJar to attach to :class:`Request `. - if isinstance(cookies, CookieJar): + if isinstance(cookies, cookielib.CookieJar): self.cookies = cookies else: - self.cookies = CookieJar() - - # Add passed cookies in. - if cookies is not None: - self.cookies.update(cookies) + self.cookies = cookiejar_from_dict(cookies) #: True if Request has been sent. self.sent = False @@ -190,15 +186,11 @@ class Request(object): # Pass settings over. response.config = self.config - # Save original response for later. - response.raw = resp - response.url = self.full_url - if resp: # Fallback to None if there's no status_code, for whatever reason. response.status_code = getattr(resp, 'status', None) - + # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) @@ -206,7 +198,7 @@ class Request(object): response.encoding = get_encoding_from_headers(response.headers) # Add new cookies from the server. - self.cookies.extract_cookies(response, self) + extract_cookies_to_jar(self.cookies, self, resp) # Save cookies in Response. response.cookies = self.cookies @@ -578,13 +570,7 @@ class Request(object): # Skip if 'cookie' header is explicitly set. if 'cookie' not in self.headers: - - # Turn it into a header. - cookie_header = self.cookies.get_header(self) - - if cookie_header: - # Attach Cookie header to request. - self.headers['Cookie'] = cookie_header + self.headers['Cookie'] = get_cookie_header(self.cookies, self) # Pre-request hook. r = dispatch_hook('pre_request', self.hooks, self) @@ -696,8 +682,8 @@ class Response(object): #: The :class:`Request ` that created the Response. self.request = None - #: A dictionary of Cookies the server sent back. - self.cookies = {} + #: A CookieJar of Cookies the server sent back. + self.cookies = None #: Dictionary of configurations for this request. self.config = {} diff --git a/requests/packages/oreos/cookiejar.py b/requests/packages/oreos/cookiejar.py deleted file mode 100644 index 9327e158..00000000 --- a/requests/packages/oreos/cookiejar.py +++ /dev/null @@ -1,193 +0,0 @@ -""" -Wrapper for CookieJar allow dict-like access. -Ideally there would be wrappers for LWPCookieJar and MozillaCookieJar as well. -""" - -import urlparse -import Cookie -from Cookie import Morsel -import cookielib -import collections - -class MockRequest: - """ - Wraps a *requests.Request* to mimic a *urllib2.Request*. - """ - def __init__(self, request): - self._r = request - - def get_type(self): - return urlparse.urlparse(self._r.full_url).scheme - - def get_host(self): - return urlparse.urlparse(self._r.full_url).netloc - - def get_origin_req_host(self): - if self._r.response.history: - r = self._r.response.history[0] - return urlparse.urlparse(r).netloc - else: - return self.get_host() - - def get_full_url(self): - return self._r.full_url - - def has_header(self, name): - return name in self._r.headers - - def get_header(self, name, default=None): - return self._r.headers.get(name, default) - - def add_unredirected_header(self, name, value): - # XXX: This is incorrect. - # To do this correctly the *requests.Request* class would need - # to keep track of headers that are only sent with the original - # request. - self._r.headers[name] = value - - def is_unverifiable(self): - # unverifiable == redirected - return bool(self._r.response.history) - -class MockResponse: - """ - Wraps a *requests.Response* to mimic a *urllib.addinfourl*. - """ - def __init__(self, response): - self._r = response - - def info(self): - return self._r.msg - - def getheaders(self, name): - self._r.msg.getheaders(name) - -class CookieJar(cookielib.CookieJar, collections.MutableMapping): - def extract_cookies(self, response, request): - if response.raw._original_response: - req = MockRequest(request) - res = MockResponse(response.raw._original_response) - cookielib.CookieJar.extract_cookies(self, res, req) - - def get_header(self, request): - r = MockRequest(request) - self.add_cookie_header(r) - return r._r.headers['Cookie'] - - def get(self, name, domain=None, path=None, default=None): - try: - return self._find(name, domain, path) - except KeyError: - return default - - def set(self, name, value, **kwargs): - if isinstance(value, Morsel): - c = morsel_to_cookielib(value) - else: - c = create_cookie(name, value, **kwargs) - self.set_cookie(c) - return c - - def update(self, other): - if isinstance(other, cookielib.CookieJar): - for cookie in other: - self.set_cookie(cookie) - else: - collections.MutableMapping.update(self, other) - - def __getitem__(self, name): - return self._find(name) - - def __setitem__(self, name, value): - self.set(name, value) - - def __delitem__(self, name): - raise NotImplementedError('cookies can be removed using *clear*') - - def _find(self, name, domain=None, path=None): - try: - if domain is None: - dompaths = self._cookies.itervalues() - else: - dompaths = [self._cookies[domain]] - for paths in dompaths: - if path is None: - pathcookies = paths.itervalues() - else: - pathcookies = [paths[path]] - for cookies in pathcookies: - for cname, cookie in cookies.iteritems(): - if name == cname: - return cookielib_to_morsel(cookie) - except KeyError: - pass - raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) - -def create_cookie(name, value, **kwargs): - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False, - ) - - badargs = set(kwargs) - set(result) - if badargs: - err = 'create_cookie() got unexpected keyword arguments: %s' - raise TypeError(err % list(badargs)) - - result.update(kwargs) - result['port_specified'] = bool(result['port']) - result['domain_specified'] = bool(result['domain']) - result['domain_initial_dot'] = result['domain'].startswith('.') - result['path_specified'] = bool(result['path']) - - return cookielib.Cookie(**result) - -def cookielib_to_morsel(cookie): - m = Morsel() - m.key = cookie.name - m.value = cookie.value - m.coded_value = Cookie._quote(cookie.value) - m.update({ - 'expires': cookie.expires or '', - 'path': cookie.path or '', - 'comment': cookie.comment or '', - 'domain': cookie.domain or '', - 'max-age': cookie.expires or '', - 'secure': cookie.secure or '', - 'httponly': cookie._rest.get('HttpOnly', ''), - 'version': cookie.version or '', - }) - return m - -def morsel_to_cookielib(morsel): - c = create_cookie( - name=morsel.key, - value=morsel.value, - version=morsel['version'] or 0, - port=None, - port_specified=False, - domain=morsel['domain'], - domain_specified=bool(morsel['domain']), - domain_initial_dot=morsel['domain'].startswith('.'), - path=morsel['path'], - path_specified=bool(morsel['path']), - secure=bool(morsel['secure']), - expires=morsel['max-age'] or morsel['expires'], - discard=False, - comment=morsel['comment'], - comment_url=bool(morsel['comment']), - rest={'HttpOnly': morsel['httponly']}, - rfc2109=False, - ) - return c diff --git a/requests/sessions.py b/requests/sessions.py index 141bbf69..b6fb5bb4 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -9,12 +9,13 @@ requests (cookies, auth, proxies). """ +from .compat import cookielib +from .cookies import cookiejar_from_dict, remove_cookie_by_name from .defaults import defaults from .models import Request from .hooks import dispatch_hook from .utils import header_expand from .packages.urllib3.poolmanager import PoolManager -from .packages.oreos.cookiejar import CookieJar def merge_kwargs(local_kwarg, default_kwarg): """Merges kwarg dictionaries. @@ -85,14 +86,10 @@ class Session(object): self.init_poolmanager() # Set up a CookieJar to be used by default - if isinstance(cookies, CookieJar): + if isinstance(cookies, cookielib.CookieJar): self.cookies = cookies else: - self.cookies = CookieJar() - - # Add passed cookies in. - if cookies is not None: - self.cookies.update(cookies) + self.cookies = cookiejar_from_dict(cookies) def init_poolmanager(self): self.poolmanager = PoolManager( @@ -186,11 +183,33 @@ class Session(object): _poolmanager=self.poolmanager ) + # merge session cookies into passed-in ones + dead_cookies = None + # passed-in cookies must become a CookieJar: + if not isinstance(cookies, cookielib.CookieJar): + args['cookies'] = cookiejar_from_dict(cookies) + # support unsetting cookies that have been passed in with None values + # this is only meaningful when `cookies` is a dict --- + # for a real CookieJar, the client should use session.cookies.clear() + if cookies is not None: + dead_cookies = [name for name in cookies if cookies[name] is None] + # merge the session's cookies into the passed-in cookies: + for cookie in self.cookies: + args['cookies'].set_cookie(cookie) + # remove the unset cookies from the jar we'll be using with the current request + # (but not from the session's own store of cookies): + if dead_cookies is not None: + for name in dead_cookies: + remove_cookie_by_name(args['cookies'], name) + # Merge local kwargs with session kwargs. for attr in self.__attrs__: + # we already merged cookies: + if attr == 'cookies': + continue + session_val = getattr(self, attr, None) local_val = args.get(attr) - args[attr] = merge_kwargs(local_val, session_val) # Arguments manipulation hook. @@ -210,7 +229,8 @@ class Session(object): r.send(prefetch=prefetch) # Send any cookies back up the to the session. - self.cookies.update(r.response.cookies) + for cookie in r.response.cookies: + self.cookies.set_cookie(cookie) # Return the response. return r.response diff --git a/requests/utils.py b/requests/utils.py index 925547a7..ecfee47b 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -18,8 +18,11 @@ import zlib from netrc import netrc, NetrcParseError from .compat import parse_http_list as _parse_list_header -from .compat import quote, cookielib, SimpleCookie, is_py2, urlparse +from .compat import quote, is_py2, urlparse from .compat import basestring, bytes, str +from .cookies import RequestsCookieJar, cookiejar_from_dict + +_hush_pyflakes = (RequestsCookieJar,) CERTIFI_BUNDLE_PATH = None try: @@ -97,25 +100,6 @@ def get_netrc_auth(url): pass - -def dict_from_string(s): - """Returns a MultiDict with Cookies.""" - - cookies = dict() - - try: - c = SimpleCookie() - c.load(s) - - for k, v in list(c.items()): - cookies.update({k: v.value}) - # This stuff is not to be trusted. - except Exception: - pass - - return cookies - - def guess_filename(obj): """Tries to guess the filename of the given object.""" name = getattr(obj, 'name', None) @@ -290,24 +274,6 @@ def dict_from_cookiejar(cj): return cookie_dict -def cookiejar_from_dict(cookie_dict): - """Returns a CookieJar from a key/value dictionary. - - :param cookie_dict: Dict of key/values to insert into CookieJar. - """ - - # return cookiejar if one was passed in - if isinstance(cookie_dict, cookielib.CookieJar): - return cookie_dict - - # create cookiejar - cj = cookielib.CookieJar() - - cj = add_dict_to_cookiejar(cj, cookie_dict) - - return cj - - def add_dict_to_cookiejar(cj, cookie_dict): """Returns a CookieJar from a key/value dictionary. @@ -315,31 +281,9 @@ def add_dict_to_cookiejar(cj, cookie_dict): :param cookie_dict: Dict of key/values to insert into CookieJar. """ - for k, v in list(cookie_dict.items()): - - cookie = cookielib.Cookie( - version=0, - name=k, - value=v, - port=None, - port_specified=False, - domain='', - domain_specified=False, - domain_initial_dot=False, - path='/', - path_specified=True, - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False - ) - - # add cookie to cookiejar + cj2 = cookiejar_from_dict(cookie_dict) + for cookie in cj2: cj.set_cookie(cookie) - return cj diff --git a/tests/test_cookies.py b/tests/test_cookies.py new file mode 100755 index 00000000..ecca5c5c --- /dev/null +++ b/tests/test_cookies.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import json +import os +import tempfile +import unittest + +# Path hack. +sys.path.insert(0, os.path.abspath('..')) +import requests +from requests.compat import cookielib + +# More hacks +sys.path.append('.') +from test_requests import httpbin, TestBaseMixin + +class CookieTests(TestBaseMixin, unittest.TestCase): + + def test_cookies_from_response(self): + """Basic test that we correctly parse received cookies in the Response object.""" + r = requests.get(httpbin('cookies', 'set', 'myname', 'myvalue')) + + # test deprecated dictionary interface + self.assertEqual(r.cookies['myname'], 'myvalue') + # test CookieJar interface + jar = r.cookies + self.assertEqual(len(jar), 1) + cookie_from_jar = list(jar)[0] + self.assertCookieHas(cookie_from_jar, name='myname', value='myvalue') + + q = requests.get(httpbin('cookies'), cookies=jar) + self.assertEqual(json.loads(q.text)['cookies'], {'myname': 'myvalue'}) + + def test_crossdomain_cookies(self): + """Cookies should not be sent to domains they didn't originate from.""" + r = requests.get("http://github.com") + c = r.cookies + # github should send us cookies + self.assertGreaterEqual(len(c), 1) + + # github cookies should not be sent to httpbin.org: + r2 = requests.get(httpbin('cookies'), cookies=c) + self.assertEqual(json.loads(r2.text)['cookies'], {}) + + # let's do this again using the session object + s = requests.session() + s.get("http://github.com") + self.assertGreaterEqual(len(s.cookies), 1) + r = s.get(httpbin('cookies')) + self.assertEqual(json.loads(r.text)['cookies'], {}) + # we can set a cookie and get exactly that same-domain cookie back: + r = s.get(httpbin('cookies', 'set', 'myname', 'myvalue')) + self.assertEqual(json.loads(r.text)['cookies'], {'myname': 'myvalue'}) + + def test_overwrite(self): + """Cookies should get overwritten when appropriate.""" + r = requests.get(httpbin('cookies', 'set', 'shimon', 'yochai')) + cookies = r.cookies + requests.get(httpbin('cookies', 'set', 'elazar', 'shimon'), cookies=cookies) + r = requests.get(httpbin('cookies'), cookies=cookies) + self.assertEqual(json.loads(r.text)['cookies'], + {'shimon': 'yochai', 'elazar': 'shimon'}) + # overwrite the value of 'shimon' + r = requests.get(httpbin('cookies', 'set', 'shimon', 'gamaliel'), cookies=cookies) + self.assertEqual(len(cookies), 2) + r = requests.get(httpbin('cookies'), cookies=cookies) + self.assertEqual(json.loads(r.text)['cookies'], + {'shimon': 'gamaliel', 'elazar': 'shimon'}) + +class LWPCookieJarTest(TestBaseMixin, unittest.TestCase): + """Check store/load of cookies to FileCookieJar's, specifically LWPCookieJar's.""" + + COOKIEJAR_CLASS = cookielib.LWPCookieJar + + def setUp(self): + # blank the file + self.cookiejar_file = tempfile.NamedTemporaryFile() + self.cookiejar_filename = self.cookiejar_file.name + cookiejar = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar.save() + + def tearDown(self): + try: + self.cookiejar_file.close() + except OSError: + pass + + def test_cookiejar_persistence(self): + """Test that we can save cookies to a FileCookieJar.""" + cookiejar = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar.load() + # initially should be blank + self.assertEqual(len(cookiejar), 0) + + response = requests.get(httpbin('cookies', 'set', 'key', 'value'), cookies=cookiejar) + self.assertEqual(len(cookiejar), 1) + cookie = list(cookiejar)[0] + self.assertEqual(json.loads(response.text)['cookies'], {'key': 'value'}) + self.assertCookieHas(cookie, name='key', value='value') + + # save and reload the cookies from the file: + cookiejar.save(ignore_discard=True) + cookiejar_2 = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar_2.load(ignore_discard=True) + self.assertEqual(len(cookiejar_2), 1) + cookie_2 = list(cookiejar_2)[0] + # this cookie should have been saved with the correct domain restriction: + self.assertCookieHas(cookie_2, name='key', value='value', + domain='httpbin.org', path='/') + + # httpbin sets session cookies, so if we don't ignore the discard attribute, + # there should be no cookie: + cookiejar_3 = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar_3.load() + self.assertEqual(len(cookiejar_3), 0) + + def test_crossdomain(self): + """Test persistence of the domains associated with the cookies.""" + cookiejar = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar.load() + self.assertEqual(len(cookiejar), 0) + + # github sets a cookie + requests.get("http://github.com", cookies=cookiejar) + num_github_cookies = len(cookiejar) + self.assertGreaterEqual(num_github_cookies, 1) + # httpbin sets another + requests.get(httpbin('cookies', 'set', 'key', 'value'), cookies=cookiejar) + num_total_cookies = len(cookiejar) + self.assertGreaterEqual(num_total_cookies, 2) + self.assertGreater(num_total_cookies, num_github_cookies) + + # save and load + cookiejar.save(ignore_discard=True) + cookiejar_2 = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar_2.load(ignore_discard=True) + self.assertEqual(len(cookiejar_2), num_total_cookies) + r = requests.get(httpbin('cookies'), cookies=cookiejar_2) + self.assertEqual(json.loads(r.text)['cookies'], {'key': 'value'}) + +class MozCookieJarTest(LWPCookieJarTest): + """Same test, but substitute MozillaCookieJar.""" + + COOKIEJAR_CLASS = cookielib.MozillaCookieJar + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_requests.py b/tests/test_requests.py index 2b702360..5b897a79 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -10,7 +10,6 @@ sys.path.insert(0, os.path.abspath('..')) import json import os -import sys import unittest import pickle @@ -52,8 +51,15 @@ class TestSetup(object): # time.sleep(1) _httpbin = True +class TestBaseMixin(object): -class RequestsTestSuite(TestSetup, unittest.TestCase): + def assertCookieHas(self, cookie, **kwargs): + """Assert that a cookie has various specified properties.""" + for attr, expected_value in kwargs.iteritems(): + message = 'Failed comparison for %s' % (attr,) + self.assertEqual(getattr(cookie, attr), expected_value, message) + +class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): """Requests test cases.""" def test_entry_points(self): @@ -632,24 +638,24 @@ class RequestsTestSuite(TestSetup, unittest.TestCase): # Those cookies persist transparently. c = json.loads(r.text).get('cookies') - assert c == _c + self.assertEqual(c, _c) # Double check. r = get(httpbin('cookies'), cookies={}, session=s) c = json.loads(r.text).get('cookies') - assert c == _c + self.assertEqual(c, _c) # Remove a cookie by setting it's value to None. r = get(httpbin('cookies'), cookies={'bessie': None}, session=s) c = json.loads(r.text).get('cookies') del _c['bessie'] - assert c == _c + self.assertEqual(c, _c) # Test session-level cookies. s = requests.session(cookies=_c) r = get(httpbin('cookies'), session=s) c = json.loads(r.text).get('cookies') - assert c == _c + self.assertEqual(c, _c) # Have the server set a cookie. r = get(httpbin('cookies', 'set', 'k', 'v'), allow_redirects=True, session=s) @@ -698,9 +704,13 @@ class RequestsTestSuite(TestSetup, unittest.TestCase): ds = pickle.loads(pickle.dumps(s)) self.assertEqual(s.headers, ds.headers) - self.assertEqual(s.cookies, ds.cookies) self.assertEqual(s.auth, ds.auth) + # Cookie doesn't have a good __eq__, so verify manually: + self.assertEqual(len(ds.cookies), 1) + for cookie in ds.cookies: + self.assertCookieHas(cookie, name='a-cookie', value='cookie-value') + def test_unpickled_session_requests(self): s = requests.session() r = get(httpbin('cookies', 'set', 'k', 'v'), allow_redirects=True, session=s) From 492bda1343b6413eead297c4b4acf26f68b2a3f2 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Wed, 25 Apr 2012 01:40:35 -0700 Subject: [PATCH 06/12] add a test for redirection --- tests/test_cookies.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_cookies.py b/tests/test_cookies.py index ecca5c5c..9cf10e8f 100755 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -69,6 +69,18 @@ class CookieTests(TestBaseMixin, unittest.TestCase): self.assertEqual(json.loads(r.text)['cookies'], {'shimon': 'gamaliel', 'elazar': 'shimon'}) + def test_redirects(self): + """Test that cookies set by a 302 page are correctly processed.""" + r = requests.get(httpbin('cookies', 'set', 'redirects', 'work')) + self.assertEqual(r.history[0].status_code, 302) + expected_cookies = {'redirects': 'work'} + self.assertEqual(json.loads(r.text)['cookies'], expected_cookies) + + r2 = requests.get(httpbin('cookies', 'set', 'very', 'well'), cookies=r.cookies) + expected_cookies = {'redirects': 'work', 'very': 'well'} + self.assertEqual(json.loads(r2.text)['cookies'], expected_cookies) + self.assertIs(r.cookies, r2.cookies) + class LWPCookieJarTest(TestBaseMixin, unittest.TestCase): """Check store/load of cookies to FileCookieJar's, specifically LWPCookieJar's.""" From 1de14e6a93d2f84b1aaea198785c05b2e51d933b Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Wed, 25 Apr 2012 09:20:19 -0700 Subject: [PATCH 07/12] fix usage of compat.urlparse (this code path appears to have been exercised only by secure cookies) --- requests/cookies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/cookies.py b/requests/cookies.py index 1e795581..bb680317 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -31,15 +31,15 @@ class MockRequest(object): self._new_headers = {} def get_type(self): - return urlparse.urlparse(self._r.full_url).scheme + return urlparse(self._r.full_url).scheme def get_host(self): - return urlparse.urlparse(self._r.full_url).netloc + return urlparse(self._r.full_url).netloc def get_origin_req_host(self): if self._r.response.history: r = self._r.response.history[0] - return urlparse.urlparse(r).netloc + return urlparse(r).netloc else: return self.get_host() From 404ecaf95bbaaa42013d7ae67b96c7593296d8cf Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Wed, 25 Apr 2012 13:40:17 -0700 Subject: [PATCH 08/12] clarified documentation from e-mail discussion --- AUTHORS.rst | 2 +- requests/cookies.py | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 6d4f9793..2f7de0e1 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -96,4 +96,4 @@ Patches and Suggestions - Michael Newman - Jonty Wareing - Shivaram Lingamneni -- Miguel (dhagrow) +- Miguel Turner diff --git a/requests/cookies.py b/requests/cookies.py index bb680317..7b50b4f9 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -87,14 +87,20 @@ class MockResponse(object): self._headers.getheaders(name) def extract_cookies_to_jar(jar, request, response): - """Extract the cookies from the headers of `response`, into `jar`.""" - if response._original_response is None: - # TODO why would this happen? - return - req = MockRequest(request) - # pull out the HTTPMessage with the headers and put it in the mock: - res = MockResponse(response._original_response.msg) - jar.extract_cookies(res, req) + """Extract the cookies from the response into a CookieJar. + + :param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar) + :param request: our own requests.Request object + :param response: urllib3.HTTPResponse object + """ + # the _original_response field is the wrapped httplib.HTTPResponse object, + # and in safe mode, it may be None if the request didn't actually complete. + # in that case, just skip the cookie extraction. + if response._original_response is not None: + req = MockRequest(request) + # pull out the HTTPMessage with the headers and put it in the mock: + res = MockResponse(response._original_response.msg) + jar.extract_cookies(res, req) def get_cookie_header(jar, request): """Produce an appropriate Cookie header string to be sent with `request`, or None.""" From ff169d32fee661a40fd600a86ea6fae931bd99c8 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Thu, 26 Apr 2012 12:20:42 -0700 Subject: [PATCH 09/12] fix and test for a bug dhagrow reported --- requests/models.py | 4 +++- tests/test_cookies.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 00d5cc40..ff0ef019 100644 --- a/requests/models.py +++ b/requests/models.py @@ -570,7 +570,9 @@ class Request(object): # Skip if 'cookie' header is explicitly set. if 'cookie' not in self.headers: - self.headers['Cookie'] = get_cookie_header(self.cookies, self) + cookie_header = get_cookie_header(self.cookies, self) + if cookie_header is not None: + self.headers['Cookie'] = cookie_header # Pre-request hook. r = dispatch_hook('pre_request', self.hooks, self) diff --git a/tests/test_cookies.py b/tests/test_cookies.py index 9cf10e8f..6c7cd13a 100755 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -81,6 +81,11 @@ class CookieTests(TestBaseMixin, unittest.TestCase): self.assertEqual(json.loads(r2.text)['cookies'], expected_cookies) self.assertIs(r.cookies, r2.cookies) + def test_none_cookie(self): + """Regression test: don't send a Cookie header with a string value of 'None'!""" + page = json.loads(requests.get(httpbin('headers')).text) + self.assertNotIn('Cookie', page['headers']) + class LWPCookieJarTest(TestBaseMixin, unittest.TestCase): """Check store/load of cookies to FileCookieJar's, specifically LWPCookieJar's.""" From 6eb349f5d20e119ce3340d9163472dc51f8031b6 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Thu, 26 Apr 2012 12:21:09 -0700 Subject: [PATCH 10/12] remove unused import in test --- tests/test_requests_async.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_requests_async.py b/tests/test_requests_async.py index 1d282616..3a5a762e 100755 --- a/tests/test_requests_async.py +++ b/tests/test_requests_async.py @@ -12,7 +12,6 @@ import select has_poll = hasattr(select, "poll") from requests import async -import envoy sys.path.append('.') from test_requests import httpbin, RequestsTestSuite, SERVICES From 7832f2ae7e3915c2fe238b0f648176328909395a Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Mon, 30 Apr 2012 04:00:48 -0700 Subject: [PATCH 11/12] add laurentb's test case for cookie handling on redirects --- tests/test_requests_ext.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_requests_ext.py b/tests/test_requests_ext.py index 1e4d89b0..883bdceb 100644 --- a/tests/test_requests_ext.py +++ b/tests/test_requests_ext.py @@ -104,8 +104,27 @@ class RequestsTestSuite(unittest.TestCase): 'php') assert r.ok + def test_cookies_on_redirects(self): + """Test interaction between cookie handling and redirection.""" + # get a cookie for tinyurl.com ONLY + s = requests.session() + s.get(url='http://tinyurl.com/preview.php?disable=1') + # we should have set a cookie for tinyurl: preview=0 + self.assertIn('preview', s.cookies) + self.assertEqual(s.cookies['preview'], '0') + self.assertEqual(list(s.cookies)[0].name, 'preview') + self.assertEqual(list(s.cookies)[0].domain, 'tinyurl.com') + # get cookies on another domain + r2 = s.get(url='http://httpbin.org/cookies') + # the cookie is not there + self.assertNotIn('preview', json.loads(r2.text)['cookies']) + # this redirects to another domain, httpbin.org + # cookies of the first domain should NOT be sent to the next one + r3 = s.get(url='http://tinyurl.com/7zp3jnr') + assert r3.url == 'http://httpbin.org/cookies' + self.assertNotIn('preview', json.loads(r2.text)['cookies']) if __name__ == '__main__': unittest.main() From 11a3eaec265735b63569bb165047d39df5ba465f Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Tue, 1 May 2012 16:32:40 -0700 Subject: [PATCH 12/12] add Python 3 compatibility --- requests/cookies.py | 4 ++-- tests/test_requests.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/cookies.py b/requests/cookies.py index 7b50b4f9..0e0dd67f 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -259,6 +259,6 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None): cookiejar = RequestsCookieJar() if cookie_dict is not None: - for name, value in cookie_dict.iteritems(): - cookiejar.set_cookie(create_cookie(name, value)) + for name in cookie_dict: + cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) return cookiejar diff --git a/tests/test_requests.py b/tests/test_requests.py index 5b897a79..fb0d75c0 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -55,7 +55,7 @@ class TestBaseMixin(object): def assertCookieHas(self, cookie, **kwargs): """Assert that a cookie has various specified properties.""" - for attr, expected_value in kwargs.iteritems(): + for attr, expected_value in kwargs.items(): message = 'Failed comparison for %s' % (attr,) self.assertEqual(getattr(cookie, attr), expected_value, message)