From 0b6dc091f50965250d9c421d8aadaa701186a1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Mon, 8 Aug 2011 18:11:33 +0200 Subject: [PATCH 001/121] Added support for URLs with path not encoded. --- requests/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index a67691dd..bb15dc9b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -185,7 +185,7 @@ class Request(object): # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: parent_url_components = urlparse(self.url) - url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, url) + url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is 303: @@ -231,9 +231,10 @@ class Request(object): def _build_url(self): """Build the actual URL to use""" - # Support for unicode domain names. + # Support for unicode domain names and paths. parsed_url = list(urlparse(self.url)) parsed_url[1] = parsed_url[1].encode('idna') + parsed_url[2] = urllib.quote(urllib.unquote(parsed_url[2])) self.url = urlunparse(parsed_url) if self._enc_params: From aa0f78b740b6098f29594242b6b923113d0659b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 15:14:08 +0200 Subject: [PATCH 002/121] Better support of utf-8 paths. --- requests/models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/requests/models.py b/requests/models.py index 619d099f..51100290 100644 --- a/requests/models.py +++ b/requests/models.py @@ -247,10 +247,12 @@ class Request(object): """Build the actual URL to use""" # Support for unicode domain names and paths. - parsed_url = list(urlparse(self.url)) - parsed_url[1] = parsed_url[1].encode('idna') - parsed_url[2] = urllib.quote(urllib.unquote(parsed_url[2])) - self.url = urlunparse(parsed_url) + scheme, netloc, path, params, query, fragment = urlparse(self.url) + netloc = netloc.encode('idna') + if isinstance(path, unicode): + path = path.encode('utf-8') + path = urllib.quote(urllib.unquote(path)) + self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) if self._enc_params: if urlparse(self.url).query: From 32a861b2eddcfd73049f245984a12b88b364dadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 15:34:44 +0200 Subject: [PATCH 003/121] Fixed memory leak (see http://bugs.python.org/issue1208304) --- requests/models.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 51100290..6d97e35e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -169,7 +169,8 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) response.read = resp.read - response.close = resp.close + response._resp = resp + response._close = resp.close except AttributeError: pass @@ -397,6 +398,11 @@ class Response(object): raise self.error + def close(self): + if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'): + self._resp.fp._sock.recv = None + self._close() + class AuthManager(object): """Requests Authentication Manager.""" From 7b79cea7384a2dd3df90ac748bac3c4a89ebce24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 15:41:59 +0200 Subject: [PATCH 004/121] Use urljoin instead of manual concatenation. --- requests/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 51100290..57ee5dd8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -12,7 +12,7 @@ import socket import zlib from urllib2 import HTTPError -from urlparse import urlparse, urlunparse +from urlparse import urlparse, urlunparse, urljoin from datetime import datetime from .config import settings @@ -198,8 +198,7 @@ class Request(object): # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: - parent_url_components = urlparse(self.url) - url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, urllib.quote(urllib.unquote(url))) + url = urljoin(r.url, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is 303: From e7e395549f8b60777f00c244dabd2bd9f89271b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 17:28:58 +0200 Subject: [PATCH 005/121] Handle too many redirects. --- requests/exceptions.py | 3 +++ requests/models.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index eff7512a..c08c6148 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -21,3 +21,6 @@ class URLRequired(RequestException): class InvalidMethod(RequestException): """An inappropriate method was attempted.""" + +class TooManyRedirects(RequestException): + """Too many redirects.""" diff --git a/requests/models.py b/requests/models.py index ca72802f..e08bce25 100644 --- a/requests/models.py +++ b/requests/models.py @@ -20,7 +20,7 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects REDIRECT_STATI = (301, 302, 303, 307) @@ -192,6 +192,9 @@ class Request(object): (self.allow_redirects)) ): + if not len(history) < 30: + raise TooManyRedirects() + history.append(r) url = r.headers['location'] From cf94c96a68904b0c541425bb2eb0ee816d6a4e4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 17:30:19 +0200 Subject: [PATCH 006/121] Always close connection during redirections. --- requests/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requests/models.py b/requests/models.py index 6d97e35e..926bb94b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -192,6 +192,8 @@ class Request(object): (self.allow_redirects)) ): + r.close() + history.append(r) url = r.headers['location'] From 3bc9a181a16418eb9486a158ec7d07256186ec40 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 14 Aug 2011 21:16:04 -0400 Subject: [PATCH 007/121] basic FAQ --- docs/index.rst | 17 ++++++++++++----- docs/user/faq.rst | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 docs/user/faq.rst diff --git a/docs/index.rst b/docs/index.rst index e1628d42..e2bc2b92 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,7 +13,8 @@ Requests is an :ref:`ISC Licensed ` HTTP library, written in Python, for hu Most existing Python modules for sending HTTP requests are extremely verbose and cumbersome. Python's builtin **urllib2** module provides most of the HTTP capabilities you should need, but the api is thoroughly **broken**. -It requires an *enormous* amount of work (even method overrides) to perform the simplest of tasks. +It requires an *enormous* amount of work (even method overrides) to perform +the simplest of tasks. Things shouldn’t be this way. Not in Python. @@ -41,28 +42,34 @@ Testimonals `Twitter, Inc `_ uses Requests internally. **Daniel Greenfeld** - Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to @kennethreitz's request library. Today has been AWESOME. + Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to + @kennethreitz's request library. Today has been AWESOME. **Kenny Meyers** - Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful, simple, Pythonic. + Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful, + simple, Pythonic. **Rich Leland** Requests is awesome. That is all. **Steve Pike** - I can never remember how to do it the regular way. ``import requests; requests.get()`` is just so easy! + I can never remember how to do it the regular way. + ``import requests; requests.get()`` is just so easy! User Guide ---------- -This part of the documentation, which is mostly prose, begins with some background information about Requests, then focuses on step-by-step instructions for getting the most out of Requests. +This part of the documentation, which is mostly prose, begins with some +background information about Requests, then focuses on step-by-step +instructions for getting the most out of Requests. .. toctree:: :maxdepth: 2 user/intro user/install + user/faq .. user/quickstart user/advanced diff --git a/docs/user/faq.rst b/docs/user/faq.rst new file mode 100644 index 00000000..2ccb6a7b --- /dev/null +++ b/docs/user/faq.rst @@ -0,0 +1,43 @@ +.. _faq: + +Frequently Asked Questions +========================== + +This part of the documentation covers common questions about Requests. + +Why not Httplib2? +----------------- + +Chris Adams gave an excellent summary on +`Hacker News `_: + + httplib2 is part of why you should use requests: it's far more respectable + as a client but not as well documented and it still takes way too much code + for basic operations. I appreciate what httplib2 is trying to do, that + there's a ton of hard low-level annoyances in building a modern HTTP + client, but really, just use requests instead. Kenneth Reitz is very + motivated and he gets the degree to which simple things should be simple + whereas httplib2 feels more like an academic exercise than something + people should use to build production systems[1]. + + Disclosure: I'm listed in the requests AUTHORS file but can claim credit + for, oh, about 0.0001% of the awesomeness. + + 1. http://code.google.com/p/httplib2/issues/detail?id=96 is a good example: + an annoying bug which affect many people, there was a fix available for + months, which worked great when I applied it in a fork and pounded a couple + TB of data through it, but it took over a year to make it into trunk and + even longer to make it onto PyPI where any other project which required " + httplib2" would get the working version. + + +Python 3 Support? +----------------- + +It's on the way. + + +Keep-alive Support? +------------------- + +It's on the way. \ No newline at end of file From d50742f059ee40abb1d410c3b0d1115a8ea8d8fb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 14 Aug 2011 22:07:15 -0400 Subject: [PATCH 008/121] massive documentation update --- docs/{user => community}/faq.rst | 0 docs/community/support.rst | 37 ++++++++++++ docs/community/updates.rst | 31 ++++++++++ docs/dev/authors.rst | 5 ++ docs/index.rst | 17 +++++- docs/user/quickstart.rst | 100 +++++++++++-------------------- 6 files changed, 123 insertions(+), 67 deletions(-) rename docs/{user => community}/faq.rst (100%) create mode 100644 docs/community/support.rst create mode 100644 docs/community/updates.rst create mode 100644 docs/dev/authors.rst diff --git a/docs/user/faq.rst b/docs/community/faq.rst similarity index 100% rename from docs/user/faq.rst rename to docs/community/faq.rst diff --git a/docs/community/support.rst b/docs/community/support.rst new file mode 100644 index 00000000..53f3c819 --- /dev/null +++ b/docs/community/support.rst @@ -0,0 +1,37 @@ +.. _support: + +Support +======= + +If you have a questions or issues about Requests, there are serveral options: + +Send a Tweet +------------ + +If your question is less than 140 characters, feel free to send a tweet to +`@kennethreitz `_. + + +File an Issue +------------- + +If you notice some unexpected behavior in Requests, or want to see support +for a new feature, +`file an issue on GitHub `_. + + +E-mail +------ + +I'm more than happy to answer any personal or in-depth questions about +Requests. Feel free to email +`requests@kennethreitz.com `_. + + +IRC +--- + +The official Freenode channel for Requests is +`#python-requests `_ + +I'm also available as **kennethreitz** on Freenode. \ No newline at end of file diff --git a/docs/community/updates.rst b/docs/community/updates.rst new file mode 100644 index 00000000..942ccac1 --- /dev/null +++ b/docs/community/updates.rst @@ -0,0 +1,31 @@ +.. _updates: + +Updates +======= + +If you'd like to stay up to date on the community and development of Requests, +there are serveral options: + +GitHub +------ + +The best way to track the development of Requests is through +`the GitHub repo `_. + +Twitter +------- + +I often tweet about new features and releases of Requests. + +Follow `@kennethreitz `_ for updates. + + + +Mailing List +------------ + +There's a low-volume mailing list for Requests. To subscribe to the +mailing list, send an email to +`requests@librelist.org `_. + + diff --git a/docs/dev/authors.rst b/docs/dev/authors.rst new file mode 100644 index 00000000..5b3cc85b --- /dev/null +++ b/docs/dev/authors.rst @@ -0,0 +1,5 @@ +Authors +======= + + +.. include:: ../../AUTHORS \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index e2bc2b92..8bef62bb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -69,11 +69,23 @@ instructions for getting the most out of Requests. user/intro user/install - user/faq -.. user/quickstart + user/quickstart user/advanced +Community Guide +----------------- + +This part of the documentation, which is mostly prose, details the +Requests ecosystem and community. + +.. toctree:: + :maxdepth: 2 + + community/faq + community/support + community/updates + API Documentation ----------------- @@ -97,3 +109,4 @@ you. dev/internals dev/todo + dev/authors diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index fea07662..75ea128a 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -1,84 +1,54 @@ -Feature Overview -================ - -Requests is designed to solve a 90% use case — making simple requests. While most -HTTP libraries are extremely extensible, they often attempt to support the entire HTTP Spec. -This often leads to extremely messy and cumbersome APIs, as is the case with urllib2. Requests abandons support for edge-cases, and focuses on the essentials. - - -.. _features: - -Requests Can: -------------- - -- Make **GET**, **POST**, **PUT**, **DELETE**, and **HEAD** requests. -- Handle HTTP and HTTPS Requests -- Add Request headers (with a simple dictionary) -- URLEncode your Form Data (with a simple dictionary) -- Add Multi-part File Uploads (with a simple dictionary) -- Handle CookieJars (with a single parameter) -- Add HTTP Authentication (with a single parameter) -- Handle redirects (with history) -- Automatically decompress GZip'd responses -- Support Unicode URLs -- Gracefully timeout -- Interface with Eventlet & Gevent - - -Requests Can't: ---------------- - -- Handle Caching -- Handle Keep-Alives - +.. _quickstart: Quickstart ========== +.. module:: requests -GET Request ------------ +Eager to get started? This page gives a good introduction in how to get started with Requests. This assumes you already have Tablib installed. If you do not, head over to the :ref:`Installation ` section. + +First, make sure that: + +* Tablib is :ref:`installed ` +* Tablib is :ref:`up-to-date ` -Adding Parameters ------------------ +Lets gets started with some simple use cases and examples. - -Adding Headers --------------- - - - -HTTP Basic Auth ---------------- - - -Tracking Redirects +Make a GET Request ------------------ +Making a standard request with Requests is very simple. + +Let's get GitHub's public timeline :: + + r = requests.get('https://github.com/timeline.json') + +Now, we have a :class:`Response` object. We can get all the information +we need from this. + + +Response Content +---------------- + +We can read the content of the server's response:: + + >>> r.content + '[{"repository":{"open_issues":0,"url":"https://github.com/... -HTTP POST (Form Data) +Response Status Codes --------------------- +We can check the response status code:: -HTTP POST (Binary Data) ------------------------ + >>> r.status_code + 200 +Requests also comes with a built-in status code lookup object for easy +reference:: -HTTP POST (Multipart Files) ---------------------------- - - -HTTP PUT --------- - - -HTTP DELETE ------------ - - -HTTP HEAD ---------- + >>> r.status_code == requests.codes.ok + True \ No newline at end of file From 414b9dde840f2624b247f42eec2c11e5ee5c6717 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 14 Aug 2011 22:08:01 -0400 Subject: [PATCH 009/121] not tablib.. --- docs/user/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 75ea128a..768468db 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -5,7 +5,7 @@ Quickstart .. module:: requests -Eager to get started? This page gives a good introduction in how to get started with Requests. This assumes you already have Tablib installed. If you do not, head over to the :ref:`Installation ` section. +Eager to get started? This page gives a good introduction in how to get started with Requests. This assumes you already have Requests installed. If you do not, head over to the :ref:`Installation ` section. First, make sure that: From 2cf0dad5738b877fffc2cc3c6013d4152dfc1860 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 14 Aug 2011 22:12:50 -0400 Subject: [PATCH 010/121] models.. --- docs/user/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 768468db..02d66f68 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -3,7 +3,7 @@ Quickstart ========== -.. module:: requests +.. module:: requests.models Eager to get started? This page gives a good introduction in how to get started with Requests. This assumes you already have Requests installed. If you do not, head over to the :ref:`Installation ` section. From f137db6803d4feadf0616dfb74059df2d6586dc1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 14 Aug 2011 22:20:03 -0400 Subject: [PATCH 011/121] raise_for_status quick start --- docs/user/quickstart.rst | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 02d66f68..06d5f286 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -25,8 +25,7 @@ Let's get GitHub's public timeline :: r = requests.get('https://github.com/timeline.json') -Now, we have a :class:`Response` object. We can get all the information -we need from this. +Now, we have a :class:`Response` object called ``r``. We can get all the information we need from this. Response Content @@ -38,7 +37,6 @@ We can read the content of the server's response:: '[{"repository":{"open_issues":0,"url":"https://github.com/... - Response Status Codes --------------------- @@ -51,4 +49,24 @@ Requests also comes with a built-in status code lookup object for easy reference:: >>> r.status_code == requests.codes.ok - True \ No newline at end of file + True + +If we made a bad request, we can raise it with +:class:`Response.raise_for_status()`:: + + >>> _r = requests.get('http://httpbin.org/status/404') + >>> _r.status_code + 404 + + >>> _r.raise_for_status() + Traceback (most recent call last): + File "requests/models.py", line 394, in raise_for_status + raise self.error + urllib2.HTTPError: HTTP Error 404: NOT FOUND + +But, sice our ``status_code`` was ``200``, when we call it:: + + >>> r.raise_for_status() + None + +All is well. \ No newline at end of file From 2357b13824952b4380c0fd2939eae827595f6ad3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 14 Aug 2011 22:30:30 -0400 Subject: [PATCH 012/121] Response Headers quickstart --- docs/user/quickstart.rst | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 06d5f286..c28ad1c6 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -69,4 +69,39 @@ But, sice our ``status_code`` was ``200``, when we call it:: >>> r.raise_for_status() None -All is well. \ No newline at end of file +All is well. + + +Response Headers +---------------- + +We can view the server's response headers with a simple Python dictionary +interface:: + + >>> r.headers + { + 'status': '200 OK', + 'content-encoding': 'gzip', + 'transfer-encoding': 'chunked', + 'connection': 'close', + 'server': 'nginx/1.0.4', + 'x-runtime': '148ms', + 'etag': '"e1ca502697e5c9317743dc078f67693f"', + 'content-type': 'application/json; charset=utf-8' + } + +The dictionary is special, though: it's made just for HTTP headers. According to `RFC 2616 `_, HTTP +Headers are case-insensitive. + +So, we can access the headers using any capitalization we want:: + + >>> r.headers['Content-Type'] + 'application/json; charset=utf-8' + + >>> r.headers.get('content-type') + 'application/json; charset=utf-8' + +If a header doesn't exist in the Response, its value defaults to ``None``:: + + >>> r.headers['X-Random'] + None \ No newline at end of file From 49e5df1ff9771810108a346cf8329c196cb099b2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 14 Aug 2011 22:32:35 -0400 Subject: [PATCH 013/121] whoops --- docs/user/quickstart.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index c28ad1c6..4669e810 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -9,8 +9,8 @@ Eager to get started? This page gives a good introduction in how to get started First, make sure that: -* Tablib is :ref:`installed ` -* Tablib is :ref:`up-to-date ` +* Requests is :ref:`installed ` +* Requests is :ref:`up-to-date ` Lets gets started with some simple use cases and examples. From c510d21e4ee8affe66ad0f5c1de32c659bf04fc3 Mon Sep 17 00:00:00 2001 From: moliware Date: Mon, 15 Aug 2011 11:58:57 +0200 Subject: [PATCH 014/121] headers.update is not the same that add headers one by one with add_header --- requests/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 08f3e321..3c0d0517 100644 --- a/requests/models.py +++ b/requests/models.py @@ -300,7 +300,8 @@ class Request(object): req = _Request(url, data=self._enc_data, method=self.method) if self.headers: - req.headers.update(self.headers) + for k,v in self.headers.iteritems(): + req.add_header(k, v) if not self.sent or anyway: From 647cdd38476f6b21887d3ebcd6e0c79a6c97baa1 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Mon, 15 Aug 2011 16:51:33 +0200 Subject: [PATCH 015/121] Use socket.setdefaulttimeout(timeout) only if timeout argument doesn't exist (Python <2.6) - https://github.com/kennethreitz/requests/issues/112 --- AUTHORS | 3 ++- requests/models.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 38528a61..196f0cd2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -31,4 +31,5 @@ Patches and Suggestions - Tamás Gulácsi - Rubén Abad - Peter Manser -- Jeremy Selie \ No newline at end of file +- Jeremy Selie +- Jens Diemer \ No newline at end of file diff --git a/requests/models.py b/requests/models.py index 08f3e321..5f778691 100644 --- a/requests/models.py +++ b/requests/models.py @@ -38,7 +38,8 @@ class Request(object): params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, allow_redirects=False, proxies=None): - socket.setdefaulttimeout(timeout) + #socket.setdefaulttimeout(timeout) + self.timeout = timeout #: Request URL. self.url = url @@ -306,7 +307,17 @@ class Request(object): try: opener = self._get_opener() - resp = opener(req) + try: + resp = opener(req, timeout=self.timeout) + except TypeError, err: + # timeout argument is new since Python v2.6 + if not "timeout" in str(err): + raise + # set global socket timeout + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(self.timeout) + resp = opener(req) + socket.setdefaulttimeout(old_timeout) if self.cookiejar is not None: self.cookiejar.extract_cookies(resp, req) From 88f79b74ce70815b343f5eeff871f0ee9e8cec67 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Mon, 15 Aug 2011 17:03:37 +0200 Subject: [PATCH 016/121] use settings.timeout_fallback bool for socket.setdefaulttimeout() fallback --- requests/config.py | 1 + requests/models.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/requests/config.py b/requests/config.py index 1ba438fd..89e40c98 100644 --- a/requests/config.py +++ b/requests/config.py @@ -57,3 +57,4 @@ settings = Settings() settings.base_headers = {'User-Agent': 'python-requests.org'} settings.accept_gzip = True settings.proxies = None +settings.timeout_fallback = True # Use socket.setdefaulttimeout() as fallback? diff --git a/requests/models.py b/requests/models.py index 5f778691..845a778e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -38,7 +38,7 @@ class Request(object): params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, allow_redirects=False, proxies=None): - #socket.setdefaulttimeout(timeout) + #: Float describ the timeout of the request. (Use socket.setdefaulttimeout() as fallback) self.timeout = timeout #: Request URL. @@ -313,11 +313,17 @@ class Request(object): # timeout argument is new since Python v2.6 if not "timeout" in str(err): raise - # set global socket timeout - old_timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(self.timeout) + + if settings.timeout_fallback: + # fall-back and use global socket timeout (This is not thread-safe!) + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(self.timeout) + resp = opener(req) - socket.setdefaulttimeout(old_timeout) + + if settings.timeout_fallback: + # restore gobal timeout + socket.setdefaulttimeout(old_timeout) if self.cookiejar is not None: self.cookiejar.extract_cookies(resp, req) From ca428504d485b0495b5e794cdd1b47bbb0bd890f Mon Sep 17 00:00:00 2001 From: Tom Hogans Date: Mon, 15 Aug 2011 16:01:26 -0400 Subject: [PATCH 017/121] Added request.session functionality --- AUTHORS | 3 ++- requests/session.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 requests/session.py diff --git a/AUTHORS b/AUTHORS index 38528a61..acaa7583 100644 --- a/AUTHORS +++ b/AUTHORS @@ -31,4 +31,5 @@ Patches and Suggestions - Tamás Gulácsi - Rubén Abad - Peter Manser -- Jeremy Selie \ No newline at end of file +- Jeremy Selie +- Tom Hogans diff --git a/requests/session.py b/requests/session.py new file mode 100644 index 00000000..72505424 --- /dev/null +++ b/requests/session.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +""" +requests.session +~~~~~~~~~~~~~~~ + +This module provides a Session object to manage and persist settings across +requests (cookies, auth, proxies). + +""" + +import requests.api +import cookielib + +class Session(object): + __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies'] + + def __init__(self, **kwargs): + # Set up a CookieJar to be used by default + self.cookies = cookielib.FileCookieJar() + # Map args from kwargs to instance-local variables + map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v), + kwargs.iterkeys(), kwargs.itervalues()) + # Map and wrap requests.api methods + self._map_api_methods() + + def _map_api_methods(self): + """ Reads each available method from requests.api and decorates + them with a wrapper that inserts any instance-local attributes + (from __attrs__) that have been set, combining them with **kwargs """ + def pass_args(func): + def wrapper_func(*args, **kwargs): + inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() + if k in self.__attrs__) + # Combine instance-local values with kwargs values, with + # priority to values in kwargs + kwargs = dict(inst_attrs.items() + kwargs.items()) + return func(*args, **kwargs) + return wrapper_func + # Map and decorate each function available in requests.api + map(lambda fn: setattr(self, fn, pass_args(getattr(requests.api, fn))), + requests.api.__all__) + + From 08a821381aa87803e925eb1e9c61269e4186fa80 Mon Sep 17 00:00:00 2001 From: alopatin Date: Mon, 15 Aug 2011 22:53:22 -0300 Subject: [PATCH 018/121] Fixed misc spelling mistakes. --- requests/status_codes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/status_codes.py b/requests/status_codes.py index 28ce00b8..e8023142 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -22,7 +22,7 @@ _codes = { # Redirection. 300: ('multiple_choices',), - 301: ('moved_pemanently', 'moved'), + 301: ('moved_permanently', 'moved'), 302: ('found',), 302: ('see_other', 'other'), 304: ('not_modified',), @@ -62,7 +62,7 @@ _codes = { 499: ('client_closed_request',), # Server Error. - 500: ('iternal_server_error', 'server_error'), + 500: ('internal_server_error', 'server_error'), 501: ('not_implemented',), 502: ('bad_gateway',), 503: ('service_unavailable', 'unavailable'), From 8f72c48548ef033b31f944e4ed4ada1e753d21d2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 15 Aug 2011 22:17:55 -0400 Subject: [PATCH 019/121] added alex to authors --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 196f0cd2..ccb95e24 100644 --- a/AUTHORS +++ b/AUTHORS @@ -32,4 +32,5 @@ Patches and Suggestions - Rubén Abad - Peter Manser - Jeremy Selie -- Jens Diemer \ No newline at end of file +- Jens Diemer +- Alex <@alopatin> \ No newline at end of file From c2fd5686950f49128c9d282bf28d440a71225c6b Mon Sep 17 00:00:00 2001 From: Tom Hogans Date: Tue, 16 Aug 2011 01:38:39 -0400 Subject: [PATCH 020/121] Added tests for requests.session --- test_requests.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test_requests.py b/test_requests.py index ff98c412..d6f948ee 100755 --- a/test_requests.py +++ b/test_requests.py @@ -13,6 +13,7 @@ except ImportError: import requests +from requests.session import Session HTTPBIN_URL = 'http://httpbin.org/' @@ -455,5 +456,28 @@ class RequestsTestSuite(unittest.TestCase): self.assertEquals(len(r.history), 3) + def test_session_HTTP_200_OK_GET(self): + s = Session() + r = s.get(httpbin('/')) + self.assertEqual(r.status_code, 200) + + def test_session_HTTPS_200_OK_GET(self): + s = Session() + r = s.get(httpsbin('/')) + self.assertEqual(r.status_code, 200) + + def test_session_persistent_headers(self): + heads = {'User-agent': 'Mozilla/5.0'} + s = Session() + s.headers = heads + # Make 2 requests from Session object, should send header both times + r1 = s.get(httpbin('user-agent') + assert heads['User-agent'] in r1.content + r2 = s.get(httpbin('user-agent') + assert heads['User-agent'] in r2.content + self.assertEqual(r.status_code, 200) + + + if __name__ == '__main__': unittest.main() From 0ed641a26ec2200de00e4bbf3d170c767375351e Mon Sep 17 00:00:00 2001 From: Tom Hogans Date: Tue, 16 Aug 2011 02:00:11 -0400 Subject: [PATCH 021/121] Added docs regarding requests.session to docs/user/advanced.rst --- docs/user/advanced.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index e69de29b..066ebdef 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -0,0 +1,22 @@ +.. _advanced: + +Advanced Usage +============== + +This document covers more advanced features. + +Session Objects +=============== + +.. module:: requests.session + +The Session object allows you to persist certain parameters across requests. It also establishes a CookieJar by default and passes it along in any requests made from the Session instance. For a complete list of allowed parameters, please see the *__attrs__* field in *requests/session.py*. :: + + from requests.session import Session + + s = Session() + s.get("http://httpbin.org/cookies/set/sessioncookie/123456789") + r = s.get("http://httpbin.org/cookies") + print r.content + +Note: Certain parameters are best set at the request.config level (i.e. a global proxy, user agent header). From f319006813c76454d5a0dfccebc6d73f02de88f1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 22:15:03 -0400 Subject: [PATCH 022/121] session => sessions --- requests/{session.py => sessions.py} | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) rename requests/{session.py => sessions.py} (91%) diff --git a/requests/session.py b/requests/sessions.py similarity index 91% rename from requests/session.py rename to requests/sessions.py index 72505424..9303a825 100644 --- a/requests/session.py +++ b/requests/sessions.py @@ -23,22 +23,25 @@ class Session(object): kwargs.iterkeys(), kwargs.itervalues()) # Map and wrap requests.api methods self._map_api_methods() - + + def __repr__(self): + return '' % (id(self)) + def _map_api_methods(self): """ Reads each available method from requests.api and decorates them with a wrapper that inserts any instance-local attributes (from __attrs__) that have been set, combining them with **kwargs """ def pass_args(func): def wrapper_func(*args, **kwargs): - inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() + inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() if k in self.__attrs__) - # Combine instance-local values with kwargs values, with + # Combine instance-local values with kwargs values, with # priority to values in kwargs kwargs = dict(inst_attrs.items() + kwargs.items()) return func(*args, **kwargs) return wrapper_func # Map and decorate each function available in requests.api map(lambda fn: setattr(self, fn, pass_args(getattr(requests.api, fn))), - requests.api.__all__) + requests.api.__all__) From 853383a8d3e58ba30c6682cd7ddc0f2d82f42b18 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 22:16:22 -0400 Subject: [PATCH 023/121] docstring fix --- requests/sessions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 9303a825..5f62a78f 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -28,9 +28,10 @@ class Session(object): return '' % (id(self)) def _map_api_methods(self): - """ Reads each available method from requests.api and decorates - them with a wrapper that inserts any instance-local attributes - (from __attrs__) that have been set, combining them with **kwargs """ + """Reads each available method from requests.api and decorates + them with a wrapper, which inserts any instance-local attributes + (from __attrs__) that have been set, combining them with **kwargs. + """ def pass_args(func): def wrapper_func(*args, **kwargs): inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() From 5bb18810d4a9bad0255628dde7d3d2825ca5113a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 22:19:49 -0400 Subject: [PATCH 024/121] memory location repr for requests-client --- requests/sessions.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/requests/sessions.py b/requests/sessions.py index 5f62a78f..f8a6883f 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -16,22 +16,26 @@ class Session(object): __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies'] def __init__(self, **kwargs): + # Set up a CookieJar to be used by default self.cookies = cookielib.FileCookieJar() + # Map args from kwargs to instance-local variables map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v), kwargs.iterkeys(), kwargs.itervalues()) + # Map and wrap requests.api methods self._map_api_methods() def __repr__(self): - return '' % (id(self)) + return '' % (id(self)) def _map_api_methods(self): """Reads each available method from requests.api and decorates them with a wrapper, which inserts any instance-local attributes (from __attrs__) that have been set, combining them with **kwargs. """ + def pass_args(func): def wrapper_func(*args, **kwargs): inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() @@ -41,6 +45,7 @@ class Session(object): kwargs = dict(inst_attrs.items() + kwargs.items()) return func(*args, **kwargs) return wrapper_func + # Map and decorate each function available in requests.api map(lambda fn: setattr(self, fn, pass_args(getattr(requests.api, fn))), requests.api.__all__) From efba606e574707968ec2b35985b8cbb78213cb71 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 22:20:26 -0400 Subject: [PATCH 025/121] doctoring for Session --- requests/sessions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requests/sessions.py b/requests/sessions.py index f8a6883f..6681e3cc 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -13,8 +13,11 @@ import requests.api import cookielib class Session(object): + """A Requests session.""" + __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies'] + def __init__(self, **kwargs): # Set up a CookieJar to be used by default @@ -27,9 +30,11 @@ class Session(object): # Map and wrap requests.api methods self._map_api_methods() + def __repr__(self): return '' % (id(self)) + def _map_api_methods(self): """Reads each available method from requests.api and decorates them with a wrapper, which inserts any instance-local attributes From 519b9cee969e7364654a05ed466db253d35370d3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 22:55:10 -0400 Subject: [PATCH 026/121] relative imports --- requests/sessions.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 6681e3cc..1ce94251 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -9,13 +9,16 @@ requests (cookies, auth, proxies). """ -import requests.api import cookielib +from . import api + + + class Session(object): """A Requests session.""" - __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies'] + __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks'] def __init__(self, **kwargs): @@ -34,6 +37,13 @@ class Session(object): def __repr__(self): return '' % (id(self)) + def __enter__(self): + return self + + def __exit__(self, *args): + # print args + pass + def _map_api_methods(self): """Reads each available method from requests.api and decorates @@ -52,7 +62,11 @@ class Session(object): return wrapper_func # Map and decorate each function available in requests.api - map(lambda fn: setattr(self, fn, pass_args(getattr(requests.api, fn))), - requests.api.__all__) + map(lambda fn: setattr(self, fn, pass_args(getattr(api, fn))), + api.__all__) +def session(**kwargs): + """Returns a :class:`Session` for context-managment.""" + + return Session(**kwargs) \ No newline at end of file From fc04d8a368b373945f2b7d701a683d1167ff4101 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 22:55:18 -0400 Subject: [PATCH 027/121] cleanups --- requests/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requests/config.py b/requests/config.py index 89e40c98..ce464484 100644 --- a/requests/config.py +++ b/requests/config.py @@ -53,8 +53,12 @@ class Settings(object): return None return object.__getattribute__(self, key) + settings = Settings() + settings.base_headers = {'User-Agent': 'python-requests.org'} settings.accept_gzip = True settings.proxies = None -settings.timeout_fallback = True # Use socket.setdefaulttimeout() as fallback? + +#: Use socket.setdefaulttimeout() as fallback? +settings.timeout_fallback = True From d0cb56932c251667420e4876f99f5fa6b5c8a9a6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 23:57:44 -0400 Subject: [PATCH 028/121] new hooks system #118 --- requests/api.py | 13 +++++++++---- requests/hooks.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 requests/hooks.py diff --git a/requests/api.py b/requests/api.py index 1a2ce300..a1a39387 100644 --- a/requests/api.py +++ b/requests/api.py @@ -12,15 +12,16 @@ This module impliments the Requests API. """ import config -from .models import Request, Response, AuthManager, AuthObject, auth_manager +from .models import Request, Response, AuthObject from .status_codes import codes +from .hooks import dispatch_hook __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, - timeout=None, allow_redirects=False, proxies=None): + timeout=None, allow_redirects=False, proxies=None, hooks=None): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -37,7 +38,7 @@ def request(method, url, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - r = Request( + args = dict( method = method, url = url, data = data, @@ -45,12 +46,16 @@ def request(method, url, headers = headers, cookiejar = cookies, files = files, - auth = auth or auth_manager.get_auth(url), + auth = auth, timeout = timeout or config.settings.timeout, allow_redirects = allow_redirects, proxies = proxies or config.settings.proxies ) + args = dispatch_hook('args', hooks, args) + + r = Request(**args) + r.send() return r.response diff --git a/requests/hooks.py b/requests/hooks.py new file mode 100644 index 00000000..7e9f23a4 --- /dev/null +++ b/requests/hooks.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +""" +requests.hooks +~~~~~~~~~~~~~~ + +This module provides the capabilities for the Requests hooks system. +""" + +import warnings + +def dispatch_hook(key, hooks, hook_data): + """""" + + hooks = hooks or dict() + + if key in hooks: + try: + return hooks.get(key).__call__(hook_data) or hook_data + + except Exception, why: + warnings.warn(str(why)) + + + return hook_data From 86a06c5c184272ffd8e6fd4a33b427400713f739 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 23:58:07 -0400 Subject: [PATCH 029/121] auth_manager internal --- requests/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/core.py b/requests/core.py index 7b5f4b20..e02a75ed 100644 --- a/requests/core.py +++ b/requests/core.py @@ -19,7 +19,7 @@ __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' -from models import HTTPError, auth_manager +from models import HTTPError from api import * from exceptions import * from status_codes import codes From 9322d13f610fc26f63cc51c8c4b0a907d7392153 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 16 Aug 2011 23:58:55 -0400 Subject: [PATCH 030/121] cleanups --- test_requests.py | 44 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/test_requests.py b/test_requests.py index d6f948ee..dd923471 100755 --- a/test_requests.py +++ b/test_requests.py @@ -13,7 +13,7 @@ except ImportError: import requests -from requests.session import Session +from requests.sessions import Session HTTPBIN_URL = 'http://httpbin.org/' @@ -132,22 +132,21 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(r.status_code, 200) - def test_AUTH_HTTPS_200_OK_GET(self): + def test_AUTH_HTTP_200_OK_GET(self): for service in SERVICES: auth = ('user', 'pass') url = service('basic-auth', 'user', 'pass') - r = requests.get(url, auth=auth) + r = requests.get(url, auth=auth) + # print r.__dict__ self.assertEqual(r.status_code, 200) + r = requests.get(url) self.assertEqual(r.status_code, 200) - # reset auto authentication - requests.auth_manager.empty() - def test_POSTBIN_GET_POST_FILES(self): @@ -245,21 +244,6 @@ class RequestsTestSuite(unittest.TestCase): r.content.decode('ascii') - def test_autoauth(self): - - http_auth = ('user', 'pass') - requests.auth_manager.add_auth('httpbin.org', http_auth) - - r = requests.get(httpbin('basic-auth', 'user', 'pass')) - self.assertEquals(r.status_code, 200) - - - requests.auth_manager.add_auth('httpbin.ep.io', http_auth) - - r = requests.get(httpsbin('basic-auth', 'user', 'pass')) - self.assertEquals(r.status_code, 200) - - def test_unicode_get(self): for service in SERVICES: @@ -457,27 +441,35 @@ class RequestsTestSuite(unittest.TestCase): def test_session_HTTP_200_OK_GET(self): + s = Session() r = s.get(httpbin('/')) self.assertEqual(r.status_code, 200) + def test_session_HTTPS_200_OK_GET(self): + s = Session() r = s.get(httpsbin('/')) self.assertEqual(r.status_code, 200) + def test_session_persistent_headers(self): + heads = {'User-agent': 'Mozilla/5.0'} + s = Session() s.headers = heads # Make 2 requests from Session object, should send header both times - r1 = s.get(httpbin('user-agent') + r1 = s.get(httpbin('user-agent')) + assert heads['User-agent'] in r1.content - r2 = s.get(httpbin('user-agent') + r2 = s.get(httpbin('user-agent')) + assert heads['User-agent'] in r2.content - self.assertEqual(r.status_code, 200) - - + self.assertEqual(r2.status_code, 200) + + if __name__ == '__main__': unittest.main() From 6b11cf21f3d703315f4ae82ee493ee52a57d0b30 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:00:47 -0400 Subject: [PATCH 031/121] todo --- requests/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index a2775550..693aaa44 100644 --- a/requests/models.py +++ b/requests/models.py @@ -128,6 +128,7 @@ class Request(object): if self.auth: if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): + # TODO: REMOVE THIS COMPLETELY auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) self.auth.handler = self.auth.handler(auth_manager) auth_manager.add_auth(self.url, self.auth) @@ -314,14 +315,14 @@ class Request(object): # timeout argument is new since Python v2.6 if not "timeout" in str(err): raise - + if settings.timeout_fallback: # fall-back and use global socket timeout (This is not thread-safe!) old_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(self.timeout) - + resp = opener(req) - + if settings.timeout_fallback: # restore gobal timeout socket.setdefaulttimeout(old_timeout) From 7322aa5b6442af61a7c0d96c33c7e253215b6f95 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:02:18 -0400 Subject: [PATCH 032/121] newlines --- requests/api.py | 3 ++- requests/models.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requests/api.py b/requests/api.py index a1a39387..d81a56d7 100644 --- a/requests/api.py +++ b/requests/api.py @@ -23,7 +23,8 @@ def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=False, proxies=None, hooks=None): - """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. + """Constructs and sends a :class:`Request `. + Returns :class:`Response ` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. diff --git a/requests/models.py b/requests/models.py index 693aaa44..3c757ae2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -38,7 +38,8 @@ class Request(object): params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, allow_redirects=False, proxies=None): - #: Float describ the timeout of the request. (Use socket.setdefaulttimeout() as fallback) + #: Float describ the timeout of the request. + # (Use socket.setdefaulttimeout() as fallback) self.timeout = timeout #: Request URL. From 8d2ab68a9716670fb44e6be729aa9d72fa433876 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:08:06 -0400 Subject: [PATCH 033/121] space --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 3c757ae2..2a699259 100644 --- a/requests/models.py +++ b/requests/models.py @@ -39,7 +39,7 @@ class Request(object): allow_redirects=False, proxies=None): #: Float describ the timeout of the request. - # (Use socket.setdefaulttimeout() as fallback) + # (Use socket.setdefaulttimeout() as fallback) self.timeout = timeout #: Request URL. From 17d3e1d498cd8afade26a31ec313739a71550eed Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:08:39 -0400 Subject: [PATCH 034/121] whitespace --- requests/models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/requests/models.py b/requests/models.py index 2a699259..90b356f9 100644 --- a/requests/models.py +++ b/requests/models.py @@ -44,23 +44,31 @@ class Request(object): #: Request URL. self.url = url + #: Dictonary of HTTP Headers to attach to the :class:`Request `. self.headers = headers + #: Dictionary of files to multipart upload (``{filename: content}``). self.files = files + #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. self.method = method + #: Dictionary or byte of request body data to attach to the #: :class:`Request `. self.data = None + #: Dictionary or byte of querystring data to attach to the #: :class:`Request `. self.params = None + #: True if :class:`Request ` is part of a redirect chain (disables history #: and HTTPError storage). self.redirect = redirect + #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) self.allow_redirects = allow_redirects + # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = proxies @@ -75,10 +83,13 @@ class Request(object): auth = AuthObject(*auth) if not auth: auth = auth_manager.get_auth(self.url) + #: :class:`AuthObject` to attach to :class:`Request `. self.auth = auth + #: CookieJar to attach to :class:`Request `. self.cookiejar = cookiejar + #: True if Request has been sent. self.sent = False From 24da41005c5d36922516d22aa9d596ec88a1a8d8 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:09:38 -0400 Subject: [PATCH 035/121] hmmm --- requests/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 90b356f9..a48e67ea 100644 --- a/requests/models.py +++ b/requests/models.py @@ -260,7 +260,7 @@ class Request(object): def _build_url(self): - """Build the actual URL to use""" + """Build the actual URL to use.""" # Support for unicode domain names. parsed_url = list(urlparse(self.url)) @@ -286,6 +286,7 @@ class Request(object): :param anyway: If True, request will be sent, even if it has already been sent. """ + self._checks() success = False @@ -495,8 +496,10 @@ class AuthManager(object): def reduce_uri(self, uri, default_port=True): """Accept authority or URI and extract only the authority and path.""" + # note HTTP URLs do not have a userinfo component parts = urllib2.urlparse.urlsplit(uri) + if parts[1]: # URI scheme = parts[0] From 783e540d6dc63c3fdebdcc94d163a05a54127dd6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:09:47 -0400 Subject: [PATCH 036/121] sigh --- requests/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requests/models.py b/requests/models.py index a48e67ea..d32193fd 100644 --- a/requests/models.py +++ b/requests/models.py @@ -510,7 +510,9 @@ class AuthManager(object): scheme = None authority = uri path = '/' + host, port = urllib2.splitport(authority) + if default_port and port is None and scheme is not None: dport = {"http": 80, "https": 443, From 5ba43d0fe5ff5b1705e94ae1b7f2abfe744c56d4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:16:57 -0400 Subject: [PATCH 037/121] hook plans --- requests/hooks.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/requests/hooks.py b/requests/hooks.py index 7e9f23a4..272dc07d 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -5,6 +5,21 @@ requests.hooks ~~~~~~~~~~~~~~ This module provides the capabilities for the Requests hooks system. + +Available hooks: + +``args``: + A dictionary of the arguments being sent to Request(). + +``pre-request``: + The Request object, directly before being sent. + +``post-request``: + The Request object, directly after being sent. + +``response``: + The response generated from a Request. + """ import warnings From e102c1b4ca68c19920dfb835f89719985f532b2c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:17:58 -0400 Subject: [PATCH 038/121] RTD doesn't work w/ this --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 899bc588..8987bb8c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -155,7 +155,7 @@ html_sidebars = { #html_split_index = False # If true, links to the reST sources are added to the pages. -html_show_sourcelink = True +html_show_sourcelink = False # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = False From 8ef0e88220dd5c87e6489a174ce4d87654a6b2dc Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:27:41 -0400 Subject: [PATCH 039/121] attach hooks to Request --- requests/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/api.py b/requests/api.py index d81a56d7..e6ff715c 100644 --- a/requests/api.py +++ b/requests/api.py @@ -50,12 +50,12 @@ def request(method, url, auth = auth, timeout = timeout or config.settings.timeout, allow_redirects = allow_redirects, - proxies = proxies or config.settings.proxies + proxies = proxies or config.settings.proxies, ) args = dispatch_hook('args', hooks, args) - r = Request(**args) + r = Request(hooks=hooks, **args) r.send() From 3e8a14e133d8c2dc35a76e0a2a2ae3151c171c26 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:27:53 -0400 Subject: [PATCH 040/121] pre_request hook --- requests/hooks.py | 4 ++-- requests/models.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/requests/hooks.py b/requests/hooks.py index 272dc07d..c5a24c24 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -11,10 +11,10 @@ Available hooks: ``args``: A dictionary of the arguments being sent to Request(). -``pre-request``: +``pre_request``: The Request object, directly before being sent. -``post-request``: +``post_request``: The Request object, directly after being sent. ``response``: diff --git a/requests/models.py b/requests/models.py index d32193fd..daf48ea8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -16,6 +16,7 @@ from urlparse import urlparse, urlunparse, urljoin from datetime import datetime from .config import settings +from .hooks import dispatch_hook from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode @@ -36,7 +37,7 @@ class Request(object): def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, - allow_redirects=False, proxies=None): + allow_redirects=False, proxies=None, hooks=None): #: Float describ the timeout of the request. # (Use socket.setdefaulttimeout() as fallback) @@ -93,6 +94,9 @@ class Request(object): #: True if Request has been sent. self.sent = False + #: Dictionary of event hook callbacks. + self.hooks = hooks + # Header manipulation and defaults. @@ -323,10 +327,18 @@ class Request(object): try: opener = self._get_opener() try: + + # pre-request hook. + self.__dict__.update( + dispatch_hook('pre_request', + self.hooks, self.__dict__) + ) + resp = opener(req, timeout=self.timeout) + except TypeError, err: # timeout argument is new since Python v2.6 - if not "timeout" in str(err): + if not 'timeout' in str(err): raise if settings.timeout_fallback: From 2a0eff9c7130371874a07ec74806cc703ebad2cc Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:33:21 -0400 Subject: [PATCH 041/121] docstrings --- requests/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/hooks.py b/requests/hooks.py index c5a24c24..59c82696 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -25,7 +25,7 @@ Available hooks: import warnings def dispatch_hook(key, hooks, hook_data): - """""" + """Dipatches a hook dictionary on a given peice of data.""" hooks = hooks or dict() From 7836536459b6a15796c004440d3f927281808455 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:33:28 -0400 Subject: [PATCH 042/121] post_request hook --- requests/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requests/models.py b/requests/models.py index daf48ea8..f9670a4c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -370,6 +370,9 @@ class Request(object): self.sent = self.response.ok + self.__dict__.update( + dispatch_hook('post_request', self.hooks, self.__dict__)) + return self.sent From 281ab5a8f9403135ffd05ff40f177529decd749e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:40:28 -0400 Subject: [PATCH 043/121] move hooks into api layer only --- requests/api.py | 12 ++++++++++++ requests/models.py | 10 ---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/requests/api.py b/requests/api.py index e6ff715c..c5a81c37 100644 --- a/requests/api.py +++ b/requests/api.py @@ -53,12 +53,24 @@ def request(method, url, proxies = proxies or config.settings.proxies, ) + # Arguments manipulation hook. args = dispatch_hook('args', hooks, args) + r = Request(hooks=hooks, **args) + # Pre-request hook. + r = dispatch_hook('pre_request', hooks, r) + + # Send the HTTP Request. r.send() + # Post-request hook. + r = dispatch_hook('post_request', hooks, r) + + # Response manipulation hook. + r.response = dispatch_hook('response', hooks, r.response) + return r.response diff --git a/requests/models.py b/requests/models.py index f9670a4c..c802f030 100644 --- a/requests/models.py +++ b/requests/models.py @@ -16,7 +16,6 @@ from urlparse import urlparse, urlunparse, urljoin from datetime import datetime from .config import settings -from .hooks import dispatch_hook from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode @@ -328,12 +327,6 @@ class Request(object): opener = self._get_opener() try: - # pre-request hook. - self.__dict__.update( - dispatch_hook('pre_request', - self.hooks, self.__dict__) - ) - resp = opener(req, timeout=self.timeout) except TypeError, err: @@ -370,9 +363,6 @@ class Request(object): self.sent = self.response.ok - self.__dict__.update( - dispatch_hook('post_request', self.hooks, self.__dict__)) - return self.sent From 855c27370790c0ba781d20f776a907fa5e2995ae Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:40:37 -0400 Subject: [PATCH 044/121] whitespace --- requests/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/hooks.py b/requests/hooks.py index 59c82696..2938029b 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -24,6 +24,7 @@ Available hooks: import warnings + def dispatch_hook(key, hooks, hook_data): """Dipatches a hook dictionary on a given peice of data.""" @@ -36,5 +37,4 @@ def dispatch_hook(key, hooks, hook_data): except Exception, why: warnings.warn(str(why)) - return hook_data From 51038497f6551e132bee06c1220daa69a7b7a71d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 00:40:49 -0400 Subject: [PATCH 045/121] better --- requests/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/requests/api.py b/requests/api.py index c5a81c37..a433b0ed 100644 --- a/requests/api.py +++ b/requests/api.py @@ -56,7 +56,6 @@ def request(method, url, # Arguments manipulation hook. args = dispatch_hook('args', hooks, args) - r = Request(hooks=hooks, **args) # Pre-request hook. From 8c737069a53933566403ad115234e9e6c809b3d6 Mon Sep 17 00:00:00 2001 From: Rick Mak Date: Wed, 17 Aug 2011 12:49:19 +0800 Subject: [PATCH 046/121] Return the r.content as unicode. --- requests/models.py | 49 +++++++++++++++++++++++++++++++++- test_unicode.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 test_unicode.py diff --git a/requests/models.py b/requests/models.py index a2775550..2e35c7ec 100644 --- a/requests/models.py +++ b/requests/models.py @@ -10,6 +10,7 @@ import urllib import urllib2 import socket import zlib +import cgi from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin @@ -400,8 +401,54 @@ class Response(object): self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) except zlib.error: pass - return self._content + return self.unicode_content(self._content) + + def get_content_type(self): + content_type = self.headers.get("content-type") + content_type, params = cgi.parse_header(content_type) + return content_type, params + + def get_encoding_from_content_type(self): + content_type, params = self.get_content_type() + if "charset" in params: + return params["charset"].strip("'\"") + + def get_encodings_from_content(self, content): + if self._charset_re is None: + self._charset_re = re.compile( + r']', flags=re.I + ) + return self._charset_re.findall(content) + + def unicode_content(self, content): + """ + Returns the requested content back in unicode. + Tried: + 1. charset from content-type + 2. every encodings from + 3. fall back and replace all unicode characters + """ + # Try charset from content-type + encoding = self.get_encoding_from_content_type() + if encoding: + try: + return unicode(content, encoding) + except UnicodeError: + self.tried_encodings.append(encoding) + + # Try every encodings from + encodings = self.get_encodings_from_content(content) + for encoding in encodings: + if encoding in self.tried_encodings: + continue + try: + return unicode(content, encoding) + except UnicodeError: + self.tried_encodings.append(encoding) + + # Fall back: + return unicode(content, encoding, errors="replace") def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" diff --git a/test_unicode.py b/test_unicode.py new file mode 100644 index 00000000..38a1dfcd --- /dev/null +++ b/test_unicode.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import unittest +import cookielib + +try: + import omnijson as json +except ImportError: + import json + +import requests + + + +HTTPBIN_URL = 'http://httpbin.org/' +HTTPSBIN_URL = 'https://httpbin.ep.io/' + +# HTTPBIN_URL = 'http://staging.httpbin.org/' +# HTTPSBIN_URL = 'https://httpbin-staging.ep.io/' + + +def httpbin(*suffix): + """Returns url for HTTPBIN resource.""" + + return HTTPBIN_URL + '/'.join(suffix) + + +def httpsbin(*suffix): + """Returns url for HTTPSBIN resource.""" + + return HTTPSBIN_URL + '/'.join(suffix) + + +SERVICES = (httpbin, httpsbin) + + + +class RequestsTestSuite(unittest.TestCase): + """Requests test cases.""" + + + def setUp(self): + pass + + + def tearDown(self): + """Teardown.""" + pass + + + def test_HTTP_200_OK_GET_ON_ISO88591(self): + r = requests.get("http://www.qypedeals.de/Verzehrgutschein+für+Jellyfish") + self.assertEqual(r.status_code, 200) + self.assertIsInstance(r.content, unicode) + + def test_HTTP_200_OK_GET_ON_BIG5(self): + r = requests.get("http://google.com.hk/") + self.assertEqual(r.status_code, 200) + self.assertIsInstance(r.content, unicode) + + +if __name__ == '__main__': + unittest.main() From b0a0b509da5df358b1ab2f66f0aa6aebad27bd16 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 01:00:00 -0400 Subject: [PATCH 047/121] session at root package level --- requests/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/core.py b/requests/core.py index e02a75ed..27384397 100644 --- a/requests/core.py +++ b/requests/core.py @@ -22,5 +22,6 @@ __copyright__ = 'Copyright 2011 Kenneth Reitz' from models import HTTPError from api import * from exceptions import * +from sessions import session from status_codes import codes from config import settings \ No newline at end of file From 7e255177dc206ec22118ef98b3b857c598073bd9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 01:23:49 -0400 Subject: [PATCH 048/121] get cookies from response #116 --- requests/models.py | 8 ++++++++ requests/utils.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 requests/utils.py diff --git a/requests/models.py b/requests/models.py index c802f030..5e54f377 100644 --- a/requests/models.py +++ b/requests/models.py @@ -20,6 +20,7 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers +from .utils import dict_from_cookiejar from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod @@ -187,6 +188,12 @@ class Request(object): response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) response.read = resp.read response.close = resp.close + + if self.cookiejar: + + response.cookies = dict_from_cookiejar(self.cookiejar) + + except AttributeError: pass @@ -397,6 +404,7 @@ class Response(object): self.history = [] #: The Request that created the Response. self.request = None + self.cookies = None def __repr__(self): diff --git a/requests/utils.py b/requests/utils.py new file mode 100644 index 00000000..f308f668 --- /dev/null +++ b/requests/utils.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +""" +requests.utils +~~~~~~~~~~~~~~ + +This module provides utlity functions that are used within Requests +that are also useful for external consumption. + +""" + + +def dict_from_cookiejar(cookiejar): + """Returns a key/value dictoinary from a CookieJar.""" + + cookie_dict = {} + + for _, cookies in cookiejar._cookies.items(): + for _, cookies in cookies.items(): + for cookie in cookies.values(): + cookie_dict[cookie.name] = cookie.value + + return cookie_dict + From d9e571737716e2929386f5279ce80c130d772a5b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 01:29:58 -0400 Subject: [PATCH 049/121] cookiejar_from_dict #12 --- requests/utils.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index f308f668..0719f810 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -9,9 +9,12 @@ that are also useful for external consumption. """ +import Cookie +import cookielib + def dict_from_cookiejar(cookiejar): - """Returns a key/value dictoinary from a CookieJar.""" + """Returns a key/value dictionary from a CookieJar.""" cookie_dict = {} @@ -22,3 +25,24 @@ def dict_from_cookiejar(cookiejar): return cookie_dict + +def cookiejar_from_dict(cookie_dict, domain=None): + """Returns a CookieJar from a key/value dictoinary.""" + + # create cookiejar + cj = cookielib.CookieJar() + + for k, v in cookie_dict.items(): + + # create cookie + ck = Cookie.SimpleCookie() + ck.name = v + ck.expires = 0 + ck.path = '/' + ck.domain = domain + + # add cookie to cookiejar + cj.set_cookie(ck) + + return cj + From d0aa0175e6b32bdb5b4f6b87c07d02c0088fcdea Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 01:32:04 -0400 Subject: [PATCH 050/121] v0.6.0 --- requests/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/core.py b/requests/core.py index 27384397..68e53923 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,8 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.5.1' -__build__ = 0x000501 +__version__ = '0.6.0' +__build__ = 0x000600 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' From ec33ce9d29d591e18ded52018501942d3eb02776 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 01:32:12 -0400 Subject: [PATCH 051/121] sp. --- requests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 0719f810..716986b8 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -27,7 +27,7 @@ def dict_from_cookiejar(cookiejar): def cookiejar_from_dict(cookie_dict, domain=None): - """Returns a CookieJar from a key/value dictoinary.""" + """Returns a CookieJar from a key/value dictionary.""" # create cookiejar cj = cookielib.CookieJar() From f7024a60004db98e467260402e6c2f5338e9d619 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 02:01:13 -0400 Subject: [PATCH 052/121] super cookiejar_from_dict powwerrrs --- requests/utils.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 716986b8..e4236b0f 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -9,7 +9,6 @@ that are also useful for external consumption. """ -import Cookie import cookielib @@ -21,28 +20,45 @@ def dict_from_cookiejar(cookiejar): for _, cookies in cookiejar._cookies.items(): for _, cookies in cookies.items(): for cookie in cookies.values(): + # print cookie cookie_dict[cookie.name] = cookie.value return cookie_dict -def cookiejar_from_dict(cookie_dict, domain=None): +def cookiejar_from_dict(cookie_dict): """Returns a CookieJar from a key/value dictionary.""" + # return cookiejar if one was passed in + if isinstance(cookie_dict, cookielib.CookieJar): + return cookie_dict + # create cookiejar cj = cookielib.CookieJar() for k, v in cookie_dict.items(): - # create cookie - ck = Cookie.SimpleCookie() - ck.name = v - ck.expires = 0 - ck.path = '/' - ck.domain = domain + cookie = cookielib.Cookie( + version=0, + name=k, + value=v, + port=None, + port_specified=False, + domain='', + domain_specified=False, + domain_initial_dot=False, + path='/', + path_specified=True, + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False + ) # add cookie to cookiejar - cj.set_cookie(ck) + cj.set_cookie(cookie) return cj - From 66391e5c9da77889ebaf1c8a1cd3f84543d6943b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 02:01:19 -0400 Subject: [PATCH 053/121] ACTIVVVAATTTEEEEEEE --- requests/api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/requests/api.py b/requests/api.py index a433b0ed..9c923e50 100644 --- a/requests/api.py +++ b/requests/api.py @@ -15,7 +15,9 @@ import config from .models import Request, Response, AuthObject from .status_codes import codes from .hooks import dispatch_hook +from .utils import cookiejar_from_dict +from urlparse import urlparse __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') @@ -39,6 +41,11 @@ def request(method, url, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ + if cookies is None: + cookies = {} + + cookies = cookiejar_from_dict(cookies) + args = dict( method = method, url = url, From 467766cfe8b2f4f13f5bee1346b9ab7a8cbfcb62 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 02:25:41 -0400 Subject: [PATCH 054/121] cookie session persistence --- requests/sessions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/requests/sessions.py b/requests/sessions.py index 1ce94251..540089cc 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -12,6 +12,7 @@ requests (cookies, auth, proxies). import cookielib from . import api +from .utils import add_dict_to_cookiejar @@ -58,6 +59,14 @@ class Session(object): # Combine instance-local values with kwargs values, with # priority to values in kwargs kwargs = dict(inst_attrs.items() + kwargs.items()) + + # If a session request has a cookie_dict, inject the + # values into the existing CookieJar instead. + if isinstance(kwargs.get('cookies', None), dict): + kwargs['cookies'] = add_dict_to_cookiejar( + inst_attrs['cookies'], kwargs['cookies'] + ) + return func(*args, **kwargs) return wrapper_func From e477fce2112dc7078363c34798facbde7be4f7c4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 02:25:56 -0400 Subject: [PATCH 055/121] add_dict_to_cookiejar Signed-off-by: Kenneth Reitz --- requests/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/requests/utils.py b/requests/utils.py index e4236b0f..8ac78b4e 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -36,6 +36,14 @@ def cookiejar_from_dict(cookie_dict): # create cookiejar cj = cookielib.CookieJar() + cj = add_dict_to_cookiejar(cj, cookie_dict) + + return cj + + +def add_dict_to_cookiejar(cj, cookie_dict): + """Returns a CookieJar from a key/value dictionary.""" + for k, v in cookie_dict.items(): cookie = cookielib.Cookie( From abcf81e2a8cd5497080fad7fb4ac6b6a87eacd6c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 02:49:13 -0400 Subject: [PATCH 056/121] docs update --- docs/user/advanced.rst | 46 +++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 066ebdef..97687b2e 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -3,20 +3,46 @@ Advanced Usage ============== -This document covers more advanced features. +This document covers some of Requests more advanced features. + Session Objects -=============== +--------------- -.. module:: requests.session +The Session object allows you to persist certain parameters across +requests. It also establishes a CookieJar and passes it along +to any requests made from the Session instance. -The Session object allows you to persist certain parameters across requests. It also establishes a CookieJar by default and passes it along in any requests made from the Session instance. For a complete list of allowed parameters, please see the *__attrs__* field in *requests/session.py*. :: +A session object has all the methods of the main Requests API. - from requests.session import Session +Let's persist some cookies across requests:: - s = Session() - s.get("http://httpbin.org/cookies/set/sessioncookie/123456789") - r = s.get("http://httpbin.org/cookies") - print r.content + with requests.session() as s: -Note: Certain parameters are best set at the request.config level (i.e. a global proxy, user agent header). + s.get('http://httpbin.org/cookies/set/sessioncookie/123456789') + r = s.get("http://httpbin.org/cookies") + + print r.content + + +Sessions can also be used to provide default data to the request methods:: + + headers = {'x-test': 'true'} + auth = ('user', 'pass') + + with requests.session(auth=auth, headers=headers) as c: + + # both 'x-test' and 'x-test2' are sent + c.get('http://httpbin.org/headers', header={'x-test2', 'true'}) + + +.. admonition:: Global Settings + + Certain parameters are best set at the ``request.config`` level + (e.g.. a global proxy, user agent header). + + +Event Hooks +----------- + +Requests has a hook system that allows you . This is useful for \ No newline at end of file From b7307298d28a30ed893d095338a36f7f62a7f1b9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 02:49:29 -0400 Subject: [PATCH 057/121] merge headers, don't overwrite --- requests/sessions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requests/sessions.py b/requests/sessions.py index 540089cc..50b09f61 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -67,6 +67,9 @@ class Session(object): inst_attrs['cookies'], kwargs['cookies'] ) + if kwargs.get('headers', None) and inst_attrs.get('headers', None): + kwargs['headers'].update(inst_attrs['headers']) + return func(*args, **kwargs) return wrapper_func From 77b37687b69ef21262e04c60401e0bcf712f0357 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 03:07:24 -0400 Subject: [PATCH 058/121] advanced docs --- docs/user/advanced.rst | 58 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 97687b2e..430a4720 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -45,4 +45,60 @@ Sessions can also be used to provide default data to the request methods:: Event Hooks ----------- -Requests has a hook system that allows you . This is useful for \ No newline at end of file +Requests has a hook system that you can use to manipulate portions of +the request process, or signal event handling. + +Available hooks: + +``args``: + A dictionary of the arguments being sent to Request(). + +``pre_request``: + The Request object, directly before being sent. + +``post_request``: + The Request object, directly after being sent. + +``response``: + The response generated from a Request. + + +You can assign a hook function on a per-request basis by passing a +``{hook_name: callback_function}`` dictionary to the ``hooks`` request +paramaeter:: + + hooks=dict(args=print_url) + +That ``callback_function`` will receive a chunk of data as its first +argument. + +:: + + def print_url(args): + print args['url'] + +If an error occurs while executing your callback, a warning is given. + +If the callback function returns a value, it is assumed that it is to +replace the data that was passed in. If the function doesn't return +anything, nothing else is effected. + +Let's print some request method arguments at runtime:: + + >>> requests.get('http://httpbin', hooks=dict(args=print_url)) + http://httpbin + + + +Verbose Logging +--------------- + +If you want to get a good look at what HTTP requests are being sent +by your application, you can turn on verbose logging. + +To do so, just configure Requests with a stream to write to:: + + >>> requests.settings.verbose = sys.stderr + >>> requests.get('http://httpbin.org/headers') + 2011-08-17T03:04:23.380175 GET http://httpbin.org/headers + \ No newline at end of file From 008c77c911f504cd72f7e097f1744daaadab2162 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 03:19:46 -0400 Subject: [PATCH 059/121] hijack arguments --- docs/user/advanced.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 430a4720..499c354d 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -89,6 +89,37 @@ Let's print some request method arguments at runtime:: http://httpbin +Let's hijack some arguments this time:: + + def hack_headers(args): + if not args[headers]: + args['headers'] = dict() + + args['headers'].update({'X-Testing': 'True'}) + + + return args + + hooks = dict(args=hack_headers) + headers = dict(yo=dawg) + + >>> requests.get('http://httpbin/headers', hooks=hooks, headers=headers) + { + "headers": { + "Content-Length": "", + "Accept-Encoding": "gzip", + "Yo": "dawg", + "X-Forwarded-For": "::ffff:24.127.96.129", + "Connection": "close", + "User-Agent": "python-requests.org", + "Host": "httpbin.org", + "X-Testing": "True", + "X-Forwarded-Protocol": "", + "Content-Type": "" + } + } + + Verbose Logging --------------- From 3f8f1dc00c11c501c9596a8d8008e40d4d99179f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 03:19:51 -0400 Subject: [PATCH 060/121] cleaner defaults --- requests/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requests/config.py b/requests/config.py index ce464484..a42f15d2 100644 --- a/requests/config.py +++ b/requests/config.py @@ -12,7 +12,7 @@ class Settings(object): _singleton = {} # attributes with defaults - __attrs__ = ('timeout', 'verbose') + __attrs__ = [] def __init__(self, **kwargs): super(Settings, self).__init__() @@ -59,6 +59,8 @@ settings = Settings() settings.base_headers = {'User-Agent': 'python-requests.org'} settings.accept_gzip = True settings.proxies = None +settings.verbose = None +settings.timeout = None #: Use socket.setdefaulttimeout() as fallback? settings.timeout_fallback = True From 4d0c6cefd033c39b80bd7efb74c53f91c5ec9903 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 03:21:31 -0400 Subject: [PATCH 061/121] yawns --- docs/user/advanced.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 499c354d..83792b58 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -89,7 +89,7 @@ Let's print some request method arguments at runtime:: http://httpbin -Let's hijack some arguments this time:: +Let's hijack some arguments this time with a new callback:: def hack_headers(args): if not args[headers]: @@ -97,12 +97,13 @@ Let's hijack some arguments this time:: args['headers'].update({'X-Testing': 'True'}) - return args hooks = dict(args=hack_headers) headers = dict(yo=dawg) +And give it a try:: + >>> requests.get('http://httpbin/headers', hooks=hooks, headers=headers) { "headers": { From b72eb53a7303c290b1c18c40f1f918a7ec17c58d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 03:25:31 -0400 Subject: [PATCH 062/121] changing history --- HISTORY.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 93cb1428..7d98f8f8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,10 +1,14 @@ History ------- -0.5.2 (2011-09-??) + +0.6.0 (2011-09-??) ++++++++++++++++++ -* status code reference dict? +* New callback hook system +* New persistient sessions object and context manager +* Transparent Dict-cookie handling +* status code reference object * Removed Response.cached * Added Response.request * all args are kwargs From 3992ed6ad17347c128dac1784020c5e627b3e742 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 03:58:55 -0400 Subject: [PATCH 063/121] cookies docs --- docs/user/quickstart.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 4669e810..61d5c7ba 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -104,4 +104,33 @@ So, we can access the headers using any capitalization we want:: If a header doesn't exist in the Response, its value defaults to ``None``:: >>> r.headers['X-Random'] - None \ No newline at end of file + None + + +Cookies +------- + +If a response contains some Cookies, you can get quick access to them:: + + # cookies test url + >>> url = 'http://httpbin.org/cookies/set/requests-is/awesome' + + >>> r = requests.get(url) + + >>> print r.cookies + {'requests-is': 'awesome'} + +The underlying CookieJar is also available for more advanced handing:: + + >>> r.request.cookiejar + + +To send your own cookies to the server, you can use the ``cookies`` +parameter:: + + >>> url = 'http://httpbin.org/cookies' + >>> cookies = dict(cookies_are='working') + + >>> r = requests.get(url, cookies=cookies) + >>> r.content + '{"cookies": {"cookies_are": "working"}}' From d4502aff8751bbb82e74974c8f6423865d1d210a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 04:00:12 -0400 Subject: [PATCH 064/121] consiserererr --- docs/user/quickstart.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 61d5c7ba..60402f91 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -112,9 +112,7 @@ Cookies If a response contains some Cookies, you can get quick access to them:: - # cookies test url >>> url = 'http://httpbin.org/cookies/set/requests-is/awesome' - >>> r = requests.get(url) >>> print r.cookies From 5f603ae787d6d98a8c1384fb693da21efd3c2b18 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 04:01:11 -0400 Subject: [PATCH 065/121] typo --- docs/user/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 60402f91..f9370f59 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -64,7 +64,7 @@ If we made a bad request, we can raise it with raise self.error urllib2.HTTPError: HTTP Error 404: NOT FOUND -But, sice our ``status_code`` was ``200``, when we call it:: +But, since our ``status_code`` was ``200``, when we call it:: >>> r.raise_for_status() None From e0ae2b415513268df3b56cb1d396b01f2888dcdf Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 04:03:54 -0400 Subject: [PATCH 066/121] fixes --- requests/api.py | 14 +++++++------- requests/models.py | 6 ++---- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/requests/api.py b/requests/api.py index 9c923e50..13dc4eaf 100644 --- a/requests/api.py +++ b/requests/api.py @@ -33,7 +33,7 @@ def request(method, url, :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. @@ -87,7 +87,7 @@ def get(url, **kwargs): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. @@ -103,7 +103,7 @@ def head(url, **kwargs): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. @@ -120,7 +120,7 @@ def post(url, data='', **kwargs): :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. @@ -138,7 +138,7 @@ def put(url, data='', **kwargs): :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. @@ -156,7 +156,7 @@ def patch(url, data='', **kwargs): :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. @@ -174,7 +174,7 @@ def delete(url, **kwargs): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. diff --git a/requests/models.py b/requests/models.py index 5e54f377..d6b058fb 100644 --- a/requests/models.py +++ b/requests/models.py @@ -37,7 +37,7 @@ class Request(object): def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, - allow_redirects=False, proxies=None, hooks=None): + allow_redirects=False, proxies=None): #: Float describ the timeout of the request. # (Use socket.setdefaulttimeout() as fallback) @@ -94,9 +94,6 @@ class Request(object): #: True if Request has been sent. self.sent = False - #: Dictionary of event hook callbacks. - self.hooks = hooks - # Header manipulation and defaults. @@ -404,6 +401,7 @@ class Response(object): self.history = [] #: The Request that created the Response. self.request = None + #: A dictionary of Cookies the server sent back. self.cookies = None From 3f873dc22271625c21cc85fe8d1186aa1dd465a0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 04:05:20 -0400 Subject: [PATCH 067/121] hooks in api layer only --- requests/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/api.py b/requests/api.py index 13dc4eaf..0cea63d4 100644 --- a/requests/api.py +++ b/requests/api.py @@ -63,7 +63,7 @@ def request(method, url, # Arguments manipulation hook. args = dispatch_hook('args', hooks, args) - r = Request(hooks=hooks, **args) + r = Request(**args) # Pre-request hook. r = dispatch_hook('pre_request', hooks, r) From 0c21aa55309a522d29fbca76ecb0aa40eda06abe Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 04:16:19 -0400 Subject: [PATCH 068/121] doc cleanups --- docs/api.rst | 13 +------------ docs/user/advanced.rst | 1 + docs/user/intro.rst | 9 +++++++++ 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index fd02285b..cbd24fdb 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -22,7 +22,7 @@ all return a :class:`Response ` object. .. autofunction:: put .. autofunction:: patch .. autofunction:: delete - +.. autofunction:: request ----------- @@ -50,19 +50,8 @@ These items are an internal component to Requests, and should never be seen by the end user (developer). This part of the API documentation exists for those who are extending the functionality of Requests. -Functions -~~~~~~~~~ - -.. autofunction:: request - Classes ~~~~~~~ .. autoclass:: requests.models.Request :inherited-members: - -Structures -~~~~~~~~~~ - -.. autoclass:: requests.structures.CaseInsensitiveDict - :inherited-members: \ No newline at end of file diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 83792b58..cdf1abb6 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -23,6 +23,7 @@ Let's persist some cookies across requests:: r = s.get("http://httpbin.org/cookies") print r.content + # '{"cookies": {"sessioncookie": "123456789"}}' Sessions can also be used to provide default data to the request methods:: diff --git a/docs/user/intro.rst b/docs/user/intro.rst index 64b49f10..217cb81c 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -1,3 +1,5 @@ +.. _introduction: + Introduction ============ @@ -40,14 +42,21 @@ Requests License THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +.. _interpreters: Python Interpreters ------------------- At this time, the following Python platforms are officially supported: * cPython 2.5 +* cPython 2.5.5 +* cPython 2.5.6 * cPython 2.6 +* cPython 2.6.6 +* cPython 2.6.7 * cPython 2.7 +* cPython 2.7.1 +* cPython 2.7.2 * PyPy-c 1.4 * PyPy-c 1.5 From 47a36de74e7697a3030bb0d8d6f324b868001ec4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 04:17:21 -0400 Subject: [PATCH 069/121] supported interpreters --- docs/community/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index 2ccb6a7b..b5717648 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -34,7 +34,7 @@ Chris Adams gave an excellent summary on Python 3 Support? ----------------- -It's on the way. +It's on the way. Here's a list of `supported interpreters `_. Keep-alive Support? From 0b34812afccc17c9d40c72c88f62c289d389faa9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 04:23:16 -0400 Subject: [PATCH 070/121] readme cleanup --- README.rst | 104 ++++++++++++++--------------------------------------- 1 file changed, 27 insertions(+), 77 deletions(-) diff --git a/README.rst b/README.rst index 312dfc3c..feac4c3f 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,35 @@ Requests: HTTP for Humans ========================= -Most existing Python modules for dealing HTTP requests are insane. I have to look up *everything* that I want to do. Most of my worst Python experiences are a result of the various built-in HTTP libraries (yes, even worse than Logging). +Requests is an ISC Licensed HTTP library, written in Python, for human +beings. -But this one's different. This one's going to be awesome. And simple. +Most existing Python modules for sending HTTP requests are extremely +verbose and cumbersome. Python’s builtin urllib2 module provides most of +the HTTP capabilities you should need, but the api is thoroughly broken. +It requires an enormous amount of work (even method overrides) to +perform the simplest of tasks. + +Things shouldn’t be this way. Not in Python. + +:: + + >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) + >>> r.status_code + 204 + >>> r.headers['content-type'] + 'application/json' + >>> r.content + ... + +See `the same code, without Requests `_. + +Requests allow you to send **HEAD**, **GET**, **POST**, **PUT**, +**PATCH**, and **DELETE** HTTP requests. You can add headers, form data, +multipart files, and parameters with simple Python dictionaries, and access the +response data in the same way. It's powered by :py:class:`urllib2`, but it does +all the hard work and crazy hacks for you. -Really simple. Features -------- @@ -55,80 +79,6 @@ Uh oh, we're not authorized! Let's add authentication. :: '{"authenticated": true, "user": "user"}' - -API ---- - -**Requests:** - -All request functions return a Response object (see below). - -If a {filename: fileobject} dictionary is passed in (files=...), a multipart_encode upload will be performed. -If CookieJar object is is passed in (cookies=...), the cookies will be sent with the request. - - HEAD Requests - >>> requests.head(url, params={}, headers={}, cookies=None, auth=None, timeout=None, proxies={}) - - - GET Requests - >>> requests.get(url, params={}, headers={}, cookies=None, auth=None, timeout=None, proxies={}) - - - POST Requests - >>> requests.post(url, data={}, headers={}, files={}, cookies=None, auth=None, timeout=None, allow_redirects=False, params{}, proxies={}) - - - PUT Requests - >>> requests.put(url, data={}, headers={}, files={}, cookies=None, auth=None, timeout=None, allow_redirects=False, params{}, proxies={}) - - - PATCH Requests - >>> requests.patch(url, data={}, headers={}, files={}, cookies=None, auth=None, timeout=None, allow_redirects=False, params{}, proxies={}) - - - DELETE Requests - >>> requests.delete(url, params={}, headers={}, cookies=None, auth=None, timeout=None, allow_redirects=False, params{}, proxies={}) - - - -**Responses:** - - Response.status_code - (Integer) Received HTTP Status Code Response - - Response.headers - ((CaseInsensitive) Dictionary) Received HTTP Response Headers. - - Response.content - (Bytes) Received Content. - - Response.history - (List of Responses) Redirection History. - - Response.url - (String) URL of response. Useful for detecting redirects. - - Response.ok - (Bool) True if no errors occurred during the request, and the status_code is kosher. - - Response.cached - (Bool) True if Response.content is stored within the object. - - Response.error - (HTTPError) If an HTTPError occurred (e.g. status of 404), Otherwise this is None. - - Response.raise_for_status() - Raises HTTPError if a request is not kosher. - - -**HTTP Authentication Registry:** - - You can register AuthObjects to automatically enable HTTP Authentication on requests that contain a registered base URL string. - - >>> requests.auth_manager.add_auth(url, authobject) - - - Installation ------------ From 470af42bf203327f84d5554cf99b4b7207b47abb Mon Sep 17 00:00:00 2001 From: Jeremy Selier Date: Wed, 17 Aug 2011 11:28:47 +0300 Subject: [PATCH 071/121] Fixing myself :) --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 6d2d1965..2d34a221 100644 --- a/AUTHORS +++ b/AUTHORS @@ -31,7 +31,7 @@ Patches and Suggestions - Tamás Gulácsi - Rubén Abad - Peter Manser -- Jeremy Selie +- Jeremy Selier - Jens Diemer - Alex <@alopatin> - Tom Hogans From 9471b0ab889a4684f87952cb951f7f4dc3f59dda Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 04:32:46 -0400 Subject: [PATCH 072/121] let's do this --- HISTORY.rst | 6 +++--- README.rst | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7d98f8f8..b2ae8f9f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,16 +2,16 @@ History ------- -0.6.0 (2011-09-??) +0.6.0 (2011-08-17) ++++++++++++++++++ * New callback hook system * New persistient sessions object and context manager * Transparent Dict-cookie handling -* status code reference object +* Status code reference object * Removed Response.cached * Added Response.request -* all args are kwargs +* All args are kwargs * Relative redirect support * HTTPError handling improvements * Improved https testing diff --git a/README.rst b/README.rst index feac4c3f..529d1445 100644 --- a/README.rst +++ b/README.rst @@ -5,12 +5,12 @@ Requests is an ISC Licensed HTTP library, written in Python, for human beings. Most existing Python modules for sending HTTP requests are extremely -verbose and cumbersome. Python’s builtin urllib2 module provides most of +verbose and cumbersome. Python's builtin urllib2 module provides most of the HTTP capabilities you should need, but the api is thoroughly broken. It requires an enormous amount of work (even method overrides) to perform the simplest of tasks. -Things shouldn’t be this way. Not in Python. +Things shouldn't be this way. Not in Python. :: @@ -27,7 +27,7 @@ See `the same code, without Requests `_. Requests allow you to send **HEAD**, **GET**, **POST**, **PUT**, **PATCH**, and **DELETE** HTTP requests. You can add headers, form data, multipart files, and parameters with simple Python dictionaries, and access the -response data in the same way. It's powered by :py:class:`urllib2`, but it does +response data in the same way. It's powered by urllib2, but it does all the hard work and crazy hacks for you. From 91e75f721508d2198ccb3d4e2979047249ae01f4 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 17 Aug 2011 13:40:29 +0300 Subject: [PATCH 073/121] As proposed by @davidbgk this is only consistent with \o/ :) --- requests/status_codes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/status_codes.py b/requests/status_codes.py index e8023142..c8a47a07 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -22,7 +22,7 @@ _codes = { # Redirection. 300: ('multiple_choices',), - 301: ('moved_permanently', 'moved'), + 301: ('moved_permanently', 'moved', '\\o-'), 302: ('found',), 302: ('see_other', 'other'), 304: ('not_modified',), @@ -36,7 +36,7 @@ _codes = { 401: ('unauthorized',), 402: ('payment_required', 'payment'), 403: ('forbidden',), - 404: ('not_found',), + 404: ('not_found', '-o-'), 405: ('method_not_allowed', 'not_allowed'), 406: ('not_acceptable',), 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), @@ -62,7 +62,7 @@ _codes = { 499: ('client_closed_request',), # Server Error. - 500: ('internal_server_error', 'server_error'), + 500: ('internal_server_error', 'server_error', '/o\\'), 501: ('not_implemented',), 502: ('bad_gateway',), 503: ('service_unavailable', 'unavailable'), From 5e6ecdad9f69b1ff789a17733b8edc6fd7091bd8 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Thu, 8 Sep 2011 02:38:50 +51800 Subject: [PATCH 074/121] Typo in documentation The kwarg is named `headers`, not `header`. Also, its a dict, not a set. --- docs/user/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index cdf1abb6..d145da21 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -34,7 +34,7 @@ Sessions can also be used to provide default data to the request methods:: with requests.session(auth=auth, headers=headers) as c: # both 'x-test' and 'x-test2' are sent - c.get('http://httpbin.org/headers', header={'x-test2', 'true'}) + c.get('http://httpbin.org/headers', headers={'x-test2': 'true'}) .. admonition:: Global Settings From 571d808d25d3391664a69920d4e0a5cb1418af90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Wed, 17 Aug 2011 15:31:24 +0200 Subject: [PATCH 075/121] Fixed bad merge. --- requests/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 0d7b67cb..7690f16a 100644 --- a/requests/models.py +++ b/requests/models.py @@ -230,7 +230,8 @@ class Request(object): # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - url = urljoin(r.url, urllib.quote(urllib.unquote(url))) + if not urlparse(url).netloc: + url = urljoin(r.url, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is 303: From 61d50d531cca57061335c776148c55d452ef5c3e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 09:41:24 -0400 Subject: [PATCH 076/121] Authors += Armin --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 2d34a221..dd1b3109 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,3 +35,4 @@ Patches and Suggestions - Jens Diemer - Alex <@alopatin> - Tom Hogans +- Armin Ronacher \ No newline at end of file From 007963afde4d856803e61357c64f29329a361ad9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 09:45:31 -0400 Subject: [PATCH 077/121] authors --- AUTHORS.orig | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 AUTHORS.orig diff --git a/AUTHORS.orig b/AUTHORS.orig new file mode 100644 index 00000000..dd88224b --- /dev/null +++ b/AUTHORS.orig @@ -0,0 +1,40 @@ +Requests is written and maintained by Kenneth Reitz and +various contributors: + +Development Lead +```````````````` + +- Kenneth Reitz + + +Patches and Suggestions +``````````````````````` + +- Various Pocoo Members +- Chris Adams +- Flavio Percoco Premoli +- Dj Gilcrease +- Justin Murphy +- Rob Madole +- Aram Dulyan +- Johannes Gorset +- 村山めがね (Megane Murayama) +- James Rowe +- Daniel Schauenberg +- Zbigniew Siciarz +- Daniele Tricoli 'Eriol' +- Richard Boulton +- Miguel Olivares +- Alberto Paro +- Jérémy Bethmont +- 潘旭 (Xu Pan) +- Tamás Gulácsi +- Rubén Abad +- Peter Manser +- Jeremy Selie +<<<<<<< HEAD +- Jens Diemer +- Alex <@alopatin> +======= +- Tom Hogans +>>>>>>> 0ed641a26ec2200de00e4bbf3d170c767375351e From c22b71fc5b69736f40f8e491c5a6c4db461321f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Wed, 17 Aug 2011 16:04:53 +0200 Subject: [PATCH 078/121] Handle redirection without scheme. --- requests/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requests/models.py b/requests/models.py index cc9cc6f0..931bafd8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -225,6 +225,11 @@ class Request(object): url = r.headers['location'] + # Handle redirection without scheme (see: RFC 1808 Section 4) + if url.startswith('//'): + parsed_rurl = urlparse(r.url) + url = '%s:%s' % (parsed_rurl.scheme, url) + # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: From 96e7a60b8d8a930d3257b87e3159b5abbbf385d5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 20:13:44 -0400 Subject: [PATCH 079/121] Allow any method, if so inclined #124 --- AUTHORS | 3 ++- requests/models.py | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index dd1b3109..5bd399a2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,4 +35,5 @@ Patches and Suggestions - Jens Diemer - Alex <@alopatin> - Tom Hogans -- Armin Ronacher \ No newline at end of file +- Armin Ronacher +- Shrikant Sharat Kandula \ No newline at end of file diff --git a/requests/models.py b/requests/models.py index d6b058fb..0b22f711 100644 --- a/requests/models.py +++ b/requests/models.py @@ -32,8 +32,6 @@ class Request(object): Requests. Recommended interface is with the Requests functions. """ - _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'PATCH') - def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, @@ -116,14 +114,6 @@ class Request(object): return '' % (self.method) - def __setattr__(self, name, value): - if (name == 'method') and (value): - if not value in self._METHODS: - raise InvalidMethod() - - object.__setattr__(self, name, value) - - def _checks(self): """Deterministic checks for consistency.""" From efbbb84315505d78aff709106c53073e5111a9fe Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 20:17:22 -0400 Subject: [PATCH 080/121] Added Mikko Ohtamaa to AUTHORS #124 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 5bd399a2..ff0c44b6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -36,4 +36,5 @@ Patches and Suggestions - Alex <@alopatin> - Tom Hogans - Armin Ronacher -- Shrikant Sharat Kandula \ No newline at end of file +- Shrikant Sharat Kandula +- Mikko Ohtamaa \ No newline at end of file From 820e2c73e62b94b43fc957a7d4d168050c7eabf5 Mon Sep 17 00:00:00 2001 From: Rick Mak Date: Thu, 18 Aug 2011 14:22:54 +0800 Subject: [PATCH 081/121] Import re --- requests/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/models.py b/requests/models.py index 2e35c7ec..a2e4c2eb 100644 --- a/requests/models.py +++ b/requests/models.py @@ -11,6 +11,7 @@ import urllib2 import socket import zlib import cgi +import re from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin From d26f3333e4d2ac65fd82bd413bc5759a0808e259 Mon Sep 17 00:00:00 2001 From: Rick Mak Date: Thu, 18 Aug 2011 14:42:35 +0800 Subject: [PATCH 082/121] Fix tried_encodings scope --- requests/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index a2e4c2eb..a8aa6cb5 100644 --- a/requests/models.py +++ b/requests/models.py @@ -430,23 +430,24 @@ class Response(object): 2. every encodings from 3. fall back and replace all unicode characters """ + tried_encodings = [] # Try charset from content-type encoding = self.get_encoding_from_content_type() if encoding: try: return unicode(content, encoding) except UnicodeError: - self.tried_encodings.append(encoding) + tried_encodings.append(encoding) # Try every encodings from encodings = self.get_encodings_from_content(content) for encoding in encodings: - if encoding in self.tried_encodings: + if encoding in tried_encodings: continue try: return unicode(content, encoding) except UnicodeError: - self.tried_encodings.append(encoding) + tried_encodings.append(encoding) # Fall back: return unicode(content, encoding, errors="replace") From 2f55393593fb0d8818fa014bc004dfed73564110 Mon Sep 17 00:00:00 2001 From: Den Shabalin Date: Fri, 19 Aug 2011 19:18:26 +0300 Subject: [PATCH 083/121] Fixes an issue #128: ``Response not working with lxml''. This error happend due to lxml's attempt to do .geturl() call on the response object. __getattr__ didn't raise AttributeError so ``response.geturl'' returned None and ``response.geturl()'' resulted into ``TypeError: 'NoneType' object is not callable'' seen in the issue. --- requests/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 0b22f711..5e0a4d3b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -416,7 +416,8 @@ class Response(object): except zlib.error: pass return self._content - + else: + raise AttributeError def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" From ce4927e5d36e02bfb9de064f87e6027fff8c1a1f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 19:39:53 -0400 Subject: [PATCH 084/121] added Den Shabalin to AUTHORS --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index ff0c44b6..61874982 100644 --- a/AUTHORS +++ b/AUTHORS @@ -37,4 +37,5 @@ Patches and Suggestions - Tom Hogans - Armin Ronacher - Shrikant Sharat Kandula -- Mikko Ohtamaa \ No newline at end of file +- Mikko Ohtamaa +- Den Shabalin \ No newline at end of file From d9a349c1a1451f7c6d1dbea0b637b35107eb8de5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 20:12:16 -0400 Subject: [PATCH 085/121] 303 status code --- requests/status_codes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/status_codes.py b/requests/status_codes.py index c8a47a07..a809de6a 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -24,7 +24,7 @@ _codes = { 300: ('multiple_choices',), 301: ('moved_permanently', 'moved', '\\o-'), 302: ('found',), - 302: ('see_other', 'other'), + 303: ('see_other', 'other'), 304: ('not_modified',), 305: ('use_proxy',), 306: ('switch_proxy',), From c75eaec8527b32a5b1bdf95c1c70f34a33f508d6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 20:12:33 -0400 Subject: [PATCH 086/121] integrate codes into models.py --- requests/models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/requests/models.py b/requests/models.py index 7e8d51b6..2d7fc8fe 100644 --- a/requests/models.py +++ b/requests/models.py @@ -22,9 +22,11 @@ from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers from .utils import dict_from_cookiejar from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .status_codes import codes -REDIRECT_STATI = (301, 302, 303, 307) +REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) + class Request(object): @@ -202,13 +204,13 @@ class Request(object): while ( ('location' in r.headers) and ((self.method in ('GET', 'HEAD')) or - (r.status_code is 303) or + (r.status_code is codes.see_other) or (self.allow_redirects)) ): r.close() - if not len(history) < 30: + if not len(history) < settings.max_redirects: raise TooManyRedirects() history.append(r) @@ -226,7 +228,7 @@ class Request(object): url = urljoin(r.url, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 - if r.status_code is 303: + if r.status_code is codes.see_other: method = 'GET' else: method = self.method From abc5e279319425e226121165d15052542acfaa9a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 20:12:41 -0400 Subject: [PATCH 087/121] config max_redirects --- requests/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/config.py b/requests/config.py index a42f15d2..39be2ed4 100644 --- a/requests/config.py +++ b/requests/config.py @@ -61,6 +61,7 @@ settings.accept_gzip = True settings.proxies = None settings.verbose = None settings.timeout = None +settings.max_redirects = 30 #: Use socket.setdefaulttimeout() as fallback? settings.timeout_fallback = True From 53a53668d90f4defdc5ece7ed79f7cbac8bed5f3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 20:59:16 -0400 Subject: [PATCH 088/121] no more roadmap --- README.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.rst b/README.rst index 529d1445..3f6455ed 100644 --- a/README.rst +++ b/README.rst @@ -100,11 +100,5 @@ Contribute If you'd like to contribute, simply fork `the repository`_, commit your changes to the **develop** branch (or branch off of it), and send a pull request. Make sure you add yourself to AUTHORS_. - -Roadmap -------- - -- Sphinx Documentation - .. _`the repository`: http://github.com/kennethreitz/requests .. _AUTHORS: http://github.com/kennethreitz/requests/blob/master/AUTHORS From 5f16a8a6b1684095f78b14533e1fe492265ce442 Mon Sep 17 00:00:00 2001 From: Alejandro Giacometti Date: Sat, 20 Aug 2011 23:05:39 +0200 Subject: [PATCH 089/121] Added readline method to response object. --- requests/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/models.py b/requests/models.py index 2d7fc8fe..3358e120 100644 --- a/requests/models.py +++ b/requests/models.py @@ -176,6 +176,7 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) response.read = resp.read + response.readline = resp.readline response._resp = resp response._close = resp.close From c58ddff3a4b782012894e148bf471e0ebd146d64 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 18:24:58 -0400 Subject: [PATCH 090/121] history update --- HISTORY.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index b2ae8f9f..b09392b5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,17 @@ History ------- +0.6.1 (2011-08-20) +++++++++++++++++++ + +* Enhanced status codes experience ``\o/`` +* Set a maximum number of redirects (``settings.max_redirects``) +* Full Unicode URL support +* Support for protocol-less redirects. +* Allow for arbitrary request types. +* Bugfixes + + 0.6.0 (2011-08-17) ++++++++++++++++++ From 8b640979ee06a03a57e72ba3ed73fcb73efc4c74 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 18:25:32 -0400 Subject: [PATCH 091/121] v0.6.1 --- requests/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/core.py b/requests/core.py index 68e53923..8ba34a2f 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,8 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.6.0' -__build__ = 0x000600 +__version__ = '0.6.1' +__build__ = 0x000601 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' From 8bfa909ff2bcf8b51f19a5d612f82301f9afce3f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 18:34:33 -0400 Subject: [PATCH 092/121] merge remnant (oops) --- AUTHORS.orig | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 AUTHORS.orig diff --git a/AUTHORS.orig b/AUTHORS.orig deleted file mode 100644 index dd88224b..00000000 --- a/AUTHORS.orig +++ /dev/null @@ -1,40 +0,0 @@ -Requests is written and maintained by Kenneth Reitz and -various contributors: - -Development Lead -```````````````` - -- Kenneth Reitz - - -Patches and Suggestions -``````````````````````` - -- Various Pocoo Members -- Chris Adams -- Flavio Percoco Premoli -- Dj Gilcrease -- Justin Murphy -- Rob Madole -- Aram Dulyan -- Johannes Gorset -- 村山めがね (Megane Murayama) -- James Rowe -- Daniel Schauenberg -- Zbigniew Siciarz -- Daniele Tricoli 'Eriol' -- Richard Boulton -- Miguel Olivares -- Alberto Paro -- Jérémy Bethmont -- 潘旭 (Xu Pan) -- Tamás Gulácsi -- Rubén Abad -- Peter Manser -- Jeremy Selie -<<<<<<< HEAD -- Jens Diemer -- Alex <@alopatin> -======= -- Tom Hogans ->>>>>>> 0ed641a26ec2200de00e4bbf3d170c767375351e From 7fc8c7ccfd46b2ac37dfc4d8aa1225b139287de1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:17:25 -0400 Subject: [PATCH 093/121] move encoding detection out of request object. --- requests/models.py | 77 +++++++++++++++------------------------------- 1 file changed, 24 insertions(+), 53 deletions(-) diff --git a/requests/models.py b/requests/models.py index 3cbf1029..b12112ac 100644 --- a/requests/models.py +++ b/requests/models.py @@ -10,8 +10,7 @@ import urllib import urllib2 import socket import zlib -import cgi -import re + from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin @@ -22,9 +21,9 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .utils import dict_from_cookiejar -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .utils import dict_from_cookiejar, get_unicode_from_response from .status_codes import codes +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) @@ -420,73 +419,43 @@ class Response(object): def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" + return not self.error def __getattr__(self, name): - """Read and returns the full stream when accessing to :attr: `content`""" + """Read and returns the full stream when accessing to + :attr: `content` + """ + if name == 'content': if self._content is not None: return self._content + + # Read the contents. self._content = self.read() + + # Decode GZip'd content. if self.headers.get('content-encoding', '') == 'gzip': try: self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) except zlib.error: pass - return self.unicode_content(self._content) - + # Decode unicode content. + try: + self._content = get_unicode_from_response(self) + # Don't trust this stuff. + except UserWarning, e: + print e + + + return self._content + else: raise AttributeError - - def get_content_type(self): - content_type = self.headers.get("content-type") - content_type, params = cgi.parse_header(content_type) - return content_type, params - def get_encoding_from_content_type(self): - content_type, params = self.get_content_type() - if "charset" in params: - return params["charset"].strip("'\"") - - def get_encodings_from_content(self, content): - if self._charset_re is None: - self._charset_re = re.compile( - r']', flags=re.I - ) - return self._charset_re.findall(content) - - def unicode_content(self, content): - """ - Returns the requested content back in unicode. - Tried: - 1. charset from content-type - 2. every encodings from - 3. fall back and replace all unicode characters - """ - tried_encodings = [] - # Try charset from content-type - encoding = self.get_encoding_from_content_type() - if encoding: - try: - return unicode(content, encoding) - except UnicodeError: - tried_encodings.append(encoding) - - # Try every encodings from - encodings = self.get_encodings_from_content(content) - for encoding in encodings: - if encoding in tried_encodings: - continue - try: - return unicode(content, encoding) - except UnicodeError: - tried_encodings.append(encoding) - - # Fall back: - return unicode(content, encoding, errors="replace") def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" @@ -499,6 +468,8 @@ class Response(object): self._resp.fp._sock.recv = None self._close() + + class AuthManager(object): """Requests Authentication Manager.""" From 8fbb1e6d97cda90d588d4263a18906a52d147fba Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:17:41 -0400 Subject: [PATCH 094/121] move encoding methods into utils for external consumption --- requests/utils.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/requests/utils.py b/requests/utils.py index 8ac78b4e..72705734 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -9,7 +9,9 @@ that are also useful for external consumption. """ +import cgi import cookielib +import re def dict_from_cookiejar(cookiejar): @@ -70,3 +72,62 @@ def add_dict_to_cookiejar(cj, cookie_dict): cj.set_cookie(cookie) return cj + + +def get_encodings_from_content(content): + """Returns encodings from given content string.""" + + charset_re = re.compile(r']', flags=re.I) + + return charset_re.findall(content) + + + +def get_encoding_from_headers(headers): + """Returns encodings from given HTTP Header Dict.""" + + content_type = headers.get('content-type') + content_type, params = cgi.parse_header(content_type) + + if 'charset' in params: + return params['charset'].strip("'\"") + + +def get_unicode_from_response(r): + """Returns the requested content back in unicode. + + Tried: + 1. charset from content-type + 2. every encodings from + 3. fall back and replace all unicode characters + """ + + tried_encodings = [] + + # Try charset from content-type + encoding = get_encoding_from_headers(r.headers) + + if encoding: + try: + print '!' + return unicode(r.content, encoding) + except UnicodeError: + tried_encodings.append(encoding) + + # Try every encodings from + encodings = get_encodings_from_content(r.content) + + for encoding in encodings: + if encoding in tried_encodings: + continue + try: + + return unicode(r.content, encoding) + except (UnicodeError, TypeError): + tried_encodings.append(encoding) + + # Fall back: + try: + return unicode(r.content, encoding, errors='replace') + except TypeError: + return r.content From 4922daf4989d921dd873b93f5ec08e02ad2222b6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:22:18 -0400 Subject: [PATCH 095/121] added Rick Mak to AUTHORS --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 61874982..1cf7d350 100644 --- a/AUTHORS +++ b/AUTHORS @@ -38,4 +38,5 @@ Patches and Suggestions - Armin Ronacher - Shrikant Sharat Kandula - Mikko Ohtamaa -- Den Shabalin \ No newline at end of file +- Den Shabalin +- Rick Mak \ No newline at end of file From 01411e37d65111d9bf09ac7ba82338664b595a0f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:22:48 -0400 Subject: [PATCH 096/121] AUTHORS += Alejandro Giacometti --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 61874982..ed833ff5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -38,4 +38,5 @@ Patches and Suggestions - Armin Ronacher - Shrikant Sharat Kandula - Mikko Ohtamaa -- Den Shabalin \ No newline at end of file +- Den Shabalin +- Alejandro Giacometti \ No newline at end of file From 882ea76aa994918d3a62d9f31ad88533edd3ba03 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:26:56 -0400 Subject: [PATCH 097/121] cleanups, response.fo --- requests/models.py | 52 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/requests/models.py b/requests/models.py index 3358e120..26436892 100644 --- a/requests/models.py +++ b/requests/models.py @@ -11,6 +11,7 @@ import urllib2 import socket import zlib + from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin from datetime import datetime @@ -20,9 +21,9 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .utils import dict_from_cookiejar -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .utils import dict_from_cookiejar, get_unicode_from_response from .status_codes import codes +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) @@ -132,9 +133,16 @@ class Request(object): _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar)) if self.auth: - if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): + if not isinstance(self.auth.handler, + (urllib2.AbstractBasicAuthHandler, + urllib2.AbstractDigestAuthHandler)): + # TODO: REMOVE THIS COMPLETELY - auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) + auth_manager.add_password( + self.auth.realm, self.url, + self.auth.username, + self.auth.password) + self.auth.handler = self.auth.handler(auth_manager) auth_manager.add_auth(self.url, self.auth) @@ -166,7 +174,10 @@ class Request(object): def _build_response(self, resp, is_error=False): - """Build internal :class:`Response ` object from given response.""" + """Build internal :class:`Response ` object + from given response. + """ + def build(resp): @@ -176,8 +187,7 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) response.read = resp.read - response.readline = resp.readline - response._resp = resp + response.fo = resp response._close = resp.close if self.cookiejar: @@ -419,24 +429,44 @@ class Response(object): def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" + return not self.error def __getattr__(self, name): - """Read and returns the full stream when accessing to :attr: `content`""" + """Read and returns the full stream when accessing to + :attr: `content` + """ + if name == 'content': if self._content is not None: return self._content + + # Read the contents. self._content = self.read() + + # Decode GZip'd content. if self.headers.get('content-encoding', '') == 'gzip': try: self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) except zlib.error: pass + + # Decode unicode content. + try: + self._content = get_unicode_from_response(self) + # Don't trust this stuff. + except UserWarning, e: + print e + + return self._content + else: raise AttributeError + + def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" if self.error: @@ -444,10 +474,12 @@ class Response(object): def close(self): - if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'): - self._resp.fp._sock.recv = None + if self.fo.fp is not None and hasattr(self.fo.fp, '_sock'): + self.fo.fp._sock.recv = None self._close() + + class AuthManager(object): """Requests Authentication Manager.""" From 7bd47777a941237036ba8e15a2aaf2e93cc8e467 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:40:33 -0400 Subject: [PATCH 098/121] no separate test suite --- test_unicode.py | 66 ------------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 test_unicode.py diff --git a/test_unicode.py b/test_unicode.py deleted file mode 100644 index 38a1dfcd..00000000 --- a/test_unicode.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import with_statement - -import unittest -import cookielib - -try: - import omnijson as json -except ImportError: - import json - -import requests - - - -HTTPBIN_URL = 'http://httpbin.org/' -HTTPSBIN_URL = 'https://httpbin.ep.io/' - -# HTTPBIN_URL = 'http://staging.httpbin.org/' -# HTTPSBIN_URL = 'https://httpbin-staging.ep.io/' - - -def httpbin(*suffix): - """Returns url for HTTPBIN resource.""" - - return HTTPBIN_URL + '/'.join(suffix) - - -def httpsbin(*suffix): - """Returns url for HTTPSBIN resource.""" - - return HTTPSBIN_URL + '/'.join(suffix) - - -SERVICES = (httpbin, httpsbin) - - - -class RequestsTestSuite(unittest.TestCase): - """Requests test cases.""" - - - def setUp(self): - pass - - - def tearDown(self): - """Teardown.""" - pass - - - def test_HTTP_200_OK_GET_ON_ISO88591(self): - r = requests.get("http://www.qypedeals.de/Verzehrgutschein+für+Jellyfish") - self.assertEqual(r.status_code, 200) - self.assertIsInstance(r.content, unicode) - - def test_HTTP_200_OK_GET_ON_BIG5(self): - r = requests.get("http://google.com.hk/") - self.assertEqual(r.status_code, 200) - self.assertIsInstance(r.content, unicode) - - -if __name__ == '__main__': - unittest.main() From a609f726e633018ebd0f11d59d7c63f54945b1be Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:43:36 -0400 Subject: [PATCH 099/121] settings.allow_unicode --- requests/config.py | 1 + requests/models.py | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/requests/config.py b/requests/config.py index 39be2ed4..5a7fcd3f 100644 --- a/requests/config.py +++ b/requests/config.py @@ -62,6 +62,7 @@ settings.proxies = None settings.verbose = None settings.timeout = None settings.max_redirects = 30 +settings.allow_unicode = True #: Use socket.setdefaulttimeout() as fallback? settings.timeout_fallback = True diff --git a/requests/models.py b/requests/models.py index 26436892..d3483149 100644 --- a/requests/models.py +++ b/requests/models.py @@ -453,12 +453,8 @@ class Response(object): pass # Decode unicode content. - try: + if settings.allow_unicode: self._content = get_unicode_from_response(self) - # Don't trust this stuff. - except UserWarning, e: - print e - return self._content From f038ec3b6e9e3dc9ba118eb34f60db295c6a5534 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:44:24 -0400 Subject: [PATCH 100/121] may have multiple encoding values --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index d3483149..8692bbb9 100644 --- a/requests/models.py +++ b/requests/models.py @@ -446,7 +446,7 @@ class Response(object): self._content = self.read() # Decode GZip'd content. - if self.headers.get('content-encoding', '') == 'gzip': + if 'gzip' in self.headers.get('content-encoding', ''): try: self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) except zlib.error: From 49066861863495f8e401ea22a25d01b07ccbab0e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:46:50 -0400 Subject: [PATCH 101/121] utils.decode_gzip --- requests/models.py | 4 ++-- requests/utils.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 8692bbb9..51524608 100644 --- a/requests/models.py +++ b/requests/models.py @@ -21,7 +21,7 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .utils import dict_from_cookiejar, get_unicode_from_response +from .utils import dict_from_cookiejar, get_unicode_from_response, decode_gzip from .status_codes import codes from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects @@ -448,7 +448,7 @@ class Response(object): # Decode GZip'd content. if 'gzip' in self.headers.get('content-encoding', ''): try: - self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) + self._content = decode_gzip(self._content) except zlib.error: pass diff --git a/requests/utils.py b/requests/utils.py index 72705734..beed3c1f 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -12,6 +12,7 @@ that are also useful for external consumption. import cgi import cookielib import re +import zlib def dict_from_cookiejar(cookiejar): @@ -131,3 +132,9 @@ def get_unicode_from_response(r): return unicode(r.content, encoding, errors='replace') except TypeError: return r.content + + +def decode_gzip(content): + """Return gzip-decoded string.""" + + return zlib.decompress(content, 16+zlib.MAX_WBITS) \ No newline at end of file From c19b2c517256ef2a8cd5915d9d5f44612731881d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:47:43 -0400 Subject: [PATCH 102/121] settings.decode_unicode --- requests/config.py | 2 +- requests/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/config.py b/requests/config.py index 5a7fcd3f..794109c5 100644 --- a/requests/config.py +++ b/requests/config.py @@ -62,7 +62,7 @@ settings.proxies = None settings.verbose = None settings.timeout = None settings.max_redirects = 30 -settings.allow_unicode = True +settings.decode_unicode = True #: Use socket.setdefaulttimeout() as fallback? settings.timeout_fallback = True diff --git a/requests/models.py b/requests/models.py index 51524608..db8a60e3 100644 --- a/requests/models.py +++ b/requests/models.py @@ -453,7 +453,7 @@ class Response(object): pass # Decode unicode content. - if settings.allow_unicode: + if settings.decode_unicode: self._content = get_unicode_from_response(self) return self._content From 86cc274cf01e38fd482c9e8be2e4c7f621340468 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:49:23 -0400 Subject: [PATCH 103/121] remote read/close methods from response --- requests/models.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/requests/models.py b/requests/models.py index db8a60e3..12551edc 100644 --- a/requests/models.py +++ b/requests/models.py @@ -186,9 +186,7 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) - response.read = resp.read response.fo = resp - response._close = resp.close if self.cookiejar: @@ -443,7 +441,7 @@ class Response(object): return self._content # Read the contents. - self._content = self.read() + self._content = self.fo.read() # Decode GZip'd content. if 'gzip' in self.headers.get('content-encoding', ''): @@ -462,19 +460,12 @@ class Response(object): raise AttributeError - def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" if self.error: raise self.error - def close(self): - if self.fo.fp is not None and hasattr(self.fo.fp, '_sock'): - self.fo.fp._sock.recv = None - self._close() - - class AuthManager(object): """Requests Authentication Manager.""" From 7f35ec7665de834c012c218df279aa28c68443bd Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:52:18 -0400 Subject: [PATCH 104/121] r.fo bugfix --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 12551edc..a34a4a4b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -217,7 +217,7 @@ class Request(object): (self.allow_redirects)) ): - r.close() + r.fo.close() if not len(history) < settings.max_redirects: raise TooManyRedirects() From 873d128c5a75bb862a6a857c9a72aa7c916659af Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:58:00 -0400 Subject: [PATCH 105/121] docstrings for utils module --- requests/utils.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index beed3c1f..0f6e8428 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -15,12 +15,15 @@ import re import zlib -def dict_from_cookiejar(cookiejar): - """Returns a key/value dictionary from a CookieJar.""" +def dict_from_cookiejar(cj): + """Returns a key/value dictionary from a CookieJar. + + :param cj: CookieJar object to extract cookies from. + """ cookie_dict = {} - for _, cookies in cookiejar._cookies.items(): + for _, cookies in cj._cookies.items(): for _, cookies in cookies.items(): for cookie in cookies.values(): # print cookie @@ -30,7 +33,10 @@ def dict_from_cookiejar(cookiejar): def cookiejar_from_dict(cookie_dict): - """Returns a CookieJar from a key/value dictionary.""" + """Returns a CookieJar from a key/value dictionary. + + :param cookie_dict: Dict of key/values to insert into CookieJar. + """ # return cookiejar if one was passed in if isinstance(cookie_dict, cookielib.CookieJar): @@ -45,7 +51,11 @@ def cookiejar_from_dict(cookie_dict): def add_dict_to_cookiejar(cj, cookie_dict): - """Returns a CookieJar from a key/value dictionary.""" + """Returns a CookieJar from a key/value dictionary. + + :param cj: CookieJar to insert cookies into. + :param cookie_dict: Dict of key/values to insert into CookieJar. + """ for k, v in cookie_dict.items(): @@ -76,7 +86,10 @@ def add_dict_to_cookiejar(cj, cookie_dict): def get_encodings_from_content(content): - """Returns encodings from given content string.""" + """Returns encodings from given content string. + + :param content: bytestring to extract encodings from. + """ charset_re = re.compile(r']', flags=re.I) @@ -85,7 +98,10 @@ def get_encodings_from_content(content): def get_encoding_from_headers(headers): - """Returns encodings from given HTTP Header Dict.""" + """Returns encodings from given HTTP Header Dict. + + :param headers: dictionary to extract encoding from. + """ content_type = headers.get('content-type') content_type, params = cgi.parse_header(content_type) @@ -97,6 +113,8 @@ def get_encoding_from_headers(headers): def get_unicode_from_response(r): """Returns the requested content back in unicode. + :param r: Reponse object to get unicode content from. + Tried: 1. charset from content-type 2. every encodings from @@ -135,6 +153,9 @@ def get_unicode_from_response(r): def decode_gzip(content): - """Return gzip-decoded string.""" + """Return gzip-decoded string. + + :param content: bytestring to gzip-decode. + """ return zlib.decompress(content, 16+zlib.MAX_WBITS) \ No newline at end of file From c5d787d6fd57b3cea6731c2afd5ba9de05cda720 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:59:21 -0400 Subject: [PATCH 106/121] cleanups --- requests/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index a34a4a4b..6d27425d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -266,8 +266,8 @@ class Request(object): Otherwise, assumes the data is already encoded appropriately, and returns it twice. - """ + if hasattr(data, 'items'): result = [] for k, vs in data.items(): @@ -375,7 +375,6 @@ class Request(object): self._build_response(why, is_error=True) - else: self._build_response(resp) self.response.ok = True @@ -386,7 +385,6 @@ class Request(object): return self.sent - class Response(object): """The core :class:`Response ` object. All :class:`Request ` objects contain a From d0fdf0eb38da6afaddc8bca88af3fa32fac367cf Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 19:59:41 -0400 Subject: [PATCH 107/121] no longer required --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 6d27425d..caf809ce 100644 --- a/requests/models.py +++ b/requests/models.py @@ -53,7 +53,7 @@ class Request(object): #: Dictionary of files to multipart upload (``{filename: content}``). self.files = files - #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. + #: HTTP Method to use. self.method = method #: Dictionary or byte of request body data to attach to the From 13db806861188ae15a5fb23bd39610fdb0aba4f2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 20:04:11 -0400 Subject: [PATCH 108/121] docstringin' it up --- requests/models.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index caf809ce..568ee5a1 100644 --- a/requests/models.py +++ b/requests/models.py @@ -393,31 +393,46 @@ class Response(object): """ def __init__(self): - #: Raw content of the response, in bytes. + #: Content of the response, in bytes or unicode (if available). #: If ``content-encoding`` of response was set to ``gzip``, the #: response data will be automatically deflated. - self._content = None + self.content = None + + # Hack for Sphinx. + del self.content + #: Integer Code of responded HTTP Status. self.status_code = None #: Case-insensitive Dictionary of Response Headers. #: For example, ``headers['content-encoding']`` will return the #: value of a ``'Content-Encoding'`` response header. self.headers = CaseInsensitiveDict() + + #: File-like object representation of response (for advanced usage). + self.fo = None + #: Final URL location of Response. self.url = None + #: True if no :attr:`error` occured. self.ok = False + #: Resulting :class:`HTTPError` of request, if one occured. self.error = None + #: A list of :class:`Response ` objects from #: the history of the Request. Any redirect responses will end #: up here. self.history = [] + #: The Request that created the Response. self.request = None + #: A dictionary of Cookies the server sent back. self.cookies = None + self._content = None + def __repr__(self): return '' % (self.status_code) From 1f3e53b32ec603c8859dacc2dbe17b875fcd2fe3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 20:05:38 -0400 Subject: [PATCH 109/121] yay --- requests/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 568ee5a1..2ca2c75c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -394,8 +394,6 @@ class Response(object): def __init__(self): #: Content of the response, in bytes or unicode (if available). - #: If ``content-encoding`` of response was set to ``gzip``, the - #: response data will be automatically deflated. self.content = None # Hack for Sphinx. @@ -403,6 +401,7 @@ class Response(object): #: Integer Code of responded HTTP Status. self.status_code = None + #: Case-insensitive Dictionary of Response Headers. #: For example, ``headers['content-encoding']`` will return the #: value of a ``'Content-Encoding'`` response header. From 596aceb185c3ca013e83db70eeeff3014d8e3a40 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 22:47:03 -0400 Subject: [PATCH 110/121] big docs update --- docs/api.rst | 56 +++++++++++++++++++++++++++++-------- requests/api.py | 70 +++++++++++----------------------------------- requests/core.py | 6 ++-- requests/models.py | 26 ++++++++--------- requests/utils.py | 6 +++- 5 files changed, 82 insertions(+), 82 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index cbd24fdb..e3338b66 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -13,34 +13,48 @@ important right here and provide links to the canonical documentation. Main Interface -------------- -All of Request's functionality can be accessed by these 6 methods. They -all return a :class:`Response ` object. +All of Request's functionality can be accessed by these 7 methods. +They all return an instance of the :class:`Response ` object. +.. autofunction:: request .. autofunction:: head .. autofunction:: get .. autofunction:: post .. autofunction:: put .. autofunction:: patch .. autofunction:: delete -.. autofunction:: request + ----------- -.. autoclass:: requests.models.Response +.. autoclass:: Response :inherited-members: -Exceptions ----------- -.. autoexception:: HTTPError +Utilities +--------- -.. autoexception:: RequestException +These functions are used internally, but may be useful outside of +Requests. -.. autoexception:: requests.models.AuthenticationError -.. autoexception:: requests.models.URLRequired -.. autoexception:: requests.models.InvalidMethod +.. module:: requests.utils + +Cookies +~~~~~~~ + +.. autofunction:: dict_from_cookiejar +.. autofunction:: cookiejar_from_dict +.. autofunction:: add_dict_to_cookiejar + +Encodings +~~~~~~~~~ + +.. autofunction:: get_encodings_from_content +.. autofunction:: get_encoding_from_headers +.. autofunction:: get_unicode_from_response +.. autofunction:: decode_gzip Internals @@ -50,8 +64,26 @@ These items are an internal component to Requests, and should never be seen by the end user (developer). This part of the API documentation exists for those who are extending the functionality of Requests. +Exceptions +~~~~~~~~~~ + +.. module:: requests + +.. autoexception:: HTTPError + +.. autoexception:: RequestException + +.. autoexception:: AuthenticationError +.. autoexception:: URLRequired +.. autoexception:: InvalidMethod +.. autoexception:: TooManyRedirects + + + Classes ~~~~~~~ -.. autoclass:: requests.models.Request +.. autoclass:: requests.Request :inherited-members: + + diff --git a/requests/api.py b/requests/api.py index 0cea63d4..5ded386c 100644 --- a/requests/api.py +++ b/requests/api.py @@ -25,8 +25,8 @@ def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=False, proxies=None, hooks=None): - """Constructs and sends a :class:`Request `. - Returns :class:`Response ` object. + """Constructs and sends a :class:`Request `. + Returns :class:`Response ` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. @@ -41,6 +41,8 @@ def request(method, url, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ + method = method.upper() + if cookies is None: cookies = {} @@ -85,50 +87,31 @@ def get(url, **kwargs): """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('GET', url, **kwargs) + return request('get', url, **kwargs) def head(url, **kwargs): - """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('HEAD', url, **kwargs) + return request('head', url, **kwargs) def post(url, data='', **kwargs): - """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('POST', url, data=data, **kwargs) + return request('post', url, data=data, **kwargs) def put(url, data='', **kwargs): @@ -136,17 +119,10 @@ def put(url, data='', **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('PUT', url, data=data, **kwargs) + return request('put', url, data=data, **kwargs) def patch(url, data='', **kwargs): @@ -154,31 +130,17 @@ def patch(url, data='', **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('PATCH', url, **kwargs) + return request('patch', url, **kwargs) def delete(url, **kwargs): - """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('DELETE', url, **kwargs) + return request('delete', url, **kwargs) diff --git a/requests/core.py b/requests/core.py index 8ba34a2f..c4366647 100644 --- a/requests/core.py +++ b/requests/core.py @@ -19,9 +19,11 @@ __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' -from models import HTTPError +from models import HTTPError, Request, Response from api import * from exceptions import * from sessions import session from status_codes import codes -from config import settings \ No newline at end of file +from config import settings + +import utils \ No newline at end of file diff --git a/requests/models.py b/requests/models.py index 2ca2c75c..92335181 100644 --- a/requests/models.py +++ b/requests/models.py @@ -31,7 +31,7 @@ REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) class Request(object): - """The :class:`Request ` object. It carries out all functionality of + """The :class:`Request ` object. It carries out all functionality of Requests. Recommended interface is with the Requests functions. """ @@ -47,7 +47,7 @@ class Request(object): #: Request URL. self.url = url - #: Dictonary of HTTP Headers to attach to the :class:`Request `. + #: Dictonary of HTTP Headers to attach to the :class:`Request `. self.headers = headers #: Dictionary of files to multipart upload (``{filename: content}``). @@ -57,14 +57,14 @@ class Request(object): self.method = method #: Dictionary or byte of request body data to attach to the - #: :class:`Request `. + #: :class:`Request `. self.data = None #: Dictionary or byte of querystring data to attach to the - #: :class:`Request `. + #: :class:`Request `. self.params = None - #: True if :class:`Request ` is part of a redirect chain (disables history + #: True if :class:`Request ` is part of a redirect chain (disables history #: and HTTPError storage). self.redirect = redirect @@ -77,7 +77,7 @@ class Request(object): self.data, self._enc_data = self._encode_params(data) self.params, self._enc_params = self._encode_params(params) - #: :class:`Response ` instance, containing + #: :class:`Response ` instance, containing #: content and metadata of HTTP Response, once :attr:`sent `. self.response = Response() @@ -86,10 +86,10 @@ class Request(object): if not auth: auth = auth_manager.get_auth(self.url) - #: :class:`AuthObject` to attach to :class:`Request `. + #: :class:`AuthObject` to attach to :class:`Request `. self.auth = auth - #: CookieJar to attach to :class:`Request `. + #: CookieJar to attach to :class:`Request `. self.cookiejar = cookiejar #: True if Request has been sent. @@ -174,7 +174,7 @@ class Request(object): def _build_response(self, resp, is_error=False): - """Build internal :class:`Response ` object + """Build internal :class:`Response ` object from given response. """ @@ -386,9 +386,9 @@ class Request(object): class Response(object): - """The core :class:`Response ` object. All - :class:`Request ` objects contain a - :class:`response ` attribute, which is an instance + """The core :class:`Response ` object. All + :class:`Request ` objects contain a + :class:`response ` attribute, which is an instance of this class. """ @@ -419,7 +419,7 @@ class Response(object): #: Resulting :class:`HTTPError` of request, if one occured. self.error = None - #: A list of :class:`Response ` objects from + #: A list of :class:`Response ` objects from #: the history of the Request. Any redirect responses will end #: up here. self.history = [] diff --git a/requests/utils.py b/requests/utils.py index 0f6e8428..d96310b4 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -116,9 +116,13 @@ def get_unicode_from_response(r): :param r: Reponse object to get unicode content from. Tried: + 1. charset from content-type - 2. every encodings from + + 2. every encodings from ```` + 3. fall back and replace all unicode characters + """ tried_encodings = [] From 28ee9788a071ac1314ed920ab5cb4af74ceee307 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 22:51:24 -0400 Subject: [PATCH 111/121] Request --- requests/api.py | 2 +- requests/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/api.py b/requests/api.py index 5ded386c..0928d8c8 100644 --- a/requests/api.py +++ b/requests/api.py @@ -41,7 +41,7 @@ def request(method, url, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - method = method.upper() + method = str(method).upper() if cookies is None: cookies = {} diff --git a/requests/models.py b/requests/models.py index 92335181..41214816 100644 --- a/requests/models.py +++ b/requests/models.py @@ -424,7 +424,7 @@ class Response(object): #: up here. self.history = [] - #: The Request that created the Response. + #: The :class:`Request ` that created the Response. self.request = None #: A dictionary of Cookies the server sent back. From b45b329861ed77c264df4271e38771eec6238e64 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 22:55:30 -0400 Subject: [PATCH 112/121] history --- HISTORY.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index b09392b5..33d56b93 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,11 @@ History ------- +* Automatic decoding of unicode, when available. +* New ``decode_unicode`` setting +* Removal of ``r.read/close`` methods +* New ``r.fo`` interface for advanced response usage. + 0.6.1 (2011-08-20) ++++++++++++++++++ From d4bdd84364b602624096b1407c5d10936c6c9068 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 23:00:17 -0400 Subject: [PATCH 113/121] new FAQs --- docs/community/faq.rst | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index b5717648..7b49725d 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -5,6 +5,21 @@ Frequently Asked Questions This part of the documentation covers common questions about Requests. +Encoded Data? +------------- + +Requests automatically decompresses GZip'ed responses, and decodes +response conten tinto unicode when encoding information is available. + + +Custom User-Agents? +------------------- + +Requests allows you to easily override User-Agent strings, along with +any other HTTP Header. + + + Why not Httplib2? ----------------- @@ -40,4 +55,11 @@ It's on the way. Here's a list of `supported interpreters `_. Keep-alive Support? ------------------- -It's on the way. \ No newline at end of file +It's on the way. + + +Proxy Support? +-------------- + +You bet! + From 6a96652396515631bb72ba7c363e96c27edd769a Mon Sep 17 00:00:00 2001 From: Den Shabalin Date: Sun, 21 Aug 2011 11:57:03 +0300 Subject: [PATCH 114/121] Refactor response.content into a property. This makes content attribute discoverable through dir(response) and also guards from errors like forgetting to raise AttributeError in __getattr__. Also you don't have to create fake self.content to fool Sphinx any more. --- requests/models.py | 56 +++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/requests/models.py b/requests/models.py index 41214816..03b853c8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -396,9 +396,6 @@ class Response(object): #: Content of the response, in bytes or unicode (if available). self.content = None - # Hack for Sphinx. - del self.content - #: Integer Code of responded HTTP Status. self.status_code = None @@ -430,8 +427,6 @@ class Response(object): #: A dictionary of Cookies the server sent back. self.cookies = None - self._content = None - def __repr__(self): return '' % (self.status_code) @@ -443,33 +438,34 @@ class Response(object): return not self.error - def __getattr__(self, name): - """Read and returns the full stream when accessing to - :attr: `content` - """ - - if name == 'content': - if self._content is not None: - return self._content - - # Read the contents. - self._content = self.fo.read() - - # Decode GZip'd content. - if 'gzip' in self.headers.get('content-encoding', ''): - try: - self._content = decode_gzip(self._content) - except zlib.error: - pass - - # Decode unicode content. - if settings.decode_unicode: - self._content = get_unicode_from_response(self) - + @property + def content(self): + """Content of the response, in bytes or unicode + (if available).""" + + if self._content is not None: return self._content - else: - raise AttributeError + # Read the contents. + self._content = self.fo.read() + + # Decode GZip'd content. + if 'gzip' in self.headers.get('content-encoding', ''): + try: + self._content = decode_gzip(self._content) + except zlib.error: + pass + + # Decode unicode content. + if settings.decode_unicode: + self._content = get_unicode_from_response(self) + + return self._content + + + @content.setter + def content(self, value): + self._content = value def raise_for_status(self): From 8d5108b142dd8d9de27559ee8364967b13f2b62a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 21 Aug 2011 06:58:34 -0400 Subject: [PATCH 115/121] faq --- docs/community/faq.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index 7b49725d..b0921956 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -3,13 +3,16 @@ Frequently Asked Questions ========================== -This part of the documentation covers common questions about Requests. +This part of the documentation answers common questions about Requests. Encoded Data? ------------- -Requests automatically decompresses GZip'ed responses, and decodes -response conten tinto unicode when encoding information is available. +Requests automatically decompresses gzip-encoded responses, and does +it's best to decodes response content to unicode when possible. + +You can get direct access to the raw reasponse (and even the socket), +if needed as well. Custom User-Agents? From 1143d4237f5e433a6181e6f7e1abcb1e7031120a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 21 Aug 2011 06:59:13 -0400 Subject: [PATCH 116/121] AUTHORS += 'Den Shabalin' --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index f79dd801..76c24373 100644 --- a/AUTHORS +++ b/AUTHORS @@ -41,3 +41,4 @@ Patches and Suggestions - Den Shabalin - Alejandro Giacometti - Rick Mak +- Den Shabalin \ No newline at end of file From c8110222a532821d27b0ade3ebdd7d0cb9545f80 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 21 Aug 2011 07:01:29 -0400 Subject: [PATCH 117/121] no need to set content avoids 2.5 workaround --- requests/models.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/requests/models.py b/requests/models.py index 03b853c8..262a564d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -442,7 +442,7 @@ class Response(object): def content(self): """Content of the response, in bytes or unicode (if available).""" - + if self._content is not None: return self._content @@ -463,11 +463,6 @@ class Response(object): return self._content - @content.setter - def content(self, value): - self._content = value - - def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" if self.error: From a86af4477ec377223c10ddd6bf2fe90f508e1b60 Mon Sep 17 00:00:00 2001 From: Den Shabalin Date: Sun, 21 Aug 2011 14:09:03 +0300 Subject: [PATCH 118/121] AUTHORS -= Den Shabalin (I've already been mentioned in the list @ line 41.) --- AUTHORS | 1 - 1 file changed, 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 76c24373..f79dd801 100644 --- a/AUTHORS +++ b/AUTHORS @@ -41,4 +41,3 @@ Patches and Suggestions - Den Shabalin - Alejandro Giacometti - Rick Mak -- Den Shabalin \ No newline at end of file From 317c5b693a16e5e638d9253dd2913d93f5389e00 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 21 Aug 2011 07:12:54 -0400 Subject: [PATCH 119/121] _content --- requests/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 262a564d..5983c237 100644 --- a/requests/models.py +++ b/requests/models.py @@ -393,8 +393,8 @@ class Response(object): """ def __init__(self): - #: Content of the response, in bytes or unicode (if available). - self.content = None + + self._content = None #: Integer Code of responded HTTP Status. self.status_code = None @@ -441,7 +441,8 @@ class Response(object): @property def content(self): """Content of the response, in bytes or unicode - (if available).""" + (if available). + """ if self._content is not None: return self._content From e8dbecb4dbed3e835021977cf43f225a76bc5ec3 Mon Sep 17 00:00:00 2001 From: jbergstroem Date: Tue, 23 Aug 2011 11:29:36 +0200 Subject: [PATCH 120/121] Add test_requests.py to manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 94c50f70..39fbb994 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README.rst LICENSE HISTORY.rst \ No newline at end of file +include README.rst LICENSE HISTORY.rst test_requests.py From 75027f9557e61f2416af1794a269758604a7454c Mon Sep 17 00:00:00 2001 From: jbergstroem Date: Tue, 23 Aug 2011 11:37:51 +0200 Subject: [PATCH 121/121] Typos, nits and some 80w fixes --- docs/community/faq.rst | 3 +-- docs/community/updates.rst | 1 - docs/user/advanced.rst | 4 ++-- docs/user/install.rst | 9 ++++++--- docs/user/intro.rst | 12 +++++++++--- docs/user/quickstart.rst | 12 ++++++++---- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index b0921956..b6efc786 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -11,7 +11,7 @@ Encoded Data? Requests automatically decompresses gzip-encoded responses, and does it's best to decodes response content to unicode when possible. -You can get direct access to the raw reasponse (and even the socket), +You can get direct access to the raw response (and even the socket), if needed as well. @@ -22,7 +22,6 @@ Requests allows you to easily override User-Agent strings, along with any other HTTP Header. - Why not Httplib2? ----------------- diff --git a/docs/community/updates.rst b/docs/community/updates.rst index 942ccac1..e6e1559f 100644 --- a/docs/community/updates.rst +++ b/docs/community/updates.rst @@ -20,7 +20,6 @@ I often tweet about new features and releases of Requests. Follow `@kennethreitz `_ for updates. - Mailing List ------------ diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d145da21..ae7827c9 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -66,7 +66,7 @@ Available hooks: You can assign a hook function on a per-request basis by passing a ``{hook_name: callback_function}`` dictionary to the ``hooks`` request -paramaeter:: +parameter:: hooks=dict(args=print_url) @@ -134,4 +134,4 @@ To do so, just configure Requests with a stream to write to:: >>> requests.settings.verbose = sys.stderr >>> requests.get('http://httpbin.org/headers') 2011-08-17T03:04:23.380175 GET http://httpbin.org/headers - \ No newline at end of file + diff --git a/docs/user/install.rst b/docs/user/install.rst index cdc6d86c..dc4aa627 100644 --- a/docs/user/install.rst +++ b/docs/user/install.rst @@ -3,7 +3,8 @@ Installation ============ -This part of the documentation covers the installation of Requests. The first step to using any software package is getting it properly installed. +This part of the documentation covers the installation of Requests. +The first step to using any software package is getting it properly installed. Distribute & Pip @@ -24,7 +25,8 @@ But, you really `shouldn't do that `_:: +If the Cheeseshop is down, you can also install Requests from Kenneth Reitz's +personal `Cheeseshop mirror `_:: $ pip install -i http://pip.kreitz.co/simple requests @@ -48,6 +50,7 @@ Or, download the `zipball ` section. +Eager to get started? This page gives a good introduction in how to get started +with Requests. This assumes you already have Requests installed. If you do not, +head over to the :ref:`Installation ` section. First, make sure that: @@ -25,7 +27,8 @@ Let's get GitHub's public timeline :: r = requests.get('https://github.com/timeline.json') -Now, we have a :class:`Response` object called ``r``. We can get all the information we need from this. +Now, we have a :class:`Response` object called ``r``. We can get all the +information we need from this. Response Content @@ -90,7 +93,8 @@ interface:: 'content-type': 'application/json; charset=utf-8' } -The dictionary is special, though: it's made just for HTTP headers. According to `RFC 2616 `_, HTTP +The dictionary is special, though: it's made just for HTTP headers. According to +`RFC 2616 `_, HTTP Headers are case-insensitive. So, we can access the headers using any capitalization we want:: @@ -118,7 +122,7 @@ If a response contains some Cookies, you can get quick access to them:: >>> print r.cookies {'requests-is': 'awesome'} -The underlying CookieJar is also available for more advanced handing:: +The underlying CookieJar is also available for more advanced handling:: >>> r.request.cookiejar