Merge branch 'develop' into urlencode

.gitignore

@@ -5,4 +5,5 @@ nosetests.xml
 pylint.txt
 *.pyc
 docs/_build
-toy.py
+toy.py
+.gitignore

AUTHORS

@@ -28,3 +28,6 @@ Patches and Suggestions
 - Alberto Paro
 - Jérémy Bethmont
 - 潘旭 (Xu Pan)
+- Tamás Gulácsi
+- Rubén Abad
+- Peter Manser

HISTORY.rst

@@ -1,10 +1,17 @@
 History
 -------
 
-0.5.1 (?)
-+++++++++
+0.5.1 (2011-07-23)
+++++++++++++++++++
 
 * International Domain Name Support!
+* Access headers without fetching entire body (``read()``)
+* Use lists as dicts for parameters
+* Add Forced Basic Authentication
+* Forced Basic is default authentication type
+* ``python-requests.org`` default User-Agent header
+* CaseInsensitiveDict lower-case caching
+* Response.history bugfix
 
 
 0.5.0 (2011-06-21)

README.rst

@@ -43,7 +43,7 @@ HTTPS? Basic Authentication? ::
 
 Uh oh, we're not authorized! Let's add authentication. ::
 
-    >>> r = requests.get(https://httpbin.ep.io/basic-auth/user/pass', auth=('user', 'pass'))
+    >>> r = requests.get('https://httpbin.ep.io/basic-auth/user/pass', auth=('user', 'pass'))
 
     >>> r.status_code
     200

debian/changelog

@@ -0,0 +1,5 @@
+python-requests (0.4.1-0) testing; urgency=low
+
+  * Initial Debian package
+
+ -- Bruno Clermont <bruno.clermont@gmail.com>  Thu, 26 May 2011 16:25:00 -0500

debian/compat

@@ -0,0 +1 @@
+5

debian/control

@@ -0,0 +1,13 @@
+Source: python-requests
+Section: python
+Priority: optional
+Maintainer: Bruno Clermont <bruno.clermont@gmail.com>
+Homepage: https://github.com/bclermont/requests
+Bugs: https://github.com/bclermont/requests/issues
+Build-Depends: debhelper, python-support
+
+Package: python-requests
+Architecture: all
+Depends: ${python:Depends}, python-support
+Provides: ${python:Provides}
+Description: Python HTTP Requests for Humans.

debian/docs

@@ -0,0 +1 @@
+docs/user/*.rst

debian/pyversions

@@ -0,0 +1 @@
+2.4-

debian/rules

@@ -0,0 +1,42 @@
+#!/usr/bin/make -f
+
+# Verbose mode
+#export DH_VERBOSE=1
+
+clean:
+	dh_testdir
+	dh_testroot
+
+	rm -rf build requests.egg-info
+#	find django-sentry/ -name *.pyc | xargs rm -f
+
+	dh_clean
+
+build:
+	dh_testdir
+
+	python setup.py build
+
+install:
+	dh_testdir
+	dh_installdirs
+
+	python setup.py install --root $(CURDIR)/debian/python-requests
+
+binary-indep: install
+
+binary-arch: install
+	dh_install
+	dh_installdocs
+#	dh_installchangelogs
+	dh_compress
+	dh_fixperms
+	dh_pysupport
+	dh_gencontrol
+	dh_installdeb
+	dh_md5sums
+	dh_builddeb -- -Z lzma -z9
+
+binary: binary-indep binary-arch
+.PHONY: build clean binary-indep binary-arch binary
+

requests/config.py

@@ -53,4 +53,6 @@ class Settings(object):
                 return None
         return object.__getattribute__(self, key)
 
-settings = Settings()
+settings = Settings()
+settings.base_headers = {'User-Agent': 'python-requests.org'}
+settings.accept_gzip = True

requests/core.py

@@ -12,8 +12,8 @@ This module implements the main Requests system.
 """
 
 __title__ = 'requests'
-__version__ = '0.5.0'
-__build__ = 0x000500
+__version__ = '0.5.1'
+__build__ = 0x000501
 __author__ = 'Kenneth Reitz'
 __license__ = 'ISC'
 __copyright__ = 'Copyright 2011 Kenneth Reitz'

requests/models.py

@@ -16,7 +16,7 @@ from urlparse import urlparse, urlunparse
 from datetime import datetime
 
 from .config import settings
-from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler
+from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler
 from .structures import CaseInsensitiveDict
 from .packages.poster.encode import multipart_encode
 from .packages.poster.streaminghttp import register_openers, get_handlers
@@ -81,6 +81,23 @@ class Request(object):
         self.sent = False
 
 
+        # Header manipulation and defaults.
+
+        if settings.accept_gzip:
+            settings.base_headers.update({'Accept-Encoding': 'gzip'})
+
+        if headers:
+            headers = CaseInsensitiveDict(self.headers)
+        else:
+            headers = CaseInsensitiveDict()
+
+        for (k, v) in settings.base_headers.items():
+            if k not in headers:
+                headers[k] = v
+
+        self.headers = headers
+
+
     def __repr__(self):
         return '<Request [%s]>' % (self.method)
 
@@ -165,10 +182,7 @@ class Request(object):
 
         r = build(resp)
 
-        if r.status_code in REDIRECT_STATI:
-            self.redirect = True
-
-        if self.redirect:
+        if r.status_code in REDIRECT_STATI and not self.redirect:
 
             while (
                 ('location' in r.headers) and
@@ -211,18 +225,19 @@ class Request(object):
         """Encode parameters in a piece of data.
 
         If the data supplied is a dictionary, encodes each parameter in it, and
-        returns the dictionary of encoded parameters, and a urlencoded version
-        of that.
+        returns a list of tuples containing the encoded parameters, and a urlencoded
+        version of that.
 
         Otherwise, assumes the data is already encoded appropriately, and
         returns it twice.
 
         """
         if hasattr(data, 'items'):
-            result = {}
-            for (k, v) in data.items():
-                result[k.encode('utf-8') if isinstance(k, unicode) else k] \
-                     = v.encode('utf-8') if isinstance(v, unicode) else v
+            result = []
+            for k, vs in data.items():
+                for v in isinstance(vs, list) and vs or [vs]:
+                    result.append((k.encode('utf-8') if isinstance(k, unicode) else k,
+                                   v.encode('utf-8') if isinstance(v, unicode) else v))
             return result, urllib.urlencode(result, doseq=True)
         else:
             return data, data
@@ -535,17 +550,18 @@ class AuthObject(object):
 
     _handlers = {
         'basic': HTTPBasicAuthHandler,
+        'forced_basic': HTTPForcedBasicAuthHandler,
         'digest': HTTPDigestAuthHandler,
         'proxy_basic': urllib2.ProxyBasicAuthHandler,
         'proxy_digest': urllib2.ProxyDigestAuthHandler
     }
 
-    def __init__(self, username, password, handler='basic', realm=None):
+    def __init__(self, username, password, handler='forced_basic', realm=None):
         self.username = username
         self.password = password
         self.realm = realm
 
         if isinstance(handler, basestring):
-            self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler)
+            self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler)
         else:
             self.handler = handler

requests/monkeys.py

@@ -9,7 +9,7 @@ Urllib2 Monkey patches.
 """
 
 import urllib2
-
+import re
 
 class Request(urllib2.Request):
     """Hidden wrapper around the urllib2.Request object. Allows for manual
@@ -26,8 +26,9 @@ class Request(urllib2.Request):
 
         return urllib2.Request.get_method(self)
 
-class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
 
+class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
+    """HTTP Redirect handler."""
     def http_error_301(self, req, fp, code, msg, headers):
         pass
 
@@ -36,10 +37,13 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
 
 
 class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+    """HTTP Basic Auth Handler with authentication loop fixes."""
 
     def __init__(self, *args, **kwargs):
         urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
         self.retried_req = None
+        self.retried = 0
+
 
     def reset_retry_count(self):
         # Python 2.6.5 will call this on 401 or 407 errors and thus loop
@@ -47,6 +51,7 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
         # http_error_auth_reqed instead.
         pass
 
+
     def http_error_auth_reqed(self, auth_header, host, req, headers):
         # Reset the retry counter once for each request.
         if req is not self.retried_req:
@@ -59,6 +64,59 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
 
 
 
+class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler):
+    """HTTP Basic Auth Handler with forced Authentication."""
+
+    auth_header = 'Authorization'
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    def __init__(self,  *args, **kwargs):
+        HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
+
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self._http_error_auth_reqed('www-authenticate', url, req, headers)
+        self.reset_retry_count()
+        return response
+
+    http_error_404 = http_error_401
+
+
+    def _http_error_auth_reqed(self, authreq, host, req, headers):
+
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+
+            mo = self.rx.search(authreq)
+
+            if mo:
+                scheme, quote, realm = mo.groups()
+
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+
+                    if response and response.code not in (401, 404):
+                        self.retried = 0
+                    return response
+        else:
+            response = self.retry_http_basic_auth(host, req, 'Realm')
+
+            if response and response.code not in (401, 404):
+                self.retried = 0
+            return response
+
+
+
 class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler):
 
     def __init__(self, *args, **kwargs):

requests/structures.py

@@ -9,20 +9,39 @@ Datastructures that power Requests.
 """
 
 class CaseInsensitiveDict(dict):
-    """Case-insensitive Dictionary for :class:`Response <models.Response>` Headers.
+    """Case-insensitive Dictionary
 
     For example, ``headers['content-encoding']`` will return the
     value of a ``'Content-Encoding'`` response header."""
 
-    def _lower_keys(self):
-        return map(str.lower, self.keys())
+    @property
+    def lower_keys(self):
+        if not hasattr(self, '_lower_keys') or not self._lower_keys:
+            self._lower_keys = dict((k.lower(), k) for k in self.iterkeys())
+        return self._lower_keys
 
+    def _clear_lower_keys(self):
+        if hasattr(self, '_lower_keys'):
+            self._lower_keys.clear()
+
+    def __setitem__(self, key, value):
+        dict.__setitem__(self, key, value)
+        self._clear_lower_keys()
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        self._lower_keys.clear()
 
     def __contains__(self, key):
-        return key.lower() in self._lower_keys()
-
+        return key.lower() in self.lower_keys
 
     def __getitem__(self, key):
         # We allow fall-through here, so values default to None
         if key in self:
-            return self.items()[self._lower_keys().index(key.lower())][1]
+            return dict.__getitem__(self, self.lower_keys[key.lower()])
+
+    def get(self, key, default=None):
+        if key in self:
+            return self[key]
+        else:
+            return default

test_requests.py

@@ -6,7 +6,10 @@ from __future__ import with_statement
 import unittest
 import cookielib
 
-import omnijson as json
+try:
+    import omnijson as json
+except ImportError:
+    import json
 
 import requests
 
@@ -323,8 +326,41 @@ class RequestsTestSuite(unittest.TestCase):
 
     def test_idna(self):
         r = requests.get(u'http://➡.ws/httpbin')
-        self.assertEqual(r.url, HTTPBIN_URL)
+        assert 'httpbin' in r.url
 
+    def test_urlencoded_get_query_multivalued_param(self):
+        r = requests.get(httpbin('get'), params=dict(test=['foo','baz']))
+        self.assertEquals(r.status_code, 200)
+        self.assertEquals(r.url, httpbin('get?test=foo&test=baz'))
+
+    def test_urlencoded_post_querystring_multivalued(self):
+        r = requests.post(httpbin('post'), params=dict(test=['foo','baz']))
+        self.assertEquals(r.status_code, 200)
+        self.assertEquals(r.headers['content-type'], 'application/json')
+        self.assertEquals(r.url, httpbin('post?test=foo&test=baz'))
+        rbody = json.loads(r.content)
+        self.assertEquals(rbody.get('form'), {}) # No form supplied
+        self.assertEquals(rbody.get('data'), '')
+
+    def test_urlencoded_post_query_multivalued_and_data(self):
+        r = requests.post(httpbin('post'), params=dict(test=['foo','baz']),
+                          data=dict(test2="foobar",test3=['foo','baz']))
+        self.assertEquals(r.status_code, 200)
+        self.assertEquals(r.headers['content-type'], 'application/json')
+        self.assertEquals(r.url, httpbin('post?test=foo&test=baz'))
+        rbody = json.loads(r.content)
+        self.assertEquals(rbody.get('form'), dict(test2='foobar',test3='foo'))
+        self.assertEquals(rbody.get('data'), '')
+
+
+    def test_redirect_history(self):
+        r = requests.get(httpbin('redirect', '3'))
+        self.assertEquals(r.status_code, 200)
+        self.assertEquals(len(r.history), 3)
+
+        r = requests.get(httpsbin('redirect', '3'))
+        self.assertEquals(r.status_code, 200)
+        self.assertEquals(len(r.history), 3)
 
 if __name__ == '__main__':
     unittest.main()