diff --git a/AUTHORS.rst b/AUTHORS.rst index 48cd155b..e4a325bf 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -178,4 +178,5 @@ Patches and Suggestions - Moinuddin Quadri (`@moin18 `_) - Matt Kohl (`@mattkohl `_) - Jonathan Vanasco (`@jvanasco `_) +- David Fontenot (`@davidfontenot `_) diff --git a/README.rst b/README.rst index d276e42a..9ff629e7 100644 --- a/README.rst +++ b/README.rst @@ -28,7 +28,7 @@ Requests: HTTP for Humans Requests is the only *Non-GMO* HTTP library for Python, safe for human consumption. -**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, +**Warning:** Recreational use of the Python standard library for HTTP may result in dangerous side-effects, including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death. @@ -50,7 +50,7 @@ Behold, the power of Requests: See `the similar code, sans Requests `_. -.. image:: http://docs.python-requests.org/en/master/_static/requests-sidebar.png +.. image:: https://raw.githubusercontent.com/kennethreitz/requests/master/docs/_static/requests-logo-small.png :target: http://docs.python-requests.org/ diff --git a/docs/_static/requests-logo-small.png b/docs/_static/requests-logo-small.png new file mode 100644 index 00000000..afadeaa4 Binary files /dev/null and b/docs/_static/requests-logo-small.png differ diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index fc9a659a..fe113734 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -40,6 +40,7 @@

More Kenneth Reitz projects:

    +
  • edmsynths.com
  • pipenv
  • pep8.org
  • httpbin.org
    • diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 4c05a538..2fb8062b 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -42,6 +42,7 @@

    More Kenneth Reitz projects:

      +
    • edmsynths.com
    • pipenv
    • pep8.org
    • httpbin.org
      • diff --git a/docs/index.rst b/docs/index.rst index c1d78c87..b7843e70 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -68,21 +68,21 @@ PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington Post, Twitter, SoundCloud, Kippt, Sony, and Federal U.S. Institutions that prefer to be unnamed claim to use Requests internally. -**Armin Ronacher** - Requests is the perfect example how beautiful an API can be with the - right level of abstraction. +**Armin Ronacher**— + *Requests is the perfect example how beautiful an API can be with the + right level of abstraction.* -**Matt DeBoard** - I'm going to get `@kennethreitz `_'s Python requests module tattooed - on my body, somehow. The whole thing. +**Matt DeBoard**— + *I'm going to get `@kennethreitz `_'s Python requests module tattooed + on my body, somehow. The whole thing.* -**Daniel Greenfeld** - Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to - `@kennethreitz `_'s request library. Today has been AWESOME. +**Daniel Greenfeld**— + *Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to + `@kennethreitz `_'s request library. Today has been AWESOME.* -**Kenny Meyers** - Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful, - simple, Pythonic. +**Kenny Meyers**— + *Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful, + simple, Pythonic.* Requests is one of the most downloaded Python packages of all time, pulling in over 11,000,000 downloads every month. All the cool kids are doing it! diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 3c63394d..0806ff6d 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -227,7 +227,7 @@ By default, ``verify`` is set to True. Option ``verify`` only applies to host ce You can also specify a local cert to use as client side certificate, as a single file (containing the private key and the certificate) or as a tuple of both -file's path:: +files' paths:: >>> requests.get('https://kennethreitz.org', cert=('/path/client.cert', '/path/client.key')) diff --git a/requests/api.py b/requests/api.py index 16fd1e94..856f0b3a 100644 --- a/requests/api.py +++ b/requests/api.py @@ -19,7 +19,7 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -100,7 +100,7 @@ def post(url, data=None, json=None, **kwargs): """Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -114,7 +114,7 @@ def put(url, data=None, **kwargs): """Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -128,7 +128,7 @@ def patch(url, data=None, **kwargs): """Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object diff --git a/requests/models.py b/requests/models.py index d1a9c868..01b60afe 100644 --- a/requests/models.py +++ b/requests/models.py @@ -659,11 +659,23 @@ class Response(object): return '' % (self.status_code) def __bool__(self): - """Returns true if :attr:`status_code` is 'OK'.""" + """Returns True if :attr:`status_code` is less than 400. + + This attribute checks if the status code of the response is between + 400 and 600 to see if there was a client error or a server error. If + the status code, is between 200 and 400, this will return True. This + is **not** a check to see if the response code is ``200 OK``. + """ return self.ok def __nonzero__(self): - """Returns true if :attr:`status_code` is 'OK'.""" + """Returns True if :attr:`status_code` is less than 400. + + This attribute checks if the status code of the response is between + 400 and 600 to see if there was a client error or a server error. If + the status code, is between 200 and 400, this will return True. This + is **not** a check to see if the response code is ``200 OK``. + """ return self.ok def __iter__(self): @@ -672,6 +684,13 @@ class Response(object): @property def ok(self): + """Returns True if :attr:`status_code` is less than 400. + + This attribute checks if the status code of the response is between + 400 and 600 to see if there was a client error or a server error. If + the status code, is between 200 and 400, this will return True. This + is **not** a check to see if the response code is ``200 OK``. + """ try: self.raise_for_status() except HTTPError: diff --git a/requests/structures.py b/requests/structures.py index 05d2b3f5..0b1bd1e7 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,9 +8,12 @@ Data structures that power Requests. """ import collections +import time from .compat import OrderedDict +current_time = getattr(time, 'monotonic', time.time) + class CaseInsensitiveDict(collections.MutableMapping): """A case-insensitive ``dict``-like object. @@ -103,3 +106,89 @@ class LookupDict(dict): def get(self, key, default=None): return self.__dict__.get(key, default) + + +class TimedCacheManaged(object): + """ + Wrap a function call in a timed cache + """ + def __init__(self, fnc): + self.fnc = fnc + self.cache = TimedCache() + + def __call__(self, *args, **kwargs): + key = args[0] + found = None + try: + found = self.cache[key] + except KeyError: + found = self.fnc(key, **kwargs) + self.cache[key] = found + + return found + + +class TimedCache(collections.MutableMapping): + """ + Evicts entries after expiration_secs. If none are expired and maxlen is hit, + will evict the oldest cached entry + """ + def __init__(self, maxlen=32, expiration_secs=60): + """ + :param maxlen: most number of entries to hold on to + :param expiration_secs: the number of seconds to hold on + to entries + """ + self.maxlen = maxlen + self.expiration_secs = expiration_secs + self._dict = OrderedDict() + + def __repr__(self): + return '' % \ + (self.maxlen, len(self._dict), self.expiration_secs) + + def __iter__(self): + return ((key, value[1]) for key, value in self._dict.items()) + + def __delitem__(self, item): + del self._dict[item] + + def __getitem__(self, key): + """ + Look up an item in the cache. If the item + has already expired, it will be invalidated and not returned + + :param key: which entry to look up + :return: the value in the cache, or None + """ + occurred, value = self._dict[key] + now = int(current_time()) + + if now - occurred > self.expiration_secs: + del self._dict[key] + raise KeyError(key) + else: + return value + + def __setitem__(self, key, value): + """ + Locates the value at lookup key, if cache is full, will evict the + oldest entry + + :param key: the key to search the cache for + :param value: the value to be added to the cache + """ + now = int(current_time()) + + while len(self._dict) >= self.maxlen: + self._dict.popitem(last=False) + + self._dict[key] = (now, value) + + def __len__(self): + """:return: the length of the cache""" + return len(self._dict) + + def clear(self): + """Clears the cache""" + return self._dict.clear() diff --git a/requests/utils.py b/requests/utils.py index e5ecd350..f49d8215 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -28,7 +28,7 @@ from .compat import ( quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, proxy_bypass, urlunparse, basestring, integer_types) from .cookies import RequestsCookieJar, cookiejar_from_dict -from .structures import CaseInsensitiveDict +from .structures import CaseInsensitiveDict, TimedCache, TimedCacheManaged from .exceptions import ( InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError) @@ -92,14 +92,16 @@ def super_len(o): else: if hasattr(o, 'seek') and total_length is None: # StringIO and BytesIO have seek but no useable fileno + try: + # seek to end of file + o.seek(0, 2) + total_length = o.tell() - # seek to end of file - o.seek(0, 2) - total_length = o.tell() - - # seek back to current position to support - # partially read file-like objects - o.seek(current_position or 0) + # seek back to current position to support + # partially read file-like objects + o.seek(current_position or 0) + except (OSError, IOError): + total_length = 0 if total_length is None: total_length = 0 @@ -577,6 +579,16 @@ def set_environ(env_name, value): os.environ[env_name] = old_value +@TimedCacheManaged +def _proxy_bypass_cached(netloc): + """ + Looks for netloc in the cache, if not found, will call proxy_bypass + for the netloc and store its result in the cache + + :rtype: bool + """ + return proxy_bypass(netloc) + def should_bypass_proxies(url, no_proxy): """ Returns whether we should bypass proxies or not. @@ -624,7 +636,7 @@ def should_bypass_proxies(url, no_proxy): # legitimate problems. with set_environ('no_proxy', no_proxy_arg): try: - bypass = proxy_bypass(netloc) + bypass = _proxy_bypass_cached(netloc) except (TypeError, socket.gaierror): bypass = False @@ -847,7 +859,7 @@ def rewind_body(prepared_request): try: body_seek(prepared_request._body_position) except (IOError, OSError): - raise UnrewindableBodyError("An error occured when rewinding request " + raise UnrewindableBodyError("An error occurred when rewinding request " "body for redirect.") else: raise UnrewindableBodyError("Unable to rewind request body for redirect.") diff --git a/tests/test_requests.py b/tests/test_requests.py index cd4c68db..829ab3eb 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1546,7 +1546,7 @@ class TestRequests: def tell(self): return 0 - def seek(self, pos): + def seek(self, pos, whence=0): raise OSError() def __iter__(self): @@ -1560,7 +1560,7 @@ class TestRequests: with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'error occured when rewinding request body' in str(e) + assert 'error occurred when rewinding request body' in str(e) def test_rewind_body_failed_tell(self): class BadFileObj: diff --git a/tests/test_structures.py b/tests/test_structures.py index e4d2459f..a28e041e 100644 --- a/tests/test_structures.py +++ b/tests/test_structures.py @@ -2,7 +2,7 @@ import pytest -from requests.structures import CaseInsensitiveDict, LookupDict +from requests.structures import CaseInsensitiveDict, LookupDict, TimedCache, TimedCacheManaged class TestCaseInsensitiveDict: @@ -74,3 +74,75 @@ class TestLookupDict: @get_item_parameters def test_get(self, key, value): assert self.lookup_dict.get(key) == value + + +class TestTimedCache(object): + @pytest.fixture(autouse=True) + def setup(self): + self.any_value = 'some value' + self.expiration_secs = 60 + self.cache = TimedCache(expiration_secs=self.expiration_secs) + yield + self.cache.clear() + + def test_get(self): + self.cache['a'] = self.any_value + assert self.cache['a'] is self.any_value + + def test_repr(self): + repr = str(self.cache) + assert repr == '' + + def test_get_expired_item(self, mocker): + self.cache = TimedCache(maxlen=1, expiration_secs=self.expiration_secs) + + mocker.patch('requests.structures.current_time', lambda: 0) + self.cache['a'] = self.any_value + mocker.patch('requests.structures.current_time', lambda: self.expiration_secs + 1) + assert self.cache.get('a') is None + + def test_evict_first_entry_when_full(self, mocker): + self.cache = TimedCache(maxlen=2, expiration_secs=2) + mocker.patch('requests.structures.current_time', lambda: 0) + self.cache['a'] = self.any_value + mocker.patch('requests.structures.current_time', lambda: 1) + self.cache['b'] = self.any_value + mocker.patch('requests.structures.current_time', lambda: 3) + self.cache['c'] = self.any_value + assert len(self.cache) is 2 + with pytest.raises(KeyError, message='Expected key not found'): + self.cache['a'] + assert self.cache['b'] is self.any_value + assert self.cache['c'] is self.any_value + + def test_delete_item_removes_item(self): + self.cache['a'] = self.any_value + del self.cache['a'] + with pytest.raises(KeyError, message='Expected key not found'): + self.cache['a'] + + def test_iterating_hides_timestamps(self): + self.cache['a'] = 1 + self.cache['b'] = 2 + expected = [('a', 1), ('b', 2)] + actual = [(key, val) for key, val in self.cache] + assert expected == actual + + +class TestTimedCacheManagedDecorator(object): + def test_caches_repeated_calls(self, mocker): + mocker.patch('requests.structures.current_time', lambda: 0) + + nonlocals = {'value': 0} + + @TimedCacheManaged + def some_method(x): + nonlocals['value'] = nonlocals['value'] + x + return nonlocals['value'] + + first_result = some_method(1) + assert first_result is 1 + second_result = some_method(1) + assert second_result is 1 + third_result = some_method(2) + assert third_result is 3