commit 2c7f0d65c8b34a8b7610a6e304e1c2fee94ca9ac Author: Kyle L. Jensen Date: Wed Jun 27 11:23:49 2012 -0400 Initial import. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6bf6524 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +build +dist +env +*.py[c,o] +*.bak +*.swp +*.egg-info +.DS_Store diff --git a/AUTHORS.txt b/AUTHORS.txt new file mode 100644 index 0000000..2a53309 --- /dev/null +++ b/AUTHORS.txt @@ -0,0 +1 @@ +Kyle Jensen - https://github.com/kljensen/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fb2615f --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2012 Kyle L. Jensen + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2d8e941 --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +.PHONY: all pep8 pyflakes clean dev + +GITIGNORES=$(shell cat .gitignore |tr "\\n" ",") + +all: pep8 + +pep8: .gitignore env + @bin/virtual-env-exec pep8 . --exclude=$(GITIGNORES) + +pyflakes: env + @bin/virtual-env-exec pyflakes reqcache tests + +pylint: env + @bin/virtual-env-exec pylint reqcache 2>&1 |less + +dev: env env/.pip + +env: + @virtualenv --distribute env + +env/.pip: env cfg/requirements.txt + @bin/virtual-env-exec pip install -r cfg/requirements.txt + @bin/virtual-env-exec pip install -e . + @touch env/.pip + +test: env/.pip + @bin/virtual-env-exec testify tests + +shell: + @bin/virtual-env-exec ipython + +devclean: + @rm -rf env + +clean: + @rm -rf build dist env diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..c822886 --- /dev/null +++ b/README.rst @@ -0,0 +1,39 @@ +py-reqcache: Caching for Python's Request Package +========================= + +py-reqcache is a Python_ package that provides caching for +the Requests_ HTTP library. It's based on the wonderful +Requests-Cache_ libary by Roman Haritonov and uses the +backends from that project. The main difference is that +this package uses the Requests_ API hooks instead of +monkeypatching. + + +Example usage +---------- + + import requests + import reqcache + + c = reqcache.ReqCache("foo", "memory") + + r = requests.get('http://github.com', hooks=c.hooks) + print getattr(r, "from_cache", False) + + r = requests.get('http://github.com', hooks=c.hooks) + print getattr(r, "from_cache", False) + + +Contribute +---------- + +#. Fork the project on github to start making your changes +#. Send pull requests with your bug fixes or features +#. Submit and create issues on github + + +References +---------- +.. _Python: http://www.python.org/ +.. _Requests: http://www.python-requests.org +.. _Requests-Cache: https://github.com/reclosedev/requests-cache diff --git a/bin/virtual-env-exec b/bin/virtual-env-exec new file mode 100755 index 0000000..2818e35 --- /dev/null +++ b/bin/virtual-env-exec @@ -0,0 +1,3 @@ +#!/bin/sh +source env/bin/activate +$@ diff --git a/cfg/requirements.txt b/cfg/requirements.txt new file mode 100644 index 0000000..42e4f52 --- /dev/null +++ b/cfg/requirements.txt @@ -0,0 +1,7 @@ +testify +pep8 +pyflakes +ipython +pylint +requests +requests-cache \ No newline at end of file diff --git a/reqcache/__init__.py b/reqcache/__init__.py new file mode 100644 index 0000000..fbf3354 --- /dev/null +++ b/reqcache/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +""" +reqcache +~~~~~~~~ + +:copyright: (c) 2012 by Firstname Lastname. +:license: ISC, see LICENSE for more details. + +""" + +__title__ = 'reqcache' +__version__ = '0.0.1' +__description__ = 'Python Requests Caching' +__url__ = 'https://github.com/kljensen/py-reqcache' +__build__ = 0 +__author__ = 'Kyle Jensen' +__license__ = 'ISC' +__copyright__ = 'Copyright 2012 Kyle Jensen' + + +from .models import ReqCache diff --git a/reqcache/models.py b/reqcache/models.py new file mode 100644 index 0000000..5b91b44 --- /dev/null +++ b/reqcache/models.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +""" +This module contains the primary objects for caching responses +for the Python Requests library. + +:copyright: (c) 2012 by Kyle Jensen. +:license: ISC, see LICENSE for more details. +""" +import hashlib +import datetime +import requests +from requests_cache import backends + + +class ReqCache(object): + + def __init__(self, cache_name, backend, + allowable_codes=(200,), + allowable_methods=('GET',), + expire_after=None, + **backend_options): + + super(ReqCache, self).__init__() + self.cache_name = cache_name + self.backend = backend + self.allowable_codes = allowable_codes + self.allowable_methods = allowable_methods + self.expire_after = expire_after + + try: + self.cache = backends.registry[self.backend]( + cache_name, + **backend_options + ) + except KeyError: + raise ValueError('Unsupported backend "%s" try one of: %s' % + (backend, ', '.join(backends.registry.keys()))) + + @staticmethod + def reqresp_to_key(r): + """ + Accepts a Request or Reponse object, returns a string key for + storing the cached response in a dictionary-like object. + """ + if isinstance(r, requests.Response): + request = r.request + else: + request = r + + key = "method={0} url={1}".format( + request.method, + request.full_url, + ) + + if request.method in ("POST", "PUT"): + data = request._encode_params(getattr(r, 'data', {})) + key = "{0} datahash={2}".format( + key, + hashlib.sha224(data).hexdigest(), + ) + return key + + def to_cache(self, response): + """ + Save a response to the cache. + """ + if (response.status_code in self.allowable_codes + and response.request.method in self.allowable_methods + and not hasattr(response, 'from_cache')): + + key = self.reqresp_to_key(response) + self.cache.save_response(key, response) + + return response + + def from_cache(self, request): + """ + Retrieve a response from the cache given a request. + """ + if request.method in self.allowable_methods: + key = self.reqresp_to_key(request) + response, timestamp = self.cache.get_response_and_time(key) + + if response: + difference = datetime.datetime.now() - timestamp + + if (self.expire_after is not None + and difference > + datetime.timedelta(minutes=self.expire_after)): + self.cache.del_cached_url(key) + else: + request.sent = True + request.response = response + request.response.request = request + request.response.from_cache = True + return request + + @property + def hooks(self): + return { + "pre_request": self.from_cache, + "response": self.to_cache, + } + +if __name__ == '__main__': + + def explain_cache_result(response): + was_cached = getattr(response, "from_cache", False) + if was_cached: + source = "cache" + else: + source = "interwebs" + + msg = "Got response from {0} for {1}".format( + source, + ReqCache.reqresp_to_key(response.request), + ) + print msg + + reqcache = ReqCache("test", "memory") + + r = requests.get('http://github.com', hooks=reqcache.hooks) + explain_cache_result(r) + + r = requests.get('http://github.com', hooks=reqcache.hooks) + explain_cache_result(r) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fc60973 --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys +from distutils.core import setup + +PACKAGES = ['reqcache'] + + +def get_init_val(val, packages=PACKAGES): + pkg_init = "%s/__init__.py" % PACKAGES[0] + value = '__%s__' % val + fn = open(pkg_init) + for line in fn.readlines(): + if line.startswith(value): + return line.split('=')[1].strip().strip("'") + + +setup( + name='py-%s' % get_init_val('title'), + version=get_init_val('version'), + description=get_init_val('description'), + long_description=open('README.rst').read(), + author=get_init_val('author'), + url=get_init_val('url'), + package_data={'': ['LICENSE', 'NOTICE']}, + license=get_init_val('license'), + packages=PACKAGES +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/models.py b/tests/models.py new file mode 100644 index 0000000..3d3ad03 --- /dev/null +++ b/tests/models.py @@ -0,0 +1,22 @@ +import testify +import requests +from reqcache.models import ReqCache + + +class ReqCacheTestCase(testify.TestCase): + + @testify.setup + def create_cache(self): + self.rcache = ReqCache("test", "memory") + + @testify.teardown + def clear_cache(self): + self.rcache = None + + def test_basic_caching(self): + + r = requests.get('http://github.com', hooks=self.rcache.hooks) + self.assertFalse(getattr(r, "from_cache", False)) + + r = requests.get('http://github.com', hooks=self.rcache.hooks) + self.assertTrue(getattr(r, "from_cache", False))