Implement request caching in Redis so that we don't always have to fetch remote pages every time we want their mf2 items

This commit is contained in:
Danielle McLean 2017-11-10 09:17:32 +11:00
parent b8a8cd62cf
commit a7f6824334
Signed by: 00dani
GPG key ID: 5A5D2D1AFF12EEC5
5 changed files with 82 additions and 5 deletions

View file

@ -41,6 +41,8 @@ django-model-utils = "*"
python-jose = "*" python-jose = "*"
django-rq = "*" django-rq = "*"
ronkyuu = "*" ronkyuu = "*"
cachecontrol = "*"
hiredis = "*"
[dev-packages] [dev-packages]

30
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "028ac3e8ca1afd917bb4d7d3e0b93c5a8b52a2700cb4e67e2e72ffae8594dfab" "sha256": "a5ea894e51ad2d925b0d24739e2c644164b1edefbd3d363c1225a76c3926c23b"
}, },
"host-environment-markers": { "host-environment-markers": {
"implementation_name": "cpython", "implementation_name": "cpython",
@ -50,6 +50,12 @@
], ],
"version": "==2.1.1" "version": "==2.1.1"
}, },
"cachecontrol": {
"hashes": [
"sha256:a9fc50e216c7c101f4ec4312f012dea501c2859cb256c7a68186a172ab71f632"
],
"version": "==0.12.3"
},
"certifi": { "certifi": {
"hashes": [ "hashes": [
"sha256:244be0d93b71e93fc0a0a479862051414d0e00e16435707e5bf5000f92e04694", "sha256:244be0d93b71e93fc0a0a479862051414d0e00e16435707e5bf5000f92e04694",
@ -218,6 +224,12 @@
], ],
"version": "==19.7.1" "version": "==19.7.1"
}, },
"hiredis": {
"hashes": [
"sha256:ca958e13128e49674aa4a96f02746f5de5973f39b57297b84d59fd44d314d5b5"
],
"version": "==0.2.0"
},
"html5lib": { "html5lib": {
"hashes": [ "hashes": [
"sha256:08a3efc117a4fc8c82c3c6d10d6f58ae266428d57ed50258a1466d2cd88de745", "sha256:08a3efc117a4fc8c82c3c6d10d6f58ae266428d57ed50258a1466d2cd88de745",
@ -284,6 +296,22 @@
], ],
"version": "==1.0.5" "version": "==1.0.5"
}, },
"msgpack-python": {
"hashes": [
"sha256:637b012c9ea021de7a7a75d6ff5e82cfef6694babd7e14bb9a3adcb2a5bd52f0",
"sha256:658c1cd5dcf7786e0e7a6d523cd0c5b33f92e139e224bd73cb3a23ada618d2dc",
"sha256:920bbbaee07ad048a4d2b4160901b19775c61ef9439f856c74509e763a326249",
"sha256:e165006f7e3d2612f1bffe2f6f042ca317d8df724d8b72a39b14c2e46c67eaae",
"sha256:95d70edd50e3d2f6ea1189f77190e4a0172626e7405ddd1689f3f64814447cba",
"sha256:7e1b12ea0134460052fabcfaa0f488ec0fc21deb14832d66236fd2870757d8f1",
"sha256:8f36890251f20d96267618cf64735759d7ef7e91bc0b86b9480547d2d1397a68",
"sha256:1e68a277e4180baa7789be36f27f0891660205f6209f78a32282d3c422873d78",
"sha256:f52d9f96df952369fe4adcb0506e10c1c92d47f653f601a66da2a26a7e7141ea",
"sha256:58c9c1d7891a35bddc6ee5dbec10d347a7ae4983169c24fc5fc8a57ae792ca76",
"sha256:1a2b19df0f03519ec7f19f826afb935b202d8979b0856c6fb3dc28955799f886"
],
"version": "==0.4.8"
},
"olefile": { "olefile": {
"hashes": [ "hashes": [
"sha256:61f2ca0cd0aa77279eb943c07f607438edf374096b66332fae1ee64a6f0f73ad" "sha256:61f2ca0cd0aa77279eb943c07f607438edf374096b66332fae1ee64a6f0f73ad"

View file

@ -1,5 +1,3 @@
import mf2py
from annoying.decorators import render_to from annoying.decorators import render_to
from django.contrib.auth import get_user_model from django.contrib.auth import get_user_model
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
@ -9,7 +7,7 @@ from django.utils.decorators import method_decorator
from django.views.generic import TemplateView from django.views.generic import TemplateView
from django.views.decorators.csrf import csrf_exempt from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST from django.views.decorators.http import require_POST
from lemoncurry import breadcrumbs, utils from lemoncurry import breadcrumbs, requests, utils
from urllib.parse import urlencode, urljoin, urlunparse, urlparse from urllib.parse import urlencode, urljoin, urlunparse, urlparse
from .. import tokens from .. import tokens
@ -68,7 +66,7 @@ class IndieView(TemplateView):
) )
scopes = params['scope'].split(' ') scopes = params['scope'].split(' ')
client = mf2py.Parser(url=params['client_id'], html_parser='html5lib') client = requests.mf2(params['client_id'])
rels = (client.to_dict()['rel-urls'] rels = (client.to_dict()['rel-urls']
.get(redirect_uri, {}) .get(redirect_uri, {})
.get('rels', ())) .get('rels', ()))

44
lemoncurry/requests.py Normal file
View file

@ -0,0 +1,44 @@
import requests
from cachecontrol.wrapper import CacheControl
from cachecontrol.cache import BaseCache
from cachecontrol.heuristics import LastModified
from datetime import datetime
from django.core.cache import cache as django_cache
from hashlib import sha256
from mf2py import Parser
class DjangoCache(BaseCache):
@classmethod
def key(cls, url):
return 'req:' + sha256(url.encode('utf-8')).hexdigest()
def get(self, url):
key = self.key(url)
return django_cache.get(key)
def set(self, url, value, expires=None):
key = self.key(url)
if expires:
lifetime = (expires - datetime.utcnow()).total_seconds()
django_cache.set(key, value, lifetime)
else:
django_cache.set(key, value)
req = CacheControl(
requests.Session(),
cache=DjangoCache(),
heuristic=LastModified(),
)
def get(url):
r = req.get(url)
r.raise_for_status()
return r
def mf2(url):
r = get(url)
return Parser(doc=r.text, url=url, html_parser='html5lib')

View file

@ -134,6 +134,11 @@ CACHES = {
'default': { 'default': {
'BACKEND': 'redis_cache.RedisCache', 'BACKEND': 'redis_cache.RedisCache',
'LOCATION': '127.0.0.1:6380', 'LOCATION': '127.0.0.1:6380',
'KEY_PREFIX': 'lemoncurry',
'OPTIONS': {
'DB': 0,
'PARSER_CLASS': 'redis.connection.HiredisParser',
},
} }
} }