Implement request caching in Redis so that we don't always have to fetch remote pages every time we want their mf2 items

This commit is contained in:
Danielle McLean 2017-11-10 09:17:32 +11:00
parent b8a8cd62cf
commit a7f6824334
Signed by: 00dani
GPG key ID: 5A5D2D1AFF12EEC5
5 changed files with 82 additions and 5 deletions

View file

@ -41,6 +41,8 @@ django-model-utils = "*"
python-jose = "*"
django-rq = "*"
ronkyuu = "*"
cachecontrol = "*"
hiredis = "*"
[dev-packages]

30
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "028ac3e8ca1afd917bb4d7d3e0b93c5a8b52a2700cb4e67e2e72ffae8594dfab"
"sha256": "a5ea894e51ad2d925b0d24739e2c644164b1edefbd3d363c1225a76c3926c23b"
},
"host-environment-markers": {
"implementation_name": "cpython",
@ -50,6 +50,12 @@
],
"version": "==2.1.1"
},
"cachecontrol": {
"hashes": [
"sha256:a9fc50e216c7c101f4ec4312f012dea501c2859cb256c7a68186a172ab71f632"
],
"version": "==0.12.3"
},
"certifi": {
"hashes": [
"sha256:244be0d93b71e93fc0a0a479862051414d0e00e16435707e5bf5000f92e04694",
@ -218,6 +224,12 @@
],
"version": "==19.7.1"
},
"hiredis": {
"hashes": [
"sha256:ca958e13128e49674aa4a96f02746f5de5973f39b57297b84d59fd44d314d5b5"
],
"version": "==0.2.0"
},
"html5lib": {
"hashes": [
"sha256:08a3efc117a4fc8c82c3c6d10d6f58ae266428d57ed50258a1466d2cd88de745",
@ -284,6 +296,22 @@
],
"version": "==1.0.5"
},
"msgpack-python": {
"hashes": [
"sha256:637b012c9ea021de7a7a75d6ff5e82cfef6694babd7e14bb9a3adcb2a5bd52f0",
"sha256:658c1cd5dcf7786e0e7a6d523cd0c5b33f92e139e224bd73cb3a23ada618d2dc",
"sha256:920bbbaee07ad048a4d2b4160901b19775c61ef9439f856c74509e763a326249",
"sha256:e165006f7e3d2612f1bffe2f6f042ca317d8df724d8b72a39b14c2e46c67eaae",
"sha256:95d70edd50e3d2f6ea1189f77190e4a0172626e7405ddd1689f3f64814447cba",
"sha256:7e1b12ea0134460052fabcfaa0f488ec0fc21deb14832d66236fd2870757d8f1",
"sha256:8f36890251f20d96267618cf64735759d7ef7e91bc0b86b9480547d2d1397a68",
"sha256:1e68a277e4180baa7789be36f27f0891660205f6209f78a32282d3c422873d78",
"sha256:f52d9f96df952369fe4adcb0506e10c1c92d47f653f601a66da2a26a7e7141ea",
"sha256:58c9c1d7891a35bddc6ee5dbec10d347a7ae4983169c24fc5fc8a57ae792ca76",
"sha256:1a2b19df0f03519ec7f19f826afb935b202d8979b0856c6fb3dc28955799f886"
],
"version": "==0.4.8"
},
"olefile": {
"hashes": [
"sha256:61f2ca0cd0aa77279eb943c07f607438edf374096b66332fae1ee64a6f0f73ad"

View file

@ -1,5 +1,3 @@
import mf2py
from annoying.decorators import render_to
from django.contrib.auth import get_user_model
from django.contrib.auth.decorators import login_required
@ -9,7 +7,7 @@ from django.utils.decorators import method_decorator
from django.views.generic import TemplateView
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from lemoncurry import breadcrumbs, utils
from lemoncurry import breadcrumbs, requests, utils
from urllib.parse import urlencode, urljoin, urlunparse, urlparse
from .. import tokens
@ -68,7 +66,7 @@ class IndieView(TemplateView):
)
scopes = params['scope'].split(' ')
client = mf2py.Parser(url=params['client_id'], html_parser='html5lib')
client = requests.mf2(params['client_id'])
rels = (client.to_dict()['rel-urls']
.get(redirect_uri, {})
.get('rels', ()))

44
lemoncurry/requests.py Normal file
View file

@ -0,0 +1,44 @@
import requests
from cachecontrol.wrapper import CacheControl
from cachecontrol.cache import BaseCache
from cachecontrol.heuristics import LastModified
from datetime import datetime
from django.core.cache import cache as django_cache
from hashlib import sha256
from mf2py import Parser
class DjangoCache(BaseCache):
@classmethod
def key(cls, url):
return 'req:' + sha256(url.encode('utf-8')).hexdigest()
def get(self, url):
key = self.key(url)
return django_cache.get(key)
def set(self, url, value, expires=None):
key = self.key(url)
if expires:
lifetime = (expires - datetime.utcnow()).total_seconds()
django_cache.set(key, value, lifetime)
else:
django_cache.set(key, value)
req = CacheControl(
requests.Session(),
cache=DjangoCache(),
heuristic=LastModified(),
)
def get(url):
r = req.get(url)
r.raise_for_status()
return r
def mf2(url):
r = get(url)
return Parser(doc=r.text, url=url, html_parser='html5lib')

View file

@ -134,6 +134,11 @@ CACHES = {
'default': {
'BACKEND': 'redis_cache.RedisCache',
'LOCATION': '127.0.0.1:6380',
'KEY_PREFIX': 'lemoncurry',
'OPTIONS': {
'DB': 0,
'PARSER_CLASS': 'redis.connection.HiredisParser',
},
}
}