Implement request caching in Redis so that we don't always have to fetch remote pages every time we want their mf2 items

Pipfile

@@ -41,6 +41,8 @@ django-model-utils = "*"
 python-jose = "*"
 django-rq = "*"
 ronkyuu = "*"
+cachecontrol = "*"
+hiredis = "*"
 
 
 [dev-packages]

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "028ac3e8ca1afd917bb4d7d3e0b93c5a8b52a2700cb4e67e2e72ffae8594dfab"
+            "sha256": "a5ea894e51ad2d925b0d24739e2c644164b1edefbd3d363c1225a76c3926c23b"
         },
         "host-environment-markers": {
             "implementation_name": "cpython",
@@ -50,6 +50,12 @@
             ],
             "version": "==2.1.1"
         },
+        "cachecontrol": {
+            "hashes": [
+                "sha256:a9fc50e216c7c101f4ec4312f012dea501c2859cb256c7a68186a172ab71f632"
+            ],
+            "version": "==0.12.3"
+        },
         "certifi": {
             "hashes": [
                 "sha256:244be0d93b71e93fc0a0a479862051414d0e00e16435707e5bf5000f92e04694",
@@ -218,6 +224,12 @@
             ],
             "version": "==19.7.1"
         },
+        "hiredis": {
+            "hashes": [
+                "sha256:ca958e13128e49674aa4a96f02746f5de5973f39b57297b84d59fd44d314d5b5"
+            ],
+            "version": "==0.2.0"
+        },
         "html5lib": {
             "hashes": [
                 "sha256:08a3efc117a4fc8c82c3c6d10d6f58ae266428d57ed50258a1466d2cd88de745",
@@ -284,6 +296,22 @@
             ],
             "version": "==1.0.5"
         },
+        "msgpack-python": {
+            "hashes": [
+                "sha256:637b012c9ea021de7a7a75d6ff5e82cfef6694babd7e14bb9a3adcb2a5bd52f0",
+                "sha256:658c1cd5dcf7786e0e7a6d523cd0c5b33f92e139e224bd73cb3a23ada618d2dc",
+                "sha256:920bbbaee07ad048a4d2b4160901b19775c61ef9439f856c74509e763a326249",
+                "sha256:e165006f7e3d2612f1bffe2f6f042ca317d8df724d8b72a39b14c2e46c67eaae",
+                "sha256:95d70edd50e3d2f6ea1189f77190e4a0172626e7405ddd1689f3f64814447cba",
+                "sha256:7e1b12ea0134460052fabcfaa0f488ec0fc21deb14832d66236fd2870757d8f1",
+                "sha256:8f36890251f20d96267618cf64735759d7ef7e91bc0b86b9480547d2d1397a68",
+                "sha256:1e68a277e4180baa7789be36f27f0891660205f6209f78a32282d3c422873d78",
+                "sha256:f52d9f96df952369fe4adcb0506e10c1c92d47f653f601a66da2a26a7e7141ea",
+                "sha256:58c9c1d7891a35bddc6ee5dbec10d347a7ae4983169c24fc5fc8a57ae792ca76",
+                "sha256:1a2b19df0f03519ec7f19f826afb935b202d8979b0856c6fb3dc28955799f886"
+            ],
+            "version": "==0.4.8"
+        },
         "olefile": {
             "hashes": [
                 "sha256:61f2ca0cd0aa77279eb943c07f607438edf374096b66332fae1ee64a6f0f73ad"

lemonauth/views/indie.py

@@ -1,5 +1,3 @@
-import mf2py
-
 from annoying.decorators import render_to
 from django.contrib.auth import get_user_model
 from django.contrib.auth.decorators import login_required
@@ -9,7 +7,7 @@ from django.utils.decorators import method_decorator
 from django.views.generic import TemplateView
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_POST
-from lemoncurry import breadcrumbs, utils
+from lemoncurry import breadcrumbs, requests, utils
 from urllib.parse import urlencode, urljoin, urlunparse, urlparse
 
 from .. import tokens
@@ -68,7 +66,7 @@ class IndieView(TemplateView):
                 )
             scopes = params['scope'].split(' ')
 
-        client = mf2py.Parser(url=params['client_id'], html_parser='html5lib')
+        client = requests.mf2(params['client_id'])
         rels = (client.to_dict()['rel-urls']
                 .get(redirect_uri, {})
                 .get('rels', ()))

lemoncurry/requests.py

@@ -0,0 +1,44 @@
+import requests
+from cachecontrol.wrapper import CacheControl
+from cachecontrol.cache import BaseCache
+from cachecontrol.heuristics import LastModified
+from datetime import datetime
+from django.core.cache import cache as django_cache
+from hashlib import sha256
+from mf2py import Parser
+
+
+class DjangoCache(BaseCache):
+    @classmethod
+    def key(cls, url):
+        return 'req:' + sha256(url.encode('utf-8')).hexdigest()
+
+    def get(self, url):
+        key = self.key(url)
+        return django_cache.get(key)
+
+    def set(self, url, value, expires=None):
+        key = self.key(url)
+        if expires:
+            lifetime = (expires - datetime.utcnow()).total_seconds()
+            django_cache.set(key, value, lifetime)
+        else:
+            django_cache.set(key, value)
+
+
+req = CacheControl(
+    requests.Session(),
+    cache=DjangoCache(),
+    heuristic=LastModified(),
+)
+
+
+def get(url):
+    r = req.get(url)
+    r.raise_for_status()
+    return r
+
+
+def mf2(url):
+    r = get(url)
+    return Parser(doc=r.text, url=url, html_parser='html5lib')

lemoncurry/settings/base.py

@@ -134,6 +134,11 @@ CACHES = {
     'default': {
         'BACKEND': 'redis_cache.RedisCache',
         'LOCATION': '127.0.0.1:6380',
+        'KEY_PREFIX': 'lemoncurry',
+        'OPTIONS': {
+            'DB': 0,
+            'PARSER_CLASS': 'redis.connection.HiredisParser',
+        },
     }
 }