mstdn-ebooks/main.py

189 lines
6.3 KiB
Python
Raw Normal View History

2018-10-08 21:11:51 -04:00
#!/usr/bin/env python3
# toot downloader version two!!
2018-10-08 21:11:51 -04:00
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from mastodon import Mastodon
from os import path
from bs4 import BeautifulSoup
import os, sqlite3, signal, sys, json, re
import requests
import functions
2018-10-08 21:11:51 -04:00
scopes = ["read:statuses", "read:accounts", "read:follows", "write:statuses", "read:notifications"]
cfg = json.load(open('config.json', 'r'))
2018-10-08 21:11:51 -04:00
2019-01-11 08:08:53 -05:00
#config.json *MUST* contain the instance URL, the instance blacklist (for dead/broken instances), and the CW text. if they're not provided, we'll fall back to defaults.
if 'site' not in cfg:
cfg['website'] = "https://botsin.space"
if 'cw' not in cfg:
2019-01-11 08:05:45 -05:00
cfg['cw'] = None
2019-01-11 08:08:53 -05:00
if 'instance_blacklist' not in cfg:
cfg["instance_blacklist"] = [
"bofa.lol",
"witches.town"
]
#if the user is using a (very!) old version that still uses the .secret files, migrate to the new method
if os.path.exists("clientcred.secret"):
2018-10-27 04:28:20 -04:00
print("Upgrading to new storage method")
cc = open("clientcred.secret").read().split("\n")
cfg['client'] = {
"id": cc[0],
"secret": cc[1]
}
cfg['secret'] = open("usercred.secret").read().rstrip("\n")
os.remove("clientcred.secret")
os.remove("usercred.secret")
if "client" not in cfg:
2019-01-11 07:08:10 -05:00
print("No application info -- registering application with {}".format(cfg['site']))
client_id, client_secret = Mastodon.create_app("mstdn-ebooks",
api_base_url=cfg['site'],
scopes=scopes,
website="https://github.com/Lynnesbian/mstdn-ebooks")
cfg['client'] = {
"id": client_id,
"secret": client_secret
}
if "secret" not in cfg:
2019-01-11 07:08:10 -05:00
print("No user credentials -- logging in to {}".format(cfg['site']))
client = Mastodon(client_id = cfg['client']['id'],
client_secret = cfg['client']['secret'],
api_base_url=cfg['site'])
2018-10-08 21:11:51 -04:00
2019-01-11 07:08:10 -05:00
print("Open this URL and authenticate to give mstdn-ebooks access to your bot's account: {}".format(client.auth_request_url(scopes=scopes)))
cfg['secret'] = client.log_in(code=input("Secret: "), scopes=scopes)
2018-10-08 21:11:51 -04:00
json.dump(cfg, open("config.json", "w+"))
2018-10-08 21:11:51 -04:00
def extract_toot(toot):
toot = functions.extract_toot(toot)
2018-11-27 06:29:50 -05:00
toot = toot.replace("@", "@\u200B") #put a zws between @ and username to avoid mentioning
return(toot)
2018-10-08 21:11:51 -04:00
client = Mastodon(
client_id=cfg['client']['id'],
client_secret = cfg['client']['secret'],
access_token=cfg['secret'],
api_base_url=cfg['site'])
2018-10-08 21:11:51 -04:00
me = client.account_verify_credentials()
following = client.account_following(me.id)
db = sqlite3.connect("toots.db")
db.text_factory=str
c = db.cursor()
c.execute("CREATE TABLE IF NOT EXISTS `toots` (id INT NOT NULL UNIQUE PRIMARY KEY, userid INT NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL) WITHOUT ROWID")
2018-10-08 21:11:51 -04:00
db.commit()
def handleCtrlC(signal, frame):
print("\nPREMATURE EVACUATION - Saving chunks")
db.commit()
sys.exit(1)
signal.signal(signal.SIGINT, handleCtrlC)
for f in following:
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone()
if last_toot != None:
last_toot = last_toot[0]
else:
last_toot = 0
print("Harvesting toots for user @{}, starting from {}".format(f.acct, last_toot))
#find the user's activitypub outbox
2018-10-27 04:28:20 -04:00
print("WebFingering...")
instance = re.search(r"^.*@(.+)", f.acct)
if instance == None:
instance = re.search(r"https?:\/\/(.*)", cfg['site']).group(1)
else:
instance = instance.group(1)
2019-01-11 08:08:53 -05:00
if instance in cfg['instance_blacklist']:
print("skipping blacklisted instance: {}".format(instance))
2018-10-25 10:33:57 -04:00
continue
2019-01-11 07:15:05 -05:00
try:
2018-10-27 08:07:38 -04:00
r = requests.get("https://{}/.well-known/host-meta".format(instance), timeout=10)
uri = re.search(r'template="([^"]+)"', r.text).group(1)
uri = uri.format(uri = "{}@{}".format(f.username, instance))
2018-10-27 08:07:38 -04:00
r = requests.get(uri, headers={"Accept": "application/json"}, timeout=10)
2018-10-27 04:28:20 -04:00
j = r.json()
if len(j['aliases']) == 1: #TODO: this is a hack on top of a hack, fix it
uri = j['aliases'][0]
else:
uri = j['aliases'][1]
2018-11-07 00:39:12 -05:00
uri = "{}/outbox?page=true".format(uri)
2018-10-27 08:07:38 -04:00
r = requests.get(uri, timeout=10)
j = r.json()
except Exception:
print("oopsy woopsy!! we made a fucky wucky!!!\n(we're probably rate limited, please hang up and try again)")
sys.exit(1)
2018-10-27 04:28:20 -04:00
pleroma = False
if 'first' in j and type(j['first']) != str:
2018-11-07 00:39:12 -05:00
print("Pleroma instance detected")
2018-10-27 04:28:20 -04:00
pleroma = True
2018-11-07 00:39:12 -05:00
j = j['first']
else:
2018-11-07 00:39:12 -05:00
print("Mastodon instance detected")
uri = "{}&min_id={}".format(uri, last_toot)
2018-11-07 00:39:12 -05:00
r = requests.get(uri)
j = r.json()
2019-01-11 07:08:10 -05:00
print("Downloading and saving toots", end='', flush=True)
2018-11-07 00:39:12 -05:00
done = False
2018-11-09 06:50:36 -05:00
try:
2018-11-07 00:39:12 -05:00
while not done and len(j['orderedItems']) > 0:
for oi in j['orderedItems']:
2018-11-28 14:36:05 -05:00
if oi['type'] != "Create":
2019-01-11 07:15:05 -05:00
continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
2018-11-28 14:36:05 -05:00
# its a toost baby
content = oi['object']['content']
if oi['object']['summary'] != None:
2019-01-11 07:15:05 -05:00
#don't download CW'd toots. if you want your bot to download and learn from CW'd toots, replace "continue" with "pass". (todo: add a config.json option for this)
2018-11-28 14:36:05 -05:00
continue
toot = extract_toot(content)
# print(toot)
try:
if pleroma:
if c.execute("SELECT COUNT(*) FROM toots WHERE id LIKE ?", (oi['object']['id'],)).fetchone()[0] > 0:
#we've caught up to the notices we've already downloaded, so we can stop now
2019-01-11 08:02:36 -05:00
#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought"
2018-11-28 14:36:05 -05:00
done = True
break
pid = re.search(r"[^\/]+$", oi['object']['id']).group(0)
2019-01-11 08:05:45 -05:00
c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)", (
pid,
2018-11-28 14:36:05 -05:00
f.id,
oi['object']['id'],
toot
2018-11-07 00:39:12 -05:00
)
2018-11-28 14:36:05 -05:00
)
pass
except:
pass #ignore any toots that don't successfully go into the DB
2018-11-07 00:39:12 -05:00
if not pleroma:
r = requests.get(j['prev'], timeout=15)
else:
r = requests.get(j['next'], timeout=15)
j = r.json()
print('.', end='', flush=True)
print(" Done!")
db.commit()
2018-11-09 06:50:36 -05:00
except:
2019-01-11 07:08:10 -05:00
print("Encountered an error! Saving toots to database and moving to next followed account.")
2018-11-09 06:50:36 -05:00
db.commit()
print("Done!")
2018-10-08 21:11:51 -04:00
db.commit()
db.execute("VACUUM") #compact db
db.commit()
db.close()