Merge branch 'master' into master
This commit is contained in:
Lynne
GitHub
commit
64b49da4eb
7 changed files with 169 additions and 185 deletions
143
main.py
143
main.py
|
|
@ -7,26 +7,43 @@
|
|||
from mastodon import Mastodon
|
||||
from os import path
|
||||
from bs4 import BeautifulSoup
|
||||
import os, sqlite3, signal, sys, json, re
|
||||
import os, sqlite3, signal, sys, json, re, shutil
|
||||
import requests
|
||||
import functions
|
||||
|
||||
scopes = ["read:statuses", "read:accounts", "read:follows", "write:statuses", "read:notifications"]
|
||||
cfg = json.load(open('config.json', 'r'))
|
||||
try:
|
||||
cfg = json.load(open('config.json', 'r'))
|
||||
except:
|
||||
shutil.copy2("config.sample.json", "config.json")
|
||||
cfg = json.load(open('config.json', 'r'))
|
||||
|
||||
#config.json *MUST* contain the instance URL, the instance blacklist (for dead/broken instances), and the CW text. if they're not provided, we'll fall back to defaults.
|
||||
if 'site' not in cfg:
|
||||
cfg['website'] = "https://botsin.space"
|
||||
if 'cw' not in cfg:
|
||||
cfg['cw'] = None
|
||||
if 'instance_blacklist' not in cfg:
|
||||
cfg["instance_blacklist"] = [
|
||||
"bofa.lol",
|
||||
"witches.town"
|
||||
]
|
||||
|
||||
#if the user is using a (very!) old version that still uses the .secret files, migrate to the new method
|
||||
if os.path.exists("clientcred.secret"):
|
||||
print("Upgrading to new storage method")
|
||||
cc = open("clientcred.secret").read().split("\n")
|
||||
cfg['client'] = {
|
||||
"id": cc[0],
|
||||
"secret": cc[1]
|
||||
}
|
||||
cfg['secret'] = open("usercred.secret").read().rstrip("\n")
|
||||
os.remove("clientcred.secret")
|
||||
os.remove("usercred.secret")
|
||||
print("Upgrading to new storage method")
|
||||
cc = open("clientcred.secret").read().split("\n")
|
||||
cfg['client'] = {
|
||||
"id": cc[0],
|
||||
"secret": cc[1]
|
||||
}
|
||||
cfg['secret'] = open("usercred.secret").read().rstrip("\n")
|
||||
os.remove("clientcred.secret")
|
||||
os.remove("usercred.secret")
|
||||
|
||||
|
||||
if "client" not in cfg:
|
||||
print("No client credentials, registering application")
|
||||
print("No application info -- registering application with {}".format(cfg['site']))
|
||||
client_id, client_secret = Mastodon.create_app("mstdn-ebooks",
|
||||
api_base_url=cfg['site'],
|
||||
scopes=scopes,
|
||||
|
|
@ -38,47 +55,18 @@ if "client" not in cfg:
|
|||
}
|
||||
|
||||
if "secret" not in cfg:
|
||||
print("No user credentials, logging in")
|
||||
print("No user credentials -- logging in to {}".format(cfg['site']))
|
||||
client = Mastodon(client_id = cfg['client']['id'],
|
||||
client_secret = cfg['client']['secret'],
|
||||
api_base_url=cfg['site'])
|
||||
|
||||
print("Open this URL: {}".format(client.auth_request_url(scopes=scopes)))
|
||||
print("Open this URL and authenticate to give mstdn-ebooks access to your bot's account: {}".format(client.auth_request_url(scopes=scopes)))
|
||||
cfg['secret'] = client.log_in(code=input("Secret: "), scopes=scopes)
|
||||
|
||||
json.dump(cfg, open("config.json", "w+"))
|
||||
|
||||
def extract_toot(toot):
|
||||
toot = toot.replace("'", "'")
|
||||
toot = toot.replace(""", '"')
|
||||
soup = BeautifulSoup(toot, "html.parser")
|
||||
|
||||
# this is the code that removes all mentions
|
||||
for mention in soup.select("span.h-card"):
|
||||
mention.a.unwrap()
|
||||
mention.span.unwrap()
|
||||
|
||||
# replace <br> with linebreak
|
||||
for lb in soup.select("br"):
|
||||
lb.insert_after("\n")
|
||||
lb.decompose()
|
||||
|
||||
# replace <p> with linebreak
|
||||
for p in soup.select("p"):
|
||||
p.insert_after("\n")
|
||||
p.unwrap()
|
||||
|
||||
# fix hashtags
|
||||
for ht in soup.select("a.hashtag"):
|
||||
ht.unwrap()
|
||||
|
||||
# fix links
|
||||
for link in soup.select("a"):
|
||||
link.insert_after(link["href"])
|
||||
link.decompose()
|
||||
|
||||
toot = soup.get_text()
|
||||
toot = toot.rstrip("\n") #remove trailing newline
|
||||
toot = functions.extract_toot(toot)
|
||||
toot = toot.replace("@", "@\u200B") #put a zws between @ and username to avoid mentioning
|
||||
return(toot)
|
||||
|
||||
|
|
@ -104,25 +92,12 @@ def handleCtrlC(signal, frame):
|
|||
|
||||
signal.signal(signal.SIGINT, handleCtrlC)
|
||||
|
||||
def get_toots_legacy(client, id):
|
||||
i = 0
|
||||
toots = client.account_statuses(id)
|
||||
while toots is not None and len(toots) > 0:
|
||||
for toot in toots:
|
||||
if toot.spoiler_text != "": continue
|
||||
if toot.reblog is not None: continue
|
||||
if toot.visibility not in ["public", "unlisted"]: continue
|
||||
t = extract_toot(toot.content)
|
||||
if t != None:
|
||||
yield {
|
||||
"toot": t,
|
||||
"id": toot.id,
|
||||
"uri": toot.uri
|
||||
}
|
||||
toots = client.fetch_next(toots)
|
||||
i += 1
|
||||
if i%20 == 0:
|
||||
print('.', end='', flush=True)
|
||||
patterns = {
|
||||
"handle": re.compile(r"^.*@(.+)"),
|
||||
"url": re.compile(r"https?:\/\/(.*)"),
|
||||
"uri": re.compile(r'template="([^"]+)"'),
|
||||
"pid": re.compile(r"[^\/]+$"),
|
||||
}
|
||||
|
||||
for f in following:
|
||||
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone()
|
||||
|
|
@ -133,28 +108,27 @@ for f in following:
|
|||
print("Harvesting toots for user @{}, starting from {}".format(f.acct, last_toot))
|
||||
|
||||
#find the user's activitypub outbox
|
||||
print("WebFingering...")
|
||||
instance = re.search(r"^.*@(.+)", f.acct)
|
||||
print("WebFingering... (do not laugh at this. WebFinger is a federated protocol. https://wikipedia.org/wiki/WebFinger)")
|
||||
instance = patterns["handle"].search(f.acct)
|
||||
if instance == None:
|
||||
instance = re.search(r"https?:\/\/(.*)", cfg['site']).group(1)
|
||||
instance = patterns["url"].search(cfg['site']).group(1)
|
||||
else:
|
||||
instance = instance.group(1)
|
||||
|
||||
if instance == "bofa.lol":
|
||||
print("rest in piece bofa, skipping")
|
||||
if instance in cfg['instance_blacklist']:
|
||||
print("skipping blacklisted instance: {}".format(instance))
|
||||
continue
|
||||
|
||||
# print("{} is on {}".format(f.acct, instance))
|
||||
|
||||
try:
|
||||
r = requests.get("https://{}/.well-known/host-meta".format(instance), timeout=10)
|
||||
uri = re.search(r'template="([^"]+)"', r.text).group(1)
|
||||
uri = patterns["uri"].search(r.text).group(1)
|
||||
uri = uri.format(uri = "{}@{}".format(f.username, instance))
|
||||
r = requests.get(uri, headers={"Accept": "application/json"}, timeout=10)
|
||||
j = r.json()
|
||||
if len(j['aliases']) == 1: #TODO: this is a hack on top of a hack, fix it
|
||||
uri = j['aliases'][0]
|
||||
else:
|
||||
uri = j['aliases'][1]
|
||||
for link in j['links']:
|
||||
if link['rel'] == 'self':
|
||||
#this is a link formatted like "https://instan.ce/users/username", which is what we need
|
||||
uri = link['href']
|
||||
uri = "{}/outbox?page=true".format(uri)
|
||||
r = requests.get(uri, timeout=10)
|
||||
j = r.json()
|
||||
|
|
@ -168,23 +142,23 @@ for f in following:
|
|||
pleroma = True
|
||||
j = j['first']
|
||||
else:
|
||||
print("Mastodon instance detected")
|
||||
print("Mastodon/Misskey instance detected")
|
||||
uri = "{}&min_id={}".format(uri, last_toot)
|
||||
r = requests.get(uri)
|
||||
j = r.json()
|
||||
|
||||
print("Downloading and parsing toots", end='', flush=True)
|
||||
print("Downloading and saving toots", end='', flush=True)
|
||||
done = False
|
||||
try:
|
||||
while not done and len(j['orderedItems']) > 0:
|
||||
for oi in j['orderedItems']:
|
||||
if oi['type'] != "Create":
|
||||
continue #not a toost. fuck outta here
|
||||
continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
|
||||
|
||||
# its a toost baby
|
||||
content = oi['object']['content']
|
||||
if oi['object']['summary'] != None and oi['object']['summary'] != "":
|
||||
#don't download CW'd toots
|
||||
#don't download CW'd toots. if you want your bot to download and learn from CW'd toots, replace "continue" with "pass". (todo: add a config.json option for this)
|
||||
continue
|
||||
toot = extract_toot(content)
|
||||
# print(toot)
|
||||
|
|
@ -192,11 +166,12 @@ for f in following:
|
|||
if pleroma:
|
||||
if c.execute("SELECT COUNT(*) FROM toots WHERE uri LIKE ?", (oi['object']['id'],)).fetchone()[0] > 0:
|
||||
#we've caught up to the notices we've already downloaded, so we can stop now
|
||||
#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought"
|
||||
done = True
|
||||
break
|
||||
pid = re.search(r"[^\/]+$", oi['object']['id']).group(0)
|
||||
c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)",
|
||||
(pid,
|
||||
pid = patterns["pid"].search(oi['object']['id']).group(0)
|
||||
c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)", (
|
||||
pid,
|
||||
f.id,
|
||||
oi['object']['id'],
|
||||
toot
|
||||
|
|
@ -205,7 +180,6 @@ for f in following:
|
|||
pass
|
||||
except:
|
||||
pass #ignore any toots that don't successfully go into the DB
|
||||
# sys.exit(0)
|
||||
if not pleroma:
|
||||
r = requests.get(j['prev'], timeout=15)
|
||||
else:
|
||||
|
|
@ -215,9 +189,8 @@ for f in following:
|
|||
print(" Done!")
|
||||
db.commit()
|
||||
except:
|
||||
print("Encountered an error! Saving toots to database and continuing.")
|
||||
print("Encountered an error! Saving toots to database and moving to next followed account.")
|
||||
db.commit()
|
||||
# db.close()
|
||||
|
||||
print("Done!")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue