Merge pull request #14 from lfuelling/master

add config option to filter for language
This commit is contained in:
Lynne 2019-02-26 18:13:43 +10:00 committed by GitHub
commit 48faf4d3b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 26 additions and 17 deletions

View file

@ -1,5 +1,6 @@
{
"lang": "en",
"site": "https://botsin.space",
"cw": null,
"learn_from_cw": false
}
}

View file

@ -16,11 +16,11 @@ def make_sentence(output):
shutil.copyfile("toots.db", "toots-copy.db") #create a copy of the database because reply.py will be using the main one
db = sqlite3.connect("toots-copy.db")
db.text_factory=str
db.text_factory = str
c = db.cursor()
if cfg['learn_from_cw']:
toots = c.execute("SELECT content FROM `toots` ORDER BY RANDOM() LIMIT 10000").fetchall()
else:
else:
toots = c.execute("SELECT content FROM `toots` WHERE cw = 0 ORDER BY RANDOM() LIMIT 10000").fetchall()
toots_str = ""
for toot in toots:
@ -88,4 +88,4 @@ def extract_toot(toot):
text = re.sub("https://([^/]+)/(@[^ ]+)", r"\2@\1", text) #put mastodon-style mentions back in
text = re.sub("https://([^/]+)/users/([^ ]+)", r"@\2@\1", text) #put pleroma-style mentions back in
text = text.rstrip("\n") #remove trailing newline
return text
return text

34
main.py
View file

@ -43,7 +43,7 @@ if os.path.exists("clientcred.secret"):
cfg['secret'] = open("usercred.secret").read().rstrip("\n")
os.remove("clientcred.secret")
os.remove("usercred.secret")
if "client" not in cfg:
print("No application info -- registering application with {}".format(cfg['site']))
@ -75,8 +75,8 @@ def extract_toot(toot):
client = Mastodon(
client_id=cfg['client']['id'],
client_secret = cfg['client']['secret'],
access_token=cfg['secret'],
client_secret = cfg['client']['secret'],
access_token=cfg['secret'],
api_base_url=cfg['site'])
me = client.account_verify_credentials()
@ -106,6 +106,18 @@ patterns = {
"pid": re.compile(r"[^\/]+$"),
}
def insert_toot(oii, acc, post, cursor): # extracted to prevent duplication
pid = patterns["pid"].search(oii['object']['id']).group(0)
cursor.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", (
pid,
1 if (oii['object']['summary'] != None and oii['object']['summary'] != "") else 0,
acc.id,
oii['object']['id'],
post
))
for f in following:
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone()
if last_toot != None:
@ -161,7 +173,7 @@ for f in following:
for oi in j['orderedItems']:
if oi['type'] != "Create":
continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
# its a toost baby
content = oi['object']['content']
toot = extract_toot(content)
@ -172,15 +184,11 @@ for f in following:
#we've caught up to the notices we've already downloaded, so we can stop now
#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought"
done = True
pid = patterns["pid"].search(oi['object']['id']).group(0)
c.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", (
pid,
1 if (oi['object']['summary'] != None and oi['object']['summary'] != "") else 0,
f.id,
oi['object']['id'],
toot
)
)
if cfg['lang']:
if oi['object']['contentMap'][cfg['lang']]: # filter for language
insert_toot(oi, f, toot, c)
else:
insert_toot(oi, f, toot, c)
pass
except:
pass #ignore any toots that don't successfully go into the DB