add config option to filter for language
This commit is contained in:
parent
b00af91305
commit
c0cf42a41a
|
@ -1,5 +1,6 @@
|
||||||
{
|
{
|
||||||
|
"lang": "en",
|
||||||
"site": "https://botsin.space",
|
"site": "https://botsin.space",
|
||||||
"cw": null,
|
"cw": null,
|
||||||
"learn_from_cw": false
|
"learn_from_cw": false
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,11 +16,11 @@ def make_sentence(output):
|
||||||
|
|
||||||
shutil.copyfile("toots.db", "toots-copy.db") #create a copy of the database because reply.py will be using the main one
|
shutil.copyfile("toots.db", "toots-copy.db") #create a copy of the database because reply.py will be using the main one
|
||||||
db = sqlite3.connect("toots-copy.db")
|
db = sqlite3.connect("toots-copy.db")
|
||||||
db.text_factory=str
|
db.text_factory = str
|
||||||
c = db.cursor()
|
c = db.cursor()
|
||||||
if cfg['learn_from_cw']:
|
if cfg['learn_from_cw']:
|
||||||
toots = c.execute("SELECT content FROM `toots` ORDER BY RANDOM() LIMIT 10000").fetchall()
|
toots = c.execute("SELECT content FROM `toots` ORDER BY RANDOM() LIMIT 10000").fetchall()
|
||||||
else:
|
else:
|
||||||
toots = c.execute("SELECT content FROM `toots` WHERE cw = 0 ORDER BY RANDOM() LIMIT 10000").fetchall()
|
toots = c.execute("SELECT content FROM `toots` WHERE cw = 0 ORDER BY RANDOM() LIMIT 10000").fetchall()
|
||||||
toots_str = ""
|
toots_str = ""
|
||||||
for toot in toots:
|
for toot in toots:
|
||||||
|
@ -88,4 +88,4 @@ def extract_toot(toot):
|
||||||
text = re.sub("https://([^/]+)/(@[^ ]+)", r"\2@\1", text) #put mastodon-style mentions back in
|
text = re.sub("https://([^/]+)/(@[^ ]+)", r"\2@\1", text) #put mastodon-style mentions back in
|
||||||
text = re.sub("https://([^/]+)/users/([^ ]+)", r"@\2@\1", text) #put pleroma-style mentions back in
|
text = re.sub("https://([^/]+)/users/([^ ]+)", r"@\2@\1", text) #put pleroma-style mentions back in
|
||||||
text = text.rstrip("\n") #remove trailing newline
|
text = text.rstrip("\n") #remove trailing newline
|
||||||
return text
|
return text
|
||||||
|
|
34
main.py
34
main.py
|
@ -43,7 +43,7 @@ if os.path.exists("clientcred.secret"):
|
||||||
cfg['secret'] = open("usercred.secret").read().rstrip("\n")
|
cfg['secret'] = open("usercred.secret").read().rstrip("\n")
|
||||||
os.remove("clientcred.secret")
|
os.remove("clientcred.secret")
|
||||||
os.remove("usercred.secret")
|
os.remove("usercred.secret")
|
||||||
|
|
||||||
|
|
||||||
if "client" not in cfg:
|
if "client" not in cfg:
|
||||||
print("No application info -- registering application with {}".format(cfg['site']))
|
print("No application info -- registering application with {}".format(cfg['site']))
|
||||||
|
@ -75,8 +75,8 @@ def extract_toot(toot):
|
||||||
|
|
||||||
client = Mastodon(
|
client = Mastodon(
|
||||||
client_id=cfg['client']['id'],
|
client_id=cfg['client']['id'],
|
||||||
client_secret = cfg['client']['secret'],
|
client_secret = cfg['client']['secret'],
|
||||||
access_token=cfg['secret'],
|
access_token=cfg['secret'],
|
||||||
api_base_url=cfg['site'])
|
api_base_url=cfg['site'])
|
||||||
|
|
||||||
me = client.account_verify_credentials()
|
me = client.account_verify_credentials()
|
||||||
|
@ -106,6 +106,18 @@ patterns = {
|
||||||
"pid": re.compile(r"[^\/]+$"),
|
"pid": re.compile(r"[^\/]+$"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def insert_toot(oii, acc, post, cursor): # extracted to prevent duplication
|
||||||
|
pid = patterns["pid"].search(oii['object']['id']).group(0)
|
||||||
|
cursor.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", (
|
||||||
|
pid,
|
||||||
|
1 if (oii['object']['summary'] != None and oii['object']['summary'] != "") else 0,
|
||||||
|
acc.id,
|
||||||
|
oii['object']['id'],
|
||||||
|
post
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
for f in following:
|
for f in following:
|
||||||
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone()
|
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone()
|
||||||
if last_toot != None:
|
if last_toot != None:
|
||||||
|
@ -161,7 +173,7 @@ for f in following:
|
||||||
for oi in j['orderedItems']:
|
for oi in j['orderedItems']:
|
||||||
if oi['type'] != "Create":
|
if oi['type'] != "Create":
|
||||||
continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
|
continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
|
||||||
|
|
||||||
# its a toost baby
|
# its a toost baby
|
||||||
content = oi['object']['content']
|
content = oi['object']['content']
|
||||||
toot = extract_toot(content)
|
toot = extract_toot(content)
|
||||||
|
@ -172,15 +184,11 @@ for f in following:
|
||||||
#we've caught up to the notices we've already downloaded, so we can stop now
|
#we've caught up to the notices we've already downloaded, so we can stop now
|
||||||
#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought"
|
#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought"
|
||||||
done = True
|
done = True
|
||||||
pid = patterns["pid"].search(oi['object']['id']).group(0)
|
if cfg['lang']:
|
||||||
c.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", (
|
if oi['object']['contentMap'][cfg['lang']]: # filter for language
|
||||||
pid,
|
insert_toot(oi, f, toot, c)
|
||||||
1 if (oi['object']['summary'] != None and oi['object']['summary'] != "") else 0,
|
else:
|
||||||
f.id,
|
insert_toot(oi, f, toot, c)
|
||||||
oi['object']['id'],
|
|
||||||
toot
|
|
||||||
)
|
|
||||||
)
|
|
||||||
pass
|
pass
|
||||||
except:
|
except:
|
||||||
pass #ignore any toots that don't successfully go into the DB
|
pass #ignore any toots that don't successfully go into the DB
|
||||||
|
|
Loading…
Reference in a new issue