dump current state (wip-ish)

.gitignore

@@ -0,0 +1,2 @@
+/data/
+/scripts/local/

DOCS

@@ -0,0 +1,3 @@
+
+https://matrix-nio.readthedocs.io/en/latest/nio.html
+https://matrix.org/docs/spec/client_server/r0.6.1

README.md

@@ -3,3 +3,8 @@
 An extensible general purpose bot for the [Matrix] network.
 
 [matrix]: https://matrix.org/
+
+# Docker container
+
+The Dockerfile creates an image with all necessary resources to run the bot, it does however not copy the bot's source files into the container.  This is done to make it easy to change the bot's source code without rebuilding the container, i.e. the container is aimed at development.  
+If you want create an image for distribution, simply change the Dockerfile to copy all relevant files (i.e. the hotdog subdir and other local Python modules) to `/var`, baking them in permanently.

data/config.example.yaml

@@ -0,0 +1,56 @@
+command_prefix: "!"
+
+matrix:
+  # The full Matrix user ID of the bot account, @name:host
+  user_id: "@example:matrix.example.org"
+  password: "You'll never guess my super secret example password!"
+  # The URL of the homeserver to connect to.
+  homeserver_url: https://matrix.example.org
+  # Optional, the session name for this bot instance, for identification only.
+  # session_name: 'bot on toast'
+  # The display name for the bot, used in chat rooms.
+  nick_name: HotDogBot
+
+storage:
+  # The path to a directory for internal bot storage
+  # containing encryption keys, sync tokens, etc.
+  store_path: "/data/store"
+
+# Location info, optionally per room.
+# This information can be used for language selection & formatting.
+# Defaults will be merged onto the room specific config.
+l6n:
+  default:
+    locale: 'de_DE.UTF-8'
+    timezone: 'Europe/Berlin'
+  rooms: []
+    # -
+    #   id: '!roomid:matrix.example.org'
+    #   name: '#hotdogbot:matrix.example.org'
+
+postillon:
+  storage: '/data/postillon.sqlite'
+
+covid:
+  storage: '/data/covid.sqlite'
+
+ddf:
+  storage: '/data/ddf.csv'
+
+feeder:
+  storage: '/data/feeder.sqlite'
+  feeds: []
+    # examplefeed:
+    #   display: Example RSS Feed
+    #   url: https://www.example.org/feeds/rss.xml
+    #   rooms: ['!roomid:matrix.example.org']
+    #   max_content_len: 666
+
+# Optional, development settings.
+dev:
+  # If active is false, the bot will ignore all messages from the dev room.
+  # If active is true, the bot will ignore all messages except in the dev room.
+  active: false
+  # If a development room is set, all messages for that room will be filtered.
+  # A room is required if active is true.
+  room: '!roomid:matrix.example.org'

data/ddf.csv

@@ -0,0 +1,220 @@
+Nr. Buch (Kosmos);Nr. Hörspiel;Nr. Buch (Random House);amerikanischer Titel;deutscher Titel;Autor;Jahr Buch (Random House);Jahr Buch (Kosmos);Jahr Hörspiel
+1;11;1;The Secret of Terror Castle;… und das Gespensterschloss;Robert Arthur;1964;1968 / 2015;1980
+8;1;2;The Mystery of the Stuttering Parrot;… und der Super-Papagei;Robert Arthur;1964;1972 / 2015;1979
+2;10;3;The Mystery of the Whispering Mummy;… und die flüsternde Mumie;Robert Arthur;1965;1969 / 2015;1980
+14;8;4;The Mystery of the Green Ghost;… und der grüne Geist;Robert Arthur;1965;1975;1979
+10;22;5;The Mystery of the Vanishing Treasure;… und der verschwundene Schatz;Robert Arthur;1966;1973 / 2015;1981
+11;18;6;The Secret of Skeleton Island;… und die Geisterinsel;Robert Arthur;1966;1973;1980
+3;5;7;The Mystery of the Fiery Eye;… und der Fluch des Rubins;Robert Arthur;1967;1970 / 2015;1979
+26;24;8;The Mystery of the Silver Spider;… und die silberne Spinne;Robert Arthur;1967;1981;1981
+4;12;9;The Mystery of the Screaming Clock;… und der seltsame Wecker;Robert Arthur;1968;1970 / 2015;1980
+13;19;10;The Mystery of the Moaning Cave;… und der Teufelsberg;William Arden;1968;1974;1980
+5;6;11;The Mystery of the Talking Skull;… und der sprechende Totenkopf;Robert Arthur;1969;1971 / 2015;1979
+6;13;12;The Mystery of the Laughing Shadow;… und der lachende Schatten;William Arden;1969;1971 / 2015;1980
+7;4;13;The Secret of the Crooked Cat;… und die schwarze Katze;William Arden;1970;1971 / 2015;1979
+9;7;14;The Mystery of the Coughing Dragon;… und der unheimliche Drache;Nick West;1970;1972 / 2015;1979
+22;20;15;The Mystery of the Flaming Footprints;… und die flammende Spur;M. V. Carey;1971;1979;1980
+12;15;16;The Mystery of the Nervous Lion;… und der rasende Löwe;Nick West;1971;1974;1980
+15;25;17;The Mystery of the Singing Serpent;… und die singende Schlange;M. V. Carey;1972;1975;1981
+16;9;18;The Mystery of the Shrinking House;… und die rätselhaften Bilder;William Arden;1972;1976;1979
+18;2;19;The Secret of Phantom Lake;… und der Phantomsee;William Arden;1973;1977;1979
+17;14;20;The Mystery of Monster Mountain;… und das Bergmonster;M. V. Carey;1973;1976;1980
+19;16;21;The Secret of the Haunted Mirror;… und der Zauberspiegel;M. V. Carey;1974;1977;1980
+20;17;22;The Mystery of the Dead Man’s Riddle;… und die gefährliche Erbschaft;William Arden;1974;1978;1980
+21;3;23;The Mystery of the Invisible Dog;… und der Karpatenhund;M. V. Carey;1975;1978;1979
+24;26;24;The Mystery of Death Trap Mine;… und die Silbermine;M. V. Carey;1976;1980;1981
+23;21;25;The Mystery of the Dancing Devil;… und der Tanzende Teufel;William Arden;1976;1979;1980
+25;23;26;The Mystery of the Headless Horse;… und das Aztekenschwert;William Arden;1977;1980;1981
+27;27;27;The Mystery of the Magic Circle;… und der magische Kreis;M. V. Carey;1978;1981;1981
+28;28;28;The Mystery of the Deadly Double;… und der Doppelgänger;William Arden;1978;1982;1982
+;29;;;Die Original-Musik der Europa-Jugendserie;Bert Brac / Betty George;;;1982 / 1996
+31;32;29;The Mystery of the Sinister Scarecrow;… und der Ameisenmensch;M. V. Carey;1979;1983;1983
+29;30;30;The Secret of Shark Reef;… und das Riff der Haie;William Arden;1979;1982;1982
+30;31;31;The Mystery of the Scar-Faced Beggar;… und das Narbengesicht;M. V. Carey;1981;1982;1983
+32;33;32;The Mystery of the Blazing Cliffs;… und die bedrohte Ranch;M. V. Carey;1981;1983;1983
+33;34;33;The Mystery of the Purple Pirate;… und der Rote Pirat;William Arden;1982;1984;1984
+34;35;34;The Mystery of the Wandering Caveman;… und der Höhlenmensch;M. V. Carey;1982;1984;1984
+35;36;35;The Mystery of the Kidnapped Whale;… und der Super-Wal;Marc Brandel;1983;1985;1985
+36;37;36;The Mystery of the Missing Mermaid;… und der heimliche Hehler;M. V. Carey;1983;1985;1985
+38;39;37;The Mystery of the Two-Toed Pigeon;… und die Perlenvögel;Marc Brandel;1984;1986;1986
+39;40;38;The Mystery of the Smashing Glass;… und der Automarder;William Arden;1984;1987;1986
+37;38;39;The Mystery of the Trail of Terror;… und der unsichtbare Gegner;M. V. Carey;1984;1986;1986
+43;44;40;The Mystery of the Rogues’ Reunion;… und der gestohlene Preis;Marc Brandel;1985;1988;1988
+42;43;41;The Mystery of the Creep-Show Crooks;… und der höllische Werwolf;M. V. Carey;1985;1988;1988
+44;45;42;The Mystery of Wreckers’ Rock;… und das Gold der Wikinger;William Arden;1986;1989;1989
+45;46;43;The Mystery of the Cranky Collector;… und der schrullige Millionär;M. V. Carey;1987;1989;1989
+;;44;The Mystery of the Ghost Train;;M. V. Carey;unvollendet;;
+41;42;FYF#1;The Case of the Weeping Coffin;… und der weinende Sarg;Megan Stine;1985;1988;1987
+40;41;FYF#2;The Case of the Dancing Dinosaur;… und das Volk der Winde;Rose Estes;1985;1987;1987
+TSE1#2;TSE1#2;FYF#7;The Case of the House of Horrors;House of Horrors – Haus der Angst;Megan & H. William Stine;1986;2011;2011
+TSE2#3;TSE2#3;FYF#8;The Case of the Savage Statue;Savage Statue – Grausame Göttin;M. V. Carey;1987;2014;
+49;53;CB#1;Hot Wheels;… und die Automafia;William Arden;1989;1991;1991
+48;47;CB#2;Murder To Go;… und der giftige Gockel;Megan & H. William Stine;1989;1990;1990
+47;48;CB#3;Rough Stuff;… und die gefährlichen Fässer;G. H. Stone;1989;1990;1990
+46;49;CB#4;Funny Business;… und die Comic-Diebe;William McCay;1989;1990;1990
+52;51;CB#5;An Ear For Danger;… und der riskante Ritt;Marc Brandel;1989;1991;1991
+51;50;CB#6;Thriller Diller;… und der verschwundene Filmstar;Megan & H. William Stine;1989;1991;1991
+50;52;CB#7;Reel Trouble;… und die Musikpiraten;G. H. Stone;1989;1991;1991
+TSE2#2;TSE2#2;CB#8;Shoot the Works;Shoot the works – Im Visier;William McCay;1990;2014;
+54;54;CB#9;Foul Play;Gefahr im Verzug;Peter Lerangis;1990;1992;1992
+53;55;CB#10;Long Shot;Gekaufte Spieler;Megan & H. William Stine;1990;1992;1992
+55;56;CB#11;Fatal Error;Angriff der Computer-Viren;G. H. Stone;1990;1992;1992
+TSE1#1;TSE1#1;CB#12;Brain Wash;Brainwash – Gefangene Gedanken;Peter Lerangis;-;2011;2011
+TSE1#3;TSE1#3;CB#13;High Strung;High Strung – Unter Hochspannung;G. H. Stone;-;2011;2011
+56;57;;;Tatort Zirkus;Brigitte-Johanna Henkel-Waidhofer;;1993;1994
+57;58;;;… und der verrückte Maler;Brigitte-Johanna Henkel-Waidhofer;;1993;1994
+58;59;;;Giftiges Wasser;Brigitte-Johanna Henkel-Waidhofer;;1993;1994
+59;60;;;Dopingmixer;Brigitte-Johanna Henkel-Waidhofer;;1994;1994
+60;61;;;… und die Rache des Tigers;Brigitte-Johanna Henkel-Waidhofer;;1994;1995
+61;62;;;Spuk im Hotel;Brigitte-Johanna Henkel-Waidhofer;;1994;1995
+62;63;;;Fußball-Gangster;Brigitte-Johanna Henkel-Waidhofer;;1995;1995
+63;64;;;Geisterstadt;Brigitte-Johanna Henkel-Waidhofer;;1995;1995
+64;65;;;Diamantenschmuggel;Brigitte-Johanna Henkel-Waidhofer;;1995;1995
+65;66;;;… und die Schattenmänner;Brigitte-Johanna Henkel-Waidhofer;;1995;1995
+66;67;;;… und das Geheimnis der Särge;Brigitte-Johanna Henkel-Waidhofer;;1996;1996
+67;68;;;… und der Schatz im Bergsee;Brigitte-Johanna Henkel-Waidhofer;;1996;1996
+68;69;;;Späte Rache;Brigitte-Johanna Henkel-Waidhofer;;1996;1996
+69;70;;;Schüsse aus dem Dunkel;Brigitte-Johanna Henkel-Waidhofer;;1996;1996
+70;71;;;Die verschwundene Seglerin;Brigitte-Johanna Henkel-Waidhofer;;1996;1996
+71;72;;;Dreckiger Deal;Brigitte-Johanna Henkel-Waidhofer;;1996;1996
+72;75;;;Die Spur des Raben;André Marx;;1997;1997
+73;73;;;Poltergeist;André Marx;;1997;1997
+74;74;;;… und das brennende Schwert;André Marx;;1997;1997
+75;77;;;Pistenteufel;Ben Nevis;;1997;1997
+76;76;;;Stimmen aus dem Nichts;André Minninger;;1997;1997
+77;78;;;Das leere Grab;André Marx;;1997;1998
+78;81;;;Verdeckte Fouls;Ben Nevis;;1998;1998
+79;79;;;Im Bann des Voodoo;André Minninger;;1998;1998
+80;80;;;Geheimsache Ufo / Geheimakte Ufo;André Marx;;1998;1998
+81;83;;;Meuterei auf hoher See;André Marx;;1998;1999
+82;84;;;Musik des Teufels;André Marx;;1998;1999
+83;82;;;Die Karten des Bösen;André Minninger;;1998;1998
+84;87;;;Wolfsgesicht;Katharina Fischer;;1999;1999
+85;86;;;Nacht in Angst;André Marx;;1999;1999
+86;85;;;Feuerturm;Ben Nevis;;1999;1999
+87;89;;;Tödliche Spur;André Marx;;1999;2000
+88;88;;;Vampir im Internet;André Minninger;;1999;1999
+89;93;;;… und das Geisterschiff;André Marx;;2000;2000
+90;92;;;Todesflug;Ben Nevis;;2000;2000
+91;91;;;Labyrinth der Götter;André Marx;;2000;2000
+92;96;;;… und der rote Rächer;Katharina Fischer;;2000;2001
+93;94;;;Das schwarze Monster;André Marx;;2000;2000
+94;95;;;Botschaft von Geisterhand;André Marx;;2000;2001
+95;97;;;Insektenstachel;André Minninger;;2001;2001
+96;99;;;Rufmord;André Minninger;;2001;2001
+97;98;;;Tal des Schreckens;Ben Nevis;;2001;2001
+98;102;;;Doppelte Täuschung;André Marx;;2001;2002
+99;101;;;… und das Hexenhandy / … und das Hexen-Handy;André Minninger;;2001;2001
+100;100;;;Toteninsel: Das Rätsel der Sphinx, Das vergessene Volk, Der Fluch der Gräber;André Marx;;2001;2001
+101;103;;;Das Erbe des Meisterdiebs / Das Erbe des Meisterdiebes;André Marx;;2002;2002
+102;104;;;Gift per E-Mail;Ben Nevis;;2002;2002
+103;105;;;… und der Nebelberg / Der Nebelberg;André Marx;;2002;2002
+104;106;;;Der Mann ohne Kopf;André Minninger;;2002;2002
+105;107;;;… und der Schatz der Mönche;Ben Nevis;;2002;2003
+106;108;;;Die sieben Tore;André Marx;;2002;2003
+107;109;;;Gefährliches Quiz;Marco Sonnleitner;;2003;2003
+108;110;;;Panik im Park;Marco Sonnleitner;;2003;2003
+109;111;;;Die Höhle des Grauens;Ben Nevis;;2003;2003
+110;113;;;Das Auge des Drachen;André Marx;;2003;2003
+111;112;;;Schlucht der Dämonen;Marco Sonnleitner;;2003;2003
+112;114;;;Die Villa der Toten;André Marx;;2003;2004
+113;90;;;Der Feuerteufel;André Marx;;1999;2000
+114;117;;;Der finstere Rivale;André Marx;;2004;2004
+115;116;;;Codename: Cobra;Marco Sonnleitner;;2004;2004
+116;115;;;Auf tödlichem Kurs;Ben Nevis;;2004;2004
+117;120;;;Der schwarze Skorpion;Marco Sonnleitner;;2004;2005
+118;118;;;Das düstere Vermächtnis;Ben Nevis;;2004;2004
+119;119;;;Der geheime Schlüssel;André Marx;;2004;2004
+120;121;;;Spur ins Nichts;André Marx;;2005;2008
+121;122;;;… und der Geisterzug;Astrid Vollenbruch;;2005;2008
+122;123;;;Fußballfieber;Marco Sonnleitner;;2005;2008
+123;126;;;Schrecken aus dem Moor;Marco Sonnleitner;;2005;2008
+124;124;;;Geister-Canyon;Ben Nevis;;2005;2008
+125;125;;;Feuermond: Das Rätsel der Meister, Der Pfad der Täuschung, Die Nacht der Schatten;André Marx;;2005;2008
+126;129;;;SMS aus dem Grab;Ben Nevis;;2006;2009
+127;127;;;Schwarze Madonna;Astrid Vollenbruch;;2006;2008
+128;128;;;Schatten über Hollywood;Astrid Vollenbruch;;2006;2009
+129;132;;;Spuk im Netz;Astrid Vollenbruch;;2006;2009
+130;130;;;Der Fluch des Drachen;André Marx;;2006;2009
+131;131;;;Haus des Schreckens;Marco Sonnleitner;;2006;2009
+132;135;;;Fluch des Piraten;Ben Nevis;;2007;2009
+133;133;;;Fels der Dämonen;Marco Sonnleitner;;2007;2009
+134;134;;;Der tote Mönch;Marco Sonnleitner;;2007;2009
+135;138;;;Die geheime Treppe;Marco Sonnleitner;;2007;2010
+136;136;;;… und das versunkene Dorf;André Marx;;2007;2010
+137;137;;;Pfad der Angst;Astrid Vollenbruch;;2007;2010
+138;141;;;… und die Fußball-Falle;Marco Sonnleitner;;2008;2010
+139;139;;;Das Geheimnis der Diva;Astrid Vollenbruch;;2008;2010
+140;140;;;Stadt der Vampire;Marco Sonnleitner;;2008;2010
+141;144;;;Zwillinge der Finsternis;Marco Sonnleitner;;2008;2011
+142;142;;;Tödliches Eis;Kari Erlhoff;;2008;2010
+143;143;;;… und die Poker-Hölle;Marco Sonnleitner;;2008;2010
+144;147;;;Grusel auf Campbell Castle;Marco Sonnleitner;;2009;2011
+145;145;;;… und die Rache der Samurai;Ben Nevis;;2009;2011
+146;146;;;Der Biss der Bestie;Kari Erlhoff;;2009;2011
+147;151;;;Schwarze Sonne;Marco Sonnleitner;;2009;2012
+148;148;;;… und die feurige Flut;Kari Erlhoff;;2009;2011
+149;149;;;Der namenlose Gegner;Kari Erlhoff;;2009;2011
+150;150;;;Geisterbucht: Rashuras Schatz, Flammendes Wasser, Der brennende Kristall;Astrid Vollenbruch;;2010;2011
+151;153;;;… und das Fußballphantom;Marco Sonnleitner;;2010;2012
+152;152;;;Skateboardfieber;Ben Nevis;;2010;2012
+153;154;;;Botschaft aus der Unterwelt;Kari Erlhoff;;2010;2012
+154;155;;;… und der Meister des Todes;Kari Erlhoff;;2010;2012
+155;156;;;Im Netz des Drachen;Marco Sonnleitner;;2010;2012
+156;157;;;Im Zeichen der Schlangen;Hendrik Buchna;;2011;2012
+157;159;;;Nacht der Tiger;Marco Sonnleitner;;2011;2013
+158;158;;;… und der Feuergeist;Marco Sonnleitner;;2011;2012
+159;162;;;… und der schreiende Nebel;Hendrik Buchna;;2011;2013
+160;160;;;Geheimnisvolle Botschaften;Christoph Dittert;;2011;2013
+161;161;;;Die blutenden Bilder;Kari Erlhoff;;2011;2013
+162;164;;;Fußball-Teufel;Marco Sonnleitner;;2012;2013
+163;163;;;… und der verschollene Pilot;Ben Nevis;;2012;2013
+164;165;;;Im Schatten des Giganten;Kari Erlhoff;;2012;2013
+165;168;;;GPS-Gangster;Marco Sonnleitner;;2012;2014
+166;167;;;… und das blaue Biest;Hendrik Buchna;;2012;2014
+167;166;;;… und die brennende Stadt;Christoph Dittert;;2012;2014
+168;171;;;… und das Phantom aus dem Meer;Marco Sonnleitner;;2013;2014
+169;170;;;Straße des Grauens;Kari Erlhoff;;2013;2014
+170;169;;;Die Spur des Spielers;André Marx;;2013;2014
+171;174;;;… und das Tuch der Toten;Marco Sonnleitner;;2013;2015
+172;172;;;… und der Eisenmann;Ben Nevis;;2013;2014
+173;173;;;Dämon der Rache;Hendrik Buchna;;2013;2015
+174;176;;;… und der gestohlene Sieg;Marco Sonnleitner;;2014;2015
+175;175;;;Schattenwelt: Teuflisches Duell, Angriff in der Nacht, Die dunkle Macht;Christoph Dittert, Kari Erlhoff, Hendrik Buchna;;2014;2015
+176;177;;;Der Geist des Goldgräbers;André Marx;;2014;2015
+177;178;;;Der gefiederte Schrecken;Christoph Dittert;;2014;2015
+178;179;;;Die Rache des Untoten;Marco Sonnleitner;;2014;2016
+179;180;;;… und die flüsternden Puppen;André Minninger;;2015;2016
+180;182;;;Im Haus des Henkers;Marco Sonnleitner;;2015;2016
+181;181;;;Das Kabinett des Zauberers;André Marx;;2015;2016
+182;183;;;… und der letzte Song;Ben Nevis;;2015;2016
+183;184;;;… und der Hexengarten;Kari Erlhoff;;2015;2016
+184;187;;;… und das silberne Amulett;Marco Sonnleitner;;2016;2017
+185;186;;;Insel des Vergessens;André Marx;;2016;2017
+186;185;;;… und der Mann ohne Augen;Christoph Dittert;;2016;2017
+187;188;;;Signale aus dem Jenseits;André Minninger;;2016;2017
+188;189;;;… und der unsichtbare Passagier;Hendrik Buchna;;2016;2017
+189;190;;;… und die Kammer der Rätsel;Ben Nevis;;2016;2017
+190;191;;;Verbrechen im Nichts;Kari Erlhoff;;2017;2018
+191;192;;;Im Bann des Drachen;Christoph Dittert;;2017;2018
+192;193;;;Schrecken aus der Tiefe;Marco Sonnleitner;;2017;2018
+193;194;;;… und die Zeitreisende;André Minninger;;2017;2018
+194;195;;;Im Reich der Ungeheuer;Hendrik Buchna;;2017;2018
+195;196;;;Geheimnis des Bauchredners;André Marx;;2017;2018
+196;199;;;… und der grüne Kobold;Marco Sonnleitner;;2018;2019
+197;197;;;Im Auge des Sturms;Kari Erlhoff;;2018;2019
+198;198;;;Die Legende der Gaukler;Christoph Dittert;;2018;2019
+199;201;;;Höhenangst;André Minninger;;2018;2019
+200;200;;;Feuriges Auge: Der verschwundene Detektiv, Die silberne Hand, Der Tempel der Gerechtigkeit;André Marx;;2018;2019
+201;202;;;Das weiße Grab;Ben Nevis;;2018;2019
+202;203;;;Tauchgang ins Ungewisse;Kari Erlhoff;;2019;2020
+203;204;;;Der dunkle Wächter;Ben Nevis;;2019;2020
+204;205;;;Das rätselhafte Erbe;Marco Sonnleitner;;2019;2020
+205;206;;;… und der Mottenmann;Christoph Dittert;;2019;2020
+206;207;;;Die falschen Detektive;Ben Nevis;;2019;2020
+207;;;;Kreaturen der Nacht;Marco Sonnleitner;;2020;
+208;;;;… und die schweigende Grotte;Christoph Dittert;;2020;
+209;;;;Kelch des Schicksals;Kari Erlhoff;;2020;
+210;;;;… und der Jadekönig;André Marx;;2020;
+211;;;;Der Fluch der Medusa;Marco Sonnleitner;;2020;
+212;;;;… und der weiße Leopard;Hendrik Buchna;;2020;

docker/.dockerignore

@@ -0,0 +1,3 @@
+*
+!entrypoint.sh
+!requirements.txt

docker/.dockerimage

@@ -0,0 +1 @@
+tikki/matrix-hotdog

docker/Dockerfile

@@ -0,0 +1,30 @@
+# FROM a24bb4013296
+FROM docker.io/library/alpine:latest
+
+RUN apk upgrade --no-cache \
+    && apk add --virtual build-deps --no-cache \
+        python3-dev \
+        gcc \
+        openssl-dev \
+        musl-dev \
+        olm-dev \
+    && apk add --no-cache \
+        python3 \
+        py3-pip \
+        py3-cffi \
+        openssl \
+        musl \
+        musl-locales \
+        olm \
+    && pip install -U pip matrix-nio[e2e] \
+    && apk del --rdepends build-deps \
+    && rm -rf /root/.cache
+
+COPY requirements.txt /
+RUN pip install -Ur /requirements.txt
+
+COPY entrypoint.sh /
+
+VOLUME ["/data"]
+
+CMD "/entrypoint.sh"

docker/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eu
+
+export PYTHONPATH=/var
+export MUSL_LOCPATH=/usr/share/i18n/locales/musl
+python3 -m hotdog --config /data/config.yaml

docker/requirements.txt

@@ -0,0 +1,5 @@
+feedparser==6.*
+matrix-nio[e2e]
+pyyaml
+requests
+youtube_dl

feeder/__init__.py

@@ -0,0 +1,3 @@
+from .feeder import Feeder, all_posts
+from .models import Feed, Post
+from .store import Store

feeder/feeder.py

@@ -0,0 +1,57 @@
+import asyncio
+import logging
+from typing import *
+
+from .models import Feed, FeedId, Post, PostId
+from .store import Store
+
+log = logging.getLogger(__name__)
+
+
+class Feeder:
+    def __init__(self, store: Store, feeds: Iterable[Feed] = None):
+        self.feeds: Dict[str, Feed] = {}
+        self.store: Store = store
+        self.news: Mapping[FeedId, Set[PostId]] = {}
+
+        if feeds:
+            self.add_feeds(feeds)
+
+    def add_feeds(self, feeds: Iterable[Feed]):
+        self.feeds.update({f.id: f for f in feeds})
+        self.store.sync_feeds(self.feeds)
+
+    async def update_all(self) -> Mapping[FeedId, Set[PostId]]:
+        new_post_ids = self.news = dict(
+            zip(
+                self.feeds,
+                await asyncio.gather(*(self.update(id) for id in self.feeds)),
+            )
+        )
+        self.store.sync_feeds(self.feeds)
+        return new_post_ids
+
+    async def update(self, feed_id) -> Set[PostId]:
+        feed = self.feeds[feed_id]
+        post_ids = feed.post_ids
+        feed.load()
+        return feed.post_ids - post_ids
+
+    def posts(self, feed_id: FeedId, post_ids: Sequence[PostId]) -> Sequence[Post]:
+        return self.store.posts(feed_id, post_ids)
+
+
+async def all_posts(feed_url: str, throttle: int = 10) -> AsyncIterable[Post]:
+    """Yield all posts from the given feed URL and all following pages.
+
+    A feed can be split into multiple pages.
+    The Feed's normal load function ignores them.  This function follows
+    them and returns all Posts from all pages.
+    """
+    feed = Feed(id=feed_url, url="", next_url=feed_url)
+    while (feed := feed.load_next()) :
+        log.debug(f"New feed page: {feed}")
+        for post in feed.posts:
+            yield post
+        log.debug(f"Waiting for {throttle} seconds ...")
+        await asyncio.sleep(throttle)

feeder/models.py

@@ -0,0 +1,109 @@
+import asyncio
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from hashlib import md5
+from typing import *
+
+import feedparser
+
+USER_AGENT = "curl/7.64.1"
+
+log = logging.getLogger(__name__)
+
+FeedId = str
+PostId = str
+
+
+@dataclass
+class Feed:
+    id: FeedId
+    url: str
+    title: Optional[str] = None
+    posts: List["Post"] = field(default_factory=list)
+    etag: Optional[str] = None
+    modified: Optional[str] = None
+    active: bool = True
+    next_url: Optional[str] = None
+
+    @property
+    def post_ids(self) -> Set[PostId]:
+        return {p.id for p in self.posts}
+
+    def load(self) -> None:
+        """Load all posts from the current feed URL."""
+        log.debug(f"Loading {self.url} ...")
+        r = feedparser.parse(
+            self.url, agent=USER_AGENT, etag=self.etag, modified=self.modified
+        )
+        log.debug(f"Loaded {self.url}: {r.get('status')} {r.headers}")
+        if r.get("status") is None:
+            log.error(f"Feed could not be loaded: {self.id}: {self.url}")
+            return
+        elif r.get("status") == 301:
+            log.warning(f"Feed URL changed: {self.id}: {r.href}")
+            self.url = r.href
+        elif r.get("status") == 410:
+            log.error(f"Feed is gone: {self.id}")
+            self.active = False
+            return
+
+        if "etag" in r:
+            self.etag = r.etag
+        if "modified" in r:
+            self.modified = r.modified
+        if "title" in r.feed:
+            self.title = r.feed.title
+
+        posts = [Post.from_entry(e) for e in r.entries]
+        for post in posts:
+            if post.date is None:
+                post.date = pubdate(r.feed)
+        posts.sort(key=lambda e: e.date, reverse=True)
+        self.posts = posts
+
+        for link in r.feed.get("links", []):
+            if link.get("rel") == "next":
+                self.next_url = link.get("href")
+                break
+        else:
+            self.next_url = None
+
+    def load_next(self) -> Optional["Feed"]:
+        if not self.next_url:
+            return None
+        feed = Feed(self.id, self.next_url)
+        feed.load()
+        return feed
+
+
+@dataclass
+class Post:
+    id: PostId
+    content: Optional[str] = None
+    date: Optional[datetime] = None
+    link: Optional[str] = None
+    title: Optional[str] = None
+
+    @classmethod
+    def from_entry(cls, entry):
+        content = entry.get("summary", "")
+        title = entry.get("title", "")
+        return cls(
+            id=(
+                entry.get("id")
+                or entry.get("link")
+                or md5(f"{title}|{content}".encode()).hexdigest()
+            ),
+            date=pubdate(entry),
+            content=content,
+            title=title,
+            link=entry.get("link"),
+        )
+
+
+def pubdate(entry) -> Optional[datetime]:
+    date = entry.get("published_parsed") or entry.get("updated_parsed")
+    if date is None:
+        return None
+    return datetime(*date[:6], tzinfo=timezone.utc)

feeder/store.py

@@ -0,0 +1,123 @@
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from typing import *
+
+from .models import Feed, Post
+
+log = logging.getLogger(__name__)
+
+
+class Store:
+    def __init__(self, dbpath: Optional[str] = None):
+        self.dbpath = dbpath
+        self.connection: Optional[sqlite3.Connection] = None
+
+    def connect(self, path: Optional[str] = None) -> None:
+        if path:
+            self.dbpath = path
+        if self.connection is not None:
+            return self.connection
+        log.debug("Connecting to %s", self.dbpath)
+        self.connection = sqlite3.connect(
+            self.dbpath, isolation_level=None
+        )  # auto commit
+        self.init()
+
+    def disconnect(self) -> None:
+        conn = self.connection
+        if conn:
+            conn.close()
+
+    def init(self) -> None:
+        conn = self.connection
+        conn.execute(
+            """
+            create table if not exists feed (
+                id text primary key not null,
+                url text unique not null,
+                active integer not null,
+                etag text,
+                modified text
+            )
+            """
+        )
+        conn.execute(
+            """
+            create table if not exists post (
+                id text primary key not null,
+                feed_id text not null references feed(id) on delete cascade,
+                content text,
+                date text,
+                link text,
+                title text
+            )
+            """
+        )
+
+    def sync_feeds(self, feeds: Dict[str, Feed]) -> None:
+        """Write the current state of feeds to store, and load existing info back."""
+        conn = self.connection
+        conn.executemany(
+            """
+            insert into feed(id, url, active)
+            values(?, ?, 1)
+            on conflict(id) do update set url=?, active=?, etag=?, modified=?
+            """,
+            (
+                (f.id, f.url, f.url, 1 if f.active else 0, f.etag, f.modified)
+                for f in feeds.values()
+            ),
+        )
+
+        conn.executemany(
+            """
+            insert into post(id, feed_id, content, date, link, title)
+            values(?, ?, ?, ?, ?, ?)
+            on conflict do nothing
+            """,
+            (
+                (p.id, f.id, p.content, p.date, p.link, p.title)
+                for f in feeds.values()
+                for p in f.posts
+            ),
+        )
+
+        sql = "select id, url, active from feed"
+        for row in conn.execute(sql):
+            id, url, active = row
+            if id not in feeds:
+                feeds[id] = Feed(id, url)
+            else:
+                if active:
+                    if feeds[id].url != url:
+                        log.warning(f"Feed URL changed: {id}: {url}")
+                        feeds[id].url = url
+                else:
+                    log.warning(f"Feed is marked inactive: {id}")
+                    del feeds[id]
+
+        post_ids = {f.id: f.post_ids for f in feeds.values()}
+        sql = """
+            select post.id, feed_id from post
+            join feed on feed.id=feed_id
+            where feed.active=1
+        """
+        for row in conn.execute(sql):
+            post_id, feed_id = row
+            if post_id not in post_ids[feed_id]:
+                post_ids[feed_id].add(post_id)
+                feeds[feed_id].posts.append(Post(post_id))
+
+    def posts(self, feed_id, post_ids) -> Sequence[Post]:
+        qs = ",".join(["?"] * len(post_ids))
+        sql = f"""
+            select id, content, date, link, title from post
+            where feed_id=? and id in ({qs})
+        """
+        conn = self.connection
+        posts = [Post(*row) for row in conn.execute(sql, (feed_id, *post_ids))]
+        for post in posts:
+            if post.date is not None:
+                post.date = datetime.fromisoformat(post.date)
+        return posts

hotdog/__main__.py

@@ -0,0 +1,31 @@
+import argparse
+import asyncio
+import logging
+
+from .bot import Bot
+from .config import Config
+
+log = logging.getLogger(__name__)
+
+
+async def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True, help="Path to config.yaml")
+    args = parser.parse_args()
+
+    config = Config(args.config)
+    logging.basicConfig(
+        format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
+        level=config.loglevel,
+    )
+    logging.getLogger("peewee").setLevel("INFO")  # XXX too spamy
+
+    bot = Bot(config)
+
+    await bot.run()
+
+
+try:
+    asyncio.run(main())
+except KeyboardInterrupt:
+    log.info("Shutdown by user.")

hotdog/bot.py

@@ -0,0 +1,299 @@
+import asyncio
+import logging
+from datetime import datetime
+from importlib import import_module
+from pathlib import Path
+from time import time as now
+from types import ModuleType as Plugin
+from typing import *
+
+from aiohttp import ClientConnectionError, ServerDisconnectedError
+from nio import AsyncClient, AsyncClientConfig, Event, InviteMemberEvent, JoinError
+from nio import LoginError as LoginErrorResponse
+from nio import MatrixRoom, RoomMessageText, UnknownEvent
+
+from .models import Job, JobCallback, Message, Reaction
+
+log = logging.getLogger(__name__)
+
+
+class LoginError(RuntimeError):
+    pass
+
+
+MessageHandler = Callable[[Message], Awaitable]
+
+# XXX add the facility to store/buffer responses, e.g. if a timer triggers
+#     a new message that should be sent to a room but the bot is disconnected
+#     at the moment, the bot should just buffer it and send it when it's back
+#     online.
+
+# XXX have a mechanism that first collects all responders and then calls them.
+#     this would allow to let responders know about each other and interact if
+#     necessary, e.g. urlinfo sees that youtube-info is already handling it.
+
+
+class Bot:
+    def __init__(self, config):
+        self.client: AsyncClient = None
+        self.config = config
+        self.plugins = {}
+        self.message_handlers: List[MessageHandler] = []
+        self.timers: List[Job] = []
+        self.shared: Dict[str, Any] = {}  # Shared memory for the plug-ins.
+
+    def on_message(self, callback: MessageHandler):
+        self.message_handlers.append(callback)
+
+    def on_command(self, command: Union[str, Container[str]], callback: MessageHandler):
+        commands = command = {command} if type(command) is str else command
+
+        async def guard(message):
+            if message.command not in commands:
+                return
+            await callback(message)
+
+        guard.__qualname__ = f"{callback.__module__}.{callback.__qualname__}"
+
+        self.message_handlers.append(guard)
+
+    def add_timer(
+        self,
+        *,
+        title: str,
+        callback: JobCallback,
+        every: Optional[float] = None,
+        next_at: Optional[datetime] = None,
+        next_in: Optional[float] = None,
+        jitter: float = 0,
+    ):
+        # We require "every", or "next_at" (x)or "next_in".
+        assert every or next_at or next_in
+        assert not (next_at and next_in)
+        job = Job(
+            app=self,
+            title=title,
+            every=every,
+            func=callback,
+            jitter=jitter,
+        )
+        job.next = (
+            now() + next_in
+            if next_in
+            else next_at.timestamp()
+            if next_at
+            else job._next()
+        )
+        self.timers.append(job)
+
+    def _init(self):
+        command_plugins = load_plugins("command")
+
+        self.client = AsyncClient(
+            self.config.homeserver_url,
+            self.config.user_id,
+            device_id=self.config.device_id,
+            store_path=self.config.store_path.as_posix(),
+            config=AsyncClientConfig(
+                max_limit_exceeded=0,
+                max_timeouts=0,
+                store_sync_tokens=True,
+                encryption_enabled=True,
+            ),
+        )
+        add_event_callback = self.client.add_event_callback
+        add_event_callback(self._on_any_event, Event)
+        add_event_callback(self._on_message, RoomMessageText)
+        # add_event_callback(self._on_invite, InviteMemberEvent)  # XXX make join-on-invite configurable
+        add_event_callback(self._on_unknown, UnknownEvent)
+
+        for name, mod in command_plugins.items():
+            log.debug(f"Initializing plugin: {name}")
+            try:
+                mod.init(self)
+            except Exception as err:
+                log.exception("Initialization failed: %s", name, exc_info=err)
+            else:
+                self.plugins[name] = mod
+
+    async def _on_unknown(self, room: MatrixRoom, event: UnknownEvent):
+        # See if we can transform an Unknown event into something we DO know.
+        if event.type == "m.reaction":
+            await self._on_reaction(room, Reaction.from_dict(event.source))
+
+    async def _on_reaction(self, room: MatrixRoom, event: Reaction):
+        # XXX allow clients to register handlers for these events
+        pass
+
+    async def _on_any_event(self, *args):
+        log.debug("New event: %s", repr(args))
+
+    async def _on_message(self, room: MatrixRoom, event: RoomMessageText):
+        if (self.config.is_dev and room.room_id != self.config.dev_room) or (
+            not self.config.is_dev and room.room_id == self.config.dev_room
+        ):
+            return
+
+        is_own_message = event.sender == self.client.user
+        if is_own_message:
+            return
+
+        log.info(f"#{room.display_name} <{room.user_name(event.sender)}> {event.body}")
+
+        msg = Message(self, event.body, room, event)
+
+        tasks = {}
+        for h in self.message_handlers:
+            try:
+                coro = h(msg)
+            except Exception as err:
+                log.exception("Error calling message handler: %s", h, exc_info=err)
+            else:
+                tasks[h] = asyncio.create_task(coro)
+
+        timeout = 10
+        fut = asyncio.gather(*tasks.values(), return_exceptions=True)
+        try:
+            await asyncio.wait_for(fut, timeout)
+        except asyncio.TimeoutError as err:
+            await swallow(fut)
+
+        for h, t in tasks.items():
+            assert t.done()
+            try:
+                err = t.exception()
+            except asyncio.CancelledError:
+                log.error("Message handler took too long to finished: %s", h)
+            if err is not None:
+                log.exception("Error in message handler: %s", h, exc_info=err)
+
+    async def _on_invite(self, room: MatrixRoom, event: InviteMemberEvent):
+        if self.config.is_dev:
+            return
+
+        log.debug(f"Received invite from user: {event.sender}: {room.room_id}")
+
+        res = await self.client.join(room.room_id)
+        if type(res) == JoinError:
+            log.error(f"Could not join room: {room.room_id}: {res.message}")
+        else:
+            log.info(f"Joined room: {room.room_id}")
+
+    async def _login(self):
+        # if self.config.access_token:
+        #     self.client.restore_login(
+        #         self.config.user_id, self.config.device_id, self.config.access_token
+        #     )
+        # if self.config.password:
+        # else:
+        #     resp = await self.client.login(token=self.config.access_token)
+        resp = await self.client.login(
+            password=self.config.password, device_name=self.config.device_name
+        )
+
+        if isinstance(resp, LoginErrorResponse):
+            raise LoginError(f"Could not log in: {resp.message}")
+
+        if not self.config.device_id:
+            self.config.device_id_path.write_text(self.client.device_id)
+
+        await self.client.set_displayname(self.config.display_name)
+        await self.client.update_device(
+            self.client.device_id, {"display_name": self.config.device_name}
+        )
+
+    async def _run_timers(self) -> NoReturn:
+        client = self.client
+        await asyncio.sleep(2)
+        while True:
+            await asyncio.sleep(0.2)
+            # if not client.logged_in:
+            #     continue
+            for job in self.timers:
+                if job.next is not None and job.next <= now():
+                    job.next = None
+                    try:
+                        coro = job.func(job)
+                    except Exception as err:
+                        log.exception(
+                            "Disabled job with error: %s", job.title, exc_info=err
+                        )
+                        job.task = None
+                    else:
+                        job.task = asyncio.create_task(coro)
+                if job.task is not None:
+                    task = job.task
+                    if task.done():
+                        job.task = None
+                        try:
+                            await task
+                        except Exception as err:
+                            log.exception(
+                                "Disabled job with error: %s", job.title, exc_info=err
+                            )
+                        else:
+                            job.next = job._next()
+                            log.debug(f"Task scheduled: {job.title}: {job.next}")
+
+    async def _run_client(self) -> NoReturn:
+        client = self.client
+        while True:
+            try:
+                if not client.logged_in:
+                    await self._login()
+
+                log.info(f"Logged in as {client.user}")
+                await client.sync_forever(timeout=30_000, full_state=True)
+
+                log.warning("Shutting down.")
+                return
+
+            except LoginError as err:
+                log.error(f"Could not log in: {err}")
+                return
+            except asyncio.exceptions.TimeoutError:
+                log.warning("Connection timed out.")
+            except ClientConnectionError as err:
+                log.warning(f"Could not connect to server: {err}")
+            except ServerDisconnectedError as err:
+                log.exception("Disconnected from server.", exc_info=err)
+            finally:
+                await client.close()
+                client.access_token = ""
+
+            waitfor = 15
+            log.info(f"Retrying in {waitfor}s...")
+            await asyncio.sleep(waitfor)
+
+    async def run(self) -> NoReturn:
+        self._init()
+        await asyncio.gather(
+            self._run_timers(),
+            self._run_client(),
+        )
+
+
+def load_plugins(pkg: str) -> Mapping[str, Plugin]:
+    modules = {}
+    for modpath in (Path(__file__).parent / pkg).glob(f"*.py"):
+        modname = modpath.with_suffix("").name
+        try:
+            mod = import_module(f".{pkg}.{modname}", __package__)
+        except Exception as err:
+            log.exception("Error loading plugin: %s", modpath, exc_info=err)
+        else:
+            if not hasattr(mod, "init"):
+                log.error(f"Invalid plugin: {modpath}")
+            else:
+                log.debug(f"Loaded plugin: {modpath}")
+                modules[modname] = mod
+    return modules
+
+
+async def swallow(future: asyncio.Future):
+    """Get rid of a Future."""
+    future.cancel()
+    try:
+        await future  # Should always immediately raise.
+    except asyncio.CancelledError:
+        pass

hotdog/command/README.md

@@ -0,0 +1,20 @@
+A plugin is any module that defines an `init` function that takes a `Bot`
+instance as first argument.  
+The function can then register any message handlers or timers on the bot,
+or add any dependencies it requires.
+
+Example:
+
+```py
+from ..functions import reply
+
+HELP = """Responds to your hello.
+!hello
+"""
+
+def init(bot):
+    bot.on_command("hello", reply_world)
+
+async def reply_world(message):
+    await reply(message, "world")
+```

hotdog/command/aoderb.py

@@ -0,0 +1,31 @@
+import random
+import re
+
+from ..functions import reply
+from ..models import Message
+
+HELP = """Entscheidet zwischen A und B.
+@me: <A> oder <B>?
+"""
+
+
+def init(bot):
+    bot.on_message(handle)
+
+
+async def handle(message: Message):
+    if not (
+        message.text.endswith("?") and "oder" in message.tokens and message.is_for_me
+    ):
+        return
+
+    _, text = message.text.split(None, 1)
+    args = text[:-1].split(" oder ")
+    if ":" in args[0]:
+        args[0] = args[0].split(":", 1)[1]
+    elif "-" in args[0]:
+        args[0] = args[0].split("-", 1)[1]
+    choice = random.choice(args).strip(" ,.:")
+    if not choice:
+        return
+    await reply(message, plain=choice, with_name=True)

hotdog/command/covid.py

@@ -0,0 +1,370 @@
+import logging
+import re
+import sqlite3
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from html import escape
+from typing import *
+
+import requests
+
+from ..functions import localizedtz, react, reply
+from ..models import Job, Message
+from ..tz import cest
+
+log = logging.getLogger(__name__)
+
+
+def init(bot):
+    if "covid.store" not in bot.shared:
+        bot.shared["covid.store"] = Store(bot.config.get("covid.storage"))
+        bot.shared["covid.store"].connect()
+
+    bot.on_message(handle)
+
+    one_minute = 60
+    one_hour = 3600
+    bot.add_timer(
+        title="update covid store",
+        every=6 * one_hour,
+        callback=update_store,
+        jitter=30 * one_minute,
+    )
+
+
+# https://npgeo-corona-npgeo-de.hub.arcgis.com/datasets/917fc37a709542548cc3be077a786c17_0
+parse_last_update = re.compile(
+    r"(?P<day>\d{1,2})\.(?P<month>\d{1,2})\.(?P<year>\d{4}), (?P<hour>\d{2}):(?P<minute>\d{2}) Uhr"
+).fullmatch
+
+api_url = "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_Landkreisdaten/FeatureServer/0/query"
+
+
+class Store:
+    def __init__(self, dbpath: Optional[str] = None):
+        self.dbpath = dbpath
+        self.connection: Optional[sqlite3.Connection] = None
+
+    def connect(self, path: Optional[str] = None) -> None:
+        if path:
+            self.dbpath = path
+        if self.connection is not None:
+            return self.connection
+        log.debug("Connecting to %s", self.dbpath)
+        conn = self.connection = sqlite3.connect(
+            self.dbpath, isolation_level=None
+        )  # auto commit
+        conn.row_factory = sqlite3.Row  # Enable access row data by column name.
+        self.init()
+
+    def init(self) -> None:
+        conn = self.connection
+        sql = """
+            create table if not exists county ( -- Landkreis
+                id integer primary key not null,
+                state_id integer not null references state(id),
+                name text unique not null
+            );;
+
+            create table if not exists state ( -- Bundesland
+                id integer primary key not null,
+                name text unique not null
+            );;
+
+            create table if not exists county_probe (
+                county_id integer not null references county(id),
+                ts integer not null,
+                cases integer not null,
+                cases7_per_100k real not null,
+                deaths integer not null,
+                population integer not null
+            );;
+
+            create unique index if not exists
+                county_probe_index
+                on county_probe(county_id, ts);;
+
+            create table if not exists state_probe (
+                state_id integer not null references state(id),
+                ts integer not null,
+                cases7_per_100k real not null,
+                population integer not null
+            );;
+
+            create unique index if not exists
+                state_probe_index
+                on state_probe(state_id, ts);;
+
+            drop view if exists current;;
+
+            create view if not exists
+                current
+            as select
+                county.name as county_name,
+                county_probe.cases as county_cases,
+                county_probe.cases7_per_100k as county_cases7_per_100k,
+                county_probe.deaths as county_deaths,
+                county_probe.population as county_population,
+                county_probe.ts as ts,
+                state.name as state_name,
+                state_probe.cases7_per_100k as state_cases7_per_100k,
+                state_probe.population as state_population
+            from county_probe
+            inner join county on county.id=county_probe.county_id
+            inner join state on state.id=county.state_id
+            inner join state_probe on state_probe.state_id
+            where
+                county_probe.ts=state_probe.ts
+                and state_probe.state_id=county.state_id
+            ;;
+
+            create trigger if not exists
+                current_insert
+                instead of insert
+                on current
+            begin
+                insert into
+                    state (name)
+                    values (new.state_name)
+                    on conflict do nothing;
+                insert into
+                    county (name, state_id)
+                    values (
+                        new.county_name,
+                        (select id from state where name=new.state_name)
+                    )
+                    on conflict do nothing;
+                insert into
+                    state_probe (state_id, ts, cases7_per_100k, population)
+                    values (
+                        (select id from state where name=new.state_name),
+                        new.ts,
+                        new.state_cases7_per_100k,
+                        new.state_population
+                    )
+                    on conflict do nothing;
+                insert into
+                    county_probe (county_id, ts, cases, cases7_per_100k, deaths, population)
+                    values (
+                        (select id from county where name=new.county_name),
+                        new.ts,
+                        new.county_cases,
+                        new.county_cases7_per_100k,
+                        new.county_deaths,
+                        new.county_population
+                    )
+                    on conflict do nothing;
+            end;;
+        """
+        for s in sql.split(";;"):
+            s = s.strip()
+            if s:
+                conn.execute(s)
+
+    def add(self, probes: Iterable["Probe"]):
+        rows = iter(p.as_row() for p in probes)
+        for first_row in rows:
+            break
+        else:
+            return
+
+        sql = f"""
+            insert into current({",".join(first_row.keys())})
+            values({",".join(["?"] * len(first_row))})
+        """
+
+        self.connection.execute(sql, tuple(first_row.values()))
+        self.connection.executemany(sql, (tuple(r.values()) for r in rows))
+
+    def _select(self, condition="", params=[]) -> Iterable["Probe"]:
+        sql = f"select * from current {condition}"
+        for row in self.connection.execute(sql, params):
+            yield Probe.from_row(row)
+
+    def find_one(self, term) -> Optional["Probe"]:
+        cond = """
+            where county_name like ?
+                or state_name like ?
+            order by ts desc
+            limit 1
+        """
+        for probe in self._select(cond, (term, term)):
+            return probe
+
+
+@dataclass
+class Probe:
+    # County fields
+    cases: int
+    deaths: int
+    county_name: str
+    ts: datetime
+    cases7_per_100k: float
+    # recovered: str
+    population: str  # Einwohnerzahl Landkreis
+
+    # State fields:
+    state_name: str
+    state_population: int  # Einwohnerzahl Bundesland
+    state_cases7_per_100k: float
+
+    @property
+    def death_rate(self) -> float:
+        return self.deaths / self.cases * 100
+
+    @property
+    def cases_per_100k(self) -> float:
+        return self.cases / self.population * 100_000
+
+    @property
+    def cases_per_population(self) -> float:
+        return self.cases / self.population
+
+    _json_fields = {
+        # County fields
+        "cases": "cases",
+        "deaths": "deaths",
+        "county": "county_name",
+        "last_update": "ts",  # Needs to be converted to a timestamp
+        "cases7_per_100k": "cases7_per_100k",
+        # "recovered": "recovered",
+        "EWZ": "population",
+        # State fields:
+        "BL": "state_name",
+        "EWZ_BL": "state_population",
+        "cases7_bl_per_100k": "state_cases7_per_100k",
+    }
+
+    @classmethod
+    def from_api_json(cls, data: Mapping[str, Any]):
+        match = parse_last_update(data["last_update"])  # "25.10.2020, 00:00 Uhr"
+        if not match:
+            raise ValueError(f"Could not parse last_updated: {data['last_update']}")
+
+        dt = datetime(
+            int(match["year"]),
+            int(match["month"]),
+            int(match["day"]),
+            int(match["hour"]),
+            int(match["minute"]),
+        )
+        dt = dt.replace(tzinfo=cest(dt))  # The timezone is from the entry date.
+
+        d = {ck: data[jk] for jk, ck in cls._json_fields.items()}
+        d["ts"] = dt
+        return cls(**d)
+
+    @classmethod
+    def from_row(cls, data):
+        return cls(
+            county_name=data["county_name"],
+            cases=data["county_cases"],
+            cases7_per_100k=data["county_cases7_per_100k"],
+            deaths=data["county_deaths"],
+            population=data["county_population"],
+            ts=datetime.fromtimestamp(data["ts"], tz=timezone.utc),
+            state_name=data["state_name"],
+            state_cases7_per_100k=data["state_cases7_per_100k"],
+            state_population=data["state_population"],
+        )
+
+    def as_row(self):
+        return {
+            "county_name": self.county_name,
+            "county_cases": self.cases,
+            "county_cases7_per_100k": self.cases7_per_100k,
+            "county_deaths": self.deaths,
+            "county_population": self.population,
+            "ts": int(self.ts.timestamp()),
+            "state_name": self.state_name,
+            "state_cases7_per_100k": self.state_cases7_per_100k,
+            "state_population": self.state_population,
+        }
+
+
+def api_params(fields="*"):
+    return {
+        "where": "9999=9999",
+        "outFields": ",".join(fields),
+        "f": "json",
+        "returnGeometry": "false",
+    }
+
+
+def load_data() -> Iterable[Probe]:
+    # import json
+
+    # with open("/data/covid.resp") as fp:
+    #     data = json.load(fp)
+
+    log.debug("Loading data from API.")
+    r = requests.get(
+        api_url,
+        params=api_params(Probe._json_fields),
+        timeout=(10, 10),
+        headers={"user-agent": "hotdog/v1 covid"},
+    )
+    r.raise_for_status()
+
+    data = r.json()
+    log.debug(f'Found {len(data["features"])} entries.')
+    for row in data["features"]:
+        yield Probe.from_api_json(row["attributes"])
+
+
+async def update_store(job: Job):
+    store: Store = job.app.shared["covid.store"]
+    store.add(load_data())
+
+
+def pfloat(n):
+    return f"{n:_.02f}".replace(".", ",")
+
+
+def as_html(probe, tzname: str, lc: str) -> str:
+    # now = datetime.now(tz=timezone.utc)
+    # since = now - probe.ts
+    # fmt = "%A" if since.days < 7 else "%x"
+    date = localizedtz(probe.ts, "%A, %x", tzname=tzname, lc=lc)
+    return (
+        f"<i>Zahlen von {date}</i>: "
+        + ", ".join(
+            [
+                f"🌍 {escape(probe.county_name)}",
+                f"😷 {probe.population:_}",
+                f"🦠 {probe.cases:_}",
+                f"☠️ {probe.deaths:_}",
+                f"📈 {pfloat(probe.cases7_per_100k)} (<i>7-Tage-Inzidenz</i>)",
+            ]
+        )
+        + " — "
+        + ", ".join(
+            [
+                f"🌍 {escape(probe.state_name)}",
+                f"😷 {probe.state_population:_}",
+                f"📈 {pfloat(probe.state_cases7_per_100k)}",
+            ]
+        )
+    )
+
+
+async def handle(message: Message):
+    if message.command not in ("cov", "covid") or not message.args:
+        return
+
+    # if message.args.get(0) == "!force-reload":
+    #     await react(message, "⚡️")
+    #     await update_store(Job(message.app, "", update_store))
+    #     await react(message, "👍")
+    #     return
+
+    store: Store = message.app.shared["covid.store"]
+    term = "%".join(message.args)
+    if (probe := store.find_one(f"%{term}%")) :
+        roomconf = message.app.config.l6n[message.room.room_id]
+        await reply(
+            message,
+            html=as_html(probe, tzname=roomconf["timezone"], lc=roomconf["locale"]),
+        )
+    else:
+        await reply(message, "No such county or state.", in_thread=True)

hotdog/command/ddf.py

@@ -0,0 +1,102 @@
+import re
+from dataclasses import dataclass
+from html import escape
+from random import choice
+from typing import *
+
+from ..functions import reply
+from ..models import Message
+
+HELP = """Die drei ??? Folgenindex
+!ddf [episode #|title]
+"""
+
+
+def init(bot):
+    if "ddf.eps" not in bot.shared:
+        with open(bot.config.get("ddf.storage")) as fp:
+            bot.shared["ddf.db"] = db = load_db(fp)
+            bot.shared["ddf.eps"] = tuple(set(db.values()))
+
+    bot.on_command({"ddf", "???"}, handle)
+
+
+def load_db(fp):
+    db = {}
+    for ep in Episode.from_csv(fp):
+        if ep.nr_europa:
+            db[ep.nr_europa.lower()] = ep
+        elif ep.nr_kosmos:
+            db[ep.nr_kosmos.lower()] = ep
+        if ep.title_de:
+            db[ep.title_de.lower()] = ep
+        if ep.title_us:
+            db[ep.title_us.lower()] = ep
+    return db
+
+
+@dataclass(frozen=True)
+class Episode:
+    nr_kosmos: str
+    nr_europa: str
+    nr_randho: str
+    title_us: str
+    title_de: str
+    autor: str
+    year_randho: str
+    year_kosmos: str
+    year_europa: str
+
+    @classmethod
+    def from_csv(cls, fp) -> Iterable["Episode"]:
+        fp = iter(fp)
+        next(fp)  # skip the first line, it contains the header
+        for line in fp:
+            line = line.strip()
+            if not line:
+                continue
+            yield cls(*line.split(";"))
+
+
+async def handle(message: Message):
+    bot = message.app
+    args = message.args
+
+    ep: Episode = None
+    if not args:
+        eps = bot.shared["ddf.eps"]
+        ep = choice(eps)
+    else:
+        arg = args.str(0).lower()
+        db = bot.shared["ddf.db"]
+        if arg in db:
+            ep = db[arg]
+        else:
+            for key in db:
+                if arg in key:
+                    ep = db[key]
+                    break
+
+    if not ep:
+        return
+
+    nr = year = src = None
+    if ep.nr_europa:
+        src = ""
+        nr = ep.nr_europa
+        year = ep.year_europa
+    elif ep.nr_kosmos:
+        src = "Buch"
+        nr = ep.nr_kosmos
+        year = ep.year_kosmos
+
+    if not nr or not year:
+        return
+
+    html = f"(<i>{src}<i>:) " if src else ""
+    html += f"<b>#{escape(nr)}</b> <i>Die drei ???</i>: <b>{escape(ep.title_de)}</b>"
+    if ep.title_us:
+        html += f" (US: <i>{escape(ep.title_us)}</i>)"
+    html += f" ({escape(year)})"
+
+    await reply(message, html=html)

hotdog/command/dm.py

@@ -0,0 +1,27 @@
+from random import choice
+
+from ..functions import react, reply
+from ..models import Message
+
+HELP = """
+!dm
+"""
+
+
+def init(bot):
+    bot.on_command("dm", handle)
+
+
+answers = (
+    "deine mutter",
+    "deine mama",
+    "deine mudda",
+    "deine muddi",
+    "dei muddi",
+    "du miefst",
+    "drogeriemarkt",
+)
+
+
+async def handle(message: Message):
+    await reply(message, choice(answers))

hotdog/command/feed.py

@@ -0,0 +1,131 @@
+import asyncio
+import logging
+from datetime import datetime, timezone
+from html import escape
+
+import feeder
+import postillon
+
+from ..functions import clamp, localizedtz, reply, send_message, strip_tags
+from ..models import Job, Message
+
+log = logging.getLogger(__name__)
+
+
+def init(bot):
+    bot.on_command("feed", handle)
+
+    if "feeder" not in bot.shared:
+        feeds = (
+            feeder.Feed(fid, f["url"], title=f["display"])
+            for fid, f in bot.config.get("feeder.feeds").items()
+        )
+        feedstore = feeder.Store(bot.config.get("feeder.storage"))
+        feedstore.connect()
+        bot.shared["feeder"] = feeder.Feeder(feedstore, feeds)
+
+    if "poststore" not in bot.shared:
+        bot.shared["poststore"] = postillon.Store(bot.config.get("postillon.storage"))
+        bot.shared["poststore"].connect()
+
+    one_minute = 60
+    one_hour = 3600
+    bot.add_timer(
+        title="update feeds",
+        every=one_hour,
+        callback=update_feeds,
+        jitter=10 * one_minute,
+    )
+
+
+async def handle(message: Message):
+    feed_id = message.args.str(0)
+    count = clamp(1, message.args.int(1) or 3, 10)
+
+    bot = message.app
+    feeder = bot.shared["feeder"]
+    if feed_id not in feeder.feeds:
+        return
+    posts = feeder.posts(feed_id, [p.id for p in feeder.feeds[feed_id].posts[:count]])
+
+    feedconf = bot.config.get("feeder.feeds")[feed_id]
+    roomconf = bot.config.l6n[message.room.room_id]
+    for post in posts:
+        text = post_as_html(
+            post,
+            tzname=roomconf["timezone"],
+            lc=roomconf["locale"],
+            max_content_len=feedconf.get("max_content_len", 300),
+        )
+        await reply(message, html=text)
+
+
+def handle_postillon(bot, posts):
+    """Special handling for Postillon posts to store them in the Postillon DB as well."""
+    poststore = bot.shared["poststore"]
+    for post in posts:
+        poststore.add(postillon.split_post(post))
+
+
+async def update_feeds(job: Job):
+    max_posts = 2
+    bot = job.app
+    feeder = bot.shared["feeder"]
+    feeds = bot.config.get("feeder.feeds")
+    rooms = {fid: f.get("rooms", []) for fid, f in feeds.items()}
+    news = await feeder.update_all()
+    sends = []
+    mores = []
+    for feed_id, post_ids in news.items():
+        posts = feeder.posts(feed_id, post_ids)
+        log.debug(f"new posts: {feed_id}: {len(posts)}")
+        if feed_id == "post":
+            handle_postillon(bot, posts)
+        prefix = f"[feed:{feed_id}]"
+        for room_id in rooms[feed_id]:
+            roomconf = bot.config.l6n[room_id]
+            selected = posts[:max_posts] if len(posts) > max_posts + 1 else posts
+            for post in selected:
+                text = post_as_html(
+                    post,
+                    tzname=roomconf["timezone"],
+                    lc=roomconf["locale"],
+                    max_content_len=feeds[feed_id].get("max_content_len", 300),
+                )
+                text = f"{prefix} {text}"
+                sends.append(send_message(bot.client, room_id, html=text))
+            if len(posts) > len(selected):
+                more = len(posts) - len(selected)
+                mores.append([bot.client, room_id, f"{prefix} {more} more"])
+    await asyncio.gather(*sends)
+    await asyncio.gather(*(send_message(*more) for more in mores))
+
+
+def post_as_html(post, tzname: str, lc: str, *, max_content_len: int = 300):
+    parts = []
+    if post.date:
+        now = datetime.now(tz=timezone.utc)
+        since = now - post.date
+        fmt = ""
+        if since.days < 1:
+            fmt = "%X"
+        else:
+            if since.days < 7:
+                fmt += "%A, "
+            fmt += "%x %X"
+        parts.append("(" + localizedtz(post.date, fmt, tzname=tzname, lc=lc) + ")")
+    if post.title:
+        parts.append(f'<a href="{escape(post.link)}">{escape(post.title)}</a>')
+    elif post.link:
+        parts.append(f'<a href="{escape(post.link)}">{escape(post.link)}</a>')
+    if post.content and max_content_len > 0:
+        if parts:
+            parts.append("—")
+        content = ""
+        for word in strip_tags(post.content).split(" "):
+            if len(content + f" {word}") > max_content_len - 3:
+                content += " […]"
+                break
+            content += f" {word}"
+        parts.append(escape(content))
+    return " ".join(parts)

hotdog/command/help.py

@@ -0,0 +1,38 @@
+import re
+from html import escape
+
+from ..functions import reply
+from ..models import Message
+
+
+def init(bot):
+    bot.on_command({"help", "usage"}, handle)
+
+
+async def handle(message: Message):
+    bot = message.app
+    plugins = {k: p for k, p in bot.plugins.items() if hasattr(p, "HELP")}
+
+    pname = message.args.str(0)
+    if pname not in plugins:
+        modnames = ", ".join(f"<code>{m}</code>" for m in sorted(plugins.keys()))
+        await reply(message, html=f"Help is available for: {modnames}")
+        return
+
+    me = f"@{message.my_name}: "
+    pfix = bot.config.command_prefix
+
+    plugin = plugins[pname]
+    usage: str = plugin.HELP
+    usage = re.sub(
+        "^(!|@me: )", lambda m: pfix if m[1] == "!" else me, usage, flags=re.M
+    )
+
+    lines = usage.splitlines(False)
+    usage = (
+        f"<i>{escape(lines[0])}</i><br>\n<code>"
+        + "<br>\n".join(escape(l) for l in lines[1:])
+        + "</code>"
+    )
+
+    await reply(message, html=usage, in_thread=True)

hotdog/command/orakel.py

@@ -0,0 +1,322 @@
+from random import choice, choices, randint, random
+
+from ..functions import reply
+from ..models import Message
+
+HELP = """Gibt Antworten auf Fragen.
+@me: <eine Frage>?
+"""
+
+
+ka = (
+    "Keine Ahnung.",
+    "Das weiß ich leider nicht.",
+    "Ich weiß es nicht.",
+    "Ich weiß es auch nicht.",
+    "Frag besser jemanden, der sich damit auskennt.",
+    "Da wendest du dich besser eine Fachkraft.",
+    "Woher soll ich das wissen?",
+    "Das weiß niemand so genau ...",
+    "Das kann dir keiner beantworten ...",
+    "Eine Frage, so alt wie die Menschheit!",
+    "Das ist eine gute Frage.",
+    "Wer weiß ...",
+    "Boa, keine Ahnung, ey!",
+    "Seh ich so aus, als ob ich wüsste ich das?",
+    "Wie kommst du darauf, dass ich das wüsste?",
+    "Bin ich die Auskunft, oder was?",
+    "Ich bin doch nicht die Auskunft!",
+    "Das ist mir doch egal ...",
+    "Das solltest du besser mit dir selbst ausmachen.",
+    "Das beantwortest du dir am besten mal selbst.",
+    "Ich glaube, das weißt du selbst ganz genau.",
+    "Das weißt du doch selbst ganz genau.",
+    "Schau tief in dich, dort findest du die Antwort.",
+    "Du kämpfst wie eine Kuh!",
+    "Die Antwort ist ... 42.",
+    "Sechs mal Neun.",
+    "Das fragst du besser deine Mama.",
+)
+
+ja = (
+    "Ja",
+    "Ja!",
+    "Ja.",
+    "Jap.",
+    "Jo.",
+    "Jau!",
+    "Joa.",
+    "Joooaaaa.",
+    "Jooo",
+    "Klar!",
+    "Klaaaar.",
+    "Ja, klar.",
+    "Klar doch.",
+    "Aber klar doch.",
+    "Na sicher.",
+    "Sicher doch.",
+    "Joa, warum nicht.",
+    "Jo, warum denn nicht.",
+    "Auf jeden Fall!",
+    "Besser wär's.",
+)
+
+nein = (
+    "Nein",
+    "Nee",
+    "Besser nicht.",
+    "Nein!",
+    "Niemals!",
+    "Auf keinen Fall!",
+)
+
+vielleicht = (
+    "Frag mich später noch mal.",
+    "Das weiß ich nicht genau.",
+    "Ja, vielleicht.",
+    "Nein, vielleicht besser nicht.",
+)
+
+
+def random_room_participant(message: Message) -> str:
+    users = [
+        u
+        for uid, u in message.room.users.items()
+        if uid not in (message.event.sender, message.app.client.user)
+    ]
+    if not users:
+        return ""
+    user = choice(users)
+    return user.display_name or user.user_id
+
+
+numnames = [
+    "null",
+    "ein",
+    "zwei",
+    "drei",
+    "vier",
+    "fünf",
+    "sechs",
+    "sieben",
+    "acht",
+    "neun",
+    "zehn",
+    "elf",
+    "zwölf",
+]
+
+praepos = (
+    "vor",
+    "hinter",
+    "über",
+    "unter",
+    "auf",
+    "neben",  # careful, this cannot be contracted with dativ
+)
+
+wwords = {
+    "wann": (
+        # Time
+        lambda m: f"{choice(['Um ', ''])}{randint(0, 23)} Uhr",
+        lambda m: f"{choice(['Um ', ''])}{randint(0, 23)}:{randint(1, 59):02} Uhr",
+        lambda m: f"{choice(['Um ', ''])}{randint(0, 23)} Uhr und {randint(1, 59)} Minuten",
+        lambda m: f"{choice(['Um ', ''])}{randint(1, 59)} Minuten {choice(['vor','nach'])} {randint(0, 12)}",
+        lambda m: f"{choice(['Um ', ''])}Viertel {choice(['vor','nach'])} {randint(0, 12)}",
+        lambda m: f"Um halb {randint(1, 12)}",
+        "Um Mitternacht",
+        lambda m: f"Am {choice(['Nach', 'Vor'])}mittag",
+        # Date
+        "Morgen",
+        "Übermorgen",
+        "Gestern",
+        "Vorgestern",
+        lambda m: f"{choice(['Nächst', 'Vorig'])}e Woche",
+        lambda m: f"{choice(['Vor', 'In'])} {randint(2, 12)} {choice(['Wochen', 'Monaten'])}",
+        lambda m: f"{choice(['Vor', 'In'])} {randint(2, 9999)} Jahren",
+        lambda m: f"{choice(['Vor', 'In'])} {randint(2, 999)} Millionen Jahren",
+        lambda m: f"Am {randint(1, 28)}.{randint(1, 12)}.{randint(9, 9999)}",
+        # Date-time
+        lambda m: f"{choice(['Vor', 'In'])} {randint(2, 17)} Jahren, {randint(2,360)} Tagen, {randint(2,23)} Stunden, {randint(2,59)} Minuten und {randint(2,59)} Sekunden",
+        lambda m: f"Am {randint(1, 28)}.{randint(1, 12)}.{randint(9, 9999)} um {randint(0, 23)}:{randint(1, 59):02} Uhr",
+    ),
+    "warum": (),
+    "was": (),
+    "wer": (
+        "Deine Mama!",
+        "Die Polizei.",
+        "Die drei Fragezeichen!",
+        "TKKG.",
+        "Tim und Struppi.",
+        "Die fünf Freunde.",
+        "Der Osterhase.",
+        "Die Ferienbande! Yaaaaay.",
+        "Der Weihnachtsmann.",
+        "Frau Holle.",
+        "Gott!",
+        "Irgendein Gott.",
+        "Du!",
+        "Du selbst.",
+        "Ein Höhlentroll mit einem dicken Prügel.",
+        "Eine höhere Macht.",
+        "Ein unbekanntes Wesen.",
+        "Ein unbekanntes Wesen aus einer unbekannten Galaxie.",
+        "Aliens!",
+        "Graf Zahl.",
+        "Die Russen!",
+        "Die USA!",
+        "Die Reichsbürger!",
+        "Die Regierung.",
+        "Die Presse.",
+        "Die Illuminaten!",
+        "Die internationale Verschwörung e.V.!",
+        "Dein Nachbar.",
+        "Die Kanzlerin.",
+        "Der Präsident.",
+        "Der Vorsitzende vom Schützenverein.",
+        "Der Postbote.",
+        "Die Behörden.",
+        "Die Wissenschaft.",
+        "Die sechs von der Müllabfuhr.",
+        "Der, der grade hinter dir steht und dir zuguckt.",
+        "Die, die sich hinter deinem Schrank versteckt hat.",
+        "Der Typ der in der Ecke steht.",
+        "Die Anderen; immer die Anderen.",
+        lambda m: random_room_participant(m) or "Irgendwer.",
+    ),
+    "wie": (
+        "Das weiß ich leider auch nicht, vielleicht mit einer Telefonlawine?",
+        "Am besten gar nicht.",
+        "Das ist schwierig zu sagen.",
+        *ka,
+    ),
+    "wieso": (),
+    "wieviel": (
+        "Nichts",
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + choice(numnames[2:]),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + f"{23 * random():n}",
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(1, 10)),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(7, 23)),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(17, 42)),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(23, 99)),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(100, 1_000_000)),
+        "Unendlich!",
+    ),
+    "wieviele": (
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + choice(numnames[2:]),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + f"{23 * random():n}",
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(1, 10)),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(7, 23)),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(17, 42)),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(23, 99)),
+        lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(100, 1_000_000)),
+        "Ein paar.",
+        "Nur ein paar.",
+        "Nur ein paar wenige.",
+        "Wirklich viele.",
+        "Einige.",
+        "Ne ganze Menge.",
+        "Zu viele!",
+        "Ziemlich viele!",
+        "Unendlich viele!",
+    ),
+    "wo": (
+        lambda m: f"Genau {choice(praepos)} dir!",
+        lambda m: f"{choice(praepos).title()} dir!",
+        lambda m: f"{choice(praepos).title()} dem Bett.",
+        lambda m: f"{choice(praepos).title()} dem Kopfkissen.",
+        lambda m: f"{choice(praepos).title()} dem Couche-Kissen.",
+        lambda m: f"{choice(praepos).title()} dem Tisch.",
+        lambda m: f"{choice(praepos).title()} der Kommode.",
+        lambda m: f"{choice(praepos).title()} dem Stuhl.",
+        "Droben auf dem Berge.",
+        "In der Garage.",
+        "Im Klo.",
+        "Auf der Straße.",
+        "In der Gasse.",
+        "In den Wolken.",
+        "Vorm Fernseher.",
+        "Im Cyberspace.",
+        "Genau vor deiner Nase.",
+        "Auf deinem Kopf.",
+        "Unter deinem Hintern.",
+        "Du sitzt drauf!",
+        "Im Gefängnis.",
+        "Im Bundestag.",
+        "Auf der dunklen Seite des Mondes.",
+        "In einer weit entfernten Galaxie.",
+        "Auf dem Mond.",
+        "Auf der ISS.",
+        "Im Inneren der Erde.",
+    ),
+    "woher": (),
+    "wohin": (),
+    "weshalb": (),
+    "welche": (),
+    "welcher": (),
+    "welches": (),
+    "um wieviel uhr": (
+        lambda m: f"Um {randint(0, 23)} Uhr",
+        lambda m: f"Um {randint(0, 23)}:{randint(1, 59):02} Uhr",
+        lambda m: f"Um {randint(0, 23)} Uhr und {randint(1, 59)} Minuten",
+        lambda m: f"Um {randint(1, 59)} Minuten {choice(['vor','nach'])} {randint(0, 12)}",
+        lambda m: f"Um Viertel {choice(['vor','nach'])} {randint(0, 12)}",
+        lambda m: f"Um halb {randint(1, 12)}",
+    ),
+    "seit wann": (
+        lambda m: f"Seit {randint(0, 23)} Uhr",
+        lambda m: f"Seit {randint(0, 23)}:{randint(1, 59):02} Uhr",
+        lambda m: f"Seit {randint(0, 23)} Uhr und {randint(1, 59)} Minuten",
+        lambda m: f"Seit {randint(1, 59)} Minuten {choice(['vor','nach'])} {randint(0, 12)}",
+        lambda m: f"Seit Viertel {choice(['vor','nach'])} {randint(0, 12)}",
+        lambda m: f"Seit halb {randint(1, 12)}",
+        # Date
+        "Seit Gestern",
+        "Seit Vorgestern",
+        lambda m: f"Seit vorige Woche",
+        lambda m: f"Seit {randint(2, 12)} {choice(['Wochen', 'Monaten'])}",
+        lambda m: f"Seit {randint(2, 9999)} Jahren",
+        lambda m: f"Seit {randint(2, 999)} Millionen Jahren",
+        lambda m: f"Seit dem {randint(1, 28)}.{randint(1, 12)}.{randint(9, 9999)}",
+        # Date-time
+        lambda m: f"Seit {randint(2, 17)} Jahren, {randint(2,360)} Tagen, {randint(2,23)} Stunden, {randint(2,59)} Minuten und {randint(2,59)} Sekunden",
+        lambda m: f"Seit dem {randint(1, 28)}.{randint(1, 12)}.{randint(9, 9999)} um {randint(0, 23)}:{randint(1, 59):02} Uhr",
+    ),
+}
+
+
+# wer
+# wem
+# seit wann
+# um wieviel uhr
+
+
+def init(bot):
+    bot.on_message(handle)
+
+
+async def handle(message: Message):
+    if "oder" in message.tokens or not (  # avoid conflicts with aoderb.py
+        message.text.endswith("?") and message.is_for_me
+    ):
+        return
+
+    if message.words.startswith("um wieviel uhr"):
+        wword = "um wieviel uhr"
+    elif message.words.startswith("seit wann"):
+        wword = "seit wann"
+    else:
+        wword = message.words.str(0).rstrip(",.?!").lower()
+        if wword in ("und", "aber"):
+            wword = message.words.str(1).rstrip(",.?!").lower()
+    if wword in wwords:
+        reps = (
+            choices([wwords[wword], ka], weights=[100, 10])[0] if wwords[wword] else ka
+        )
+        template = choice(reps)
+    else:
+        template = choice(choices([ja, nein, vielleicht], weights=[100, 100, 15])[0])
+
+    text = template(message) if callable(template) else template
+
+    await reply(message, text, with_name=True)

hotdog/command/post.py

@@ -0,0 +1,127 @@
+from datetime import datetime, timezone
+
+import postillon
+
+from ..functions import clamp, localizedtz, reply
+from ..models import Message
+
+HELP = """Postillon Newsticker.
+!post [how many|search terms ...]
+!post find [search terms ...]
+!post random [how many]
+!post more
+"""
+
+
+def init(bot):
+    bot.on_command("post", handle)
+
+    bot.shared.setdefault("post.finds", {})
+    if "post.store" not in bot.shared:
+        bot.shared["post.store"] = postillon.Store(bot.config.get("postillon.storage"))
+        bot.shared["post.store"].connect()
+
+
+def post_as_plain(post, tzname: str, lc: str):
+    parts = []
+    if post.content:
+        parts.append(f"+++ {post.content} +++")
+    if post.date:
+        now = datetime.now(tz=timezone.utc)
+        since = now - post.date
+        fmt = ""
+        if since.days < 1:
+            fmt = "%X"
+        else:
+            if since.days < 7:
+                fmt += "%A, "
+            fmt += "%x %X"
+        parts.append("(" + localizedtz(post.date, fmt, tzname=tzname, lc=lc) + ")")
+    return " ".join(parts)
+
+
+def post_as_html(post, tzname: str, lc: str):
+    parts = []
+    if post.content:
+        content = post.content.strip()
+        sep = ": " if ": " in content else "? " if "? " in content else None
+        if not sep:
+            parts.append(f"+++ {post.content} +++")
+        else:
+            q, a = content.split(sep, 2)
+            parts.append(f"+++ <b>{q}{sep}</b>{a} +++")
+    if post.date:
+        now = datetime.now(tz=timezone.utc)
+        since = now - post.date
+        fmt = ""
+        if since.days < 1:
+            fmt = "%X"
+        else:
+            if since.days < 7:
+                fmt += "%A, "
+            fmt += "%x %X"
+        parts.append("(" + localizedtz(post.date, fmt, tzname=tzname, lc=lc) + ")")
+    return " ".join(parts)
+
+
+async def handle(message: Message):
+    bot = message.app
+    poststore = bot.shared["post.store"]
+    finds = bot.shared["post.finds"]  # used to page through find results
+    max_results = 5
+
+    mode = message.args.str(0)
+    args = message.args
+    if mode in ("find", "random", "more"):
+        args = args[1:]
+    else:
+        # Time for magic!
+        if not args:
+            if finds.get(message.room.room_id, (None, 0))[0] is not None:
+                mode = "more"
+            else:
+                mode = "random"
+        else:
+            if len(args) == 1 and args.str(0).isdecimal():
+                mode = "random"
+            else:
+                mode = "find"
+
+    if mode == "find":
+        term = "%".join(args)
+        posts = poststore.search(f"%{term}%")
+        finds[message.room.room_id] = (term, 0)
+    elif mode == "random":
+        finds[message.room.room_id] = (None, 0)
+        count = clamp(1, args.int(0), max_results)
+        posts = [poststore.random_post() for _ in range(count)]
+    elif mode == "more":
+        term, page = finds.get(message.room.room_id, (None, 0))
+        if term is None:
+            return
+        page += 1
+        posts = poststore.search(f"%{term}%", skip=page * max_results)
+        finds[message.room.room_id] = (term, page)
+    else:
+        return
+
+    roomconf = bot.config.l6n[message.room.room_id]
+    i = None
+    for i, post in enumerate(posts):
+        if i >= max_results:
+            await reply(
+                message,
+                html=f"<i>Für weitere Ergebnisse</i>: <code>{bot.config.command_prefix}post more</code>",
+                in_thread=True,
+            )
+            break
+        text = post_as_html(
+            post,
+            tzname=roomconf["timezone"],
+            lc=roomconf["locale"],
+        )
+        await reply(message, html=text)
+    else:
+        finds[message.room.room_id] = (None, 0)
+    if i is None:
+        await reply(message, html="<i>Keine weiteren Ergebnisse.</i>", in_thread=True)

hotdog/command/prost.py

@@ -0,0 +1,39 @@
+from random import choice
+
+from ..functions import react
+from ..models import Message
+
+
+def init(bot):
+    bot.on_message(handle)
+
+
+emojis = {
+    "☕️",
+    "🍵",
+    "🍶",
+    "🍷",
+    "🍸",
+    "🍹",
+    "🍺",
+    "🍻",
+    "🍼",
+    "🍾",
+    "🥂",
+    "🥃",
+    "🥛",
+    "🥤",
+    "🧃",
+    "🧉",
+}
+
+
+async def handle(message: Message):
+    vs16 = "\ufe0f"  # see https://en.wikipedia.org/wiki/Variation_Selectors_%28Unicode_block%29
+    is_drinkmoji = message.text.rstrip(vs16) in emojis
+    if not (message.tokens.str(0).startswith("PR!") or is_drinkmoji):
+        return
+
+    remoji = message.text if is_drinkmoji else choice(tuple(emojis))
+
+    await react(message, remoji)

hotdog/command/reminder.py

@@ -0,0 +1,20 @@
+from ..functions import reply
+from ..models import Message
+
+# @hotdog: remind <who> <time> <what>
+# .remind me at 14:30 take out the milk
+# .remind me to take out the milk tomorrow
+
+# today -> next 6 hour step
+# tomorrow -> at 10
+# in X {days|hours|minutes|seconds}
+# {on|at} d.m.y [hh:mm]
+
+
+def init(bot):
+    bot.on_message(handle)
+
+
+async def handle(message: Message):
+    if message.command != "remind" or not message.args.startswith("me"):
+        return

hotdog/command/retour.py

@@ -0,0 +1,26 @@
+from random import choice
+
+from ..functions import reply
+from ..models import Message
+
+
+def init(bot):
+    bot.on_message(handle)
+
+
+replies = (
+    "Du auch.",
+    "Selber.",
+    "Danke, gleichfalls.",
+)
+
+
+async def handle(message: Message):
+    if "oder" in message.tokens or not (
+        message.words.startswith("du bist")
+        and message.is_for_me
+        and not message.text.endswith("?")  # avoid conflicts with orakel.py
+    ):
+        return
+
+    await reply(message, choice(replies), with_name=True)

hotdog/command/roll.py

@@ -0,0 +1,59 @@
+import re
+from collections import defaultdict
+from random import randint
+
+from ..functions import reply
+from ..models import Message
+
+HELP = """Würfelt eine Summe aus.
+!roll [how many]d<sides> [ ... ]
+"""
+
+parse_die = re.compile("(?P<num>\d*)[dwDW](?P<sides>\d+)").fullmatch
+
+
+def init(bot):
+    bot.on_command("roll", handle)
+
+
+async def handle(message: Message):
+    if len(message.args) == 1 and message.args[0].isdecimal():
+        dice = [(1, int(message.args[0]))]
+    else:
+        dice = []
+        num = None
+        for arg in message.args:
+            if (match := parse_die(arg)) :
+                if num and match["num"]:
+                    return
+                num, sides = int(match["num"] or num or 1), int(match["sides"])
+                if not 1 <= num < 1000 or not 2 <= sides <= 100:
+                    return
+                dice.append((num, sides))
+                num = None
+            elif arg.isdecimal():
+                if num is not None:
+                    return
+                num = int(arg)
+            else:
+                return
+
+    if not 0 < len(dice) < 20:
+        return
+
+    rolls = defaultdict(list)
+    for num, sides in dice:
+        for _ in range(num):
+            rolls[sides].append(randint(1, sides))
+
+    total = sum(sum(vs) for sides, vs in rolls.items())
+    if len(rolls) == 1:
+        sides = list(rolls.keys())[0]
+        text = f"{total} ({len(rolls[sides])}d{sides})"
+    else:
+        dicestr = " + ".join(
+            f"{len(vs)}d{sides} ({sum(vs)})"
+            for sides, vs in sorted(rolls.items(), key=lambda i: i[0], reverse=1)
+        )
+        text = f"{total} = {dicestr}"
+    await reply(message, plain=text, in_thread=True)

hotdog/command/urlinfo.py

@@ -0,0 +1,199 @@
+import codecs
+import re
+from functools import lru_cache
+from html import escape
+from html.parser import HTMLParser
+from random import randint
+from time import time as now
+from typing import *
+
+import requests
+
+from ..functions import reply
+from ..models import Message
+
+HELP = """Return information about an online HTTP resource.
+!u[rl] <url>
+"""
+
+
+def init(bot):
+    bot.on_command({"u", "url"}, handle)
+
+
+match_url = re.compile(
+    # r"https?://(?:[a-zA-Z]|[0-9]|[$-_~@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
+    r"https?://\S+"
+).fullmatch
+
+
+class TitleParser(HTMLParser):
+    """Parse the first <title> from HTML"""
+
+    # XXX check if it's the <head>'s title we're in, but beware that head can be implicit
+    def __init__(self):
+        super().__init__()
+        self.__is_title = False
+        self.__found = False
+        self.__title = ""
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "title":
+            self.__is_title = True
+
+    def handle_endtag(self, tag):
+        if tag == "title":
+            self.__found = True
+            self.__is_title = False
+
+    def handle_data(self, data):
+        if self.__is_title and not self.__found:
+            self.__title += data
+
+    @property
+    def title(self) -> Optional[str]:
+        return self.__title if self.__found else None
+
+
+def get_encodings_from_content(content: str) -> List[str]:
+    """Returns encodings from given content string."""
+
+    charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
+    pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
+    xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')
+
+    return (
+        charset_re.findall(content)
+        + pragma_re.findall(content)
+        + xml_re.findall(content)
+    )
+
+
+def stream_decode_response_unicode(content: Iterable[bytes]) -> Iterable[str]:
+    """Stream decodes a iterator."""
+
+    decoder = None
+    for chunk in content:
+        if decoder is None:
+            encodings = get_encodings_from_content(
+                chunk.decode("utf-8", errors="replace")
+            ) + ["utf-8"]
+            decoder = codecs.getincrementaldecoder(encodings[0])(errors="replace")
+
+        rv = decoder.decode(chunk)
+        if rv:
+            yield rv
+    rv = decoder.decode(b"", final=True)
+    if rv:
+        yield rv
+
+
+def title(content: Iterable[str]) -> Optional[str]:
+    t = TitleParser()
+    for chunk in content:
+        t.feed(chunk)
+        if t.title is not None:
+            break
+    return t.title
+
+
+def capped(content: Iterable[str], read_max: int) -> Iterable[str]:
+    read = 0
+    for chunk in content:
+        read += len(chunk)
+        yield chunk
+        if read >= read_max:
+            break
+
+
+@lru_cache(maxsize=5)
+def load_info(url: str, cachetoken) -> Optional[Mapping[str, Any]]:
+    """The cachetoken is just there to bust the LRU cache after a while."""
+    try:
+        r = requests.get(
+            url,
+            stream=True,
+            timeout=(3, 3),
+            headers={"user-agent": "hotdog/v1 urlinfo"},
+        )
+    except Exception:
+        return None
+
+    content_type = r.headers.get("Content-Type", "")
+    is_html = content_type.startswith("text/html") or url.lower().endswith(
+        (".html", ".htm")
+    )
+    if is_html:
+        one_kb = 2 ** 10
+        # chunks = r.iter_content(chunk_size=30 * one_kb, decode_unicode=True)
+        chunks = stream_decode_response_unicode(r.iter_content(chunk_size=30 * one_kb))
+        html_title = title(capped(chunks, read_max=200 * one_kb))
+    else:
+        html_title = None
+
+    filename = None
+    dispo = r.headers.get("Content-Disposition", "").split(";")
+    if len(dispo) == 2 and dispo[0] == "attachment":
+        dispo = dispo[1].strip().split("=", 2)
+        if len(dispo) == 2 and dispo[0] == "filename":
+            filename = dispo[1].strip()
+
+    return {
+        "code": r.status_code,
+        "url": r.url,
+        "elapsed_ms": int(r.elapsed.total_seconds() * 1_000),
+        "reason": r.reason,
+        "type": r.headers.get("Content-Type"),
+        "size": (
+            int(r.headers["Content-Length"]) if "Content-Length" in r.headers else None
+        ),
+        "title": html_title,
+        "filename": filename,
+    }
+
+
+def cachetoken(quant_m=15):
+    """Return a cache token with the given time frame"""
+    return int(now() / 60 / quant_m)
+
+
+def pretty_size(size: int) -> str:
+    qs = "", "K", "M", "G", "T", "P"
+    for q in qs:
+        if size < 1024 or q == qs[-1]:
+            break
+        size /= 1000
+    if not q:
+        return f"{size} B"
+    return f"{size:_.02f} {q}B"
+
+
+async def handle(message: Message):
+    url = message.args.str(0)
+    if not match_url(url):
+        return
+
+    info = load_info(url, cachetoken())
+    if not info:
+        return
+
+    details = []
+    if info["type"]:
+        details.append(f"<i>Media type</i>: {escape(info['type'])}")
+    if info["size"]:
+        details.append(f"<i>Size</i>: {pretty_size(info['size'])}")
+    details.append(f"<i>Status</i>: {info['code']}")
+    if info["reason"]:
+        details[-1] += f" ({escape(info['reason'])})"
+    if info["url"] != url:
+        details.append(
+            f"""<i>Redirected to</i>: <a href="{escape(info['url'])}">{escape(info['url'])}</a>"""
+        )
+    if info["filename"] and info["filename"] != url.rsplit("/", 2)[-1]:
+        details.append(f"<i>Filename</i>: {escape(info['filename'])}")
+    details.append(f"<i>TTFB</i>: {info['elapsed_ms']:_} ms")
+
+    text = f"<b>{escape(info['title'])}</b> — " if info["title"] else ""
+    text += "; ".join(details)
+
+    await reply(message, html=text, in_thread=True)

hotdog/command/wikipedia.py

@@ -0,0 +1,458 @@
+import asyncio
+import logging
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from functools import lru_cache, partial
+from html import escape
+from time import time as now
+from typing import *
+
+import requests
+
+from ..functions import localizedtz, react, reply, strip_tags
+from ..models import Message
+
+log = logging.getLogger(__name__)
+
+HELP = """Look up articles on Wikipedia.
+!w[p|ikipedia] [lang (ISO 639)] <search terms ...>
+"""
+
+
+def init(bot):
+    bot.shared[__name__] = {"last": 0}
+
+    bot.on_command({"w", "wp", "wikipedia"}, handle)
+
+
+api_url = "https://{lang}.wikipedia.org/w/api.php"
+
+# see https://de.wikipedia.org/wiki/Liste_der_Wikipedias_nach_Sprachen
+langs = {
+    "aa",
+    "ab",
+    "ace",
+    "af",
+    "ak",
+    "als",
+    "am",
+    "an",
+    "ang",
+    "ar",
+    "arc",
+    "as",
+    "ast",
+    "av",
+    "ay",
+    "az",
+    "ba",
+    "bar",
+    "bat-smg",
+    "bcl",
+    "be",
+    "bg",
+    "bh",
+    "bi",
+    "bjn",
+    "bm",
+    "bn",
+    "bo",
+    "bpy",
+    "br",
+    "bs",
+    "bug",
+    "bxr",
+    "ca",
+    "cbk-zam",
+    "cdo",
+    "ce",
+    "ceb",
+    "ch",
+    "cho",
+    "chr",
+    "chy",
+    "ckb",
+    "co",
+    "cr",
+    "crh",
+    "cs",
+    "csb",
+    "cu",
+    "cv",
+    "cy",
+    "da",
+    "de",
+    "diq",
+    "dsb",
+    "dv",
+    "dz",
+    "ee",
+    "el",
+    "eml",
+    "en",
+    "eo",
+    "es",
+    "et",
+    "eu",
+    "ext",
+    "fa",
+    "ff",
+    "fi",
+    "fiu-vro",
+    "fj",
+    "fo",
+    "fr",
+    "frp",
+    "frr",
+    "fur",
+    "fy",
+    "ga",
+    "gag",
+    "gan",
+    "gd",
+    "gl",
+    "glk",
+    "gn",
+    "got",
+    "gu",
+    "gv",
+    "ha",
+    "hak",
+    "haw",
+    "he",
+    "hi",
+    "hif",
+    "ho",
+    "hr",
+    "hsb",
+    "ht",
+    "hu",
+    "hy",
+    "hz",
+    "ia",
+    "id",
+    "ie",
+    "ig",
+    "ii",
+    "ik",
+    "ilo",
+    "io",
+    "is",
+    "it",
+    "iu",
+    "ja",
+    "jbo",
+    "jv",
+    "ka",
+    "kaa",
+    "kab",
+    "kbd",
+    "kg",
+    "kj",
+    "kk",
+    "kl",
+    "km",
+    "kn",
+    "ko",
+    "koi",
+    "kr[",
+    "krc",
+    "ks",
+    "ksh",
+    "ku",
+    "kv",
+    "kw",
+    "ky",
+    "la",
+    "lad",
+    "lb",
+    "lbe",
+    "lez",
+    "lg",
+    "li",
+    "lij",
+    "lmo",
+    "ln",
+    "lo",
+    "lt",
+    "ltg",
+    "lv",
+    "map-bms",
+    "mdf",
+    "mg",
+    "mh",
+    "mhr",
+    "mi",
+    "mk",
+    "ml",
+    "mn",
+    "mr",
+    "mrj",
+    "ms",
+    "mt",
+    "mus",
+    "mwl",
+    "my",
+    "myv",
+    "mzn",
+    "na",
+    "nah",
+    "nap",
+    "nds",
+    "nds-nl",
+    "ne",
+    "new",
+    "ng",
+    "nl",
+    "nn",
+    "no",
+    "nov",
+    "nrm",
+    "nso",
+    "nv",
+    "oc",
+    "om",
+    "or",
+    "os",
+    "pa",
+    "pag",
+    "pam",
+    "pap",
+    "pcd",
+    "pdc",
+    "pfl",
+    "pi",
+    "pih",
+    "pl",
+    "pms",
+    "pnb",
+    "pnt",
+    "ps",
+    "pt",
+    "qu",
+    "rm",
+    "rmy",
+    "rn",
+    "ro",
+    "roa-rup",
+    "roa-tara",
+    "ru",
+    "rue",
+    "rw",
+    "sa",
+    "sc",
+    "scn",
+    "sco",
+    "sd",
+    "se",
+    "sg",
+    "sh",
+    "si",
+    "simple",
+    "sk",
+    "sl",
+    "sm",
+    "sn",
+    "so",
+    "sq",
+    "sr",
+    "srn",
+    "ss",
+    "st",
+    "stq",
+    "su",
+    "sv",
+    "sw",
+    "szl",
+    "ta",
+    "te",
+    "tet",
+    "tg",
+    "th",
+    "ti",
+    "tk",
+    "tl",
+    "tn",
+    "to",
+    "tpi",
+    "tr",
+    "ts",
+    "tt",
+    "tum",
+    "tw",
+    "ty",
+    "udm",
+    "ug",
+    "uk",
+    "ur",
+    "uz",
+    "ve",
+    "vec",
+    "vep",
+    "vi",
+    "vls",
+    "vo",
+    "wa",
+    "war",
+    "wo",
+    "wuu",
+    "xal",
+    "xh",
+    "xmf",
+    "yi",
+    "yo",
+    "za",
+    "zea",
+    "zh",
+    "zh-classical",
+    "zh-min-nan",
+    "zh-yue",
+    "zu",
+    "zu",
+}
+
+
+def searchparams(terms: Iterable[str]):
+    # see https://www.mediawiki.org/wiki/API:Search
+    return {
+        "action": "query",
+        "list": "search",
+        "srsearch": " ".join(terms),
+        "format": "json",
+        "srlimit": 3,
+    }
+
+
+def resolveparams(ids: Iterable[int]):
+    return {
+        "action": "query",
+        "prop": "info",
+        "pageids": "|".join(map(str, ids)),
+        "inprop": "url",
+        "format": "json",
+    }
+
+
+@dataclass(frozen=True)
+class Hit:
+    # ns: int
+    pageid: int
+    size: int
+    snippet: str
+    timestamp: datetime
+    title: str
+    wordcount: int
+
+    @classmethod
+    def from_json(cls, data):
+        return cls(
+            data["pageid"],
+            data["size"],
+            data["snippet"],
+            fromjsonformat(data["timestamp"]),
+            data["title"],
+            data["wordcount"],
+        )
+
+
+def fromjsonformat(s: str) -> datetime:
+    if s.endswith("Z"):
+        s = s[:-1] + "+00:00"
+    return datetime.fromisoformat(s)
+
+
+def load_api_json(session, lang, params):
+    r = session.get(
+        api_url.format(lang=lang),
+        params=params,
+        timeout=(3, 3),
+        headers={"User-Agent": "hotdog/v1 wikipedia"},
+    )
+    r.raise_for_status()
+    return r.json()
+
+
+def search(session, lang, terms):
+    data = load_api_json(session, lang, searchparams(terms))
+    return {
+        "total": data["query"]["searchinfo"]["totalhits"],
+        "hits": [Hit.from_json(d) for d in data["query"]["search"]],
+    }
+
+
+def resolve_urls(session, lang, ids: Collection[int]):
+    if not ids:
+        return {}
+    data = load_api_json(session, lang, resolveparams(ids))
+    return {
+        int(pid): p.get("canonicalurl") for pid, p in data["query"]["pages"].items()
+    }
+
+
+@lru_cache(maxsize=10)
+def search_and_resolve(lang, terms):
+    session = requests.Session()
+    result = search(session, lang, terms)
+    result["urls"] = resolve_urls(session, lang, [hit.pageid for hit in result["hits"]])
+    return result
+
+
+def snippet(s: str, max_content_len: int = 300):
+    content = ""
+    for word in s.split():
+        if not word:
+            continue
+        if len(content + f" {word}") > max_content_len - 3:
+            content += " […]"
+            break
+        content += f" {word}"
+    return content
+
+
+async def handle(message: Message):
+    bot = message.app
+    roomconf = bot.config.l6n[message.room.room_id]
+    if message.args.get(0) in langs:
+        lang = message.args[0]
+        args = message.args[1:]
+    else:
+        lang, *_ = roomconf["locale"].split("_", 1)
+        args = message.args
+
+    if not args:
+        return
+
+    await react(message, "⚡️")
+
+    # XXX no need to wait if the result is already cached - can we check that?
+
+    # Guard against API flooding...
+    timeout = 10
+    while 0 < (waitfor := timeout - (now() - bot.shared[__name__]["last"])):
+        log.debug(f"Waiting for {waitfor}s before next API call.")
+        await asyncio.sleep(waitfor)
+    bot.shared[__name__]["last"] = now()
+
+    r = search_and_resolve(lang, args)
+
+    if not r["hits"]:
+        await react(message, "❌")
+        return
+
+    lines = []
+    if r["total"] > 3:
+        lines.append(f"Found <b>{r['total']}</b> matching articles.")
+    for hit in r["hits"][:3]:
+        last_updated = localizedtz(
+            hit.timestamp, "%x %X", tzname=roomconf["timezone"], lc=roomconf["locale"]
+        )
+        teaser = snippet(escape(strip_tags(f"… {hit.snippet} …")))
+        lines.append(
+            f'<b><a href="{escape(r["urls"][hit.pageid])}">{escape(hit.title)}</a></b>: <i>{teaser}</i> (<i>{last_updated}</i>)'
+        )
+
+    await asyncio.gather(
+        reply(message, html="<br/>\n".join(lines), in_thread=True),
+        react(message, "✅"),
+    )

hotdog/command/youtube.py

@@ -0,0 +1,131 @@
+import re
+from dataclasses import dataclass, fields
+from functools import lru_cache
+from html import escape
+from time import time as now
+from typing import *
+
+import youtube_dl
+
+from ..functions import reply
+from ..models import Message
+
+HELP = """Gibt Informationen zu Youtube-Videos aus.
+Some text containing a <Youtube URL>.
+"""
+
+youtube_re = re.compile(
+    r"\byoutu(\.be/|be\.com/(embed/|v/|watch\?\w*v=))(?P<id>[0-9A-Za-z_-]{10,11})\b"
+)
+
+
+def init(bot):
+    bot.on_message(handle)
+
+
+@lru_cache(maxsize=5)
+def load_info(url, cachetoken):
+    """The cachetoken is just there to bust the LRU cache after a while."""
+    return Info.from_url(url)
+
+
+def cachetoken(quant_m=15):
+    """Return a cache token with the given time frame"""
+    return int(now() / 60 / quant_m)
+
+
+async def handle(message: Message):
+    if message.command in {"u", "url"}:
+        return
+
+    match = youtube_re.search(message.text)
+    if not match:
+        return
+
+    youtube_id = match["id"]
+
+    info = load_info(youtube_id, cachetoken())
+    info.escape_all()
+    details = [
+        f"🖋 {info.author}",
+        f"⏱ {pretty_duration(info.duration_seconds)}",
+        f"📺 {info.width}×{info.height}",
+        f"👀 {info.view_count:_}",
+    ]
+    tag = (info.categories[:1] or info.tags[:1] or [""])[0]
+    if tag:
+        details.append(f"🏷 {tag}")
+    text = f"<b>{info.title}</b> — {', '.join(details)}"
+    await reply(message, html=text)
+
+
+def pretty_duration(seconds: int) -> str:
+    hours = seconds // 3600
+    minutes = (seconds - hours * 3600) // 60
+    seconds = seconds % 60
+    return (
+        f"{hours}h{minutes:02}m{seconds:02}s" if hours else f"{minutes}m{seconds:02}s"
+    )
+
+
+class Nolog:
+    def debug(self, msg):
+        pass
+
+    def warning(self, msg):
+        pass
+
+    def error(self, msg):
+        pass
+
+
+ytdl = youtube_dl.YoutubeDL({"skip_download": True, "logger": Nolog()})
+
+
+@dataclass
+class Info:
+    author: str
+    title: str
+    description: str
+    duration_seconds: int
+    view_count: int
+    thumbnail: str
+    width: int
+    height: int
+    categories: List[str]
+    tags: List[str]
+
+    def escape_all(self):
+        for f in fields(self):
+            if f.type is str:
+                setattr(self, f.name, escape(getattr(self, f.name)))
+            elif get_origin(f.type) is list:
+                setattr(self, f.name, [escape(x) for x in getattr(self, f.name)])
+
+    @classmethod
+    def from_url(cls, url):
+        info = ytdl.extract_info(url, download=False)
+        new = cls(
+            author=info["uploader"] or info["creator"] or info["uploader_id"] or "",
+            title=info["title"] or info["alt_title"] or "",
+            description=info["description"] or "",
+            duration_seconds=info["duration"] or 0,
+            view_count=info["view_count"] or 0,
+            thumbnail=(
+                info["thumbnail"]
+                or (info["thumbnails"][-1] if info["thumbnails"] else "")
+            ),
+            categories=info["categories"] or [],
+            tags=info["tags"] or [],
+            width=info["width"] or 0,
+            height=info["height"] or 0,
+        )
+        if info["formats"]:
+            new.width, new.height = max(
+                (
+                    (f["width"] or new.width),
+                    (f["height"] or new.height),
+                )
+                for f in info["formats"]
+            )
+        return new

hotdog/config.py

@@ -0,0 +1,92 @@
+import platform
+import re
+from pathlib import Path
+from typing import *
+
+import yaml
+
+
+class ConfigError(RuntimeError):
+    pass
+
+
+class Fallbackdict(dict):
+    """Like defaultdict, but not implicitly creating entries."""
+
+    def __init__(self, default):
+        super().__init__()
+        self._default = default
+
+    def get(self, key, default=None):
+        return super().get(key, self._default if default is None else default)
+
+    def __getitem__(self, key):
+        return super().get(key, self._default)
+
+
+class Config:
+    def __init__(self, path):
+        self.load(path)
+
+    def load(self, path):
+        with open(path) as fp:
+            self.config = yaml.safe_load(fp)
+
+        self.command_prefix = self.get("command_prefix")
+
+        # Logging
+        self.loglevel = self.get("loglevel", default="INFO")
+
+        # Storage
+        self.store_path = Path(self.get("storage.store_path"))
+
+        # Matrix
+        self.user_id = self.get("matrix.user_id")
+        if not re.fullmatch("@.*:.*", self.user_id):
+            raise ConfigError("matrix.user_id must be in the form @name:domain")
+
+        self.device_id_path = self.store_path / f"{self.user_id}.device_id"
+        try:
+            self.device_id = self.device_id_path.read_text()
+        except FileNotFoundError:
+            self.device_id = ""
+
+        self.device_name = self.get(
+            "matrix.session_name", default=f"hotdog/{platform.node()}"
+        )
+        self.display_name = self.get("matrix.nick_name")
+        self.homeserver_url = self.get("matrix.homeserver_url")
+        self.password = self.get("matrix.password")
+
+        # Development
+        self.is_dev = self.get("dev.active", False)
+        self.dev_room = self.get("dev.room", "")
+
+        # Location / l6n
+        l6n_default = self.get("l6n.default")
+        l6n = Fallbackdict(l6n_default)
+        for room in self.get("l6n.rooms"):
+            l6n[room["id"]] = {**l6n_default, **room}  # XXX in py3.9 use |-operator
+        self.l6n = l6n
+
+    def get(self, path: Union[str, List[str]], default: Any = None) -> Any:
+        if isinstance(path, str):
+            path = path.split(".")
+        config = self.config
+        for part in path:
+            config = config.get(part)
+            if config is None:
+                if default is not None:
+                    return default
+                raise ConfigError(f"Config value is missing: {'.'.join(path)}")
+        return config
+
+    def set(self, path: Union[str, List[str]], value: Any):
+        if isinstance(path, str):
+            path = path.split(".")
+        config = self.config
+        for part in path[:-1]:
+            if part in config and not isinstance(config[part], dict):
+                raise ConfigError(f"Conflicting value exists: {'.'.join(path)}")
+            config = config.get(part, {})
+        config[path[-1]] = value

hotdog/functions.py

@@ -0,0 +1,181 @@
+import locale
+import logging
+import unicodedata
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from html import escape
+from html.parser import HTMLParser
+from io import StringIO
+from typing import *
+
+import nio
+
+log = logging.getLogger(__name__)
+
+tzdb = {
+    "Europe/Berlin": timezone(timedelta(hours=2), "Europe/Berlin"),
+}
+
+
+def html_nametag(uid, name):
+    return f'<a href="https://matrix.to/#/{escape(uid)}">{escape(name)}</a>'
+
+
+async def reply(
+    message,
+    plain: Optional[str] = None,
+    *,
+    html: Optional[str] = None,
+    with_name: bool = False,
+    in_thread: bool = False,
+) -> nio.RoomSendResponse:
+    """Reply to the given message in plain text or HTML.
+
+    If with_name is set the reply will be prefixed with the original
+    message's sender's name.
+    If in_thread is set the reply will reference the original message,
+    allowing a client to connect the messages unambiguously.
+    """
+    assert plain or html
+    if with_name:
+        if plain:
+            plain = f"{message.sender_name}: {plain}"
+        if html:
+            sender = html_nametag(message.event.sender, message.sender_name)
+            html = f"{sender}: {html}"
+    args = {}
+    if in_thread:
+        args["reply_to_event_id"] = message.event.event_id
+    if html:
+        args["html"] = html
+    if plain:
+        args["plain"] = plain
+    return await send_message(message.app.client, message.room.room_id, **args)
+
+
+async def react(message, emoji: str) -> nio.RoomSendResponse:
+    """Annotate a message with an Emoji.
+
+    Only a single Emoji character should be used for the reaction; compound
+    characters are ok.
+    """
+    # For details on Matrix reactions see MSC2677:
+    # - https://github.com/uhoreg/matrix-doc/blob/aggregations-reactions/proposals/2677-reactions.md
+    # - and https://github.com/matrix-org/matrix-appservice-slack/pull/522/commits/88e7076a595509b196f53369102a469cace6cc19
+    vs16 = "\ufe0f"  # see https://en.wikipedia.org/wiki/Variation_Selectors_%28Unicode_block%29
+    if not emoji.endswith(vs16):
+        if len(emoji) != 1:
+            log.warning("Reactions should only use a single emoji, got: %s", emoji)
+        else:
+            emoji = unicodedata.normalize("NFC", emoji + vs16)
+    return await message.app.client.room_send(
+        room_id=message.room.room_id,
+        message_type="m.reaction",
+        content={
+            "m.relates_to": {
+                "rel_type": "m.annotation",
+                "event_id": message.event.event_id,
+                "key": emoji,
+            }
+        },
+        ignore_unverified_devices=True,
+    )
+
+
+async def redact(message, reason: Optional[str] = None) -> nio.RoomRedactResponse:
+    """Redact a message.
+
+    This allows not only to redact text messages, but also reactions, i.e.
+    take back a reaction.
+    """
+    resp = await message.app.client.room_redact(
+        room_id=message.room.room_id, event_id=message.event.event_id, reason=reason
+    )
+    if isinstance(resp, nio.RoomRedactError):
+        raise RuntimeError(
+            f"Could not redact: {message.event.event_id}: {resp.message}"
+        )
+    return resp
+
+
+async def send_message(
+    client: nio.AsyncClient,
+    room_id: str,
+    plain: str = "",
+    *,
+    html: Optional[str] = None,
+    as_notice: bool = True,
+    reply_to_event_id: Optional[str] = None,
+) -> nio.RoomSendResponse:
+    """Send text to a matrix room.
+
+    The message can be given as either plain text or HTML.
+    If the plain text variant is not explicitly given, it will be
+    generated from the rich text variant.
+    You should always send the message as notice.  Per convention
+    bots don't react to notices, so sending only notices will avoid
+    infinite loops between multiple bots in a channel.
+    """
+    assert plain or html
+
+    content = {
+        "msgtype": "m.notice" if as_notice else "m.text",
+        "body": plain,
+    }
+
+    if html:
+        content["format"] = "org.matrix.custom.html"
+        content["formatted_body"] = html
+        if not plain:
+            content["body"] = strip_tags(html)
+
+    if reply_to_event_id:
+        content["m.relates_to"] = {"m.in_reply_to": {"event_id": reply_to_event_id}}
+
+    try:
+        return await client.room_send(
+            room_id,
+            "m.room.message",
+            content,
+            ignore_unverified_devices=True,
+        )
+    except nio.SendRetryError:
+        log.exception(f"Unable to send message to room: {room_id}")
+
+
+@contextmanager
+def localized(lc: str, category=locale.LC_ALL):
+    locale.setlocale(category, lc)
+    try:
+        yield
+    finally:
+        locale.resetlocale(category)
+
+
+def localizedtz(dt: datetime, fmt: str, lc: str, tzname: str):
+    tz = tzdb[tzname]
+    with localized(lc, locale.LC_TIME):
+        return dt.astimezone(tz).strftime(fmt)
+
+
+class TextonlyParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.__text = StringIO()
+
+    def handle_data(self, d):
+        self.__text.write(d)
+
+    @property
+    def text(self):
+        return self.__text.getvalue()
+
+
+def strip_tags(html):
+    s = TextonlyParser()
+    s.feed(html)
+    return s.text
+
+
+def clamp(lower, x, upper):
+    return max(lower, min(x, upper))

hotdog/models.py

@@ -0,0 +1,131 @@
+import asyncio
+from dataclasses import dataclass, field
+from random import random
+from time import time as now
+from typing import *
+
+import nio
+
+JobCallback = Callable[["Job"], None]
+
+
+@dataclass
+class Job:
+    app: "Bot"
+    title: str
+    func: JobCallback
+    every: Optional[float] = None
+    jitter: float = 0.0
+    next: Optional[float] = None
+    task: Optional[asyncio.Task] = None
+
+    def _next(self) -> Optional[float]:
+        if self.every is None:
+            return None
+        return now() + self.every + self._jitter(self.jitter)
+
+    @staticmethod
+    def _jitter(x: float) -> float:
+        return 2 * x * random() - x
+
+
+class Tokens(tuple):
+    def get(self, pos, default=None):
+        try:
+            return self[pos]
+        except:
+            return default
+
+    def int(self, pos, default=0):
+        try:
+            return abs(int(self[pos]))
+        except:
+            return default
+
+    def str(self, pos, default=""):
+        try:
+            return str(self[pos])
+        except:
+            return default
+
+    def startswith(self, args: Union[str, List[str]]):
+        if isinstance(args, str):
+            args = Tokens.from_str(args)
+        return self[: len(args)] == args
+
+    @classmethod
+    def from_str(cls, s):
+        return cls(t for t in s.split() if t)
+
+    def __getitem__(self, index):
+        result = super().__getitem__(index)
+        return Tokens(result) if isinstance(index, slice) else result
+
+
+@dataclass
+class Message:
+    app: "Bot"
+    text: str
+    room: nio.rooms.MatrixRoom
+    event: nio.events.room_events.RoomMessageText
+    tokens: Tokens = None
+    words: Tokens = None
+    is_for_me: bool = False
+    command: Optional[str] = None
+    args: Optional[Tokens] = None
+
+    @property
+    def sender_name(self) -> str:
+        return self.room.user_name(self.event.sender)
+
+    @property
+    def my_name(self) -> str:
+        return self.room.user_name(self.app.client.user)
+
+    @property
+    def is_direct_chat(self):
+        # "A group is an unnamed room with no canonical alias."
+        return self.room.is_group and self.room.member_count == 2
+
+    def __post_init__(self):
+        self.tokens = Tokens.from_str(self.text)
+        self.words = self.tokens
+
+        """
+        .roll d20
+        hotdog: roll d20
+        hotdog, roll d20
+        @hotdog roll d20
+        @hotdog : roll d20
+        """
+
+        first_arg = self.tokens.str(0)
+        if self.text.startswith(self.app.config.command_prefix):
+            self.command = first_arg[len(self.app.config.command_prefix) :]
+            self.args = self.tokens[1:]
+            self.words = self.args
+            return
+
+        me = self.my_name
+        if me and me.lower() == first_arg.lstrip("@").rstrip(":,!").lower():
+            self.is_for_me = True
+            if self.tokens.str(1) in ":,!":
+                args = self.tokens[2:]
+            else:
+                args = self.tokens[1:]
+            self.words = args
+            self.command, self.args = args.get(0), args[1:]
+
+
+@dataclass
+class Reaction(nio.Event):
+    related_event_id: str = field()
+    key: str = field()
+
+    @classmethod
+    def from_dict(cls, event_dict):
+        return cls(
+            event_dict,
+            event_dict["content"]["m.relates_to"]["event_id"],
+            event_dict["content"]["m.relates_to"]["key"],
+        )

hotdog/tz.py

@@ -0,0 +1,51 @@
+import calendar
+from datetime import datetime, timedelta, timezone
+from typing import Optional
+
+
+def _last_of_month(calendar_day: int, year: int, month: int) -> int:
+    last = None
+    for date, day in calendar.Calendar().itermonthdays2(year, month):
+        if day == calendar_day and date != 0:
+            last = date
+    if last is None:
+        raise RuntimeError(
+            "No such calendar day found: {calendar_day}".format(
+                calendar_day=calendar_day
+            )
+        )
+    return last
+
+
+_cest = timezone(timedelta(hours=2), "CEST")
+_cet = timezone(timedelta(hours=1), "CET")
+
+
+def cest(date: Optional[datetime] = None) -> timezone:
+    """
+    Return the timezone for Central European (Summer) Time.
+
+    Automatically selects CET/CEST based on the given date.
+
+    Since 1996, European Summer Time has been observed from the last Sunday
+    in March to the last Sunday in October.
+    """
+    if date is None:
+        date = datetime.now()
+    if date.year < 1996:
+        raise NotImplementedError()
+    march = 3
+    october = 10
+    if march < date.month < october:
+        return _cest
+    if march > date.month or date.month > october:
+        return _cet
+    past_break = (
+        date.day >= _last_of_month(calendar.SUNDAY, date.year, date.month)
+        and date.hour > 2
+    )
+    if date.month == march:
+        return _cest if past_break else _cet
+    if date.month == october:
+        return _cet if past_break else _cest
+    raise RuntimeError("Month is out of bounds: {date.month}".format(date=date))

postillon/__init__.py

@@ -0,0 +1,4 @@
+from feeder import Feed, Post, all_posts
+
+from .postbox import FEED_URL, split_post
+from .store import Store

postillon/__main__.py

@@ -0,0 +1,51 @@
+import argparse
+import asyncio
+import logging
+import os
+from typing import *
+
+import postillon
+
+log = logging.getLogger(__name__)
+
+logging.basicConfig(
+    format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
+    level=os.getenv("LOGLEVEL", "INFO"),
+)
+
+
+async def all_posts(feed_url, throttle: int = 10) -> AsyncIterable[postillon.Post]:
+    """We can't use feed's all_posts because blogger creates broken next URLs."""
+    feed = postillon.Feed(feed_url, url="", next_url=feed_url)
+    while (feed := feed.load_next()) :
+        log.debug(f"New feed page: {feed}")
+        if feed.next_url:
+            feed.next_url = feed.next_url.replace(
+                f"{postillon.FEED_URL}/-/Newsticker", postillon.FEED_URL
+            )
+        for post in feed.posts:
+            yield post
+        log.debug(f"Waiting for {throttle} seconds ...")
+        await asyncio.sleep(throttle)
+
+
+async def dump_all(dbpath: str, feed_url: str):
+    store = postillon.Store(dbpath)
+    store.connect()
+    async for post in all_posts(feed_url):
+        store.add(postillon.split_post(post))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--database", "--db", "-d", required=True, help="Path to database.sqlite"
+    )
+    parser.add_argument("--feed", "-f", default=postillon.FEED_URL, help="Feed URL")
+    args = parser.parse_args()
+
+    asyncio.run(dump_all(args.database, args.feed))
+
+
+if __name__ == "__main__":
+    main()

postillon/postbox.py

@@ -0,0 +1,44 @@
+import re
+from dataclasses import replace
+from html.parser import HTMLParser
+from io import StringIO
+from typing import *
+
+from . import Post
+
+FEED_URL = "https://www.blogger.com/feeds/746298260979647434/posts/default/-/Newsticker"
+
+
+class TextonlyParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.reset()
+        self.strict = False
+        self.convert_charrefs = True
+        self._text = StringIO()
+
+    def handle_data(self, d):
+        self._text.write(d)
+
+    @property
+    def text(self):
+        return self._text.getvalue()
+
+
+def strip_tags(html):
+    s = TextonlyParser()
+    s.feed(html)
+    return s.text
+
+
+find_tags = re.compile(r"\+\+\+ (.*?) \+\+\+").finditer
+
+
+def feed_page(page: int = 1, per_page: int = 25) -> str:
+    start = 1 + (page - 1) * per_page
+    return f"{FEED_URL}?start-index={start}&max-results={per_page}"
+
+
+def split_post(post: Post) -> Iterable[Post]:
+    for match in find_tags(strip_tags(post.content)):
+        yield replace(post, content=match[1])

postillon/store.py

@@ -0,0 +1,78 @@
+import sqlite3
+from datetime import datetime, timezone
+from typing import *
+
+from . import Post
+
+
+class Store:
+    def __init__(self, dbpath: Optional[str] = None):
+        self.dbpath = dbpath
+        self.connection: Optional[sqlite3.Connection] = None
+
+    def connect(self, path: Optional[str] = None):
+        if path:
+            self.dbpath = path
+        if self.connection is not None:
+            return self.connection
+        self.connection = sqlite3.connect(
+            self.dbpath, isolation_level=None
+        )  # auto commit
+        self.init()
+
+    def disconnect(self):
+        conn = self.connection
+        if conn:
+            conn.close()
+
+    def init(self):
+        conn = self.connection
+        conn.execute(
+            """
+            create table if not exists post (
+                id integer primary key,
+                content text unique not null,
+                source text, -- link to the source of this post
+                date integer not null
+            )
+            """
+        )
+        conn.execute(
+            """
+            create index if not exists post_date
+            on post(date)
+            """
+        )
+
+    def add(self, posts: Iterable[Post]):
+        sql = f"""
+            insert into post(content, source, date)
+            values (?, ?, ?)
+            on conflict do nothing
+        """
+        self.connection.executemany(
+            sql,
+            (
+                (p.content, p.link, int(p.date.timestamp()) if p.date else None)
+                for p in posts
+            ),
+        )
+
+    def _select(self, condition="", params=[]) -> Iterable[Post]:
+        sql = f"select id, content, date, source from post {condition}"
+        for row in self.connection.execute(sql, params):
+            id, content, date, source = row
+            if date is not None:
+                date = datetime.fromtimestamp(date, tz=timezone.utc)
+            post = Post(id, content, date, link=source)
+            yield post
+
+    def random_post(self) -> Optional[Post]:
+        cond = "where id in (select id from post order by random() limit 1)"
+        for post in self._select(cond):
+            return post
+
+    def search(self, term, skip: int = 0) -> Iterable[Post]:
+        cond = "where content like ? order by date desc limit -1 offset ?"
+        for post in self._select(cond, (term, skip)):
+            yield post

requirements.txt

@@ -0,0 +1 @@
+docker/requirements.txt

run

@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+export RUN_BIN="$0"
+export RUN_DIR=$(dirname "$(realpath "$0")")
+
+task=
+if [ -e "$RUN_DIR/scripts/$1" ]; then
+    task="$1"
+elif [ -e "$RUN_DIR/scripts/local/$1" ]; then
+    task="local/$1"
+fi
+
+[ -z "$task" ] && exit 1
+
+shift
+exec "$RUN_DIR/scripts/$task" "$@"

scripts/app

@@ -0,0 +1,22 @@
+#!/bin/sh -eu
+
+cd "$RUN_DIR"
+
+image=$(cat docker/.dockerimage)
+
+if [ -z "$image" ]; then
+    echo >&2 'image name not found.'
+    exit 2
+fi
+
+set -x
+
+"$RUN_BIN" build
+
+docker run \
+    -it --rm \
+    -v "$PWD"/data:/data:rw \
+    -v "$PWD"/hotdog:/var/hotdog:ro \
+    -v "$PWD"/postillon:/var/postillon:ro \
+    -v "$PWD"/feeder:/var/feeder:ro \
+    "$image":latest "$@"

scripts/build

@@ -0,0 +1,15 @@
+#!/bin/sh -eu
+
+cd "$RUN_DIR"
+
+contextdir="$RUN_DIR"/docker
+image=$(cat "$contextdir"/.dockerimage)
+
+if [ -z "$image" ]; then
+    echo >&2 'image name not found.'
+    exit 2
+fi
+
+set -x
+
+docker build "$@" --tag "$image":latest "$contextdir"

scripts/lint

@@ -0,0 +1,11 @@
+#!/bin/sh -e
+
+if [ "$1" = '--fix' ]; then
+  set -x
+  black .
+  isort --profile black .
+else
+  set -x
+  black --check .
+  isort --profile black --check .
+fi