ducklet/hotdog
1
0
Fork 0
Code Issues 1 Pull requests Releases Activity

dump current state (wip-ish)

Browse source
This commit is contained in:
ducklet 2020-11-01 16:31:37 +01:00
parent 0124c35472
commit 51fb1c9f26
46 changed files with 3749 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

0
hotdog/__init__.py Normal file
View file

31
hotdog/__main__.py Normal file
View file

@ -0,0 +1,31 @@
import argparse
import asyncio
import logging
from .bot import Bot
from .config import Config
log = logging.getLogger(__name__)
async def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True, help="Path to config.yaml")
args = parser.parse_args()
config = Config(args.config)
logging.basicConfig(
format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
level=config.loglevel,
)
logging.getLogger("peewee").setLevel("INFO") # XXX too spamy
bot = Bot(config)
await bot.run()
try:
asyncio.run(main())
except KeyboardInterrupt:
log.info("Shutdown by user.")

299
hotdog/bot.py Normal file
View file

@ -0,0 +1,299 @@
import asyncio
import logging
from datetime import datetime
from importlib import import_module
from pathlib import Path
from time import time as now
from types import ModuleType as Plugin
from typing import *
from aiohttp import ClientConnectionError, ServerDisconnectedError
from nio import AsyncClient, AsyncClientConfig, Event, InviteMemberEvent, JoinError
from nio import LoginError as LoginErrorResponse
from nio import MatrixRoom, RoomMessageText, UnknownEvent
from .models import Job, JobCallback, Message, Reaction
log = logging.getLogger(__name__)
class LoginError(RuntimeError):
pass
MessageHandler = Callable[[Message], Awaitable]
# XXX add the facility to store/buffer responses, e.g. if a timer triggers
# a new message that should be sent to a room but the bot is disconnected
# at the moment, the bot should just buffer it and send it when it's back
# online.
# XXX have a mechanism that first collects all responders and then calls them.
# this would allow to let responders know about each other and interact if
# necessary, e.g. urlinfo sees that youtube-info is already handling it.
class Bot:
def __init__(self, config):
self.client: AsyncClient = None
self.config = config
self.plugins = {}
self.message_handlers: List[MessageHandler] = []
self.timers: List[Job] = []
self.shared: Dict[str, Any] = {} # Shared memory for the plug-ins.
def on_message(self, callback: MessageHandler):
self.message_handlers.append(callback)
def on_command(self, command: Union[str, Container[str]], callback: MessageHandler):
commands = command = {command} if type(command) is str else command
async def guard(message):
if message.command not in commands:
return
await callback(message)
guard.__qualname__ = f"{callback.__module__}.{callback.__qualname__}"
self.message_handlers.append(guard)
def add_timer(
self,
*,
title: str,
callback: JobCallback,
every: Optional[float] = None,
next_at: Optional[datetime] = None,
next_in: Optional[float] = None,
jitter: float = 0,
):
# We require "every", or "next_at" (x)or "next_in".
assert every or next_at or next_in
assert not (next_at and next_in)
job = Job(
app=self,
title=title,
every=every,
func=callback,
jitter=jitter,
)
job.next = (
now() + next_in
if next_in
else next_at.timestamp()
if next_at
else job._next()
)
self.timers.append(job)
def _init(self):
command_plugins = load_plugins("command")
self.client = AsyncClient(
self.config.homeserver_url,
self.config.user_id,
device_id=self.config.device_id,
store_path=self.config.store_path.as_posix(),
config=AsyncClientConfig(
max_limit_exceeded=0,
max_timeouts=0,
store_sync_tokens=True,
encryption_enabled=True,
),
)
add_event_callback = self.client.add_event_callback
add_event_callback(self._on_any_event, Event)
add_event_callback(self._on_message, RoomMessageText)
# add_event_callback(self._on_invite, InviteMemberEvent) # XXX make join-on-invite configurable
add_event_callback(self._on_unknown, UnknownEvent)
for name, mod in command_plugins.items():
log.debug(f"Initializing plugin: {name}")
try:
mod.init(self)
except Exception as err:
log.exception("Initialization failed: %s", name, exc_info=err)
else:
self.plugins[name] = mod
async def _on_unknown(self, room: MatrixRoom, event: UnknownEvent):
# See if we can transform an Unknown event into something we DO know.
if event.type == "m.reaction":
await self._on_reaction(room, Reaction.from_dict(event.source))
async def _on_reaction(self, room: MatrixRoom, event: Reaction):
# XXX allow clients to register handlers for these events
pass
async def _on_any_event(self, *args):
log.debug("New event: %s", repr(args))
async def _on_message(self, room: MatrixRoom, event: RoomMessageText):
if (self.config.is_dev and room.room_id != self.config.dev_room) or (
not self.config.is_dev and room.room_id == self.config.dev_room
):
return
is_own_message = event.sender == self.client.user
if is_own_message:
return
log.info(f"#{room.display_name} <{room.user_name(event.sender)}> {event.body}")
msg = Message(self, event.body, room, event)
tasks = {}
for h in self.message_handlers:
try:
coro = h(msg)
except Exception as err:
log.exception("Error calling message handler: %s", h, exc_info=err)
else:
tasks[h] = asyncio.create_task(coro)
timeout = 10
fut = asyncio.gather(*tasks.values(), return_exceptions=True)
try:
await asyncio.wait_for(fut, timeout)
except asyncio.TimeoutError as err:
await swallow(fut)
for h, t in tasks.items():
assert t.done()
try:
err = t.exception()
except asyncio.CancelledError:
log.error("Message handler took too long to finished: %s", h)
if err is not None:
log.exception("Error in message handler: %s", h, exc_info=err)
async def _on_invite(self, room: MatrixRoom, event: InviteMemberEvent):
if self.config.is_dev:
return
log.debug(f"Received invite from user: {event.sender}: {room.room_id}")
res = await self.client.join(room.room_id)
if type(res) == JoinError:
log.error(f"Could not join room: {room.room_id}: {res.message}")
else:
log.info(f"Joined room: {room.room_id}")
async def _login(self):
# if self.config.access_token:
# self.client.restore_login(
# self.config.user_id, self.config.device_id, self.config.access_token
# )
# if self.config.password:
# else:
# resp = await self.client.login(token=self.config.access_token)
resp = await self.client.login(
password=self.config.password, device_name=self.config.device_name
)
if isinstance(resp, LoginErrorResponse):
raise LoginError(f"Could not log in: {resp.message}")
if not self.config.device_id:
self.config.device_id_path.write_text(self.client.device_id)
await self.client.set_displayname(self.config.display_name)
await self.client.update_device(
self.client.device_id, {"display_name": self.config.device_name}
)
async def _run_timers(self) -> NoReturn:
client = self.client
await asyncio.sleep(2)
while True:
await asyncio.sleep(0.2)
# if not client.logged_in:
# continue
for job in self.timers:
if job.next is not None and job.next <= now():
job.next = None
try:
coro = job.func(job)
except Exception as err:
log.exception(
"Disabled job with error: %s", job.title, exc_info=err
)
job.task = None
else:
job.task = asyncio.create_task(coro)
if job.task is not None:
task = job.task
if task.done():
job.task = None
try:
await task
except Exception as err:
log.exception(
"Disabled job with error: %s", job.title, exc_info=err
)
else:
job.next = job._next()
log.debug(f"Task scheduled: {job.title}: {job.next}")
async def _run_client(self) -> NoReturn:
client = self.client
while True:
try:
if not client.logged_in:
await self._login()
log.info(f"Logged in as {client.user}")
await client.sync_forever(timeout=30_000, full_state=True)
log.warning("Shutting down.")
return
except LoginError as err:
log.error(f"Could not log in: {err}")
return
except asyncio.exceptions.TimeoutError:
log.warning("Connection timed out.")
except ClientConnectionError as err:
log.warning(f"Could not connect to server: {err}")
except ServerDisconnectedError as err:
log.exception("Disconnected from server.", exc_info=err)
finally:
await client.close()
client.access_token = ""
waitfor = 15
log.info(f"Retrying in {waitfor}s...")
await asyncio.sleep(waitfor)
async def run(self) -> NoReturn:
self._init()
await asyncio.gather(
self._run_timers(),
self._run_client(),
)
def load_plugins(pkg: str) -> Mapping[str, Plugin]:
modules = {}
for modpath in (Path(__file__).parent / pkg).glob(f"*.py"):
modname = modpath.with_suffix("").name
try:
mod = import_module(f".{pkg}.{modname}", __package__)
except Exception as err:
log.exception("Error loading plugin: %s", modpath, exc_info=err)
else:
if not hasattr(mod, "init"):
log.error(f"Invalid plugin: {modpath}")
else:
log.debug(f"Loaded plugin: {modpath}")
modules[modname] = mod
return modules
async def swallow(future: asyncio.Future):
"""Get rid of a Future."""
future.cancel()
try:
await future # Should always immediately raise.
except asyncio.CancelledError:
pass

20
hotdog/command/README.md Normal file
View file

@ -0,0 +1,20 @@
A plugin is any module that defines an `init` function that takes a `Bot`
instance as first argument.
The function can then register any message handlers or timers on the bot,
or add any dependencies it requires.
Example:
```py
from ..functions import reply
HELP = """Responds to your hello.
!hello
"""
def init(bot):
bot.on_command("hello", reply_world)
async def reply_world(message):
await reply(message, "world")
```

31
hotdog/command/aoderb.py Normal file
View file

@ -0,0 +1,31 @@
import random
import re
from ..functions import reply
from ..models import Message
HELP = """Entscheidet zwischen A und B.
@me: <A> oder <B>?
"""
def init(bot):
bot.on_message(handle)
async def handle(message: Message):
if not (
message.text.endswith("?") and "oder" in message.tokens and message.is_for_me
):
return
_, text = message.text.split(None, 1)
args = text[:-1].split(" oder ")
if ":" in args[0]:
args[0] = args[0].split(":", 1)[1]
elif "-" in args[0]:
args[0] = args[0].split("-", 1)[1]
choice = random.choice(args).strip(" ,.:")
if not choice:
return
await reply(message, plain=choice, with_name=True)

370
hotdog/command/covid.py Normal file
View file

@ -0,0 +1,370 @@
import logging
import re
import sqlite3
from dataclasses import dataclass
from datetime import datetime, timezone
from html import escape
from typing import *
import requests
from ..functions import localizedtz, react, reply
from ..models import Job, Message
from ..tz import cest
log = logging.getLogger(__name__)
def init(bot):
if "covid.store" not in bot.shared:
bot.shared["covid.store"] = Store(bot.config.get("covid.storage"))
bot.shared["covid.store"].connect()
bot.on_message(handle)
one_minute = 60
one_hour = 3600
bot.add_timer(
title="update covid store",
every=6 * one_hour,
callback=update_store,
jitter=30 * one_minute,
)
# https://npgeo-corona-npgeo-de.hub.arcgis.com/datasets/917fc37a709542548cc3be077a786c17_0
parse_last_update = re.compile(
r"(?P<day>\d{1,2})\.(?P<month>\d{1,2})\.(?P<year>\d{4}), (?P<hour>\d{2}):(?P<minute>\d{2}) Uhr"
).fullmatch
api_url = "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_Landkreisdaten/FeatureServer/0/query"
class Store:
def __init__(self, dbpath: Optional[str] = None):
self.dbpath = dbpath
self.connection: Optional[sqlite3.Connection] = None
def connect(self, path: Optional[str] = None) -> None:
if path:
self.dbpath = path
if self.connection is not None:
return self.connection
log.debug("Connecting to %s", self.dbpath)
conn = self.connection = sqlite3.connect(
self.dbpath, isolation_level=None
) # auto commit
conn.row_factory = sqlite3.Row # Enable access row data by column name.
self.init()
def init(self) -> None:
conn = self.connection
sql = """
create table if not exists county ( -- Landkreis
id integer primary key not null,
state_id integer not null references state(id),
name text unique not null
);;
create table if not exists state ( -- Bundesland
id integer primary key not null,
name text unique not null
);;
create table if not exists county_probe (
county_id integer not null references county(id),
ts integer not null,
cases integer not null,
cases7_per_100k real not null,
deaths integer not null,
population integer not null
);;
create unique index if not exists
county_probe_index
on county_probe(county_id, ts);;
create table if not exists state_probe (
state_id integer not null references state(id),
ts integer not null,
cases7_per_100k real not null,
population integer not null
);;
create unique index if not exists
state_probe_index
on state_probe(state_id, ts);;
drop view if exists current;;
create view if not exists
current
as select
county.name as county_name,
county_probe.cases as county_cases,
county_probe.cases7_per_100k as county_cases7_per_100k,
county_probe.deaths as county_deaths,
county_probe.population as county_population,
county_probe.ts as ts,
state.name as state_name,
state_probe.cases7_per_100k as state_cases7_per_100k,
state_probe.population as state_population
from county_probe
inner join county on county.id=county_probe.county_id
inner join state on state.id=county.state_id
inner join state_probe on state_probe.state_id
where
county_probe.ts=state_probe.ts
and state_probe.state_id=county.state_id
;;
create trigger if not exists
current_insert
instead of insert
on current
begin
insert into
state (name)
values (new.state_name)
on conflict do nothing;
insert into
county (name, state_id)
values (
new.county_name,
(select id from state where name=new.state_name)
)
on conflict do nothing;
insert into
state_probe (state_id, ts, cases7_per_100k, population)
values (
(select id from state where name=new.state_name),
new.ts,
new.state_cases7_per_100k,
new.state_population
)
on conflict do nothing;
insert into
county_probe (county_id, ts, cases, cases7_per_100k, deaths, population)
values (
(select id from county where name=new.county_name),
new.ts,
new.county_cases,
new.county_cases7_per_100k,
new.county_deaths,
new.county_population
)
on conflict do nothing;
end;;
"""
for s in sql.split(";;"):
s = s.strip()
if s:
conn.execute(s)
def add(self, probes: Iterable["Probe"]):
rows = iter(p.as_row() for p in probes)
for first_row in rows:
break
else:
return
sql = f"""
insert into current({",".join(first_row.keys())})
values({",".join(["?"] * len(first_row))})
"""
self.connection.execute(sql, tuple(first_row.values()))
self.connection.executemany(sql, (tuple(r.values()) for r in rows))
def _select(self, condition="", params=[]) -> Iterable["Probe"]:
sql = f"select * from current {condition}"
for row in self.connection.execute(sql, params):
yield Probe.from_row(row)
def find_one(self, term) -> Optional["Probe"]:
cond = """
where county_name like ?
or state_name like ?
order by ts desc
limit 1
"""
for probe in self._select(cond, (term, term)):
return probe
@dataclass
class Probe:
# County fields
cases: int
deaths: int
county_name: str
ts: datetime
cases7_per_100k: float
# recovered: str
population: str # Einwohnerzahl Landkreis
# State fields:
state_name: str
state_population: int # Einwohnerzahl Bundesland
state_cases7_per_100k: float
@property
def death_rate(self) -> float:
return self.deaths / self.cases * 100
@property
def cases_per_100k(self) -> float:
return self.cases / self.population * 100_000
@property
def cases_per_population(self) -> float:
return self.cases / self.population
_json_fields = {
# County fields
"cases": "cases",
"deaths": "deaths",
"county": "county_name",
"last_update": "ts", # Needs to be converted to a timestamp
"cases7_per_100k": "cases7_per_100k",
# "recovered": "recovered",
"EWZ": "population",
# State fields:
"BL": "state_name",
"EWZ_BL": "state_population",
"cases7_bl_per_100k": "state_cases7_per_100k",
}
@classmethod
def from_api_json(cls, data: Mapping[str, Any]):
match = parse_last_update(data["last_update"]) # "25.10.2020, 00:00 Uhr"
if not match:
raise ValueError(f"Could not parse last_updated: {data['last_update']}")
dt = datetime(
int(match["year"]),
int(match["month"]),
int(match["day"]),
int(match["hour"]),
int(match["minute"]),
)
dt = dt.replace(tzinfo=cest(dt)) # The timezone is from the entry date.
d = {ck: data[jk] for jk, ck in cls._json_fields.items()}
d["ts"] = dt
return cls(**d)
@classmethod
def from_row(cls, data):
return cls(
county_name=data["county_name"],
cases=data["county_cases"],
cases7_per_100k=data["county_cases7_per_100k"],
deaths=data["county_deaths"],
population=data["county_population"],
ts=datetime.fromtimestamp(data["ts"], tz=timezone.utc),
state_name=data["state_name"],
state_cases7_per_100k=data["state_cases7_per_100k"],
state_population=data["state_population"],
)
def as_row(self):
return {
"county_name": self.county_name,
"county_cases": self.cases,
"county_cases7_per_100k": self.cases7_per_100k,
"county_deaths": self.deaths,
"county_population": self.population,
"ts": int(self.ts.timestamp()),
"state_name": self.state_name,
"state_cases7_per_100k": self.state_cases7_per_100k,
"state_population": self.state_population,
}
def api_params(fields="*"):
return {
"where": "9999=9999",
"outFields": ",".join(fields),
"f": "json",
"returnGeometry": "false",
}
def load_data() -> Iterable[Probe]:
# import json
# with open("/data/covid.resp") as fp:
# data = json.load(fp)
log.debug("Loading data from API.")
r = requests.get(
api_url,
params=api_params(Probe._json_fields),
timeout=(10, 10),
headers={"user-agent": "hotdog/v1 covid"},
)
r.raise_for_status()
data = r.json()
log.debug(f'Found {len(data["features"])} entries.')
for row in data["features"]:
yield Probe.from_api_json(row["attributes"])
async def update_store(job: Job):
store: Store = job.app.shared["covid.store"]
store.add(load_data())
def pfloat(n):
return f"{n:_.02f}".replace(".", ",")
def as_html(probe, tzname: str, lc: str) -> str:
# now = datetime.now(tz=timezone.utc)
# since = now - probe.ts
# fmt = "%A" if since.days < 7 else "%x"
date = localizedtz(probe.ts, "%A, %x", tzname=tzname, lc=lc)
return (
f"<i>Zahlen von {date}</i>: "
+ ", ".join(
[
f"🌍 {escape(probe.county_name)}",
f"😷 {probe.population:_}",
f"🦠 {probe.cases:_}",
f"☠️ {probe.deaths:_}",
f"📈 {pfloat(probe.cases7_per_100k)} (<i>7-Tage-Inzidenz</i>)",
]
)
+ " — "
+ ", ".join(
[
f"🌍 {escape(probe.state_name)}",
f"😷 {probe.state_population:_}",
f"📈 {pfloat(probe.state_cases7_per_100k)}",
]
)
)
async def handle(message: Message):
if message.command not in ("cov", "covid") or not message.args:
return
# if message.args.get(0) == "!force-reload":
# await react(message, "⚡️")
# await update_store(Job(message.app, "", update_store))
# await react(message, "👍")
# return
store: Store = message.app.shared["covid.store"]
term = "%".join(message.args)
if (probe := store.find_one(f"%{term}%")) :
roomconf = message.app.config.l6n[message.room.room_id]
await reply(
message,
html=as_html(probe, tzname=roomconf["timezone"], lc=roomconf["locale"]),
)
else:
await reply(message, "No such county or state.", in_thread=True)

102
hotdog/command/ddf.py Normal file
View file

@ -0,0 +1,102 @@
import re
from dataclasses import dataclass
from html import escape
from random import choice
from typing import *
from ..functions import reply
from ..models import Message
HELP = """Die drei ??? Folgenindex
!ddf [episode #|title]
"""
def init(bot):
if "ddf.eps" not in bot.shared:
with open(bot.config.get("ddf.storage")) as fp:
bot.shared["ddf.db"] = db = load_db(fp)
bot.shared["ddf.eps"] = tuple(set(db.values()))
bot.on_command({"ddf", "???"}, handle)
def load_db(fp):
db = {}
for ep in Episode.from_csv(fp):
if ep.nr_europa:
db[ep.nr_europa.lower()] = ep
elif ep.nr_kosmos:
db[ep.nr_kosmos.lower()] = ep
if ep.title_de:
db[ep.title_de.lower()] = ep
if ep.title_us:
db[ep.title_us.lower()] = ep
return db
@dataclass(frozen=True)
class Episode:
nr_kosmos: str
nr_europa: str
nr_randho: str
title_us: str
title_de: str
autor: str
year_randho: str
year_kosmos: str
year_europa: str
@classmethod
def from_csv(cls, fp) -> Iterable["Episode"]:
fp = iter(fp)
next(fp) # skip the first line, it contains the header
for line in fp:
line = line.strip()
if not line:
continue
yield cls(*line.split(";"))
async def handle(message: Message):
bot = message.app
args = message.args
ep: Episode = None
if not args:
eps = bot.shared["ddf.eps"]
ep = choice(eps)
else:
arg = args.str(0).lower()
db = bot.shared["ddf.db"]
if arg in db:
ep = db[arg]
else:
for key in db:
if arg in key:
ep = db[key]
break
if not ep:
return
nr = year = src = None
if ep.nr_europa:
src = ""
nr = ep.nr_europa
year = ep.year_europa
elif ep.nr_kosmos:
src = "Buch"
nr = ep.nr_kosmos
year = ep.year_kosmos
if not nr or not year:
return
html = f"(<i>{src}<i>:) " if src else ""
html += f"<b>#{escape(nr)}</b> <i>Die drei ???</i>: <b>{escape(ep.title_de)}</b>"
if ep.title_us:
html += f" (US: <i>{escape(ep.title_us)}</i>)"
html += f" ({escape(year)})"
await reply(message, html=html)

27
hotdog/command/dm.py Normal file
View file

@ -0,0 +1,27 @@
from random import choice
from ..functions import react, reply
from ..models import Message
HELP = """
!dm
"""
def init(bot):
bot.on_command("dm", handle)
answers = (
"deine mutter",
"deine mama",
"deine mudda",
"deine muddi",
"dei muddi",
"du miefst",
"drogeriemarkt",
)
async def handle(message: Message):
await reply(message, choice(answers))

131
hotdog/command/feed.py Normal file
View file

@ -0,0 +1,131 @@
import asyncio
import logging
from datetime import datetime, timezone
from html import escape
import feeder
import postillon
from ..functions import clamp, localizedtz, reply, send_message, strip_tags
from ..models import Job, Message
log = logging.getLogger(__name__)
def init(bot):
bot.on_command("feed", handle)
if "feeder" not in bot.shared:
feeds = (
feeder.Feed(fid, f["url"], title=f["display"])
for fid, f in bot.config.get("feeder.feeds").items()
)
feedstore = feeder.Store(bot.config.get("feeder.storage"))
feedstore.connect()
bot.shared["feeder"] = feeder.Feeder(feedstore, feeds)
if "poststore" not in bot.shared:
bot.shared["poststore"] = postillon.Store(bot.config.get("postillon.storage"))
bot.shared["poststore"].connect()
one_minute = 60
one_hour = 3600
bot.add_timer(
title="update feeds",
every=one_hour,
callback=update_feeds,
jitter=10 * one_minute,
)
async def handle(message: Message):
feed_id = message.args.str(0)
count = clamp(1, message.args.int(1) or 3, 10)
bot = message.app
feeder = bot.shared["feeder"]
if feed_id not in feeder.feeds:
return
posts = feeder.posts(feed_id, [p.id for p in feeder.feeds[feed_id].posts[:count]])
feedconf = bot.config.get("feeder.feeds")[feed_id]
roomconf = bot.config.l6n[message.room.room_id]
for post in posts:
text = post_as_html(
post,
tzname=roomconf["timezone"],
lc=roomconf["locale"],
max_content_len=feedconf.get("max_content_len", 300),
)
await reply(message, html=text)
def handle_postillon(bot, posts):
"""Special handling for Postillon posts to store them in the Postillon DB as well."""
poststore = bot.shared["poststore"]
for post in posts:
poststore.add(postillon.split_post(post))
async def update_feeds(job: Job):
max_posts = 2
bot = job.app
feeder = bot.shared["feeder"]
feeds = bot.config.get("feeder.feeds")
rooms = {fid: f.get("rooms", []) for fid, f in feeds.items()}
news = await feeder.update_all()
sends = []
mores = []
for feed_id, post_ids in news.items():
posts = feeder.posts(feed_id, post_ids)
log.debug(f"new posts: {feed_id}: {len(posts)}")
if feed_id == "post":
handle_postillon(bot, posts)
prefix = f"[feed:{feed_id}]"
for room_id in rooms[feed_id]:
roomconf = bot.config.l6n[room_id]
selected = posts[:max_posts] if len(posts) > max_posts + 1 else posts
for post in selected:
text = post_as_html(
post,
tzname=roomconf["timezone"],
lc=roomconf["locale"],
max_content_len=feeds[feed_id].get("max_content_len", 300),
)
text = f"{prefix} {text}"
sends.append(send_message(bot.client, room_id, html=text))
if len(posts) > len(selected):
more = len(posts) - len(selected)
mores.append([bot.client, room_id, f"{prefix} {more} more"])
await asyncio.gather(*sends)
await asyncio.gather(*(send_message(*more) for more in mores))
def post_as_html(post, tzname: str, lc: str, *, max_content_len: int = 300):
parts = []
if post.date:
now = datetime.now(tz=timezone.utc)
since = now - post.date
fmt = ""
if since.days < 1:
fmt = "%X"
else:
if since.days < 7:
fmt += "%A, "
fmt += "%x %X"
parts.append("(" + localizedtz(post.date, fmt, tzname=tzname, lc=lc) + ")")
if post.title:
parts.append(f'<a href="{escape(post.link)}">{escape(post.title)}</a>')
elif post.link:
parts.append(f'<a href="{escape(post.link)}">{escape(post.link)}</a>')
if post.content and max_content_len > 0:
if parts:
parts.append("—")
content = ""
for word in strip_tags(post.content).split(" "):
if len(content + f" {word}") > max_content_len - 3:
content += " […]"
break
content += f" {word}"
parts.append(escape(content))
return " ".join(parts)

38
hotdog/command/help.py Normal file
View file

@ -0,0 +1,38 @@
import re
from html import escape
from ..functions import reply
from ..models import Message
def init(bot):
bot.on_command({"help", "usage"}, handle)
async def handle(message: Message):
bot = message.app
plugins = {k: p for k, p in bot.plugins.items() if hasattr(p, "HELP")}
pname = message.args.str(0)
if pname not in plugins:
modnames = ", ".join(f"<code>{m}</code>" for m in sorted(plugins.keys()))
await reply(message, html=f"Help is available for: {modnames}")
return
me = f"@{message.my_name}: "
pfix = bot.config.command_prefix
plugin = plugins[pname]
usage: str = plugin.HELP
usage = re.sub(
"^(!|@me: )", lambda m: pfix if m[1] == "!" else me, usage, flags=re.M
)
lines = usage.splitlines(False)
usage = (
f"<i>{escape(lines[0])}</i><br>\n<code>"
+ "<br>\n".join(escape(l) for l in lines[1:])
+ "</code>"
)
await reply(message, html=usage, in_thread=True)

322
hotdog/command/orakel.py Normal file
View file

@ -0,0 +1,322 @@
from random import choice, choices, randint, random
from ..functions import reply
from ..models import Message
HELP = """Gibt Antworten auf Fragen.
@me: <eine Frage>?
"""
ka = (
"Keine Ahnung.",
"Das weiß ich leider nicht.",
"Ich weiß es nicht.",
"Ich weiß es auch nicht.",
"Frag besser jemanden, der sich damit auskennt.",
"Da wendest du dich besser eine Fachkraft.",
"Woher soll ich das wissen?",
"Das weiß niemand so genau ...",
"Das kann dir keiner beantworten ...",
"Eine Frage, so alt wie die Menschheit!",
"Das ist eine gute Frage.",
"Wer weiß ...",
"Boa, keine Ahnung, ey!",
"Seh ich so aus, als ob ich wüsste ich das?",
"Wie kommst du darauf, dass ich das wüsste?",
"Bin ich die Auskunft, oder was?",
"Ich bin doch nicht die Auskunft!",
"Das ist mir doch egal ...",
"Das solltest du besser mit dir selbst ausmachen.",
"Das beantwortest du dir am besten mal selbst.",
"Ich glaube, das weißt du selbst ganz genau.",
"Das weißt du doch selbst ganz genau.",
"Schau tief in dich, dort findest du die Antwort.",
"Du kämpfst wie eine Kuh!",
"Die Antwort ist ... 42.",
"Sechs mal Neun.",
"Das fragst du besser deine Mama.",
)
ja = (
"Ja",
"Ja!",
"Ja.",
"Jap.",
"Jo.",
"Jau!",
"Joa.",
"Joooaaaa.",
"Jooo",
"Klar!",
"Klaaaar.",
"Ja, klar.",
"Klar doch.",
"Aber klar doch.",
"Na sicher.",
"Sicher doch.",
"Joa, warum nicht.",
"Jo, warum denn nicht.",
"Auf jeden Fall!",
"Besser wär's.",
)
nein = (
"Nein",
"Nee",
"Besser nicht.",
"Nein!",
"Niemals!",
"Auf keinen Fall!",
)
vielleicht = (
"Frag mich später noch mal.",
"Das weiß ich nicht genau.",
"Ja, vielleicht.",
"Nein, vielleicht besser nicht.",
)
def random_room_participant(message: Message) -> str:
users = [
u
for uid, u in message.room.users.items()
if uid not in (message.event.sender, message.app.client.user)
]
if not users:
return ""
user = choice(users)
return user.display_name or user.user_id
numnames = [
"null",
"ein",
"zwei",
"drei",
"vier",
"fünf",
"sechs",
"sieben",
"acht",
"neun",
"zehn",
"elf",
"zwölf",
]
praepos = (
"vor",
"hinter",
"über",
"unter",
"auf",
"neben", # careful, this cannot be contracted with dativ
)
wwords = {
"wann": (
# Time
lambda m: f"{choice(['Um ', ''])}{randint(0, 23)} Uhr",
lambda m: f"{choice(['Um ', ''])}{randint(0, 23)}:{randint(1, 59):02} Uhr",
lambda m: f"{choice(['Um ', ''])}{randint(0, 23)} Uhr und {randint(1, 59)} Minuten",
lambda m: f"{choice(['Um ', ''])}{randint(1, 59)} Minuten {choice(['vor','nach'])} {randint(0, 12)}",
lambda m: f"{choice(['Um ', ''])}Viertel {choice(['vor','nach'])} {randint(0, 12)}",
lambda m: f"Um halb {randint(1, 12)}",
"Um Mitternacht",
lambda m: f"Am {choice(['Nach', 'Vor'])}mittag",
# Date
"Morgen",
"Übermorgen",
"Gestern",
"Vorgestern",
lambda m: f"{choice(['Nächst', 'Vorig'])}e Woche",
lambda m: f"{choice(['Vor', 'In'])} {randint(2, 12)} {choice(['Wochen', 'Monaten'])}",
lambda m: f"{choice(['Vor', 'In'])} {randint(2, 9999)} Jahren",
lambda m: f"{choice(['Vor', 'In'])} {randint(2, 999)} Millionen Jahren",
lambda m: f"Am {randint(1, 28)}.{randint(1, 12)}.{randint(9, 9999)}",
# Date-time
lambda m: f"{choice(['Vor', 'In'])} {randint(2, 17)} Jahren, {randint(2,360)} Tagen, {randint(2,23)} Stunden, {randint(2,59)} Minuten und {randint(2,59)} Sekunden",
lambda m: f"Am {randint(1, 28)}.{randint(1, 12)}.{randint(9, 9999)} um {randint(0, 23)}:{randint(1, 59):02} Uhr",
),
"warum": (),
"was": (),
"wer": (
"Deine Mama!",
"Die Polizei.",
"Die drei Fragezeichen!",
"TKKG.",
"Tim und Struppi.",
"Die fünf Freunde.",
"Der Osterhase.",
"Die Ferienbande! Yaaaaay.",
"Der Weihnachtsmann.",
"Frau Holle.",
"Gott!",
"Irgendein Gott.",
"Du!",
"Du selbst.",
"Ein Höhlentroll mit einem dicken Prügel.",
"Eine höhere Macht.",
"Ein unbekanntes Wesen.",
"Ein unbekanntes Wesen aus einer unbekannten Galaxie.",
"Aliens!",
"Graf Zahl.",
"Die Russen!",
"Die USA!",
"Die Reichsbürger!",
"Die Regierung.",
"Die Presse.",
"Die Illuminaten!",
"Die internationale Verschwörung e.V.!",
"Dein Nachbar.",
"Die Kanzlerin.",
"Der Präsident.",
"Der Vorsitzende vom Schützenverein.",
"Der Postbote.",
"Die Behörden.",
"Die Wissenschaft.",
"Die sechs von der Müllabfuhr.",
"Der, der grade hinter dir steht und dir zuguckt.",
"Die, die sich hinter deinem Schrank versteckt hat.",
"Der Typ der in der Ecke steht.",
"Die Anderen; immer die Anderen.",
lambda m: random_room_participant(m) or "Irgendwer.",
),
"wie": (
"Das weiß ich leider auch nicht, vielleicht mit einer Telefonlawine?",
"Am besten gar nicht.",
"Das ist schwierig zu sagen.",
*ka,
),
"wieso": (),
"wieviel": (
"Nichts",
lambda m: choice(["Ungefähr ", "Genau ", ""]) + choice(numnames[2:]),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + f"{23 * random():n}",
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(1, 10)),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(7, 23)),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(17, 42)),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(23, 99)),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(100, 1_000_000)),
"Unendlich!",
),
"wieviele": (
lambda m: choice(["Ungefähr ", "Genau ", ""]) + choice(numnames[2:]),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + f"{23 * random():n}",
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(1, 10)),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(7, 23)),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(17, 42)),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(23, 99)),
lambda m: choice(["Ungefähr ", "Genau ", ""]) + str(randint(100, 1_000_000)),
"Ein paar.",
"Nur ein paar.",
"Nur ein paar wenige.",
"Wirklich viele.",
"Einige.",
"Ne ganze Menge.",
"Zu viele!",
"Ziemlich viele!",
"Unendlich viele!",
),
"wo": (
lambda m: f"Genau {choice(praepos)} dir!",
lambda m: f"{choice(praepos).title()} dir!",
lambda m: f"{choice(praepos).title()} dem Bett.",
lambda m: f"{choice(praepos).title()} dem Kopfkissen.",
lambda m: f"{choice(praepos).title()} dem Couche-Kissen.",
lambda m: f"{choice(praepos).title()} dem Tisch.",
lambda m: f"{choice(praepos).title()} der Kommode.",
lambda m: f"{choice(praepos).title()} dem Stuhl.",
"Droben auf dem Berge.",
"In der Garage.",
"Im Klo.",
"Auf der Straße.",
"In der Gasse.",
"In den Wolken.",
"Vorm Fernseher.",
"Im Cyberspace.",
"Genau vor deiner Nase.",
"Auf deinem Kopf.",
"Unter deinem Hintern.",
"Du sitzt drauf!",
"Im Gefängnis.",
"Im Bundestag.",
"Auf der dunklen Seite des Mondes.",
"In einer weit entfernten Galaxie.",
"Auf dem Mond.",
"Auf der ISS.",
"Im Inneren der Erde.",
),
"woher": (),
"wohin": (),
"weshalb": (),
"welche": (),
"welcher": (),
"welches": (),
"um wieviel uhr": (
lambda m: f"Um {randint(0, 23)} Uhr",
lambda m: f"Um {randint(0, 23)}:{randint(1, 59):02} Uhr",
lambda m: f"Um {randint(0, 23)} Uhr und {randint(1, 59)} Minuten",
lambda m: f"Um {randint(1, 59)} Minuten {choice(['vor','nach'])} {randint(0, 12)}",
lambda m: f"Um Viertel {choice(['vor','nach'])} {randint(0, 12)}",
lambda m: f"Um halb {randint(1, 12)}",
),
"seit wann": (
lambda m: f"Seit {randint(0, 23)} Uhr",
lambda m: f"Seit {randint(0, 23)}:{randint(1, 59):02} Uhr",
lambda m: f"Seit {randint(0, 23)} Uhr und {randint(1, 59)} Minuten",
lambda m: f"Seit {randint(1, 59)} Minuten {choice(['vor','nach'])} {randint(0, 12)}",
lambda m: f"Seit Viertel {choice(['vor','nach'])} {randint(0, 12)}",
lambda m: f"Seit halb {randint(1, 12)}",
# Date
"Seit Gestern",
"Seit Vorgestern",
lambda m: f"Seit vorige Woche",
lambda m: f"Seit {randint(2, 12)} {choice(['Wochen', 'Monaten'])}",
lambda m: f"Seit {randint(2, 9999)} Jahren",
lambda m: f"Seit {randint(2, 999)} Millionen Jahren",
lambda m: f"Seit dem {randint(1, 28)}.{randint(1, 12)}.{randint(9, 9999)}",
# Date-time
lambda m: f"Seit {randint(2, 17)} Jahren, {randint(2,360)} Tagen, {randint(2,23)} Stunden, {randint(2,59)} Minuten und {randint(2,59)} Sekunden",
lambda m: f"Seit dem {randint(1, 28)}.{randint(1, 12)}.{randint(9, 9999)} um {randint(0, 23)}:{randint(1, 59):02} Uhr",
),
}
# wer
# wem
# seit wann
# um wieviel uhr
def init(bot):
bot.on_message(handle)
async def handle(message: Message):
if "oder" in message.tokens or not ( # avoid conflicts with aoderb.py
message.text.endswith("?") and message.is_for_me
):
return
if message.words.startswith("um wieviel uhr"):
wword = "um wieviel uhr"
elif message.words.startswith("seit wann"):
wword = "seit wann"
else:
wword = message.words.str(0).rstrip(",.?!").lower()
if wword in ("und", "aber"):
wword = message.words.str(1).rstrip(",.?!").lower()
if wword in wwords:
reps = (
choices([wwords[wword], ka], weights=[100, 10])[0] if wwords[wword] else ka
)
template = choice(reps)
else:
template = choice(choices([ja, nein, vielleicht], weights=[100, 100, 15])[0])
text = template(message) if callable(template) else template
await reply(message, text, with_name=True)

127
hotdog/command/post.py Normal file
View file

@ -0,0 +1,127 @@
from datetime import datetime, timezone
import postillon
from ..functions import clamp, localizedtz, reply
from ..models import Message
HELP = """Postillon Newsticker.
!post [how many|search terms ...]
!post find [search terms ...]
!post random [how many]
!post more
"""
def init(bot):
bot.on_command("post", handle)
bot.shared.setdefault("post.finds", {})
if "post.store" not in bot.shared:
bot.shared["post.store"] = postillon.Store(bot.config.get("postillon.storage"))
bot.shared["post.store"].connect()
def post_as_plain(post, tzname: str, lc: str):
parts = []
if post.content:
parts.append(f"+++ {post.content} +++")
if post.date:
now = datetime.now(tz=timezone.utc)
since = now - post.date
fmt = ""
if since.days < 1:
fmt = "%X"
else:
if since.days < 7:
fmt += "%A, "
fmt += "%x %X"
parts.append("(" + localizedtz(post.date, fmt, tzname=tzname, lc=lc) + ")")
return " ".join(parts)
def post_as_html(post, tzname: str, lc: str):
parts = []
if post.content:
content = post.content.strip()
sep = ": " if ": " in content else "? " if "? " in content else None
if not sep:
parts.append(f"+++ {post.content} +++")
else:
q, a = content.split(sep, 2)
parts.append(f"+++ <b>{q}{sep}</b>{a} +++")
if post.date:
now = datetime.now(tz=timezone.utc)
since = now - post.date
fmt = ""
if since.days < 1:
fmt = "%X"
else:
if since.days < 7:
fmt += "%A, "
fmt += "%x %X"
parts.append("(" + localizedtz(post.date, fmt, tzname=tzname, lc=lc) + ")")
return " ".join(parts)
async def handle(message: Message):
bot = message.app
poststore = bot.shared["post.store"]
finds = bot.shared["post.finds"] # used to page through find results
max_results = 5
mode = message.args.str(0)
args = message.args
if mode in ("find", "random", "more"):
args = args[1:]
else:
# Time for magic!
if not args:
if finds.get(message.room.room_id, (None, 0))[0] is not None:
mode = "more"
else:
mode = "random"
else:
if len(args) == 1 and args.str(0).isdecimal():
mode = "random"
else:
mode = "find"
if mode == "find":
term = "%".join(args)
posts = poststore.search(f"%{term}%")
finds[message.room.room_id] = (term, 0)
elif mode == "random":
finds[message.room.room_id] = (None, 0)
count = clamp(1, args.int(0), max_results)
posts = [poststore.random_post() for _ in range(count)]
elif mode == "more":
term, page = finds.get(message.room.room_id, (None, 0))
if term is None:
return
page += 1
posts = poststore.search(f"%{term}%", skip=page * max_results)
finds[message.room.room_id] = (term, page)
else:
return
roomconf = bot.config.l6n[message.room.room_id]
i = None
for i, post in enumerate(posts):
if i >= max_results:
await reply(
message,
html=f"<i>Für weitere Ergebnisse</i>: <code>{bot.config.command_prefix}post more</code>",
in_thread=True,
)
break
text = post_as_html(
post,
tzname=roomconf["timezone"],
lc=roomconf["locale"],
)
await reply(message, html=text)
else:
finds[message.room.room_id] = (None, 0)
if i is None:
await reply(message, html="<i>Keine weiteren Ergebnisse.</i>", in_thread=True)

39
hotdog/command/prost.py Normal file
View file

@ -0,0 +1,39 @@
from random import choice
from ..functions import react
from ..models import Message
def init(bot):
bot.on_message(handle)
emojis = {
"☕️",
"🍵",
"🍶",
"🍷",
"🍸",
"🍹",
"🍺",
"🍻",
"🍼",
"🍾",
"🥂",
"🥃",
"🥛",
"🥤",
"🧃",
"🧉",
}
async def handle(message: Message):
vs16 = "\ufe0f" # see https://en.wikipedia.org/wiki/Variation_Selectors_%28Unicode_block%29
is_drinkmoji = message.text.rstrip(vs16) in emojis
if not (message.tokens.str(0).startswith("PR!") or is_drinkmoji):
return
remoji = message.text if is_drinkmoji else choice(tuple(emojis))
await react(message, remoji)

20
hotdog/command/reminder.py Normal file
View file

@ -0,0 +1,20 @@
from ..functions import reply
from ..models import Message
# @hotdog: remind <who> <time> <what>
# .remind me at 14:30 take out the milk
# .remind me to take out the milk tomorrow
# today -> next 6 hour step
# tomorrow -> at 10
# in X {days|hours|minutes|seconds}
# {on|at} d.m.y [hh:mm]
def init(bot):
bot.on_message(handle)
async def handle(message: Message):
if message.command != "remind" or not message.args.startswith("me"):
return

26
hotdog/command/retour.py Normal file
View file

@ -0,0 +1,26 @@
from random import choice
from ..functions import reply
from ..models import Message
def init(bot):
bot.on_message(handle)
replies = (
"Du auch.",
"Selber.",
"Danke, gleichfalls.",
)
async def handle(message: Message):
if "oder" in message.tokens or not (
message.words.startswith("du bist")
and message.is_for_me
and not message.text.endswith("?") # avoid conflicts with orakel.py
):
return
await reply(message, choice(replies), with_name=True)

59
hotdog/command/roll.py Normal file
View file

@ -0,0 +1,59 @@
import re
from collections import defaultdict
from random import randint
from ..functions import reply
from ..models import Message
HELP = """Würfelt eine Summe aus.
!roll [how many]d<sides> [ ... ]
"""
parse_die = re.compile("(?P<num>\d*)[dwDW](?P<sides>\d+)").fullmatch
def init(bot):
bot.on_command("roll", handle)
async def handle(message: Message):
if len(message.args) == 1 and message.args[0].isdecimal():
dice = [(1, int(message.args[0]))]
else:
dice = []
num = None
for arg in message.args:
if (match := parse_die(arg)) :
if num and match["num"]:
return
num, sides = int(match["num"] or num or 1), int(match["sides"])
if not 1 <= num < 1000 or not 2 <= sides <= 100:
return
dice.append((num, sides))
num = None
elif arg.isdecimal():
if num is not None:
return
num = int(arg)
else:
return
if not 0 < len(dice) < 20:
return
rolls = defaultdict(list)
for num, sides in dice:
for _ in range(num):
rolls[sides].append(randint(1, sides))
total = sum(sum(vs) for sides, vs in rolls.items())
if len(rolls) == 1:
sides = list(rolls.keys())[0]
text = f"{total} ({len(rolls[sides])}d{sides})"
else:
dicestr = " + ".join(
f"{len(vs)}d{sides} ({sum(vs)})"
for sides, vs in sorted(rolls.items(), key=lambda i: i[0], reverse=1)
)
text = f"{total} = {dicestr}"
await reply(message, plain=text, in_thread=True)

199
hotdog/command/urlinfo.py Normal file
View file

@ -0,0 +1,199 @@
import codecs
import re
from functools import lru_cache
from html import escape
from html.parser import HTMLParser
from random import randint
from time import time as now
from typing import *
import requests
from ..functions import reply
from ..models import Message
HELP = """Return information about an online HTTP resource.
!u[rl] <url>
"""
def init(bot):
bot.on_command({"u", "url"}, handle)
match_url = re.compile(
# r"https?://(?:[a-zA-Z]|[0-9]|[$-_~@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
r"https?://\S+"
).fullmatch
class TitleParser(HTMLParser):
"""Parse the first <title> from HTML"""
# XXX check if it's the <head>'s title we're in, but beware that head can be implicit
def __init__(self):
super().__init__()
self.__is_title = False
self.__found = False
self.__title = ""
def handle_starttag(self, tag, attrs):
if tag == "title":
self.__is_title = True
def handle_endtag(self, tag):
if tag == "title":
self.__found = True
self.__is_title = False
def handle_data(self, data):
if self.__is_title and not self.__found:
self.__title += data
@property
def title(self) -> Optional[str]:
return self.__title if self.__found else None
def get_encodings_from_content(content: str) -> List[str]:
"""Returns encodings from given content string."""
charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')
return (
charset_re.findall(content)
+ pragma_re.findall(content)
+ xml_re.findall(content)
)
def stream_decode_response_unicode(content: Iterable[bytes]) -> Iterable[str]:
"""Stream decodes a iterator."""
decoder = None
for chunk in content:
if decoder is None:
encodings = get_encodings_from_content(
chunk.decode("utf-8", errors="replace")
) + ["utf-8"]
decoder = codecs.getincrementaldecoder(encodings[0])(errors="replace")
rv = decoder.decode(chunk)
if rv:
yield rv
rv = decoder.decode(b"", final=True)
if rv:
yield rv
def title(content: Iterable[str]) -> Optional[str]:
t = TitleParser()
for chunk in content:
t.feed(chunk)
if t.title is not None:
break
return t.title
def capped(content: Iterable[str], read_max: int) -> Iterable[str]:
read = 0
for chunk in content:
read += len(chunk)
yield chunk
if read >= read_max:
break
@lru_cache(maxsize=5)
def load_info(url: str, cachetoken) -> Optional[Mapping[str, Any]]:
"""The cachetoken is just there to bust the LRU cache after a while."""
try:
r = requests.get(
url,
stream=True,
timeout=(3, 3),
headers={"user-agent": "hotdog/v1 urlinfo"},
)
except Exception:
return None
content_type = r.headers.get("Content-Type", "")
is_html = content_type.startswith("text/html") or url.lower().endswith(
(".html", ".htm")
)
if is_html:
one_kb = 2 ** 10
# chunks = r.iter_content(chunk_size=30 * one_kb, decode_unicode=True)
chunks = stream_decode_response_unicode(r.iter_content(chunk_size=30 * one_kb))
html_title = title(capped(chunks, read_max=200 * one_kb))
else:
html_title = None
filename = None
dispo = r.headers.get("Content-Disposition", "").split(";")
if len(dispo) == 2 and dispo[0] == "attachment":
dispo = dispo[1].strip().split("=", 2)
if len(dispo) == 2 and dispo[0] == "filename":
filename = dispo[1].strip()
return {
"code": r.status_code,
"url": r.url,
"elapsed_ms": int(r.elapsed.total_seconds() * 1_000),
"reason": r.reason,
"type": r.headers.get("Content-Type"),
"size": (
int(r.headers["Content-Length"]) if "Content-Length" in r.headers else None
),
"title": html_title,
"filename": filename,
}
def cachetoken(quant_m=15):
"""Return a cache token with the given time frame"""
return int(now() / 60 / quant_m)
def pretty_size(size: int) -> str:
qs = "", "K", "M", "G", "T", "P"
for q in qs:
if size < 1024 or q == qs[-1]:
break
size /= 1000
if not q:
return f"{size} B"
return f"{size:_.02f} {q}B"
async def handle(message: Message):
url = message.args.str(0)
if not match_url(url):
return
info = load_info(url, cachetoken())
if not info:
return
details = []
if info["type"]:
details.append(f"<i>Media type</i>: {escape(info['type'])}")
if info["size"]:
details.append(f"<i>Size</i>: {pretty_size(info['size'])}")
details.append(f"<i>Status</i>: {info['code']}")
if info["reason"]:
details[-1] += f" ({escape(info['reason'])})"
if info["url"] != url:
details.append(
f"""<i>Redirected to</i>: <a href="{escape(info['url'])}">{escape(info['url'])}</a>"""
)
if info["filename"] and info["filename"] != url.rsplit("/", 2)[-1]:
details.append(f"<i>Filename</i>: {escape(info['filename'])}")
details.append(f"<i>TTFB</i>: {info['elapsed_ms']:_} ms")
text = f"<b>{escape(info['title'])}</b> — " if info["title"] else ""
text += "; ".join(details)
await reply(message, html=text, in_thread=True)

458
hotdog/command/wikipedia.py Normal file
View file

@ -0,0 +1,458 @@
import asyncio
import logging
import re
from dataclasses import dataclass
from datetime import datetime
from functools import lru_cache, partial
from html import escape
from time import time as now
from typing import *
import requests
from ..functions import localizedtz, react, reply, strip_tags
from ..models import Message
log = logging.getLogger(__name__)
HELP = """Look up articles on Wikipedia.
!w[p|ikipedia] [lang (ISO 639)] <search terms ...>
"""
def init(bot):
bot.shared[__name__] = {"last": 0}
bot.on_command({"w", "wp", "wikipedia"}, handle)
api_url = "https://{lang}.wikipedia.org/w/api.php"
# see https://de.wikipedia.org/wiki/Liste_der_Wikipedias_nach_Sprachen
langs = {
"aa",
"ab",
"ace",
"af",
"ak",
"als",
"am",
"an",
"ang",
"ar",
"arc",
"as",
"ast",
"av",
"ay",
"az",
"ba",
"bar",
"bat-smg",
"bcl",
"be",
"bg",
"bh",
"bi",
"bjn",
"bm",
"bn",
"bo",
"bpy",
"br",
"bs",
"bug",
"bxr",
"ca",
"cbk-zam",
"cdo",
"ce",
"ceb",
"ch",
"cho",
"chr",
"chy",
"ckb",
"co",
"cr",
"crh",
"cs",
"csb",
"cu",
"cv",
"cy",
"da",
"de",
"diq",
"dsb",
"dv",
"dz",
"ee",
"el",
"eml",
"en",
"eo",
"es",
"et",
"eu",
"ext",
"fa",
"ff",
"fi",
"fiu-vro",
"fj",
"fo",
"fr",
"frp",
"frr",
"fur",
"fy",
"ga",
"gag",
"gan",
"gd",
"gl",
"glk",
"gn",
"got",
"gu",
"gv",
"ha",
"hak",
"haw",
"he",
"hi",
"hif",
"ho",
"hr",
"hsb",
"ht",
"hu",
"hy",
"hz",
"ia",
"id",
"ie",
"ig",
"ii",
"ik",
"ilo",
"io",
"is",
"it",
"iu",
"ja",
"jbo",
"jv",
"ka",
"kaa",
"kab",
"kbd",
"kg",
"kj",
"kk",
"kl",
"km",
"kn",
"ko",
"koi",
"kr[",
"krc",
"ks",
"ksh",
"ku",
"kv",
"kw",
"ky",
"la",
"lad",
"lb",
"lbe",
"lez",
"lg",
"li",
"lij",
"lmo",
"ln",
"lo",
"lt",
"ltg",
"lv",
"map-bms",
"mdf",
"mg",
"mh",
"mhr",
"mi",
"mk",
"ml",
"mn",
"mr",
"mrj",
"ms",
"mt",
"mus",
"mwl",
"my",
"myv",
"mzn",
"na",
"nah",
"nap",
"nds",
"nds-nl",
"ne",
"new",
"ng",
"nl",
"nn",
"no",
"nov",
"nrm",
"nso",
"nv",
"oc",
"om",
"or",
"os",
"pa",
"pag",
"pam",
"pap",
"pcd",
"pdc",
"pfl",
"pi",
"pih",
"pl",
"pms",
"pnb",
"pnt",
"ps",
"pt",
"qu",
"rm",
"rmy",
"rn",
"ro",
"roa-rup",
"roa-tara",
"ru",
"rue",
"rw",
"sa",
"sc",
"scn",
"sco",
"sd",
"se",
"sg",
"sh",
"si",
"simple",
"sk",
"sl",
"sm",
"sn",
"so",
"sq",
"sr",
"srn",
"ss",
"st",
"stq",
"su",
"sv",
"sw",
"szl",
"ta",
"te",
"tet",
"tg",
"th",
"ti",
"tk",
"tl",
"tn",
"to",
"tpi",
"tr",
"ts",
"tt",
"tum",
"tw",
"ty",
"udm",
"ug",
"uk",
"ur",
"uz",
"ve",
"vec",
"vep",
"vi",
"vls",
"vo",
"wa",
"war",
"wo",
"wuu",
"xal",
"xh",
"xmf",
"yi",
"yo",
"za",
"zea",
"zh",
"zh-classical",
"zh-min-nan",
"zh-yue",
"zu",
"zu",
}
def searchparams(terms: Iterable[str]):
# see https://www.mediawiki.org/wiki/API:Search
return {
"action": "query",
"list": "search",
"srsearch": " ".join(terms),
"format": "json",
"srlimit": 3,
}
def resolveparams(ids: Iterable[int]):
return {
"action": "query",
"prop": "info",
"pageids": "|".join(map(str, ids)),
"inprop": "url",
"format": "json",
}
@dataclass(frozen=True)
class Hit:
# ns: int
pageid: int
size: int
snippet: str
timestamp: datetime
title: str
wordcount: int
@classmethod
def from_json(cls, data):
return cls(
data["pageid"],
data["size"],
data["snippet"],
fromjsonformat(data["timestamp"]),
data["title"],
data["wordcount"],
)
def fromjsonformat(s: str) -> datetime:
if s.endswith("Z"):
s = s[:-1] + "+00:00"
return datetime.fromisoformat(s)
def load_api_json(session, lang, params):
r = session.get(
api_url.format(lang=lang),
params=params,
timeout=(3, 3),
headers={"User-Agent": "hotdog/v1 wikipedia"},
)
r.raise_for_status()
return r.json()
def search(session, lang, terms):
data = load_api_json(session, lang, searchparams(terms))
return {
"total": data["query"]["searchinfo"]["totalhits"],
"hits": [Hit.from_json(d) for d in data["query"]["search"]],
}
def resolve_urls(session, lang, ids: Collection[int]):
if not ids:
return {}
data = load_api_json(session, lang, resolveparams(ids))
return {
int(pid): p.get("canonicalurl") for pid, p in data["query"]["pages"].items()
}
@lru_cache(maxsize=10)
def search_and_resolve(lang, terms):
session = requests.Session()
result = search(session, lang, terms)
result["urls"] = resolve_urls(session, lang, [hit.pageid for hit in result["hits"]])
return result
def snippet(s: str, max_content_len: int = 300):
content = ""
for word in s.split():
if not word:
continue
if len(content + f" {word}") > max_content_len - 3:
content += " […]"
break
content += f" {word}"
return content
async def handle(message: Message):
bot = message.app
roomconf = bot.config.l6n[message.room.room_id]
if message.args.get(0) in langs:
lang = message.args[0]
args = message.args[1:]
else:
lang, *_ = roomconf["locale"].split("_", 1)
args = message.args
if not args:
return
await react(message, "⚡️")
# XXX no need to wait if the result is already cached - can we check that?
# Guard against API flooding...
timeout = 10
while 0 < (waitfor := timeout - (now() - bot.shared[__name__]["last"])):
log.debug(f"Waiting for {waitfor}s before next API call.")
await asyncio.sleep(waitfor)
bot.shared[__name__]["last"] = now()
r = search_and_resolve(lang, args)
if not r["hits"]:
await react(message, "❌")
return
lines = []
if r["total"] > 3:
lines.append(f"Found <b>{r['total']}</b> matching articles.")
for hit in r["hits"][:3]:
last_updated = localizedtz(
hit.timestamp, "%x %X", tzname=roomconf["timezone"], lc=roomconf["locale"]
)
teaser = snippet(escape(strip_tags(f"… {hit.snippet} …")))
lines.append(
f'<b><a href="{escape(r["urls"][hit.pageid])}">{escape(hit.title)}</a></b>: <i>{teaser}</i> (<i>{last_updated}</i>)'
)
await asyncio.gather(
reply(message, html="<br/>\n".join(lines), in_thread=True),
react(message, "✅"),
)

131
hotdog/command/youtube.py Normal file
View file

@ -0,0 +1,131 @@
import re
from dataclasses import dataclass, fields
from functools import lru_cache
from html import escape
from time import time as now
from typing import *
import youtube_dl
from ..functions import reply
from ..models import Message
HELP = """Gibt Informationen zu Youtube-Videos aus.
Some text containing a <Youtube URL>.
"""
youtube_re = re.compile(
r"\byoutu(\.be/|be\.com/(embed/|v/|watch\?\w*v=))(?P<id>[0-9A-Za-z_-]{10,11})\b"
)
def init(bot):
bot.on_message(handle)
@lru_cache(maxsize=5)
def load_info(url, cachetoken):
"""The cachetoken is just there to bust the LRU cache after a while."""
return Info.from_url(url)
def cachetoken(quant_m=15):
"""Return a cache token with the given time frame"""
return int(now() / 60 / quant_m)
async def handle(message: Message):
if message.command in {"u", "url"}:
return
match = youtube_re.search(message.text)
if not match:
return
youtube_id = match["id"]
info = load_info(youtube_id, cachetoken())
info.escape_all()
details = [
f"🖋 {info.author}",
f"⏱ {pretty_duration(info.duration_seconds)}",
f"📺 {info.width}×{info.height}",
f"👀 {info.view_count:_}",
]
tag = (info.categories[:1] or info.tags[:1] or [""])[0]
if tag:
details.append(f"🏷 {tag}")
text = f"<b>{info.title}</b> — {', '.join(details)}"
await reply(message, html=text)
def pretty_duration(seconds: int) -> str:
hours = seconds // 3600
minutes = (seconds - hours * 3600) // 60
seconds = seconds % 60
return (
f"{hours}h{minutes:02}m{seconds:02}s" if hours else f"{minutes}m{seconds:02}s"
)
class Nolog:
def debug(self, msg):
pass
def warning(self, msg):
pass
def error(self, msg):
pass
ytdl = youtube_dl.YoutubeDL({"skip_download": True, "logger": Nolog()})
@dataclass
class Info:
author: str
title: str
description: str
duration_seconds: int
view_count: int
thumbnail: str
width: int
height: int
categories: List[str]
tags: List[str]
def escape_all(self):
for f in fields(self):
if f.type is str:
setattr(self, f.name, escape(getattr(self, f.name)))
elif get_origin(f.type) is list:
setattr(self, f.name, [escape(x) for x in getattr(self, f.name)])
@classmethod
def from_url(cls, url):
info = ytdl.extract_info(url, download=False)
new = cls(
author=info["uploader"] or info["creator"] or info["uploader_id"] or "",
title=info["title"] or info["alt_title"] or "",
description=info["description"] or "",
duration_seconds=info["duration"] or 0,
view_count=info["view_count"] or 0,
thumbnail=(
info["thumbnail"]
or (info["thumbnails"][-1] if info["thumbnails"] else "")
),
categories=info["categories"] or [],
tags=info["tags"] or [],
width=info["width"] or 0,
height=info["height"] or 0,
)
if info["formats"]:
new.width, new.height = max(
(
(f["width"] or new.width),
(f["height"] or new.height),
)
for f in info["formats"]
)
return new

92
hotdog/config.py Normal file
View file

@ -0,0 +1,92 @@
import platform
import re
from pathlib import Path
from typing import *
import yaml
class ConfigError(RuntimeError):
pass
class Fallbackdict(dict):
"""Like defaultdict, but not implicitly creating entries."""
def __init__(self, default):
super().__init__()
self._default = default
def get(self, key, default=None):
return super().get(key, self._default if default is None else default)
def __getitem__(self, key):
return super().get(key, self._default)
class Config:
def __init__(self, path):
self.load(path)
def load(self, path):
with open(path) as fp:
self.config = yaml.safe_load(fp)
self.command_prefix = self.get("command_prefix")
# Logging
self.loglevel = self.get("loglevel", default="INFO")
# Storage
self.store_path = Path(self.get("storage.store_path"))
# Matrix
self.user_id = self.get("matrix.user_id")
if not re.fullmatch("@.*:.*", self.user_id):
raise ConfigError("matrix.user_id must be in the form @name:domain")
self.device_id_path = self.store_path / f"{self.user_id}.device_id"
try:
self.device_id = self.device_id_path.read_text()
except FileNotFoundError:
self.device_id = ""
self.device_name = self.get(
"matrix.session_name", default=f"hotdog/{platform.node()}"
)
self.display_name = self.get("matrix.nick_name")
self.homeserver_url = self.get("matrix.homeserver_url")
self.password = self.get("matrix.password")
# Development
self.is_dev = self.get("dev.active", False)
self.dev_room = self.get("dev.room", "")
# Location / l6n
l6n_default = self.get("l6n.default")
l6n = Fallbackdict(l6n_default)
for room in self.get("l6n.rooms"):
l6n[room["id"]] = {**l6n_default, **room} # XXX in py3.9 use |-operator
self.l6n = l6n
def get(self, path: Union[str, List[str]], default: Any = None) -> Any:
if isinstance(path, str):
path = path.split(".")
config = self.config
for part in path:
config = config.get(part)
if config is None:
if default is not None:
return default
raise ConfigError(f"Config value is missing: {'.'.join(path)}")
return config
def set(self, path: Union[str, List[str]], value: Any):
if isinstance(path, str):
path = path.split(".")
config = self.config
for part in path[:-1]:
if part in config and not isinstance(config[part], dict):
raise ConfigError(f"Conflicting value exists: {'.'.join(path)}")
config = config.get(part, {})
config[path[-1]] = value

181
hotdog/functions.py Normal file
View file

@ -0,0 +1,181 @@
import locale
import logging
import unicodedata
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from html import escape
from html.parser import HTMLParser
from io import StringIO
from typing import *
import nio
log = logging.getLogger(__name__)
tzdb = {
"Europe/Berlin": timezone(timedelta(hours=2), "Europe/Berlin"),
}
def html_nametag(uid, name):
return f'<a href="https://matrix.to/#/{escape(uid)}">{escape(name)}</a>'
async def reply(
message,
plain: Optional[str] = None,
*,
html: Optional[str] = None,
with_name: bool = False,
in_thread: bool = False,
) -> nio.RoomSendResponse:
"""Reply to the given message in plain text or HTML.
If with_name is set the reply will be prefixed with the original
message's sender's name.
If in_thread is set the reply will reference the original message,
allowing a client to connect the messages unambiguously.
"""
assert plain or html
if with_name:
if plain:
plain = f"{message.sender_name}: {plain}"
if html:
sender = html_nametag(message.event.sender, message.sender_name)
html = f"{sender}: {html}"
args = {}
if in_thread:
args["reply_to_event_id"] = message.event.event_id
if html:
args["html"] = html
if plain:
args["plain"] = plain
return await send_message(message.app.client, message.room.room_id, **args)
async def react(message, emoji: str) -> nio.RoomSendResponse:
"""Annotate a message with an Emoji.
Only a single Emoji character should be used for the reaction; compound
characters are ok.
"""
# For details on Matrix reactions see MSC2677:
# - https://github.com/uhoreg/matrix-doc/blob/aggregations-reactions/proposals/2677-reactions.md
# - and https://github.com/matrix-org/matrix-appservice-slack/pull/522/commits/88e7076a595509b196f53369102a469cace6cc19
vs16 = "\ufe0f" # see https://en.wikipedia.org/wiki/Variation_Selectors_%28Unicode_block%29
if not emoji.endswith(vs16):
if len(emoji) != 1:
log.warning("Reactions should only use a single emoji, got: %s", emoji)
else:
emoji = unicodedata.normalize("NFC", emoji + vs16)
return await message.app.client.room_send(
room_id=message.room.room_id,
message_type="m.reaction",
content={
"m.relates_to": {
"rel_type": "m.annotation",
"event_id": message.event.event_id,
"key": emoji,
}
},
ignore_unverified_devices=True,
)
async def redact(message, reason: Optional[str] = None) -> nio.RoomRedactResponse:
"""Redact a message.
This allows not only to redact text messages, but also reactions, i.e.
take back a reaction.
"""
resp = await message.app.client.room_redact(
room_id=message.room.room_id, event_id=message.event.event_id, reason=reason
)
if isinstance(resp, nio.RoomRedactError):
raise RuntimeError(
f"Could not redact: {message.event.event_id}: {resp.message}"
)
return resp
async def send_message(
client: nio.AsyncClient,
room_id: str,
plain: str = "",
*,
html: Optional[str] = None,
as_notice: bool = True,
reply_to_event_id: Optional[str] = None,
) -> nio.RoomSendResponse:
"""Send text to a matrix room.
The message can be given as either plain text or HTML.
If the plain text variant is not explicitly given, it will be
generated from the rich text variant.
You should always send the message as notice. Per convention
bots don't react to notices, so sending only notices will avoid
infinite loops between multiple bots in a channel.
"""
assert plain or html
content = {
"msgtype": "m.notice" if as_notice else "m.text",
"body": plain,
}
if html:
content["format"] = "org.matrix.custom.html"
content["formatted_body"] = html
if not plain:
content["body"] = strip_tags(html)
if reply_to_event_id:
content["m.relates_to"] = {"m.in_reply_to": {"event_id": reply_to_event_id}}
try:
return await client.room_send(
room_id,
"m.room.message",
content,
ignore_unverified_devices=True,
)
except nio.SendRetryError:
log.exception(f"Unable to send message to room: {room_id}")
@contextmanager
def localized(lc: str, category=locale.LC_ALL):
locale.setlocale(category, lc)
try:
yield
finally:
locale.resetlocale(category)
def localizedtz(dt: datetime, fmt: str, lc: str, tzname: str):
tz = tzdb[tzname]
with localized(lc, locale.LC_TIME):
return dt.astimezone(tz).strftime(fmt)
class TextonlyParser(HTMLParser):
def __init__(self):
super().__init__()
self.__text = StringIO()
def handle_data(self, d):
self.__text.write(d)
@property
def text(self):
return self.__text.getvalue()
def strip_tags(html):
s = TextonlyParser()
s.feed(html)
return s.text
def clamp(lower, x, upper):
return max(lower, min(x, upper))

131
hotdog/models.py Normal file
View file

@ -0,0 +1,131 @@
import asyncio
from dataclasses import dataclass, field
from random import random
from time import time as now
from typing import *
import nio
JobCallback = Callable[["Job"], None]
@dataclass
class Job:
app: "Bot"
title: str
func: JobCallback
every: Optional[float] = None
jitter: float = 0.0
next: Optional[float] = None
task: Optional[asyncio.Task] = None
def _next(self) -> Optional[float]:
if self.every is None:
return None
return now() + self.every + self._jitter(self.jitter)
@staticmethod
def _jitter(x: float) -> float:
return 2 * x * random() - x
class Tokens(tuple):
def get(self, pos, default=None):
try:
return self[pos]
except:
return default
def int(self, pos, default=0):
try:
return abs(int(self[pos]))
except:
return default
def str(self, pos, default=""):
try:
return str(self[pos])
except:
return default
def startswith(self, args: Union[str, List[str]]):
if isinstance(args, str):
args = Tokens.from_str(args)
return self[: len(args)] == args
@classmethod
def from_str(cls, s):
return cls(t for t in s.split() if t)
def __getitem__(self, index):
result = super().__getitem__(index)
return Tokens(result) if isinstance(index, slice) else result
@dataclass
class Message:
app: "Bot"
text: str
room: nio.rooms.MatrixRoom
event: nio.events.room_events.RoomMessageText
tokens: Tokens = None
words: Tokens = None
is_for_me: bool = False
command: Optional[str] = None
args: Optional[Tokens] = None
@property
def sender_name(self) -> str:
return self.room.user_name(self.event.sender)
@property
def my_name(self) -> str:
return self.room.user_name(self.app.client.user)
@property
def is_direct_chat(self):
# "A group is an unnamed room with no canonical alias."
return self.room.is_group and self.room.member_count == 2
def __post_init__(self):
self.tokens = Tokens.from_str(self.text)
self.words = self.tokens
"""
.roll d20
hotdog: roll d20
hotdog, roll d20
@hotdog roll d20
@hotdog : roll d20
"""
first_arg = self.tokens.str(0)
if self.text.startswith(self.app.config.command_prefix):
self.command = first_arg[len(self.app.config.command_prefix) :]
self.args = self.tokens[1:]
self.words = self.args
return
me = self.my_name
if me and me.lower() == first_arg.lstrip("@").rstrip(":,!").lower():
self.is_for_me = True
if self.tokens.str(1) in ":,!":
args = self.tokens[2:]
else:
args = self.tokens[1:]
self.words = args
self.command, self.args = args.get(0), args[1:]
@dataclass
class Reaction(nio.Event):
related_event_id: str = field()
key: str = field()
@classmethod
def from_dict(cls, event_dict):
return cls(
event_dict,
event_dict["content"]["m.relates_to"]["event_id"],
event_dict["content"]["m.relates_to"]["key"],
)

51
hotdog/tz.py Normal file
View file

@ -0,0 +1,51 @@
import calendar
from datetime import datetime, timedelta, timezone
from typing import Optional
def _last_of_month(calendar_day: int, year: int, month: int) -> int:
last = None
for date, day in calendar.Calendar().itermonthdays2(year, month):
if day == calendar_day and date != 0:
last = date
if last is None:
raise RuntimeError(
"No such calendar day found: {calendar_day}".format(
calendar_day=calendar_day
)
)
return last
_cest = timezone(timedelta(hours=2), "CEST")
_cet = timezone(timedelta(hours=1), "CET")
def cest(date: Optional[datetime] = None) -> timezone:
"""
Return the timezone for Central European (Summer) Time.
Automatically selects CET/CEST based on the given date.
Since 1996, European Summer Time has been observed from the last Sunday
in March to the last Sunday in October.
"""
if date is None:
date = datetime.now()
if date.year < 1996:
raise NotImplementedError()
march = 3
october = 10
if march < date.month < october:
return _cest
if march > date.month or date.month > october:
return _cet
past_break = (
date.day >= _last_of_month(calendar.SUNDAY, date.year, date.month)
and date.hour > 2
)
if date.month == march:
return _cest if past_break else _cet
if date.month == october:
return _cet if past_break else _cest
raise RuntimeError("Month is out of bounds: {date.month}".format(date=date))