improve typing
This commit is contained in:
parent
cb7c66a8d1
commit
60d38e9b49
4 changed files with 12 additions and 9 deletions
|
|
@ -15,7 +15,7 @@ async def run_load_user_ratings_from_imdb():
|
||||||
await open_connection_pool()
|
await open_connection_pool()
|
||||||
|
|
||||||
i = 0
|
i = 0
|
||||||
async for rating in refresh_user_ratings_from_imdb():
|
async for _ in refresh_user_ratings_from_imdb():
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
log.info("✨ Imported %s new ratings.", i)
|
log.info("✨ Imported %s new ratings.", i)
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ log = logging.getLogger(__name__)
|
||||||
# p.text-muted.text-small span[name=nv] [data-value]
|
# p.text-muted.text-small span[name=nv] [data-value]
|
||||||
|
|
||||||
|
|
||||||
async def refresh_user_ratings_from_imdb(stop_on_dupe=True):
|
async def refresh_user_ratings_from_imdb(stop_on_dupe: bool = True):
|
||||||
with session() as s:
|
with session() as s:
|
||||||
s.headers["Accept-Language"] = "en-US, en;q=0.5"
|
s.headers["Accept-Language"] = "en-US, en;q=0.5"
|
||||||
|
|
||||||
|
|
@ -149,7 +149,7 @@ def movie_and_rating_from_item(item) -> tuple[Movie, Rating]:
|
||||||
ForgedRequest = namedtuple("ForgedRequest", "url headers")
|
ForgedRequest = namedtuple("ForgedRequest", "url headers")
|
||||||
|
|
||||||
|
|
||||||
async def parse_page(url) -> tuple[list[Rating], str | None]:
|
async def parse_page(url: str) -> tuple[list[Rating], str | None]:
|
||||||
ratings = []
|
ratings = []
|
||||||
|
|
||||||
soup = soup_from_url(url)
|
soup = soup_from_url(url)
|
||||||
|
|
@ -191,7 +191,7 @@ async def parse_page(url) -> tuple[list[Rating], str | None]:
|
||||||
return (ratings, next_url if url != next_url else None)
|
return (ratings, next_url if url != next_url else None)
|
||||||
|
|
||||||
|
|
||||||
async def load_ratings(user_id):
|
async def load_ratings(user_id: str):
|
||||||
next_url = user_ratings_url(user_id)
|
next_url = user_ratings_url(user_id)
|
||||||
|
|
||||||
while next_url:
|
while next_url:
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ from typing import (
|
||||||
|
|
||||||
from .types import ULID
|
from .types import ULID
|
||||||
|
|
||||||
JSON = Union[int, float, str, None, list["JSON"], dict[str, "JSON"]]
|
JSON = int | float | str | None | list["JSON"] | dict[str, "JSON"]
|
||||||
JSONObject = dict[str, JSON]
|
JSONObject = dict[str, JSON]
|
||||||
|
|
||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from hashlib import md5
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from random import random
|
from random import random
|
||||||
from time import sleep, time
|
from time import sleep, time
|
||||||
from typing import Callable, cast
|
from typing import Callable, ParamSpec, TypeVar, cast
|
||||||
|
|
||||||
import bs4
|
import bs4
|
||||||
import httpx
|
import httpx
|
||||||
|
|
@ -31,6 +31,9 @@ _ASession_T = httpx.AsyncClient
|
||||||
_Session_T = httpx.Client
|
_Session_T = httpx.Client
|
||||||
_Response_T = httpx.Response
|
_Response_T = httpx.Response
|
||||||
|
|
||||||
|
_T = TypeVar("_T")
|
||||||
|
_P = ParamSpec("_P")
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def session():
|
def session():
|
||||||
|
|
@ -88,15 +91,15 @@ async def asession():
|
||||||
|
|
||||||
def _throttle(
|
def _throttle(
|
||||||
times: int, per_seconds: float, jitter: Callable[[], float] | None = None
|
times: int, per_seconds: float, jitter: Callable[[], float] | None = None
|
||||||
) -> Callable[[Callable], Callable]:
|
) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
|
||||||
calls: deque[float] = deque(maxlen=times)
|
calls: deque[float] = deque(maxlen=times)
|
||||||
|
|
||||||
if jitter is None:
|
if jitter is None:
|
||||||
jitter = lambda: 0.0
|
jitter = lambda: 0.0
|
||||||
|
|
||||||
def decorator(func: Callable) -> Callable:
|
def decorator(func: Callable[_P, _T]) -> Callable[_P, _T]:
|
||||||
@wraps(func)
|
@wraps(func)
|
||||||
def inner(*args, **kwds):
|
def inner(*args: _P.args, **kwds: _P.kwargs):
|
||||||
# clean up
|
# clean up
|
||||||
while calls:
|
while calls:
|
||||||
if calls[0] + per_seconds > time():
|
if calls[0] + per_seconds > time():
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue