ducklet/unwind
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fix: support new user ratings page markup

Browse source 
We use __NEXT_DATA__ from the page to find the user's latest rated
movies.
We found that at least in one case (of a Video Game) the movie details
were wrong. Normally this shouldn't be a problem though because we know
all the movies already and we keep the values we already have. Otherwise
the data from __NEXT_DATA__ seems more accurate and complete.
This commit is contained in:
ducklet 2024-07-21 14:46:45 +02:00
parent d7530e6bb0
commit 380d6ff186
4 changed files with 217 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

85
tests/test_imdb.py
View file

@ -134,7 +134,7 @@ async def test_load_ratings_page_20240510(monkeypatch):
if rating.movie.imdb_id == item["imdb_id"]:
rating_dict = {key: getattr(rating.movie, key) for key in item.keys()}
return rating_dict
raise AssertionError()
raise AssertionError(f"{item["imdb_id"]} not found in page.ratings")
a_movie = {
"title": "Kung Fu Panda 4",
@ -142,8 +142,9 @@ async def test_load_ratings_page_20240510(monkeypatch):
"media_type": "Movie",
"imdb_id": "tt21692408",
"imdb_score": 59,
"imdb_votes": 36000,
"imdb_votes": 36069,
"runtime": 94,
"genres": {"Action", "Adventure", "Animation"},
}
assert a_movie == movie(a_movie)
@ -153,7 +154,8 @@ async def test_load_ratings_page_20240510(monkeypatch):
"media_type": "TV Series",
"imdb_id": "tt8888540",
"imdb_score": 64,
"imdb_votes": 6000,
"imdb_votes": 6044,
"genres": {"Drama"},
}
assert a_running_tvseries == movie(a_running_tvseries)
@ -163,29 +165,94 @@ async def test_load_ratings_page_20240510(monkeypatch):
"media_type": "TV Series",
"imdb_id": "tt0072500",
"imdb_score": 87,
"imdb_votes": 100000,
"imdb_votes": 100261,
"genres": {"Comedy"},
}
assert a_finished_tvseries == movie(a_finished_tvseries)
a_tvepisode = {
"title": "Columbo / No Time to Die",
"original_title": None,
"original_title": "Columbo / No Time to Die",
"release_year": 1992,
"media_type": "TV Episode",
"imdb_id": "tt0103987",
"imdb_score": 59,
"imdb_votes": 2100,
"imdb_votes": 2122,
"runtime": 98,
"genres": {"Crime", "Drama", "Mystery"},
}
assert a_tvepisode == movie(a_tvepisode)
a_videogame = {
"title": "Alan Wake",
"original_title": None,
"original_title": "Alan Wake",
"release_year": 2010,
"media_type": "Video Game",
"imdb_id": "tt0466662",
"imdb_score": 82,
"imdb_votes": 7300,
# The data from __NEXT_DATA__ is wrong, the actual values should be:
# "imdb_score": 82,
# "imdb_votes": 7300,
# "genres": {"Action", "Adventure", "Horror"},
"imdb_score": 67, # Wrong value, but correctly parsed from __NEXT_DATA__
"imdb_votes": 11655, # Wrong value, but correctly parsed from __NEXT_DATA__
"genres": {"Comedy", "Crime", "Drama"}, # Wrong value
}
assert a_videogame == movie(a_videogame)
@pytest.mark.asyncio
async def test_load_ratings_page_20240720(monkeypatch):
with bz2.open(fixturesdir / "ratings-ur655321-20240720.html.bz2", "rb") as f:
html = f.read()
soup = bs4.BeautifulSoup(html, "html5lib")
monkeypatch.setattr(imdb, "asoup_from_url", AsyncMock(return_value=soup))
with bz2.open(fixturesdir / "ratings-ur655321-20240720.gql.json.bz2", "rb") as f:
jsonstr = f.read()
async with imdb.asession() as s:
monkeypatch.setattr(s, "post", AsyncMock(return_value=_mock_response(jsonstr)))
page = await imdb._load_ratings_page("fakeurl", "ur655321")
assert len(page.ratings) == 100
assert page.imdb_user_id is not None
assert page.imdb_user_id == "ur655321"
assert page.imdb_user_name == "AlexUltra"
assert page.next_page_url is None, "not supported for new ratings page"
def movie(item: dict):
for rating in page.ratings:
assert rating.movie
if rating.movie.imdb_id == item["imdb_id"]:
rating_dict = {key: getattr(rating.movie, key) for key in item.keys()}
return rating_dict
raise AssertionError(f"{item["imdb_id"]} not found in page.ratings")
a_movie = {
"title": "Kung Fu Panda 4",
"release_year": 2024,
"media_type": "Movie",
"imdb_id": "tt21692408",
"imdb_score": 59,
"imdb_votes": 48018,
"runtime": 94,
}
assert a_movie == movie(a_movie)
a_running_tvseries = {
"title": "Palm Royale",
"release_year": 2024,
"media_type": "TV Series",
"imdb_id": "tt8888540",
"imdb_score": 63,
"imdb_votes": 9458,
}
assert a_running_tvseries == movie(a_running_tvseries)
a_finished_tvseries = {
"title": "Fawlty Towers",
"release_year": 1975,
"media_type": "TV Series",
"imdb_id": "tt0072500",
"imdb_score": 87,
"imdb_votes": 100860,
}
assert a_finished_tvseries == movie(a_finished_tvseries)