diff --git a/letterboxd_stats/web_scraper.py b/letterboxd_stats/web_scraper.py index 93bd48f..57288c9 100644 --- a/letterboxd_stats/web_scraper.py +++ b/letterboxd_stats/web_scraper.py @@ -4,7 +4,7 @@ from letterboxd_stats import cli import requests from lxml import html -import pickledb +import shelve URL = "https://letterboxd.com" LOGIN_PAGE = URL + "/user/login.do" @@ -24,7 +24,6 @@ } cache_path = os.path.expanduser(os.path.join(config["root_folder"], "static", "cache.db")) -tmdb_id_cache = pickledb.load(cache_path, auto_dump=True) class Downloader: @@ -95,26 +94,38 @@ def create_movie_url(title: str, operation: str): return URL + OPERATIONS_URLS[operation](title) -def get_tmdb_id(link: str, is_diary: bool): - if tmdb_id_cache.exists(link): - return tmdb_id_cache.get(link) +def _get_tmdb_id_from_web(link: str, is_diary: bool): res = requests.get(link) movie_page = html.fromstring(res.text) if is_diary: title_link = movie_page.xpath("//span[@class='film-title-wrapper']/a") if len(title_link) == 0: - return None + raise ValueError("No movie link found.") movie_link = title_link[0] movie_url = URL + movie_link.get("href") movie_page = html.fromstring(requests.get(movie_url).text) tmdb_link = movie_page.xpath("//a[@data-track-action='TMDb']") if len(tmdb_link) == 0: - return None + raise ValueError("No Movie link found") id = tmdb_link[0].get("href").split("/")[-2] - tmdb_id_cache.set(link, id) return int(id) +def get_tmdb_id(link: str, is_diary: bool): + tmdb_id_cache = shelve.open(cache_path, writeback=False, protocol=5) + if link in tmdb_id_cache: + id = tmdb_id_cache[link] + else: + try: + id = _get_tmdb_id_from_web(link, is_diary) + tmdb_id_cache[link] = id + except ValueError as e: + print(e) + id = None + tmdb_id_cache.close() + return id + + def select_optional_operation(): return cli.select_value(["Exit"] + list(MOVIE_OPERATIONS.keys()), "Select operation:") diff --git a/pyproject.toml b/pyproject.toml index 8ccc940..5ff1f83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,27 +4,26 @@ build-backend = "setuptools.build_meta" [project] name = "letterboxd_stats" -version = "0.2.4" +version = "0.2.5" authors = [{ name = "mBaratta96" }] description = "Get information about your Letterboxd activity." readme = "README.md" license = { file = "LICENSE" } requires-python = ">=3.8" classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", ] dependencies = [ - "ascii_magic~=2.3.0", - "inquirerpy~=0.3.4", - "lxml~=4.9.0", - "pandas~=1.5.1", - "pickleDB~=0.9.2", - "platformdirs~=3.0.0", - "rich~=13.3.5", - "tmdbv3api~=1.7.7", - "tomli~=2.0.1", + "ascii_magic~=2.3.0", + "inquirerpy~=0.3.4", + "lxml~=4.9.0", + "pandas~=1.5.1", + "platformdirs~=3.0.0", + "rich~=13.3.5", + "tmdbv3api~=1.7.7", + "tomli~=2.0.1", ] [project.urls]