Merge pull request #51 from mBaratta96/develop

switched to shelve

letterboxd_stats/web_scraper.py

@@ -4,7 +4,7 @@
 from letterboxd_stats import cli
 import requests
 from lxml import html
-import pickledb
+import shelve
 
 URL = "https://letterboxd.com"
 LOGIN_PAGE = URL + "/user/login.do"
@@ -24,7 +24,6 @@
 }
 
 cache_path = os.path.expanduser(os.path.join(config["root_folder"], "static", "cache.db"))
-tmdb_id_cache = pickledb.load(cache_path, auto_dump=True)
 
 
 class Downloader:
@@ -95,26 +94,38 @@ def create_movie_url(title: str, operation: str):
     return URL + OPERATIONS_URLS[operation](title)
 
 
-def get_tmdb_id(link: str, is_diary: bool):
-    if tmdb_id_cache.exists(link):
-        return tmdb_id_cache.get(link)
+def _get_tmdb_id_from_web(link: str, is_diary: bool):
     res = requests.get(link)
     movie_page = html.fromstring(res.text)
     if is_diary:
         title_link = movie_page.xpath("//span[@class='film-title-wrapper']/a")
         if len(title_link) == 0:
-            return None
+            raise ValueError("No movie link found.")
         movie_link = title_link[0]
         movie_url = URL + movie_link.get("href")
         movie_page = html.fromstring(requests.get(movie_url).text)
     tmdb_link = movie_page.xpath("//a[@data-track-action='TMDb']")
     if len(tmdb_link) == 0:
-        return None
+        raise ValueError("No Movie link found")
     id = tmdb_link[0].get("href").split("/")[-2]
-    tmdb_id_cache.set(link, id)
     return int(id)
 
 
+def get_tmdb_id(link: str, is_diary: bool):
+    tmdb_id_cache = shelve.open(cache_path, writeback=False, protocol=5)
+    if link in tmdb_id_cache:
+        id = tmdb_id_cache[link]
+    else:
+        try:
+            id = _get_tmdb_id_from_web(link, is_diary)
+            tmdb_id_cache[link] = id
+        except ValueError as e:
+            print(e)
+            id = None
+    tmdb_id_cache.close()
+    return id
+
+
 def select_optional_operation():
     return cli.select_value(["Exit"] + list(MOVIE_OPERATIONS.keys()), "Select operation:")
 

pyproject.toml

@@ -4,27 +4,26 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "letterboxd_stats"
-version = "0.2.4"
+version = "0.2.5"
 authors = [{ name = "mBaratta96" }]
 description = "Get information about your Letterboxd activity."
 readme = "README.md"
 license = { file = "LICENSE" }
 requires-python = ">=3.8"
 classifiers = [
-  "Programming Language :: Python :: 3",
-  "License :: OSI Approved :: MIT License",
-  "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
 ]
 dependencies = [
-  "ascii_magic~=2.3.0",
-  "inquirerpy~=0.3.4",
-  "lxml~=4.9.0",
-  "pandas~=1.5.1",
-  "pickleDB~=0.9.2",
-  "platformdirs~=3.0.0",
-  "rich~=13.3.5",
-  "tmdbv3api~=1.7.7",
-  "tomli~=2.0.1",
+    "ascii_magic~=2.3.0",
+    "inquirerpy~=0.3.4",
+    "lxml~=4.9.0",
+    "pandas~=1.5.1",
+    "platformdirs~=3.0.0",
+    "rich~=13.3.5",
+    "tmdbv3api~=1.7.7",
+    "tomli~=2.0.1",
 ]
 
 [project.urls]