diff --git a/CHANGELOG.md b/CHANGELOG.md index 518d6b6..9b7a7be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.3: + - Series can now also be updated by utilizing TVDB to resolve IMDB ids + - TVDB <=> IMDB resolvement only has to be done once + - Items that fail to be resolved will be blacklisted for 14 days to prevent spamming TVDB on every iteration of the tool + - TVDB authorization is a bit more complex than simply providing an API key, more about the topic is in the README.md + ## 1.2.5: - Removed deprecated legacy CLI interface from project - Deprecated and removed OMDB interfaces and implementations, this tool will no longer use the OMDB API as IMDB provides a rating dataset that is refreshed daily and thus more up to date diff --git a/README.md b/README.md index a901685..df2a0f1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Rating update tool for IMDB ratings in Plex libraries -A tool to update the IMDB ratings for Plex libraries that contain movies. +A tool to update the IMDB ratings for Plex libraries that contain movies and series. ## What does this do? @@ -12,7 +12,7 @@ This tool allows you to update the database that stores this data with the corre An advantage is that it works outside Plex by manipulating the local Plex database. Thus, no metadata refresh operations have to be done within Plex. It is faster and will not lead into the unforeseen consequences that one sometimes experiences with a Plex metadata refresh (missing or changed posters if not using a custom poster). -This tool currently only works on movies and will only allow you to select libraries that use the Plex IMDB agent (because it depends on the IMDB ids). In my library with 1800 movies it transformed entries for 698 items. In case that even tho you use the IMDB agent you still have items that are TMDB matched you can run it with an TMDB API key and it will match an IMDB rating to the TMDB item (if TMDB provides an IMDB id). +This tool currently works on movies/series that use the Plex IMDB agent as source of ratings. For the movies it will match items that use the imdb/tmdb agent. For series, it will use the tvdb to resolve the tvdb <=> imdb relationship (which can fail if the tvdb has no imdb id matched to the item). In my library with 1800 movies it transformed entries for 698 items and 1000+ entries for series. In case that even tho you use the IMDB agent you still have items that are TMDB matched you can run it with an TMDB API key and it will match an IMDB rating to the TMDB item (if TMDB provides an IMDB id). Before (Not IMDB matched) | After Match :-------------------------:|:-------------------------: @@ -43,6 +43,15 @@ docker run -dit -e RUN_EVERY_N_HOURS=12 \ -v "/mnt/data/Plex Media Server":/plexdata \ -v "/mnt/data/imdpupdaterconfig":/config \ mynttt/updatetool + +# With TMDB fallback and TVDB resolvement for series + +docker run -dit -e RUN_EVERY_N_HOURS=12 \ + -e TMDB_API_KEY=yourkey \ + -e TVDB_AUTH_STRING="tvdbusername;tvdbuserid;tvdbapikey" \ + -v "/mnt/data/Plex Media Server":/plexdata \ + -v "/mnt/data/imdpupdaterconfig":/config \ + mynttt/updatetool ``` Explained: @@ -53,6 +62,9 @@ docker run -dit -e RUN_EVERY_N_HOURS=12 \ # Optional parameter: will try to get an IMDB ID from TMDB matched items -e TMDB_API_KEY=yourkey \ + # Three items are required to auth with TVDB username, userkey, apikey + # Supply these as semicolon seperated values. Example: username;DAWIDK9CJKWFJAWKF;e33914feabd52e8192011b0ce6c8 + -e TVDB_AUTH_STRING="tvdbusername;tvdbuserkey;tvdbapikey" \ # The plex data root (that contains Plug-ins, Metadata, ... # https://support.plex.tv/articles/202915258-where-is-the-plex-media-server-data-directory-located/ -v "/mnt/data/Plex Media Server":/plexdata \ @@ -61,6 +73,8 @@ docker run -dit mynttt/updatetool ``` +[TVDB User Key](https://thetvdb.com/dashboard/account/editinfo) - [TVDB API Key](https://thetvdb.com/dashboard/account/apikey) + *"/mnt/data/Plex Media Server" and "/mnt/data/imdpupdaterconfig" are just sample paths! Set your own paths there or it will probably not work!* *On windows the \ syntax to make the command multiline will not work. You have to remove those and make the command a single line command!* @@ -81,7 +95,9 @@ docker run -dit **If the /config folder does not exist yet in appdata unraid will create it! It is important to access logs easily!** -![](img/unraidv2.PNG) +![](img/unraidv3.PNG) + +*TMDB and TVDB are optional settings that are not required for base movie imdb operations! TMDB unlocks matching for movies that have a TMBD match for whatever reason and TVDB allows to update series as well!* 6.) You can now start the container. If it has errors it will stop. The log in the config folder shows you what it does or why it crashed if that happens. @@ -100,9 +116,11 @@ Provides a watchdog that once started will run every N hours over all IMDB suppo # Created files in PWD - cache-tmdb2imdb.json - If TMDB fallback is enabled this file will contain the resolved TMDB <=> IMDB mappings. +- cache-tvdb2imdb.json - TVDB to IMDB mapping. +- cache-tvdbBlacklist.json - Items that TVDB provides no IMDB id for or that fail being looked up. The blacklist is reset every 14 days. - state-imdb.json - Set of jobs that have not finished - xml-error-{uuid}-{library}.log - List of files that could not be updated by the XML transform step (not important tbh, plex reads from the DB) -- updatetool.log - Log file +- updatetool.{increment}.log - Log file - rating_set.tsv - latest IMDB rating set - ratingSetLastUpdate - UNIX timestamp of last rating set update diff --git a/VERSION b/VERSION index 3a1f10e..a58941b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.5 \ No newline at end of file +1.3 \ No newline at end of file diff --git a/build.gradle b/build.gradle index 1c86a87..7846aba 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ plugins { id 'eclipse' } -version = '1.2.5' +version = '1.3' sourceCompatibility = '11' new File(projectDir, "VERSION").text = version; diff --git a/img/unraidv2.PNG b/img/unraidv2.PNG deleted file mode 100644 index 8c661ee..0000000 Binary files a/img/unraidv2.PNG and /dev/null differ diff --git a/img/unraidv3.PNG b/img/unraidv3.PNG new file mode 100644 index 0000000..f3c5b63 Binary files /dev/null and b/img/unraidv3.PNG differ diff --git a/src/main/java/updatetool/Main.java b/src/main/java/updatetool/Main.java index f3ea6b2..709d97e 100644 --- a/src/main/java/updatetool/Main.java +++ b/src/main/java/updatetool/Main.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -11,6 +12,7 @@ import updatetool.api.Implementation; import updatetool.common.AbstractApi; import updatetool.common.TmdbApi; +import updatetool.common.TvdbApi; import updatetool.imdb.ImdbDockerImplementation; public class Main { @@ -41,6 +43,9 @@ public enum Implementations { new String[] { "The following environment variables must be set and exported before launching this tool successfully!", "PLEX_DATA_DIR: Used for the data directory of plex", "(Optional) TMDB_API_KEY: Used to convert TMDB matched items to IMDB items. The fallback will only be available if this is set.", + "(Optional) TVDB_AUTH_STRING: Used to auth with the TVDB API. Must be entered as a ';' seperated string of username, userid, apikey", + "Example: username;DAWIDK9CJKWFJAWKF;e33914feabd52e8192011b0ce6c8", + "", "No parameters starts with the default of {every_n_hour} = 12, {cache_pruge_in_days} = 14 and {new_movie_cache_purge_threshold} = 12", "{every_n_hour} : Invoke this every n hour on all IMDB supported libraries"}); @@ -66,6 +71,8 @@ public static Implementations of(String string) { } public static void main(String[] args) throws Exception { + preLogPurge(); + Thread.setDefaultUncaughtExceptionHandler((t, e) -> { Logger.error("Uncaught " + e.getClass().getSimpleName() + " exception encountered..."); Logger.error("Please contact the maintainer of the application with the stacktrace below if you think this is unwanted behavior."); @@ -100,6 +107,30 @@ public static void main(String[] args) throws Exception { constructor.newInstance().invoke(args); } + public static void rollingLogPurge() throws IOException { + var files = Files.list(Main.PWD) + .filter(p -> p.getFileName().toString().startsWith("updatetool.")).collect(Collectors.toList()); + var keep = files.stream().max((p1, p2) -> { + try { + return Files.getLastModifiedTime(p2).compareTo(Files.getLastModifiedTime(p1)); + } catch (IOException e) {} + return 0; + }); + if(files.size() > 1) { + keep.ifPresent(f -> files.remove(f)); + for(var f : files) + Files.delete(f); + } + } + + private static void preLogPurge() throws IOException { + var files = Files.list(Main.PWD) + .filter(p -> p.getFileName().toString().startsWith("updatetool.")) + .collect(Collectors.toList()); + for(var p : files) + Files.delete(p); + } + public static void printHelp(Implementations i, boolean datahint) { if(datahint) System.out.println("Data folder: https://support.plex.tv/articles/202915258-where-is-the-plex-media-server-data-directory-located"); @@ -128,6 +159,17 @@ public static void testApiTmdb(String apikeyTmdb) throws Exception { var api = new TmdbApi(apikeyTmdb); genericApiTest(api); } + + public static void testApiTvdb(String[] credentials) { + Logger.info("Testing TVDB API authorization: username={} | userkey={} | apikey={}", credentials[0], credentials[1], credentials[2]); + try { + new TvdbApi(credentials); + } catch(IllegalArgumentException e) { + Logger.error("API Test failed: " + e.getMessage()); + Logger.error("Keys available under: https://thetvdb.com/"); + System.exit(-1); + } + } private static void genericApiTest(AbstractApi api) throws Exception { var response = api.testApi(); diff --git a/src/main/java/updatetool/api/Job.java b/src/main/java/updatetool/api/Job.java index c2343e7..4a1d531 100644 --- a/src/main/java/updatetool/api/Job.java +++ b/src/main/java/updatetool/api/Job.java @@ -2,15 +2,19 @@ import java.util.Objects; import updatetool.api.Pipeline.PipelineStage; +import updatetool.common.DatabaseSupport.Library; +import updatetool.common.DatabaseSupport.LibraryType; public abstract class Job { public String library; public String uuid; + public LibraryType libraryType; public PipelineStage stage = PipelineStage.CREATED; - public Job(String library, String uuid) { - this.library = library; - this.uuid = uuid; + public Job(Library library) { + this.library = library.name; + this.uuid = library.uuid; + this.libraryType = library.type; } public abstract String whatKindOfJob(); diff --git a/src/main/java/updatetool/common/AbstractApi.java b/src/main/java/updatetool/common/AbstractApi.java index 3568167..17aea0b 100644 --- a/src/main/java/updatetool/common/AbstractApi.java +++ b/src/main/java/updatetool/common/AbstractApi.java @@ -6,18 +6,15 @@ import java.net.http.HttpClient; import java.net.http.HttpClient.Version; import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; import java.net.http.HttpResponse.BodyHandlers; import java.time.Duration; -import java.util.Objects; public abstract class AbstractApi { - private final String apiKey; private final HttpClient client; - public AbstractApi(String apiKey) { - Objects.requireNonNull(apiKey); - this.apiKey = apiKey; + public AbstractApi() { this.client = HttpClient.newBuilder() .version(Version.HTTP_2) .connectTimeout(Duration.ofMillis(2000)) @@ -36,12 +33,20 @@ protected final HttpRequest get(String url) { throw Utility.rethrow(e); } } + + protected final HttpRequest postJson(String url, String jsonBody) { + try { + return HttpRequest.newBuilder(new URI(url)) + .header("Content-Type", "application/json") + .POST(BodyPublishers.ofString(jsonBody)) + .build(); + } catch(URISyntaxException e) { + throw Utility.rethrow(e); + } + } protected HttpResponse send(HttpRequest request) throws IOException, InterruptedException { return client.send(request, BodyHandlers.ofString()); } - protected final String apikey() { - return apiKey; - } } diff --git a/src/main/java/updatetool/common/DatabaseSupport.java b/src/main/java/updatetool/common/DatabaseSupport.java index 8787199..d95d485 100644 --- a/src/main/java/updatetool/common/DatabaseSupport.java +++ b/src/main/java/updatetool/common/DatabaseSupport.java @@ -12,18 +12,37 @@ public DatabaseSupport(SqliteDatabaseProvider provider) { this.provider = provider; } - public class LibraryItem { + public enum LibraryType { + MOVIE(1), + SERIES(2); + + private int n; + + LibraryType(int n) { + this.n = n; + } + + public static LibraryType of (int n) { + for(var s : values()) + if(s.n == n) return s; + throw new IllegalArgumentException("number not present"); + } + } + + public class Library { + public final LibraryType type; public final long id; public final int items; public final String name; public final String uuid; - private LibraryItem(ResultSet r, SqliteDatabaseProvider p) throws SQLException { + private Library(ResultSet r, SqliteDatabaseProvider p) throws SQLException { try(var handle = p.queryFor("SELECT count(*) FROM media_items WHERE library_section_id = " + r.getLong(1))) { items = handle.result().getInt(1); id = r.getLong(1); name = r.getString(2); uuid = r.getString(3); + type = LibraryType.of(r.getInt(4)); } catch(SQLException e) { throw e; } @@ -35,15 +54,23 @@ public String toString() { } } - public List requestLibraries() { - try(var handle = provider.queryFor("SELECT id, name, uuid FROM library_sections WHERE section_type = 1 AND agent = 'com.plexapp.agents.imdb'")) { - var list = new ArrayList(); + public List requestMovieLibraries() { + return requestLibrary("SELECT id, name, uuid, section_type FROM library_sections WHERE section_type = 1 AND agent = 'com.plexapp.agents.imdb'"); + } + + public List requestSeriesLibraries() { + return requestLibrary("SELECT id, name, uuid, section_type FROM library_sections WHERE section_type = 2 AND agent = 'com.plexapp.agents.thetvdb'"); + } + + private List requestLibrary(String sql) { + try(var handle = provider.queryFor(sql)) { + var list = new ArrayList(); while(handle.result().next()) - list.add(new LibraryItem(handle.result(), provider)); + list.add(new Library(handle.result(), provider)); return list; } catch (SQLException e) { throw Utility.rethrow(e); - } + } } } diff --git a/src/main/java/updatetool/common/TmdbApi.java b/src/main/java/updatetool/common/TmdbApi.java index 7de319d..a0fc837 100644 --- a/src/main/java/updatetool/common/TmdbApi.java +++ b/src/main/java/updatetool/common/TmdbApi.java @@ -4,6 +4,7 @@ import java.net.http.HttpResponse; public class TmdbApi extends AbstractApi { + private final String apiKey; public static class TMDBResponse { public final String imdb_id, title; @@ -15,7 +16,8 @@ public TMDBResponse(String imdb_id, String title) { } public TmdbApi(String apiKey) { - super(apiKey); + super(); + this.apiKey = apiKey; } public HttpResponse tmdbId2imdbId(String tmdbId) throws IOException, InterruptedException { @@ -31,7 +33,7 @@ public HttpResponse testApi() throws IOException, InterruptedException { } private String of(String tmdbId) { - return String.format("https://api.themoviedb.org/3/movie/%s?api_key=%s", tmdbId, apikey()); + return String.format("https://api.themoviedb.org/3/movie/%s?api_key=%s", tmdbId, apiKey); } @Override diff --git a/src/main/java/updatetool/common/TvdbApi.java b/src/main/java/updatetool/common/TvdbApi.java new file mode 100644 index 0000000..18390e2 --- /dev/null +++ b/src/main/java/updatetool/common/TvdbApi.java @@ -0,0 +1,72 @@ +package updatetool.common; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Map; +import com.google.gson.Gson; + +public class TvdbApi extends AbstractApi { + private static final String BASE_URL = "https://api.thetvdb.com"; + private final String authToken; + private final Gson gson = new Gson(); + + public TvdbApi(String[] credentials) throws IllegalArgumentException { + super(); + authToken = "Bearer " + auth(credentials); + } + + private class Token { String token; }; + + private String auth(String[] credentials) { + try { + var response = send( + postJson(BASE_URL + "/login", gson.toJson(Map.of( + "username", credentials[0], + "userkey", credentials[1], + "apikey", credentials[2]) + )) + ); + if(response.statusCode() != 200) + throw new IllegalArgumentException("Code " + response.statusCode() + " | " + response.body()); + return new Gson().fromJson(response.body(), Token.class).token; + } catch (IOException | InterruptedException e) { + throw Utility.rethrow(e); + } + } + + public HttpResponse seriesImdbId(String tvdbId) { + try { + return send(HttpRequest.newBuilder(new URI(String.format("%s/series/%s", BASE_URL, tvdbId))) + .GET() + .header("Authorization", authToken) + .build()); + } catch (IOException | InterruptedException | URISyntaxException e) { + throw Utility.rethrow(e); + } + } + + public HttpResponse episodeImdbId(String[] parts) { + try { + return send(HttpRequest.newBuilder(new URI(String.format("%s/series/%s/episodes/query?airedSeason=%s&airedEpisode=%s", BASE_URL, parts[0], parts[1], parts[2]))) + .GET() + .header("Authorization", authToken) + .build()); + } catch (IOException | InterruptedException | URISyntaxException e) { + throw Utility.rethrow(e); + } + } + + @Override + public HttpResponse testApi() { + throw new UnsupportedOperationException(); + } + + @Override + public String keysWhere() { + throw new UnsupportedOperationException(); + } + +} diff --git a/src/main/java/updatetool/imdb/ImdbDatabaseSupport.java b/src/main/java/updatetool/imdb/ImdbDatabaseSupport.java index 3ae8da3..d9179e4 100644 --- a/src/main/java/updatetool/imdb/ImdbDatabaseSupport.java +++ b/src/main/java/updatetool/imdb/ImdbDatabaseSupport.java @@ -55,7 +55,19 @@ public boolean equals(Object obj) { } public List requestEntries(long libraryId) { - try(var handle = provider.queryFor("SELECT id, library_section_id, guid, title, extra_data, hash, rating from metadata_items WHERE media_item_count = 1 AND library_section_id = " + libraryId)){ + return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating from metadata_items WHERE media_item_count = 1 AND library_section_id = " + libraryId); + } + + public List requestTvSeriesRoot(long libraryId) { + return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating from metadata_items WHERE media_item_count = 0 AND parent_id IS NULL AND library_section_id = " + libraryId); + } + + public List requestTvSeasonRoot(long libraryId) { + return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating from metadata_items WHERE media_item_count = 0 AND parent_id NOT NULL AND library_section_id = " + libraryId); + } + + private List requestMetadata(String query) { + try(var handle = provider.queryFor(query)){ List list = new ArrayList<>(); while(handle.result().next()) list.add(new ImdbMetadataResult(handle.result())); @@ -65,7 +77,6 @@ public List requestEntries(long libraryId) { } } - public long requestLibraryIdOfUuid(String uuid) { try(var handle = provider.queryFor("SELECT id FROM library_sections WHERE uuid = '" + uuid + "';")) { return handle.result().getLong(1); diff --git a/src/main/java/updatetool/imdb/ImdbDockerImplementation.java b/src/main/java/updatetool/imdb/ImdbDockerImplementation.java index fa7ef83..55b0ed3 100644 --- a/src/main/java/updatetool/imdb/ImdbDockerImplementation.java +++ b/src/main/java/updatetool/imdb/ImdbDockerImplementation.java @@ -5,6 +5,7 @@ import java.nio.file.Path; import java.util.ArrayDeque; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -27,11 +28,15 @@ public class ImdbDockerImplementation implements Implementation { public int RUN_EVERY_N_HOUR = 12; private String apikeyTmdb; + + //Format: ;; for ENV + private String[] apiauthTvdb; private Path plexdata; @Override public void invoke(String[] args) throws Exception { apikeyTmdb = System.getenv("TMDB_API_KEY"); + String tvdbAuth = System.getenv("TVDB_AUTH_STRING"); String data = System.getenv("PLEX_DATA_DIR"); Objects.requireNonNull(data, "Environment variable PLEX_DATA_DIR is not set"); @@ -59,12 +64,25 @@ public void invoke(String[] args) throws Exception { System.exit(-1); } - if(apikeyTmdb == null || apikeyTmdb.trim().isEmpty()) { + if(apikeyTmdb == null || apikeyTmdb.isBlank()) { Logger.info("No TMDB API key detected. Will not attempt to do an TMDB <=> IMDB ID conversion to update TMDB matched items (unless already matched previously)."); } else { Main.testApiTmdb(apikeyTmdb); Logger.info("TMDB API key enabled TMDB <=> IMDB matching. Will fetch IMDB ratings for non matched IMDB items."); } + + if(tvdbAuth == null || tvdbAuth.isBlank()) { + Logger.info("No TVDB API authorization string detected. Will not attempt to update IMDB ratings for TV Series with the TVDB agent."); + } else { + String[] info = tvdbAuth.split(";"); + if(info.length == 3) { + Main.testApiTvdb(info); + apiauthTvdb = info; + Logger.info("TVDB API authorization enabled IMDB rating update for TV Series with the TVDB agent."); + } else { + Logger.error("Invalid TVDB API authorization string given. Must contain 3 items seperated by a ';'. Will ignore TV Series with the TVDB agent."); + } + } if(args.length >= 2) { RUN_EVERY_N_HOUR = parseCommandInt(args[1], i -> i > 0, "Invalid parameter for: RUN_EVERY_N_HOUR (must be number and > 0)"); @@ -75,11 +93,25 @@ public void invoke(String[] args) throws Exception { Logger.info("Invoke every " + RUN_EVERY_N_HOUR + " hour(s)"); var state = State.recoverImdb(Main.STATE_IMDB); - var cache = ImdbTmdbCache.of(Main.PWD); + var caches = Map.of("tmdb", KeyValueStore.of(Main.PWD.resolve("cache-tmdb2imdb.json")), + "tvdb", KeyValueStore.of(Main.PWD.resolve("cache-tvdb2imdb.json")), + "tvdb-blacklist", KeyValueStore.of(Main.PWD.resolve("cache-tvdbBlacklist.json"))); + + var tvdbBlacklist = caches.get("tvdb-blacklist"); + String expire = tvdbBlacklist.lookup("__EXPIRE"); + if(expire == null) { + tvdbBlacklist.cache("__EXPIRE", Long.toString(System.currentTimeMillis()+TimeUnit.DAYS.toMillis(14))); + } else { + long l = Long.parseLong(expire); + if(l <= System.currentTimeMillis()) { + tvdbBlacklist.reset(); + tvdbBlacklist.cache("__EXPIRE", Long.toString(System.currentTimeMillis()+TimeUnit.DAYS.toMillis(14))); + } + } Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { - ImdbTmdbCache.dump(Main.PWD, cache); + caches.values().forEach(KeyValueStore::dump); } catch (Exception e) { Logger.error("Failed to save cache."); Logger.error(e); @@ -98,50 +130,56 @@ public void invoke(String[] args) throws Exception { if(!state.isEmpty()) Logger.info("Loaded " + state.size() + " unfinished job(s).\n"); - var config = new ImdbPipelineConfiguration(apikeyTmdb, plexdata.resolve("Metadata/Movies")); + var config = new ImdbPipelineConfiguration(apikeyTmdb, apiauthTvdb, plexdata.resolve("Metadata/Movies")); var scheduler = Executors.newSingleThreadScheduledExecutor(); Logger.info("Running first task..."); - scheduler.schedule(new ImdbBatchJob(config, plexdata, cache, state), 1, TimeUnit.SECONDS); + scheduler.schedule(new ImdbBatchJob(config, plexdata, caches, state), 1, TimeUnit.SECONDS); Logger.info("Scheduling next tasks to run @ every " + RUN_EVERY_N_HOUR + " hour(s)"); - scheduler.scheduleAtFixedRate(new ImdbBatchJob(config, plexdata, cache, state), 1, RUN_EVERY_N_HOUR, TimeUnit.HOURS); + scheduler.scheduleAtFixedRate(new ImdbBatchJob(config, plexdata, caches, state), 1, RUN_EVERY_N_HOUR, TimeUnit.HOURS); } private static class ImdbBatchJob implements Runnable { private final ImdbPipelineConfiguration config; private final ExecutorService service; - private final ImdbTmdbCache cache; + private final Map caches; private final Set state; private final String dbLocation; - public ImdbBatchJob(ImdbPipelineConfiguration config, Path plexdata, ImdbTmdbCache cache, Set state) { + public ImdbBatchJob(ImdbPipelineConfiguration config, Path plexdata, Map caches, Set state) { service = Executors.newFixedThreadPool(6); this.config = config; - this.cache = cache; + this.caches = caches; this.state = state; this.dbLocation = plexdata.resolve("Plug-in Support/Databases/com.plexapp.plugins.library.db").toAbsolutePath().toString(); } @Override public void run() { + try { + Main.rollingLogPurge(); + } catch (IOException e) { e.printStackTrace(); } SqliteDatabaseProvider connection = null; try { connection = new SqliteDatabaseProvider(dbLocation); - var libraries = new DatabaseSupport(connection).requestLibraries(); + var support = new DatabaseSupport(connection); + var libraries = support.requestMovieLibraries(); + if(config.resolveTvdb()) + libraries.addAll(support.requestSeriesLibraries()); var jobs = new ArrayDeque(); var db = new ImdbDatabaseSupport(connection); - var pipeline = new ImdbPipeline(db, service, cache, config, ImdbRatingDatasetFactory.requestSet()); + var pipeline = new ImdbPipeline(db, service, caches, config, ImdbRatingDatasetFactory.requestSet()); var runner = new ImdbJobRunner(); for(var lib : libraries) { - jobs.add(new ImdbJob(lib.name, lib.uuid)); - Logger.info("Library: " + lib.name + " has " + lib.items + " item(s)"); + jobs.add(new ImdbJob(lib)); + Logger.info("[{}] {} has {} item(s)", lib.type, lib.name, lib.items); } while(!jobs.isEmpty()) { var job = jobs.pop(); - Logger.info("Processing library: " + job.library + " with UUID " + job.uuid + " at stage: " + job.stage); + Logger.info("Processing [{}] {} with UUID {} at stage: {}", job.libraryType, job.library, job.uuid, job.stage); var result = runner.run(job, pipeline); Logger.info("Job returned " + result.code + " : " + result.userDefinedMessage); if(result.code == StatusCode.PASS) { - Logger.info("Job finished successfully for library " + job.library + " with UUID " + job.uuid); + Logger.info("Job finished successfully for [{}] {} with UUID {}", job.libraryType, job.library, job.uuid); state.remove(job); } if(result.code == StatusCode.API_ERROR) { @@ -157,7 +195,7 @@ public void run() { throw Utility.rethrow(result.exception); } } - ImdbTmdbCache.dump(Main.PWD, cache); + caches.values().forEach(KeyValueStore::dump); Logger.info("Completed batch successfully. Waiting till next invocation..."); Logger.info("It is now safe to suspend execution if this tool should not run 24/7."); try { connection.close(); } catch (Exception e) {} diff --git a/src/main/java/updatetool/imdb/ImdbJob.java b/src/main/java/updatetool/imdb/ImdbJob.java index 948f676..cdacffe 100644 --- a/src/main/java/updatetool/imdb/ImdbJob.java +++ b/src/main/java/updatetool/imdb/ImdbJob.java @@ -3,13 +3,14 @@ import java.util.ArrayList; import java.util.List; import updatetool.api.Job; +import updatetool.common.DatabaseSupport.Library; import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; public class ImdbJob extends Job { public List items = new ArrayList<>(); - public ImdbJob(String library, String uuid) { - super(library, uuid); + public ImdbJob(Library library) { + super(library); } @Override diff --git a/src/main/java/updatetool/imdb/ImdbPipeline.java b/src/main/java/updatetool/imdb/ImdbPipeline.java index 0614d38..3035d6c 100644 --- a/src/main/java/updatetool/imdb/ImdbPipeline.java +++ b/src/main/java/updatetool/imdb/ImdbPipeline.java @@ -5,6 +5,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -19,8 +20,10 @@ import updatetool.api.AgentResolvementStrategy; import updatetool.api.ExportedRating; import updatetool.api.Pipeline; +import updatetool.common.DatabaseSupport.LibraryType; import updatetool.common.ErrorReports; import updatetool.common.TmdbApi; +import updatetool.common.TvdbApi; import updatetool.common.Utility; import updatetool.exceptions.DatabaseLockedException; import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; @@ -28,11 +31,13 @@ import updatetool.imdb.resolvement.DefaultResolvement; import updatetool.imdb.resolvement.ImdbResolvement; import updatetool.imdb.resolvement.TmdbToImdbResolvement; +import updatetool.imdb.resolvement.TvdbToImdbResolvement; public class ImdbPipeline extends Pipeline { private static final Pattern RESOLVEMENT = Pattern.compile( "(?agents.imdb:\\/\\/tt)" + "|(?agents.themoviedb:\\/\\/)" + + "|(?agents.thetvdb:\\/\\/)" ); private static final int LIST_PARTITIONS = 16; @@ -48,30 +53,42 @@ public class ImdbPipeline extends Pipeline { public static class ImdbPipelineConfiguration { public final String tmdbApiKey; + public final String[] apiauthTvdb; public final Path metadataRoot; - public ImdbPipelineConfiguration(String tmdbApiKey, Path metadataRoot) { + public ImdbPipelineConfiguration(String tmdbApiKey, String[] apiauthTvdb, Path metadataRoot) { this.tmdbApiKey = tmdbApiKey; + this.apiauthTvdb = apiauthTvdb; this.metadataRoot = metadataRoot; } public boolean resolveTmdbConflicts() { return tmdbApiKey != null; } + + public boolean resolveTvdb() { + return apiauthTvdb != null; + } } - public ImdbPipeline(ImdbDatabaseSupport db, ExecutorService service, ImdbTmdbCache cache, ImdbPipelineConfiguration configuration, ImdbRatingDataset dataset) { + public ImdbPipeline(ImdbDatabaseSupport db, ExecutorService service, Map caches, ImdbPipelineConfiguration configuration, ImdbRatingDataset dataset) { this.db = db; this.service = service; this.configuration = configuration; this.dataset = dataset; resolve.put("IMDB", new ImdbResolvement()); - resolve.put("TMDB", configuration.resolveTmdbConflicts() ? new TmdbToImdbResolvement(cache, new TmdbApi(configuration.tmdbApiKey)) : resolveDefault); + resolve.put("TMDB", configuration.resolveTmdbConflicts() ? new TmdbToImdbResolvement(caches.get("tmdb"), new TmdbApi(configuration.tmdbApiKey)) : resolveDefault); + resolve.put("TVDB", configuration.resolveTvdb() ? new TvdbToImdbResolvement(caches.get("tvdb"), caches.get("tvdb-blacklist"), new TvdbApi(configuration.apiauthTvdb)) : resolveDefault); } @Override public void analyseDatabase(ImdbJob job) throws Exception { - var items = db.requestEntries(db.requestLibraryIdOfUuid(job.uuid)); + var lib = db.requestLibraryIdOfUuid(job.uuid); + var items = db.requestEntries(lib); + if(configuration.resolveTvdb() && job.libraryType == LibraryType.SERIES) { + items.addAll(db.requestTvSeriesRoot(lib)); + items.addAll(db.requestTvSeasonRoot(lib)); + } Logger.info("Resolving IMDB identifiers for items. Only warnings and errors will show up..."); Logger.info("Items that show up here will not be processed by further stages of the pipeline."); int skipped = 0; diff --git a/src/main/java/updatetool/imdb/ImdbTmdbCache.java b/src/main/java/updatetool/imdb/ImdbTmdbCache.java deleted file mode 100644 index d2acbee..0000000 --- a/src/main/java/updatetool/imdb/ImdbTmdbCache.java +++ /dev/null @@ -1,42 +0,0 @@ -package updatetool.imdb; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; - -public class ImdbTmdbCache { - private static final String TMDB2IMDB = "cache-tmdb2imdb.json"; - private final HashMap tmdb2imdb = new HashMap<>(); - - @SuppressWarnings("serial") - public static ImdbTmdbCache of(Path p) { - var cache = new ImdbTmdbCache(); - try { - HashMap m = new Gson().fromJson(Files.readString(p.resolve(TMDB2IMDB), StandardCharsets.UTF_8), new TypeToken>() {}.getType()); - cache.tmdb2imdb.putAll(m); - } catch(JsonSyntaxException | IOException e) {} - return cache; - } - - public static void dump(Path p, ImdbTmdbCache data) throws Exception { - Exception ex = null; - try { - Files.writeString(p.resolve(TMDB2IMDB), new Gson().toJson(data.tmdb2imdb), StandardCharsets.UTF_8); - } catch(Exception e) { ex = e; } - if(ex != null) - throw ex; - } - - public String lookupTmdb(String tmdbId) { - return tmdb2imdb.get(tmdbId); - } - - public void cacheTmdb(String tmdbId, String imdbId) { - tmdb2imdb.put(tmdbId, imdbId); - } -} diff --git a/src/main/java/updatetool/imdb/ImdbUtility.java b/src/main/java/updatetool/imdb/ImdbUtility.java index 254aa34..e55b32a 100644 --- a/src/main/java/updatetool/imdb/ImdbUtility.java +++ b/src/main/java/updatetool/imdb/ImdbUtility.java @@ -4,7 +4,8 @@ public class ImdbUtility { public static final Pattern IMDB = Pattern.compile("tt[0-9]*"); - public static final Pattern TMDB = Pattern.compile("[0-9]+"); + public static final Pattern NUMERIC = Pattern.compile("[0-9]+"); + public static final Pattern TVDB = Pattern.compile("[0-9]+(\\/[0-9]+)*"); public static String extractId(Pattern pattern, String guid) { var matcher = pattern.matcher(guid); diff --git a/src/main/java/updatetool/imdb/KeyValueStore.java b/src/main/java/updatetool/imdb/KeyValueStore.java new file mode 100644 index 0000000..0f21a75 --- /dev/null +++ b/src/main/java/updatetool/imdb/KeyValueStore.java @@ -0,0 +1,50 @@ +package updatetool.imdb; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import updatetool.common.Utility; + +public class KeyValueStore { + private final HashMap map = new HashMap<>(); + private final Path p; + + private KeyValueStore(Path p) { + this.p = p; + } + + @SuppressWarnings("serial") + public static KeyValueStore of(Path p) { + var cache = new KeyValueStore(p); + try { + HashMap m = new Gson().fromJson(Files.readString(p, StandardCharsets.UTF_8), new TypeToken>() {}.getType()); + cache.map.putAll(m); + } catch(JsonSyntaxException | IOException e) {} + return cache; + } + + public void dump() { + try { + Files.writeString(p, new Gson().toJson(map), StandardCharsets.UTF_8); + } catch(Exception e) { + throw Utility.rethrow(e); + } + } + + public String lookup(String key) { + return map.get(key); + } + + public void cache(String key, String value) { + map.put(key, value); + } + + public void reset() { + map.clear(); + } +} diff --git a/src/main/java/updatetool/imdb/resolvement/TmdbToImdbResolvement.java b/src/main/java/updatetool/imdb/resolvement/TmdbToImdbResolvement.java index 6bd7c3b..5dbb8c4 100644 --- a/src/main/java/updatetool/imdb/resolvement/TmdbToImdbResolvement.java +++ b/src/main/java/updatetool/imdb/resolvement/TmdbToImdbResolvement.java @@ -6,18 +6,19 @@ import com.google.gson.Gson; import updatetool.api.AgentResolvementStrategy; import updatetool.common.TmdbApi; +import updatetool.common.Utility; import updatetool.common.TmdbApi.TMDBResponse; import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; -import updatetool.imdb.ImdbTmdbCache; import updatetool.imdb.ImdbUtility; +import updatetool.imdb.KeyValueStore; public class TmdbToImdbResolvement implements AgentResolvementStrategy { private static final int MAX_TRIES = 3; private final Gson gson; - private final ImdbTmdbCache cache; + private final KeyValueStore cache; private final TmdbApi api; - public TmdbToImdbResolvement(ImdbTmdbCache cache, TmdbApi api) { + public TmdbToImdbResolvement(KeyValueStore cache, TmdbApi api) { this.cache = cache; this.api = api; this.gson = new Gson(); @@ -25,12 +26,12 @@ public TmdbToImdbResolvement(ImdbTmdbCache cache, TmdbApi api) { @Override public boolean resolve(ImdbMetadataResult toResolve) { - String tmdbId = ImdbUtility.extractId(ImdbUtility.TMDB, toResolve.guid); + String tmdbId = ImdbUtility.extractId(ImdbUtility.NUMERIC, toResolve.guid); if(tmdbId == null) { Logger.error("Item: {} is detected as TMDB but has no id. (guid={})", toResolve.title, toResolve.guid); return false; } - var lookup = cache.lookupTmdb(tmdbId); + var lookup = cache.lookup(tmdbId); if(lookup != null) { toResolve.imdbId = lookup; return true; @@ -40,6 +41,7 @@ public boolean resolve(ImdbMetadataResult toResolve) { public boolean resolveUncached(ImdbMetadataResult toResolve, String tmdbId) { Logger.info("Attempting to resolve TMDB identifer {} against IMDB...", tmdbId); + Exception ex = null; TMDBResponse result = null; HttpResponse response = null; @@ -56,10 +58,13 @@ public boolean resolveUncached(ImdbMetadataResult toResolve, String tmdbId) { } catch(Exception e) { Logger.warn("TMBD API request failed: [" + (i+1) + "/" + MAX_TRIES +"] : " + e.getMessage()); Logger.warn("Dumping response:" + response); - return false; + ex = e; } } - + + if(ex != null) + throw Utility.rethrow(ex); + if(result == null) { Logger.warn("TMDB API failed to deliver a valid response. Dumping last response: {}", response); return false; @@ -70,7 +75,7 @@ public boolean resolveUncached(ImdbMetadataResult toResolve, String tmdbId) { return false; } - cache.cacheTmdb(tmdbId, result.imdb_id); + cache.cache(tmdbId, result.imdb_id); toResolve.imdbId = result.imdb_id; Logger.info("Resolved and cached TMDB {} to IMDB {}.", tmdbId, result.imdb_id); diff --git a/src/main/java/updatetool/imdb/resolvement/TvdbToImdbResolvement.java b/src/main/java/updatetool/imdb/resolvement/TvdbToImdbResolvement.java new file mode 100644 index 0000000..2d12b0f --- /dev/null +++ b/src/main/java/updatetool/imdb/resolvement/TvdbToImdbResolvement.java @@ -0,0 +1,169 @@ +package updatetool.imdb.resolvement; + +import java.net.http.HttpResponse; +import java.util.function.Supplier; +import java.util.regex.Pattern; +import org.tinylog.Logger; +import com.google.gson.Gson; +import updatetool.api.AgentResolvementStrategy; +import updatetool.common.TvdbApi; +import updatetool.common.Utility; +import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; +import updatetool.imdb.ImdbUtility; +import updatetool.imdb.KeyValueStore; + +public class TvdbToImdbResolvement implements AgentResolvementStrategy { + private static final Pattern EPISODE = Pattern.compile("[0-9]+\\/[0-9]+\\/[0-9]+"); + private static final Pattern SEASON = Pattern.compile("[0-9]+\\/[0-9]+"); + private static final Pattern SERIES = Pattern.compile("[0-9]+"); + private static final int MAX_TRIES = 3; + + private final Gson gson; + private final KeyValueStore cache, blacklist; + private final TvdbApi api; + + public TvdbToImdbResolvement(KeyValueStore cache, KeyValueStore blacklist, TvdbApi api) { + this.cache = cache; + this.blacklist = blacklist; + this.api = api; + this.gson = new Gson(); + } + + private class ApiResult { + boolean success; + String message; + + public ApiResult(boolean success, String message) { + this.success = success; + this.message = message; + } + } + + private class Unmarshal { + private class Data { + public String imdbId; + } + private Data data; + } + + private class UnmarshalEpisode { + private class Data { + public String imdbId; + } + private Data[] data; + } + + private class UnmarshalError { + String Error; + } + + @Override + public boolean resolve(ImdbMetadataResult toResolve) { + String tvdbId = ImdbUtility.extractId(ImdbUtility.TVDB, toResolve.guid); + if(tvdbId == null) { + Logger.error("Item: {} is detected as TVDB but has no id. (guid={})", toResolve.title, toResolve.guid); + return false; + } + var lookup = cache.lookup(tvdbId); + if(lookup != null) { + toResolve.imdbId = lookup; + return true; + } + if(blacklist.lookup(tvdbId) != null) + return false; + return resolveUncached(toResolve, tvdbId, categorize(tvdbId)); + } + + private int categorize(String tvdbId) { + if(EPISODE.matcher(tvdbId).find()) + return 2; + if(SEASON.matcher(tvdbId).find()) + return 1; + if(SERIES.matcher(tvdbId).find()) + return 0; + throw new IllegalArgumentException("This should never happen! Input was: " + tvdbId); + } + + private boolean resolveUncached(ImdbMetadataResult toResolve, String tvdbId, int category) { + switch(category) { + case 0: + return resolveSeries(tvdbId, toResolve); + case 1: + return resolveSeason(tvdbId, toResolve); + case 2: + return resolveEpisode(tvdbId, toResolve); + default: + throw new UnsupportedOperationException(); + } + } + + private boolean resolveSeries(String tvdbId, ImdbMetadataResult toResolve) { + String[] parts = tvdbId.split("/"); + var response = query(() -> api.seriesImdbId(parts[0]), tvdbId); + if(!response.success) { + Logger.warn("TVDB API failed for {}: {}", tvdbId, response.message); + return false; + } + var data = gson.fromJson(response.message, Unmarshal.class).data; + if(data == null || data.imdbId == null || data.imdbId.isBlank()) { + Logger.warn("TMDB item {} with id {} does not have an IMDB id associated.", toResolve.title, tvdbId); + blacklist.cache(tvdbId, "x"); + return false; + } + cache.cache(tvdbId, data.imdbId); + toResolve.imdbId = data.imdbId; + return true; + } + + private boolean resolveSeason(String tvdbId, ImdbMetadataResult toResolve) { + // If ever added to IMDB could be implemented here + return false; + } + + private boolean resolveEpisode(String tvdbId, ImdbMetadataResult toResolve) { + String parts[] = tvdbId.split("/"); + var response = query(() -> api.episodeImdbId(parts), tvdbId); + if(!response.success) { + Logger.warn("TVDB API failed for {}: {}", tvdbId, response.message); + return false; + } + var data = gson.fromJson(response.message, UnmarshalEpisode.class).data; + if(data == null || data[0] == null || data[0].imdbId == null || data[0].imdbId.isBlank()) { + Logger.warn("TMDB item {} with id {} does not have an IMDB id associated.", toResolve.title, tvdbId); + blacklist.cache(tvdbId, "x"); + return false; + } + cache.cache(tvdbId, data[0].imdbId); + toResolve.imdbId = data[0].imdbId; + return true; + } + + private ApiResult query(Supplier> supplier, String tvdbId) { + HttpResponse response = null; + Exception ex = null; + + for(int i = 0; i < MAX_TRIES; i++) { + try { + response = supplier.get(); + if(response.statusCode() == 200) + return new ApiResult(true, response.body()); + if(response.statusCode() == 404) { + blacklist.cache(tvdbId, "x"); + return new ApiResult(false, response.body()); + } + Logger.warn("TVDB API returned a reply with status code != 200. Trying again... {}/{}", i+1, MAX_TRIES); + } catch(Exception e) { + ex = e; + Logger.warn("TVDB API request failed: [" + (i+1) + "/" + MAX_TRIES +"] : " + e.getMessage()); + Logger.warn("Dumping response:" + response); + } + } + + if(ex != null) + throw Utility.rethrow(ex); + + var error = gson.fromJson(response.body(), UnmarshalError.class); + String msg = error.Error == null || error.Error.isBlank() ? "Code " + response.statusCode() : "Code " + response.statusCode() + " | " + error.Error; + return new ApiResult(false, msg); + } +} diff --git a/src/main/resources/VERSION b/src/main/resources/VERSION index 3a1f10e..a58941b 100644 --- a/src/main/resources/VERSION +++ b/src/main/resources/VERSION @@ -1 +1 @@ -1.2.5 \ No newline at end of file +1.3 \ No newline at end of file diff --git a/src/main/resources/tinylog.properties b/src/main/resources/tinylog.properties index c74af70..75dd39c 100644 --- a/src/main/resources/tinylog.properties +++ b/src/main/resources/tinylog.properties @@ -1,10 +1,12 @@ -writer = console -writer.level = debug -writer.format = {[{level}|min-size=6}] - {date} @ {class-name}.{method}: {message} -writer.charset = UTF-8 +writer = console +writer.level = debug +writer.format = {[{level}|min-size=6}] - {date} @ {class-name}.{method}: {message} +writer.charset = UTF-8 -writer2 = file -writer2.file = updatetool.log -writer2.level = debug -writer2.format = {[{level}|min-size=6}] - {date} @ {class-name}.{method}: {message} -writer2.charset = UTF-8 \ No newline at end of file +writer2 = rolling file +writer2.file = updatetool.{count}.log +writer2.backups = 1 +writer2.level = debug +writer2.format = {[{level}|min-size=6}] - {date} @ {class-name}.{method}: {message} +writer2.charset = UTF-8 +writer2.policies = startup, size: 10mb \ No newline at end of file