From 3a7f56176fa27ba31dd09bd63fc0892f161dd259 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sun, 13 Mar 2022 15:39:34 -0400 Subject: [PATCH] [5] cache tmdb movies/shows --- VERSION | 2 +- modules/builder.py | 12 ++-- modules/cache.py | 150 +++++++++++++++++++++++++++++++++++++++++++ modules/config.py | 3 +- modules/meta.py | 24 +++---- modules/tmdb.py | 101 +++++++++++++++++++++++++++-- plex_meta_manager.py | 6 +- 7 files changed, 267 insertions(+), 31 deletions(-) diff --git a/VERSION b/VERSION index 11e9c31f..bc2cbf69 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.16.1-develop4 +1.16.1-develop5 diff --git a/modules/builder.py b/modules/builder.py index c4486167..bdcab5da 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1898,9 +1898,9 @@ class CollectionBuilder: try: if item is None: if is_movie: - item = self.config.TMDb.get_movie(item_id, partial="keywords") + item = self.config.TMDb.get_movie(item_id) else: - item = self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(item_id), partial="keywords") + item = self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(item_id)) if check_released: date_to_check = item.release_date if is_movie else item.first_air_date if not date_to_check or date_to_check > self.current_time: @@ -1913,7 +1913,7 @@ class CollectionBuilder: elif filter_attr == "tmdb_type": check_value = discover_types[item.type] elif filter_attr == "original_language": - check_value = item.original_language.iso_639_1 + check_value = item.language_iso else: raise Failed if (modifier == ".not" and check_value in filter_data) or (modifier == "" and check_value not in filter_data): @@ -1936,11 +1936,11 @@ class CollectionBuilder: return False elif filter_attr in ["tmdb_genre", "tmdb_keyword", "origin_country"]: if filter_attr == "tmdb_genre": - attrs = [g.name for g in item.genres] + attrs = item.genres elif filter_attr == "tmdb_keyword": - attrs = [k.name for k in item.keywords] + attrs = item.keywords elif filter_attr == "origin_country": - attrs = [c.iso_3166_1 for c in item.origin_countries] + attrs = [c.iso_3166_1 for c in item.countries] else: raise Failed if (not list(set(filter_data) & set(attrs)) and modifier == "") \ diff --git a/modules/cache.py b/modules/cache.py index f16d9387..6666e828 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -109,6 +109,56 @@ class Cache: certification TEXT, expiration_date TEXT)""" ) + cursor.execute( + """CREATE TABLE IF NOT EXISTS tmdb_movie_data ( + key INTEGER PRIMARY KEY, + tmdb_id INTEGER UNIQUE, + title TEXT, + original_title TEXT, + studio TEXT, + overview TEXT, + tagline TEXT, + imdb_id TEXT, + poster_url TEXT, + backdrop_url TEXT, + vote_count INTEGER, + vote_average REAL, + language_iso TEXT, + language_name TEXT, + genres TEXT, + keywords TEXT, + release_date TEXT, + collection_id INTEGER, + collection_name TEXT, + expiration_date TEXT)""" + ) + cursor.execute( + """CREATE TABLE IF NOT EXISTS tmdb_show_data ( + key INTEGER PRIMARY KEY, + tmdb_id INTEGER UNIQUE, + title TEXT, + original_title TEXT, + studio TEXT, + overview TEXT, + tagline TEXT, + imdb_id TEXT, + poster_url TEXT, + backdrop_url TEXT, + vote_count INTEGER, + vote_average REAL, + language_iso TEXT, + language_name TEXT, + genres TEXT, + keywords TEXT, + first_air_date TEXT, + last_air_date TEXT, + status TEXT, + type TEXT, + tvdb_id INTEGER, + countries TEXT, + seasons TEXT, + expiration_date TEXT)""" + ) cursor.execute( """CREATE TABLE IF NOT EXISTS anime_map ( key INTEGER PRIMARY KEY, @@ -361,6 +411,106 @@ class Cache: mdb.commonsense, expiration_date.strftime("%Y-%m-%d"), key_id )) + def query_tmdb_movie(self, tmdb_id, expiration): + tmdb_dict = {} + expired = None + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("SELECT * FROM tmdb_movie_data WHERE tmdb_id = ?", (tmdb_id,)) + row = cursor.fetchone() + if row: + tmdb_dict["title"] = row["title"] if row["title"] else "" + tmdb_dict["original_title"] = row["original_title"] if row["original_title"] else "" + tmdb_dict["studio"] = row["studio"] if row["studio"] else "" + tmdb_dict["overview"] = row["overview"] if row["overview"] else "" + tmdb_dict["tagline"] = row["tagline"] if row["tagline"] else "" + tmdb_dict["imdb_id"] = row["imdb_id"] if row["imdb_id"] else "" + tmdb_dict["poster_url"] = row["poster_url"] if row["poster_url"] else "" + tmdb_dict["backdrop_url"] = row["backdrop_url"] if row["backdrop_url"] else "" + tmdb_dict["vote_count"] = row["vote_count"] if row["vote_count"] else 0 + tmdb_dict["vote_average"] = row["vote_average"] if row["vote_average"] else 0 + tmdb_dict["language_iso"] = row["language_iso"] if row["language_iso"] else None + tmdb_dict["language_name"] = row["language_name"] if row["language_name"] else None + tmdb_dict["genres"] = row["genres"] if row["genres"] else "" + tmdb_dict["keywords"] = row["keywords"] if row["keywords"] else "" + tmdb_dict["release_date"] = datetime.strptime(row["release_date"], "%Y-%m-%d") if row["release_date"] else None + tmdb_dict["collection_id"] = row["collection_id"] if row["collection_id"] else None + tmdb_dict["collection_name"] = row["collection_name"] if row["collection_name"] else None + datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d") + time_between_insertion = datetime.now() - datetime_object + expired = time_between_insertion.days > expiration + return tmdb_dict, expired + + def update_tmdb_movie(self, expired, obj, expiration): + expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, expiration))) + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("INSERT OR IGNORE INTO tmdb_movie_data(tmdb_id) VALUES(?)", (obj.tmdb_id,)) + update_sql = "UPDATE tmdb_movie_data SET title = ?, original_title = ?, studio = ?, overview = ?, tagline = ?, imdb_id = ?, " \ + "poster_url = ?, backdrop_url = ?, vote_count = ?, vote_average = ?, language_iso = ?, " \ + "language_name = ?, genres = ?, keywords = ?, release_date = ?, collection_id = ?, " \ + "collection_name = ?, expiration_date = ? WHERE tmdb_id = ?" + cursor.execute(update_sql, ( + obj.title, obj.original_title, obj.studio, obj.overview, obj.tagline, obj.imdb_id, obj.poster_url, obj.backdrop_url, + obj.vote_count, obj.vote_average, obj.language_iso, obj.language_name, "|".join(obj.genres), "|".join(obj.keywords), + obj.release_date.strftime("%Y-%m-%d") if obj.release_date else None, obj.collection_id, obj.collection_name, + expiration_date.strftime("%Y-%m-%d"), obj.tmdb_id + )) + + def query_tmdb_show(self, tmdb_id, expiration): + tmdb_dict = {} + expired = None + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("SELECT * FROM tmdb_show_data WHERE tmdb_id = ?", (tmdb_id,)) + row = cursor.fetchone() + if row: + tmdb_dict["title"] = row["title"] if row["title"] else "" + tmdb_dict["original_title"] = row["original_title"] if row["original_title"] else "" + tmdb_dict["studio"] = row["studio"] if row["studio"] else "" + tmdb_dict["overview"] = row["overview"] if row["overview"] else "" + tmdb_dict["tagline"] = row["tagline"] if row["tagline"] else "" + tmdb_dict["imdb_id"] = row["imdb_id"] if row["imdb_id"] else "" + tmdb_dict["poster_url"] = row["poster_url"] if row["poster_url"] else "" + tmdb_dict["backdrop_url"] = row["backdrop_url"] if row["backdrop_url"] else "" + tmdb_dict["vote_count"] = row["vote_count"] if row["vote_count"] else 0 + tmdb_dict["vote_average"] = row["vote_average"] if row["vote_average"] else 0 + tmdb_dict["language_iso"] = row["language_iso"] if row["language_iso"] else None + tmdb_dict["language_name"] = row["language_name"] if row["language_name"] else None + tmdb_dict["genres"] = row["genres"] if row["genres"] else "" + tmdb_dict["keywords"] = row["keywords"] if row["keywords"] else "" + tmdb_dict["first_air_date"] = datetime.strptime(row["first_air_date"], "%Y-%m-%d") if row["first_air_date"] else None + tmdb_dict["last_air_date"] = datetime.strptime(row["last_air_date"], "%Y-%m-%d") if row["last_air_date"] else None + tmdb_dict["status"] = row["status"] if row["status"] else None + tmdb_dict["type"] = row["type"] if row["type"] else None + tmdb_dict["tvdb_id"] = row["tvdb_id"] if row["tvdb_id"] else None + datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d") + time_between_insertion = datetime.now() - datetime_object + expired = time_between_insertion.days > expiration + return tmdb_dict, expired + + def update_tmdb_show(self, expired, obj, expiration): + expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, expiration))) + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("INSERT OR IGNORE INTO tmdb_show_data(tmdb_id) VALUES(?)", (obj.tmdb_id,)) + update_sql = "UPDATE tmdb_show_data SET title = ?, original_title = ?, studio = ?, overview = ?, tagline = ?, imdb_id = ?, " \ + "poster_url = ?, backdrop_url = ?, vote_count = ?, vote_average = ?, language_iso = ?, " \ + "language_name = ?, genres = ?, keywords = ?, first_air_date = ?, last_air_date = ?, status = ?, " \ + "type = ?, tvdb_id = ?, countries = ?, seasons = ?, expiration_date = ? WHERE tmdb_id = ?" + cursor.execute(update_sql, ( + obj.title, obj.original_title, obj.studio, obj.overview, obj.tagline, obj.imdb_id, obj.poster_url, obj.backdrop_url, + obj.vote_count, obj.vote_average, obj.language_iso, obj.language_name, "|".join(obj.genres), "|".join(obj.keywords), + obj.first_air_date.strftime("%Y-%m-%d") if obj.first_air_date else None, + obj.last_air_date.strftime("%Y-%m-%d") if obj.last_air_date else None, + obj.status, obj.type, obj.tvdb_id, "|".join(obj.countries), "|".join(obj.seasons), + expiration_date.strftime("%Y-%m-%d"), obj.tmdb_id + )) + def query_anime_map(self, anime_id, id_type): ids = None expired = None diff --git a/modules/config.py b/modules/config.py index 6f2ad3da..566ed5fa 100644 --- a/modules/config.py +++ b/modules/config.py @@ -366,7 +366,8 @@ class ConfigFile: logger.info("Connecting to TMDb...") self.TMDb = TMDb(self, { "apikey": check_for_attribute(self.data, "apikey", parent="tmdb", throw=True), - "language": check_for_attribute(self.data, "language", parent="tmdb", default="en") + "language": check_for_attribute(self.data, "language", parent="tmdb", default="en"), + "expiration": check_for_attribute(self.data, "cache_expiration", parent="tmdb", var_type="int", default=60) }) logger.info(f"TMDb Connection {'Failed' if self.TMDb is None else 'Successful'}") else: diff --git a/modules/meta.py b/modules/meta.py index b5d0c273..3c87b648 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -303,8 +303,8 @@ class MetadataFile(DataFile): logger.ghost(f"Processing: {i}/{len(all_items)} {item.title}") tmdb_id, tvdb_id, imdb_id = library.get_ids(item) tmdb_item = config.TMDb.get_item(item, tmdb_id, tvdb_id, imdb_id, is_movie=True) - if tmdb_item and tmdb_item.collection and tmdb_item.collection.id not in exclude and tmdb_item.collection.name not in exclude: - auto_list[str(tmdb_item.collection.id)] = tmdb_item.collection.name + if tmdb_item and tmdb_item.collection_id and tmdb_item.collection_id not in exclude and tmdb_item.collection_name not in exclude: + auto_list[str(tmdb_item.collection_id)] = tmdb_item.collection_name logger.exorcise() elif auto_type == "original_language": if not all_items: @@ -313,8 +313,8 @@ class MetadataFile(DataFile): logger.ghost(f"Processing: {i}/{len(all_items)} {item.title}") tmdb_id, tvdb_id, imdb_id = library.get_ids(item) tmdb_item = config.TMDb.get_item(item, tmdb_id, tvdb_id, imdb_id, is_movie=library.type == "Movie") - if tmdb_item and tmdb_item.original_language and tmdb_item.original_language.iso_639_1 not in exclude and tmdb_item.original_language.english_name not in exclude: - auto_list[tmdb_item.original_language.iso_639_1] = tmdb_item.original_language.english_name + if tmdb_item and tmdb_item.language_iso and tmdb_item.language_iso not in exclude and tmdb_item.language_name not in exclude: + auto_list[tmdb_item.language_iso] = tmdb_item.language_name logger.exorcise() default_title_format = "<> <>s" elif auto_type == "origin_country": @@ -324,8 +324,8 @@ class MetadataFile(DataFile): logger.ghost(f"Processing: {i}/{len(all_items)} {item.title}") tmdb_id, tvdb_id, imdb_id = library.get_ids(item) tmdb_item = config.TMDb.get_item(item, tmdb_id, tvdb_id, imdb_id, is_movie=library.type == "Movie") - if tmdb_item and tmdb_item.origin_countries: - for country in tmdb_item.origin_countries: + if tmdb_item and tmdb_item.countries: + for country in tmdb_item.countries: if country.iso_3166_1 not in exclude and country.name not in exclude: auto_list[country.iso_3166_1] = country.name logger.exorcise() @@ -662,18 +662,14 @@ class MetadataFile(DataFile): genres = [] if tmdb_item: originally_available = datetime.strftime(tmdb_item.release_date if tmdb_is_movie else tmdb_item.first_air_date, "%Y-%m-%d") - if tmdb_is_movie and tmdb_item.original_title != tmdb_item.title: + + if tmdb_item.original_title != tmdb_item.title: original_title = tmdb_item.original_title - elif not tmdb_is_movie and tmdb_item.original_name != tmdb_item.name: - original_title = tmdb_item.original_name rating = tmdb_item.vote_average - if tmdb_is_movie and tmdb_item.companies: - studio = tmdb_item.companies[0].name - elif not tmdb_is_movie and tmdb_item.networks: - studio = tmdb_item.networks[0].name + studio = tmdb_item.studio tagline = tmdb_item.tagline if len(tmdb_item.tagline) > 0 else None summary = tmdb_item.overview - genres = [genre.name for genre in tmdb_item.genres] + genres = tmdb_item.genres edits = {} add_edit("title", item, meta, methods, value=title) diff --git a/modules/tmdb.py b/modules/tmdb.py index 34e99e43..2544fc76 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -57,11 +57,100 @@ discover_movie_sort = [ discover_tv_sort = ["vote_average.desc", "vote_average.asc", "first_air_date.desc", "first_air_date.asc", "popularity.desc", "popularity.asc"] discover_monetization_types = ["flatrate", "free", "ads", "rent", "buy"] + +class TMDbCountry: + def __init__(self, data): + self.iso_3166_1 = data.split(":")[0] if isinstance(data, str) else data.iso_3166_1 + self.name = data.split(":")[1] if isinstance(data, str) else data.name + + def __repr__(self): + return f"{self.iso_3166_1}:{self.name}" + + +class TMDbSeason: + def __init__(self, data): + self.season_number = data.split(":")[0] if isinstance(data, str) else data.season_number + self.name = data.split(":")[1] if isinstance(data, str) else data.name + + def __repr__(self): + return f"{self.season_number}:{self.name}" + + +class TMDBObj: + def __init__(self, tmdb, tmdb_id, ignore_cache=False): + self._tmdb = tmdb + self.tmdb_id = tmdb_id + self.ignore_cache = ignore_cache + + def _load(self, data): + self.title = data["title"] if isinstance(data, dict) else data.title + self.tagline = data["tagline"] if isinstance(data, dict) else data.tagline + self.overview = data["overview"] if isinstance(data, dict) else data.overview + self.imdb_id = data["imdb_id"] if isinstance(data, dict) else data.imdb_id + self.poster_url = data["poster_url"] if isinstance(data, dict) else data.poster_url + self.backdrop_url = data["backdrop_url"] if isinstance(data, dict) else data.backdrop_url + self.vote_count = data["vote_count"] if isinstance(data, dict) else data.vote_count + self.vote_average = data["vote_average"] if isinstance(data, dict) else data.vote_average + self.language_iso = data["language_iso"] if isinstance(data, dict) else data.original_language.iso_639_1 if data.original_language else None + self.language_name = data["language_name"] if isinstance(data, dict) else data.original_language.english_name if data.original_language else None + self.genres = data["genres"].split("|") if isinstance(data, dict) else [g.name for g in data.genres] + self.keywords = data["keywords"].split("|") if isinstance(data, dict) else [g.name for g in data.keywords] + + +class TMDbMovie(TMDBObj): + def __init__(self, tmdb, tmdb_id, ignore_cache=False): + super().__init__(tmdb, tmdb_id, ignore_cache=ignore_cache) + expired = None + data = None + if self._tmdb.config.Cache and not ignore_cache: + data, expired = self._tmdb.config.Cache.query_tmdb_movie(tmdb_id, self._tmdb.expiration) + if expired or not data: + data = self._tmdb.TMDb.movie(self.tmdb_id, partial="external_ids,keywords") + super()._load(data) + + self.original_title = data["original_title"] if isinstance(data, dict) else data.original_title + self.release_date = data["release_date"] if isinstance(data, dict) else data.release_date + self.studio = data["studio"] if isinstance(data, dict) else data.companies[0].name + self.collection_id = data["collection_id"] if isinstance(data, dict) else data.collection.id if data.collection else None + self.collection_name = data["collection_name"] if isinstance(data, dict) else data.collection.name if data.collection else None + + if self._tmdb.config.Cache and not ignore_cache: + self._tmdb.config.Cache.update_tmdb_movie(expired, self, self._tmdb.expiration) + + +class TMDbShow(TMDBObj): + def __init__(self, tmdb, tmdb_id, ignore_cache=False): + super().__init__(tmdb, tmdb_id, ignore_cache=ignore_cache) + expired = None + data = None + if self._tmdb.config.Cache and not ignore_cache: + data, expired = self._tmdb.config.Cache.query_tmdb_show(tmdb_id, self._tmdb.expiration) + if expired or not data: + data = self._tmdb.TMDb.tv_show(self.tmdb_id, partial="external_ids,keywords") + super()._load(data) + + self.original_title = data["original_title"] if isinstance(data, dict) else data.original_name + self.first_air_date = data["first_air_date"] if isinstance(data, dict) else data.first_air_date + self.last_air_date = data["last_air_date"] if isinstance(data, dict) else data.last_air_date + self.status = data["status"] if isinstance(data, dict) else data.status + self.type = data["type"] if isinstance(data, dict) else data.type + self.studio = data["studio"] if isinstance(data, dict) else data.networks[0].name + self.tvdb_id = data["tvdb_id"] if isinstance(data, dict) else data.tvdb_id + loop = data["countries"].split("|") if isinstance(data, dict) else data.origin_countries + self.countries = [TMDbCountry(c) for c in loop] + loop = data["seasons"].split("|") if isinstance(data, dict) else data.seasons + self.seasons = [TMDbSeason(s) for s in loop] + + if self._tmdb.config.Cache and not ignore_cache: + self._tmdb.config.Cache.update_tmdb_show(expired, self, self._tmdb.expiration) + + class TMDb: def __init__(self, config, params): self.config = config self.apikey = params["apikey"] self.language = params["language"] + self.expiration = params["expiration"] logger.secret(self.apikey) try: self.TMDb = TMDbAPIs(self.apikey, language=self.language, session=self.config.session) @@ -69,7 +158,7 @@ class TMDb: raise Failed(f"TMDb Error: {e}") def convert_from(self, tmdb_id, convert_to, is_movie): - item = self.get_movie(tmdb_id, partial="external_ids") if is_movie else self.get_show(tmdb_id, partial="external_ids") + item = self.get_movie(tmdb_id) if is_movie else self.get_show(tmdb_id) check_id = item.tvdb_id if convert_to == "tvdb_id" and not is_movie else item.imdb_id if not check_id: raise Failed(f"TMDb Error: No {convert_to.upper().replace('B_', 'b ')} found for TMDb ID {tmdb_id}") @@ -106,12 +195,12 @@ class TMDb: except Failed: raise Failed(f"TMDb Error: No Movie or Collection found for TMDb ID {tmdb_id}") else: return self.get_show(tmdb_id) - def get_movie(self, tmdb_id, partial=None): - try: return self.TMDb.movie(tmdb_id, partial=partial) + def get_movie(self, tmdb_id): + try: return TMDbMovie(self, tmdb_id) except TMDbException as e: raise Failed(f"TMDb Error: No Movie found for TMDb ID {tmdb_id}: {e}") - def get_show(self, tmdb_id, partial=None): - try: return self.TMDb.tv_show(tmdb_id, partial=partial) + def get_show(self, tmdb_id): + try: return TMDbShow(self, tmdb_id) except TMDbException as e: raise Failed(f"TMDb Error: No Show found for TMDb ID {tmdb_id}: {e}") def get_collection(self, tmdb_id, partial=None): @@ -219,7 +308,7 @@ class TMDb: tmdb_name = collection.name ids = [(t.id, "tmdb") for t in collection.movies] elif method == "tmdb_show": - tmdb_name = self.get_show(tmdb_id).name + tmdb_name = self.get_show(tmdb_id).title ids.append((tmdb_id, "tmdb_show")) else: person = self.get_person(tmdb_id, partial="movie_credits,tv_credits") diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 2b183f91..93a4199a 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -533,8 +533,8 @@ def library_operations(config, library): else: logger.info(f"{item.title[:25]:<25} | No IMDb ID for Guid: {item.guid}") - if library.tmdb_collections and tmdb_item and tmdb_item.collection: - tmdb_collections[tmdb_item.collection.id] = tmdb_item.collection.name + if library.tmdb_collections and tmdb_item and tmdb_item.collection_id: + tmdb_collections[tmdb_item.collection_id] = tmdb_item.collection_name def get_rating(attribute): if tmdb_item and attribute == "tmdb": @@ -565,7 +565,7 @@ def library_operations(config, library): if library.mass_genre_update: try: if tmdb_item and library.mass_genre_update == "tmdb": - new_genres = [genre.name for genre in tmdb_item.genres] + new_genres = tmdb_item.genres elif omdb_item and library.mass_genre_update == "omdb": new_genres = omdb_item.genres elif tvdb_item and library.mass_genre_update == "tvdb":