From e391a2849fc0e4db845f7bbddc475ba87ddf5230 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 22 Apr 2021 14:03:27 -0400 Subject: [PATCH] #171 Cache Anime IDs --- modules/cache.py | 66 ++++++++++++++++++++++++++++++++++------------- modules/config.py | 39 +++++++++++++++++++++------- 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/modules/cache.py b/modules/cache.py index bfdbd159..f8cdbedc 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -24,7 +24,6 @@ class Cache: imdb_id TEXT, tvdb_id TEXT, anidb_id TEXT, - mal_id TEXT, expiration_date TEXT, media_type TEXT)""" ) @@ -57,6 +56,15 @@ class Cache: type TEXT, expiration_date TEXT)""" ) + cursor.execute( + """CREATE TABLE IF NOT EXISTS anime_map ( + INTEGER PRIMARY KEY, + anidb TEXT UNIQUE, + anilist TEXT, + myanimelist TEXT, + kitsu TEXT, + expiration_date TEXT)""" + ) self.expiration = expiration self.cache_path = cache @@ -65,28 +73,24 @@ class Cache: tvdb_id, tvdb_expired = self.get_tvdb_id("show", imdb_id=imdb_id) return tmdb_id, tvdb_id - def get_tmdb_id(self, media_type, plex_guid=None, imdb_id=None, tvdb_id=None, anidb_id=None, mal_id=None): - return self.get_id_from(media_type, "tmdb_id", plex_guid=plex_guid, imdb_id=imdb_id, tvdb_id=tvdb_id, anidb_id=anidb_id, mal_id=mal_id) - - def get_imdb_id(self, media_type, plex_guid=None, tmdb_id=None, tvdb_id=None, anidb_id=None, mal_id=None): - return self.get_id_from(media_type, "imdb_id", plex_guid=plex_guid, tmdb_id=tmdb_id, tvdb_id=tvdb_id, anidb_id=anidb_id, mal_id=mal_id) + def get_tmdb_id(self, media_type, plex_guid=None, imdb_id=None, tvdb_id=None, anidb_id=None): + return self.get_id_from(media_type, "tmdb_id", plex_guid=plex_guid, imdb_id=imdb_id, tvdb_id=tvdb_id, anidb_id=anidb_id) - def get_tvdb_id(self, media_type, plex_guid=None, tmdb_id=None, imdb_id=None, anidb_id=None, mal_id=None): - return self.get_id_from(media_type, "tvdb_id", plex_guid=plex_guid, tmdb_id=tmdb_id, imdb_id=imdb_id, anidb_id=anidb_id, mal_id=mal_id) + def get_imdb_id(self, media_type, plex_guid=None, tmdb_id=None, tvdb_id=None, anidb_id=None): + return self.get_id_from(media_type, "imdb_id", plex_guid=plex_guid, tmdb_id=tmdb_id, tvdb_id=tvdb_id, anidb_id=anidb_id) - def get_anidb_id(self, media_type, plex_guid=None, tmdb_id=None, imdb_id=None, tvdb_id=None, mal_id=None): - return self.get_id_from(media_type, "anidb_id", plex_guid=plex_guid, tmdb_id=tmdb_id, imdb_id=imdb_id, tvdb_id=tvdb_id, mal_id=mal_id) + def get_tvdb_id(self, media_type, plex_guid=None, tmdb_id=None, imdb_id=None, anidb_id=None): + return self.get_id_from(media_type, "tvdb_id", plex_guid=plex_guid, tmdb_id=tmdb_id, imdb_id=imdb_id, anidb_id=anidb_id) - def get_mal_id(self, media_type, plex_guid=None, tmdb_id=None, imdb_id=None, tvdb_id=None, anidb_id=None): - return self.get_id_from(media_type, "anidb_id", plex_guid=plex_guid, tmdb_id=tmdb_id, imdb_id=imdb_id, tvdb_id=tvdb_id, anidb_id=anidb_id) + def get_anidb_id(self, media_type, plex_guid=None, tmdb_id=None, imdb_id=None, tvdb_id=None): + return self.get_id_from(media_type, "anidb_id", plex_guid=plex_guid, tmdb_id=tmdb_id, imdb_id=imdb_id, tvdb_id=tvdb_id) - def get_id_from(self, media_type, id_from, plex_guid=None, tmdb_id=None, imdb_id=None, tvdb_id=None, anidb_id=None, mal_id=None): + def get_id_from(self, media_type, id_from, plex_guid=None, tmdb_id=None, imdb_id=None, tvdb_id=None, anidb_id=None): if plex_guid: return self.get_id(media_type, "plex_guid", id_from, plex_guid) elif tmdb_id: return self.get_id(media_type, "tmdb_id", id_from, tmdb_id) elif imdb_id: return self.get_id(media_type, "imdb_id", id_from, imdb_id) elif tvdb_id: return self.get_id(media_type, "tvdb_id", id_from, tvdb_id) elif anidb_id: return self.get_id(media_type, "anidb_id", id_from, anidb_id) - elif mal_id: return self.get_id(media_type, "mal_id", id_from, mal_id) else: return None, None def get_id(self, media_type, from_id, to_id, key): @@ -132,13 +136,12 @@ class Cache: if row["imdb_id"]: ids_to_return["imdb"] = row["imdb_id"] if row["tvdb_id"]: ids_to_return["tvdb"] = int(row["tvdb_id"]) if row["anidb_id"]: ids_to_return["anidb"] = int(row["anidb_id"]) - if row["mal_id"]: ids_to_return["mal"] = int(row["mal_id"]) datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d") time_between_insertion = datetime.now() - datetime_object expired = time_between_insertion.days > self.expiration return ids_to_return, expired - def update_guid(self, media_type, plex_guid, tmdb_id, imdb_id, tvdb_id, anidb_id, mal_id, expired): + def update_guid(self, media_type, plex_guid, tmdb_id, imdb_id, tvdb_id, anidb_id, expired): expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, self.expiration))) with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row @@ -150,10 +153,9 @@ class Cache: imdb_id = ?, tvdb_id = ?, anidb_id = ?, - mal_id = ?, expiration_date = ?, media_type = ? - WHERE plex_guid = ?""", (tmdb_id, imdb_id, tvdb_id, anidb_id, mal_id, expiration_date.strftime("%Y-%m-%d"), media_type, plex_guid)) + WHERE plex_guid = ?""", (tmdb_id, imdb_id, tvdb_id, anidb_id, expiration_date.strftime("%Y-%m-%d"), media_type, plex_guid)) if imdb_id and (tmdb_id or tvdb_id): cursor.execute("INSERT OR IGNORE INTO imdb_map(imdb_id) VALUES(?)", (imdb_id,)) cursor.execute("UPDATE imdb_map SET t_id = ?, expiration_date = ?, media_type = ? WHERE imdb_id = ?", (tmdb_id if media_type == "movie" else tvdb_id, expiration_date.strftime("%Y-%m-%d"), media_type, imdb_id)) @@ -237,3 +239,31 @@ class Cache: cursor.execute("INSERT OR IGNORE INTO omdb_data(imdb_id) VALUES(?)", (omdb.imdb_id,)) update_sql = "UPDATE omdb_data SET title = ?, year = ?, content_rating = ?, genres = ?, imdb_rating = ?, imdb_votes = ?, metacritic_rating = ?, type = ?, expiration_date = ? WHERE imdb_id = ?" cursor.execute(update_sql, (omdb.title, omdb.year, omdb.content_rating, omdb.genres_str, omdb.imdb_rating, omdb.imdb_votes, omdb.metacritic_rating, omdb.type, expiration_date.strftime("%Y-%m-%d"), omdb.imdb_id)) + + def query_anime_map(self, anime_id, id_type): + ids = None + expired = None + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute(f"SELECT * FROM anime_map WHERE {id_type} = ?", (anime_id, )) + row = cursor.fetchone() + if row and row["anidb"]: + datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d") + time_between_insertion = datetime.now() - datetime_object + ids = { + "anilist": int(row["anilist"]) if row["anilist"] else None, + "anidb": int(row["anidb"]) if row["anidb"] else None, + "myanimelist": int(row["myanimelist"]) if row["myanimelist"] else None, + "kitsu": int(row["kitsu"]) if row["kitsu"] else None + } + expired = time_between_insertion.days > self.expiration + return ids, expired + + def update_anime(self, expired, anime_ids): + expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, self.expiration))) + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("INSERT OR IGNORE INTO anime_map(anidb) VALUES(?)", (anime_ids["anidb"],)) + cursor.execute("UPDATE anime_map SET anilist = ?, myanimelist = ?, kitsu = ?, expiration_date = ? WHERE anidb = ?", (anime_ids["anidb"], anime_ids["myanimelist"], anime_ids["kitsu"], expiration_date.strftime("%Y-%m-%d"), anime_ids["anidb"])) diff --git a/modules/config.py b/modules/config.py index be581127..f260a2bc 100644 --- a/modules/config.py +++ b/modules/config.py @@ -723,16 +723,36 @@ class Config: def convert_anime_ids(self, anilist_ids=None, anidb_ids=None, mal_ids=None): all_ids = [] - def collect_ids(ids, id_type): + def collect_ids(ids, id_name): if ids: if isinstance(ids, list): - all_ids.extend([{id_type: anime_id} for anime_id in ids]) + all_ids.extend([{id_name: a_id} for a_id in ids]) else: - all_ids.append({id_type: ids}) + all_ids.append({id_name: ids}) collect_ids(anilist_ids, "anilist") collect_ids(anidb_ids, "anidb") collect_ids(mal_ids, "myanimelist") - return requests.post("https://relations.yuna.moe/api/ids", json=all_ids).json() + if self.Cache: + converted_ids = [] + unconverted_ids = [] + for anime_dict in all_ids: + for id_type, anime_id in anime_dict.items(): + query_ids, update = self.Cache.query_anime_map(anime_id, id_type) + if not update and query_ids: + converted_ids.append(query_ids) + else: + unconverted_ids.append({id_type: anime_id}) + arm_ids = self.call_arm_server(unconverted_ids) + for anime_ids in arm_ids: + self.Cache.update_anime(False, anime_ids) + converted_ids.extend(arm_ids) + return converted_ids + else: + return self.call_arm_server(all_ids) + + @retry(stop_max_attempt_number=6, wait_fixed=10000) + def call_arm_server(self, ids): + return requests.post("https://relations.yuna.moe/api/ids", json=ids).json() def convert_from_imdb(self, imdb_id, language): update_tmdb = False @@ -1008,7 +1028,7 @@ class Config: try: tvdb_id = self.Trakt.convert_imdb_to_tvdb(tmdb_id) except Failed: pass - if (not tmdb_id and library.is_movie) or (not tvdb_id and not ((anidb_id or mal_id) and tmdb_id) and library.is_show): + if (not tmdb_id and library.is_movie) or (not tvdb_id and not (anidb_id and tmdb_id) and library.is_show): service_name = "TMDb ID" if library.is_movie else "TVDb ID" if self.Trakt: api_name = "TMDb or Trakt" @@ -1022,21 +1042,20 @@ class Config: else: id_name = None if anidb_id and not tmdb_id and not tvdb_id: error_message = f"Unable to convert AniDB ID: {anidb_id} to TMDb ID or TVDb ID" - elif mal_id and not tmdb_id and not tvdb_id: error_message = f"Unable to convert MyAnimeList ID: {mal_id} to TMDb ID or TVDb ID" elif id_name: error_message = f"Unable to convert {id_name} to {service_name} using {api_name}" else: error_message = f"No ID to convert to {service_name}" - if self.Cache and ((tmdb_id and library.is_movie) or ((tvdb_id or ((anidb_id or mal_id) and tmdb_id)) and library.is_show)): + if self.Cache and ((tmdb_id and library.is_movie) or ((tvdb_id or (anidb_id and tmdb_id)) and library.is_show)): if not isinstance(tmdb_id, list): tmdb_id = [tmdb_id] if not isinstance(imdb_id, list): imdb_id = [imdb_id] for i in range(len(tmdb_id)): try: imdb_value = imdb_id[i] except IndexError: imdb_value = None - util.print_end(length, f"Cache | {'^' if expired is True else '+'} | {item.guid:<46} | {tmdb_id[i] if tmdb_id[i] else 'None':<6} | {imdb_value if imdb_value else 'None':<10} | {tvdb_id if tvdb_id else 'None':<6} | {anidb_id if anidb_id else 'None':<5} | {mal_id if mal_id else 'None':<5} | {item.title}") - self.Cache.update_guid("movie" if library.is_movie else "show", item.guid, tmdb_id[i], imdb_value, tvdb_id, anidb_id, mal_id, expired) + util.print_end(length, f"Cache | {'^' if expired is True else '+'} | {item.guid:<46} | {tmdb_id[i] if tmdb_id[i] else 'None':<6} | {imdb_value if imdb_value else 'None':<10} | {tvdb_id if tvdb_id else 'None':<6} | {anidb_id if anidb_id else 'None':<5} | {item.title}") + self.Cache.update_guid("movie" if library.is_movie else "show", item.guid, tmdb_id[i], imdb_value, tvdb_id, anidb_id, expired) if tmdb_id and library.is_movie: return "movie", tmdb_id elif tvdb_id and library.is_show: return "show", tvdb_id - elif (anidb_id or mal_id) and tmdb_id: return "movie", tmdb_id + elif anidb_id and tmdb_id: return "movie", tmdb_id else: util.print_end(length, f"{'Cache | ! |' if self.Cache else 'Mapping Error:'} {item.guid:<46} | {error_message} for {item.title}") return None, None