From 0525f78f0f89f7d8456251d47b15acb5e222b737 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 10 Mar 2021 11:51:27 -0500 Subject: [PATCH] Added AniList Builders --- modules/anilist.py | 210 ++++++++++++++++++++++++++++++++++++++++++ modules/builder.py | 22 +++++ modules/config.py | 2 + modules/letterboxd.py | 2 +- modules/mal.py | 17 ++-- modules/omdb.py | 3 +- modules/util.py | 19 ++++ 7 files changed, 262 insertions(+), 13 deletions(-) create mode 100644 modules/anilist.py diff --git a/modules/anilist.py b/modules/anilist.py new file mode 100644 index 00000000..eed4c1e8 --- /dev/null +++ b/modules/anilist.py @@ -0,0 +1,210 @@ +import logging, requests, time +from modules import util +from modules.util import Failed +from retrying import retry + +logger = logging.getLogger("Plex Meta Manager") + +class AniListAPI: + def __init__(self, config): + self.config = config + self.url = "https://graphql.anilist.co" + + @retry(stop_max_attempt_number=6, wait_fixed=10000) + def post(self, query, variables): + return requests.post(self.url, json={"query": query, "variables": variables}) + + @retry(stop_max_attempt_number=2, retry_on_exception=util.retry_if_not_failed) + def send_request(self, query, variables): + response = self.post(query, variables) + json_obj = response.json() + if "errors" in json_obj: + if json_obj['errors'][0]['message'] == "Too Many Requests.": + if "Retry-After" in response.headers: + time.sleep(int(response.headers["Retry-After"])) + raise ValueError + else: + raise Failed(f"AniList Error: {json_obj['errors'][0]['message']}") + else: + time.sleep(0.4) + return json_obj + + def anilist_id(self, anilist_id): + query = "query ($id: Int) {Media(id: $id) {idMal title{romaji english}}}" + media = self.send_request(query, {"id": anilist_id})["data"]["Media"] + if media["idMal"]: + return media["idMal"], media["title"]["english" if media["title"]["english"] else "romaji"] + raise Failed(f"AniList Error: No MyAnimeList ID found for {anilist_id}") + + def get_pagenation(self, query, limit=0, variables=None): + mal_ids = [] + count = 0 + page_num = 0 + if variables is None: + variables = {"page": page_num} + else: + variables["page"] = page_num + next_page = True + while next_page: + page_num += 1 + json_obj = self.send_request(query, variables) + next_page = json_obj["data"]["Page"]["pageInfo"]["hasNextPage"] + for media in json_obj["data"]["Page"]["media"]: + if media["idMal"]: + mal_ids.append(media["idMal"]) + count += 1 + if 0 < limit == count: + break + if 0 < limit == count: + break + + return mal_ids + + def top_rated(self, limit): + query = """ + query ($page: Int) { + Page(page: $page) { + pageInfo {hasNextPage} + media(averageScore_greater: 3, sort: SCORE_DESC, type: ANIME) {idMal} + } + } + """ + return self.get_pagenation(query, limit=limit) + + def popular(self, limit): + query = """ + query ($page: Int) { + Page(page: $page) { + pageInfo {hasNextPage} + media(popularity_greater: 1000, sort: POPULARITY_DESC, type: ANIME) {idMal} + } + } + """ + return self.get_pagenation(query, limit=limit) + + def season(self, season, year, sort, limit): + query = """ + query ($page: Int, $season: String, $year: Int, $sort: String) { + Page(page: $page){ + pageInfo {hasNextPage} + media(season: $season, seasonYear: $year, type: ANIME, sort: $sort){idMal} + } + } + """ + variables = {"season": season.upper(), "year": year, "sort": "SCORE_DESC" if sort == "score" else "POPULARITY_DESC"} + return self.get_pagenation(query, limit=limit, variables=variables) + + def studio(self, studio_id): + query = """ + query ($page: Int, $id: Int) { + Studio(id: $id) { + name + media(page: $page) { + nodes {idMal type} + pageInfo {hasNextPage} + } + } + } + """ + mal_ids = [] + page_num = 0 + next_page = True + name = None + while next_page: + page_num += 1 + json_obj = self.send_request(query, {"id": studio_id, "page": page_num}) + if not name: + name = json_obj["data"]["Studio"]["name"] + next_page = json_obj["data"]["Studio"]["media"]["pageInfo"]["hasNextPage"] + for media in json_obj["data"]["Studio"]["media"]["nodes"]: + if media["idMal"] and media["type"] == "ANIME": + mal_ids.append(media["idMal"]) + return mal_ids, name + + def relations(self, anilist_id, ignore_ids=None): + query = """ + query ($id: Int) { + Media(id: $id) { + idMal + relations { + edges {node{id idMal type} relationType} + nodes {id idMal type} + } + } + } + """ + anilist_ids = [] + mal_ids = [] + name = "" + if not ignore_ids: + ignore_ids = [anilist_id] + mal_id, name = self.anilist_id(anilist_id) + mal_ids.append(mal_id) + json_obj = self.send_request(query, {"id": anilist_id}) + edges = [media["node"]["id"] for media in json_obj["data"]["Media"]["relations"]["edges"] + if media["relationType"] not in ["CHARACTER", "OTHER"] and media["node"]["type"] == "ANIME"] + for media in json_obj["data"]["Media"]["relations"]["nodes"]: + if media["idMal"] and media["id"] not in ignore_ids and media["id"] in edges and media["type"] == "ANIME": + anilist_ids.append(media["id"]) + ignore_ids.append(media["id"]) + mal_ids.append(media["idMal"]) + + for next_id in anilist_ids: + new_mal_ids, ignore_ids, _ = self.relations(next_id, ignore_ids=ignore_ids) + mal_ids.extend(new_mal_ids) + + return mal_ids, ignore_ids, name + + def validate_anilist_ids(self, anilist_ids, studio=False): + anilist_values = [] + for anilist_id in anilist_ids: + if studio: query = "query ($id: Int) {Studio(id: $id) {name}}" + else: query = "query ($id: Int) {Media(id: $id) {idMal}}" + try: + self.send_request(query, {"id": anilist_id}) + anilist_values.append(anilist_id) + except Failed as e: logger.error(e) + if len(anilist_values) > 0: + return anilist_values + raise Failed(f"AniList Error: No valid AniList IDs in {anilist_ids}") + + def get_items(self, method, data, status_message=True): + if status_message: + logger.debug(f"Data: {data}") + pretty = util.pretty_names[method] if method in util.pretty_names else method + if method == "anilist_id": + mal_id, name = self.anilist_id(data) + mal_ids = [mal_id] + if status_message: + logger.info(f"Processing {pretty}: ({data}) {name}") + elif method in ["anilist_popular", "anilist_top_rated"]: + mal_ids = self.popular(data) if method == "anilist_popular" else self.top_rated(data) + if status_message: + logger.info(f"Processing {pretty}: {data} Anime") + elif method == "anilist_season": + mal_ids = self.season(data["season"], data["year"], data["sort_by"], data["limit"]) + if status_message: + logger.info(f"Processing {pretty}: {data['limit']} Anime from {util.pretty_seasons[data['season']]} {data['year']} sorted by {util.anilist_pretty[data['sort_by']]}") + elif method in ["anilist_studio", "anilist_relations"]: + if method == "anilist_studio": mal_ids, name = self.studio(data) + else: mal_ids, _, name = self.relations(data) + if status_message: + logger.info(f"Processing {pretty}: ({data}) {name} ({len(mal_ids)} Anime)") + else: + raise Failed(f"AniList Error: Method {method} not supported") + show_ids = [] + movie_ids = [] + for mal_id in mal_ids: + try: + ids = self.config.MyAnimeListIDList.find_mal_ids(mal_id) + if "thetvdb_id" in ids and int(ids["thetvdb_id"]) > 0: show_ids.append(int(ids["thetvdb_id"])) + elif "themoviedb_id" in ids and int(ids["themoviedb_id"]) > 0: movie_ids.append(int(ids["themoviedb_id"])) + else: raise Failed(f"MyAnimeList Error: MyAnimeList ID: {mal_id} has no other IDs associated with it") + except Failed as e: + if status_message: + logger.error(e) + if status_message: + logger.debug(f"MyAnimeList IDs Found: {mal_ids}") + logger.debug(f"Shows Found: {show_ids}") + logger.debug(f"Movies Found: {movie_ids}") + return movie_ids, show_ids diff --git a/modules/builder.py b/modules/builder.py index bc7e13e5..954ed403 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -325,6 +325,8 @@ class CollectionBuilder: self.methods.append((method_name, util.get_int_list(data[m], "MyAnimeList ID"))) elif method_name in ["anidb_id", "anidb_relation"]: self.methods.append((method_name, config.AniDB.validate_anidb_list(util.get_int_list(data[m], "AniDB ID"), self.library.Plex.language))) + elif method_name in ["anilist_id", "anilist_relations", "anilist_studio"]: + self.methods.append((method_name, config.AniList.validate_anilist_ids(util.get_int_list(data[m], "AniList ID"), studio=method_name == "anilist_studio"))) elif method_name == "trakt_list": self.methods.append((method_name, config.Trakt.validate_trakt_list(util.get_list(data[m])))) elif method_name == "trakt_list_details": @@ -542,6 +544,26 @@ class CollectionBuilder: new_dictionary["limit"] = get_int(method_name, "limit", data[m], 100, maximum=1000) self.methods.append((method_name, [new_dictionary])) + elif method_name == "anilist_season": + new_dictionary = {"sort_by": "score"} + if "sort_by" not in data[m]: logger.warning("Collection Warning: anilist_season sort_by attribute not found using score as default") + elif not data[m]["sort_by"]: logger.warning("Collection Warning: anilist_season sort_by attribute is blank using score as default") + elif data[m]["sort_by"] not in ["score", "popular"]: logger.warning(f"Collection Warning: anilist_season sort_by attribute {data[m]['sort_by']} invalid must be either 'score' or 'popular' using score as default") + else: new_dictionary["sort_by"] = data[m]["sort_by"] + + if current_time.month in [12, 1, 2]: new_dictionary["season"] = "winter" + elif current_time.month in [3, 4, 5]: new_dictionary["season"] = "spring" + elif current_time.month in [6, 7, 8]: new_dictionary["season"] = "summer" + elif current_time.month in [9, 10, 11]: new_dictionary["season"] = "fall" + + if "season" not in data[m]: logger.warning(f"Collection Warning: anilist_season season attribute not found using the current season: {new_dictionary['season']} as default") + elif not data[m]["season"]: logger.warning(f"Collection Warning: anilist_season season attribute is blank using the current season: {new_dictionary['season']} as default") + elif data[m]["season"] not in util.pretty_seasons: logger.warning(f"Collection Warning: anilist_season season attribute {data[m]['season']} invalid must be either 'winter', 'spring', 'summer' or 'fall' using the current season: {new_dictionary['season']} as default") + else: new_dictionary["season"] = data[m]["season"] + + new_dictionary["year"] = get_int(method_name, "year", data[m], current_time.year, minimum=1917, maximum=current_time.year + 1) + new_dictionary["limit"] = get_int(method_name, "limit", data[m], 100, maximum=500) + self.methods.append((method_name, [new_dictionary])) else: raise Failed(f"Collection Error: {m} attribute is not a dictionary: {data[m]}") elif method_name in util.count_lists: diff --git a/modules/config.py b/modules/config.py index ad438961..0fbf1921 100644 --- a/modules/config.py +++ b/modules/config.py @@ -1,6 +1,7 @@ import logging, os, re, requests, time from modules import util from modules.anidb import AniDBAPI +from modules.anilist import AniListAPI from modules.builder import CollectionBuilder from modules.cache import Cache from modules.imdb import IMDbAPI @@ -227,6 +228,7 @@ class Config: self.TVDb = TVDbAPI(self) self.IMDb = IMDbAPI(self) self.AniDB = AniDBAPI(self) + self.AniList = AniListAPI(self) self.Letterboxd = LetterboxdAPI() util.separator() diff --git a/modules/letterboxd.py b/modules/letterboxd.py index 16b817ef..e079d841 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -1,4 +1,4 @@ -import logging, math, re, requests +import logging, requests from lxml import html from modules import util from modules.util import Failed diff --git a/modules/mal.py b/modules/mal.py index ae822b4a..063638fe 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -123,32 +123,29 @@ class MyAnimeListAPI: if "error" in response: raise Failed(f"MyAnimeList Error: {response['error']}") else: return response - def parse_mal_ids(self, data): - mal_ids = [] - if "data" in data: - for d in data["data"]: - mal_ids.append(d["node"]["id"]) - return mal_ids + def request_and_parse_mal_ids(self, url): + data = self.send_request(url) + return [d["node"]["id"] for d in data["data"]] if "data" in data else [] def get_username(self): return self.send_request(f"{self.urls['user']}/@me")["name"] def get_ranked(self, ranking_type, limit): url = f"{self.urls['ranking']}?ranking_type={ranking_type}&limit={limit}" - return self.parse_mal_ids(self.send_request(url)) + return self.request_and_parse_mal_ids(url) def get_season(self, season, year, sort_by, limit): url = f"{self.urls['season']}/{year}/{season}?sort={sort_by}&limit={limit}" - return self.parse_mal_ids(self.send_request(url)) + return self.request_and_parse_mal_ids(url) def get_suggestions(self, limit): url = f"{self.urls['suggestions']}?limit={limit}" - return self.parse_mal_ids(self.send_request(url)) + return self.request_and_parse_mal_ids(url) def get_userlist(self, username, status, sort_by, limit): final_status = "" if status == "all" else f"status={status}&" url = f"{self.urls['user']}/{username}/animelist?{final_status}sort={sort_by}&limit={limit}" - return self.parse_mal_ids(self.send_request(url)) + return self.request_and_parse_mal_ids(url) def get_items(self, method, data, status_message=True): if status_message: diff --git a/modules/omdb.py b/modules/omdb.py index ab42d66e..2cf96b4b 100644 --- a/modules/omdb.py +++ b/modules/omdb.py @@ -1,5 +1,4 @@ -import logging, math, re, requests -from lxml import html +import logging, requests from modules import util from modules.util import Failed from retrying import retry diff --git a/modules/util.py b/modules/util.py index bc99bea3..7735581c 100644 --- a/modules/util.py +++ b/modules/util.py @@ -95,6 +95,12 @@ pretty_names = { "anidb_id": "AniDB ID", "anidb_relation": "AniDB Relation", "anidb_popular": "AniDB Popular", + "anilist_id": "AniList ID", + "anilist_popular": "AniList Popular", + "anilist_relations": "AniList Relations", + "anilist_season": "AniList Season", + "anilist_studio": "AniList Studio", + "anilist_top_rated": "AniList Top Rated", "imdb_list": "IMDb List", "imdb_id": "IMDb ID", "letterboxd_list": "Letterboxd List", @@ -211,6 +217,10 @@ mal_userlist_status = [ "dropped", "plan_to_watch" ] +anilist_pretty = { + "score": "Average Score", + "popular": "Popularity" +} pretty_ids = { "anidbid": "AniDB", "imdbid": "IMDb", @@ -223,6 +233,12 @@ all_lists = [ "anidb_id", "anidb_relation", "anidb_popular", + "anilist_id", + "anilist_popular", + "anilist_relations", + "anilist_season", + "anilist_studio", + "anilist_top_rated", "imdb_list", "imdb_id", "letterboxd_list", @@ -305,6 +321,7 @@ other_attributes = [ ] dictionary_lists = [ "filters", + "anilist_season", "mal_season", "mal_userlist", "plex_collectionless", @@ -359,6 +376,8 @@ tmdb_searches = [ ] count_lists = [ "anidb_popular", + "anilist_popular", + "anilist_top_rated", "mal_all", "mal_airing", "mal_upcoming",