From 71cf71798e8d67fc08201fbf54a56bd16f15c9a6 Mon Sep 17 00:00:00 2001 From: Critical-Impact Date: Sun, 11 Jul 2021 18:12:52 +1000 Subject: [PATCH] AniDB Authentication and Tag Builder --- config/config.yml.template | 3 ++ modules/anidb.py | 58 ++++++++++++++++++++++++++++++++++---- modules/config.py | 23 ++++++++++++++- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/config/config.yml.template b/config/config.yml.template index 9ee8c3de..9a2bb7d6 100644 --- a/config/config.yml.template +++ b/config/config.yml.template @@ -89,3 +89,6 @@ mal: token_type: expires_in: refresh_token: +anidb: + username: ###### - optional + password: ###### \ No newline at end of file diff --git a/modules/anidb.py b/modules/anidb.py index d9147f84..6f05424b 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -1,4 +1,4 @@ -import logging, requests +import logging, requests,time from lxml import html from modules import util from modules.util import Failed @@ -6,20 +6,42 @@ from retrying import retry logger = logging.getLogger("Plex Meta Manager") -builders = ["anidb_id", "anidb_relation", "anidb_popular"] +builders = ["anidb_id", "anidb_relation", "anidb_popular", 'anidb_tag'] class AniDB: - def __init__(self, config): + def __init__(self, params, config): self.config = config + + # Create a session so if we login we can continue to use the same session + self.anidb_session = requests.Session() + self.urls = { "anime": "https://anidb.net/anime", "popular": "https://anidb.net/latest/anime/popular/?h=1", - "relation": "/relation/graph" + "relation": "/relation/graph", + "anidb_tag": "https://anidb.net/tag", + "login": "https://anidb.net/perl-bin/animedb.pl" } + if params and "username" in params and "password" in params: + result = str(self._login(params["username"], params["password"]).content) + + # Login response does not use proper status codes so we have to check the content of the document + if "Wrong username/password" in result: + raise Failed("AniDB Error: Login failed") + @retry(stop_max_attempt_number=6, wait_fixed=10000) def _request(self, url, language): - return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + return html.fromstring(self.anidb_session.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + + def _login(self, username, password): + data = { + "show": "main", + "xuser": username, + "xpass": password, + "xdoautologin": "on" + } + return self.anidb_session.post(self.urls["login"], data, headers={"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 x64"}) def _popular(self, language): response = self._request(self.urls["popular"], language) @@ -47,12 +69,36 @@ class AniDB: return anidb_values raise Failed(f"AniDB Error: No valid AniDB IDs in {anidb_list}") + def _tag(self, tag, limit, language): + anidb_ids = [] + next_page = True + current_url = self.urls["anidb_tag"] + "/" + str(tag) + while next_page: + logger.debug(f"Sending request to {current_url}") + response = self._request(current_url, language) + int_list = util.get_int_list(response.xpath("//td[@class='name main anime']/a/@href"), "AniDB ID") + anidb_ids.extend(int_list) + next_page_list = response.xpath("//li[@class='next']/a/@href") + logger.debug(f"next page list {next_page_list}") + if len(next_page_list) != 0 and len(anidb_ids) <= limit: + logger.debug(f"Loading next anidb page") + time.sleep(2)# Sleep as we are paging through anidb and don't want the ban hammer + current_url = "https://anidb.net" + next_page_list[0] + else: + logger.debug(f"Got to last page") + next_page = False + anidb_ids = anidb_ids[:limit] + return anidb_ids + def get_items(self, method, data, language): pretty = util.pretty_names[method] if method in util.pretty_names else method anidb_ids = [] if method == "anidb_popular": logger.info(f"Processing {pretty}: {data} Anime") anidb_ids.extend(self._popular(language)[:data]) + elif method == "anidb_tag": + anidb_ids = self._tag(data["tag"], data["limit"], language) + logger.info(f"Processing {pretty}: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag ID: {data['tag']}") else: logger.info(f"Processing {pretty}: {data}") if method == "anidb_id": anidb_ids.append(data) @@ -63,4 +109,4 @@ class AniDB: logger.debug(f"{len(anidb_ids)} AniDB IDs Found: {anidb_ids}") logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids + return movie_ids, show_ids \ No newline at end of file diff --git a/modules/config.py b/modules/config.py index b097d5ad..f3394a25 100644 --- a/modules/config.py +++ b/modules/config.py @@ -108,6 +108,7 @@ class Config: if "omdb" in new_config: new_config["omdb"] = new_config.pop("omdb") if "trakt" in new_config: new_config["trakt"] = new_config.pop("trakt") if "mal" in new_config: new_config["mal"] = new_config.pop("mal") + if "anidb" in new_config: new_config["anidb"] = new_config.pop("anidb") yaml.round_trip_dump(new_config, open(self.config_path, "w", encoding="utf-8"), indent=ind, block_seq_indent=bsi) self.data = new_config except yaml.scanner.ScannerError as e: @@ -270,9 +271,29 @@ class Config: else: logger.warning("mal attribute not found") + util.separator() + + self.AniDB = None + anidb_username = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False) + anidb_password = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False) + if "anidb" in self.data and anidb_username and anidb_password: + logger.info("Connecting to AniDB...") + self.anidb = {} + try: + self.anidb["username"] = check_for_attribute(self.data, "username", parent="anidb", throw=True) + self.anidb["password"] = check_for_attribute(self.data, "password", parent="anidb", throw=True) + self.AniDB = AniDB(self.anidb, self) + except Failed as e: + logger.error(e) + logger.info(f"My Anime List Connection {'Failed' if self.MyAnimeList is None else 'Successful'}") + else: + logger.info("Using guest authentication for AniDB") + self.AniDB = AniDB(None, self) + + util.separator() + self.TVDb = TVDb(self) self.IMDb = IMDb(self) - self.AniDB = AniDB(self) self.Convert = Convert(self) self.AniList = AniList(self) self.Letterboxd = Letterboxd(self)