Merge pull request #331 from Critical-Impact/anidb-authentication

AniDB Authentication and Tag Builder

config/config.yml.template

@@ -89,3 +89,6 @@ mal:
     token_type:
     expires_in:
     refresh_token:
+anidb:
+  username: ###### - optional
+  password: ######

modules/anidb.py

@@ -1,4 +1,4 @@
-import logging, requests
+import logging, requests,time
 from lxml import html
 from modules import util
 from modules.util import Failed
@@ -6,20 +6,42 @@ from retrying import retry
 
 logger = logging.getLogger("Plex Meta Manager")
 
-builders = ["anidb_id", "anidb_relation", "anidb_popular"]
+builders = ["anidb_id", "anidb_relation", "anidb_popular", 'anidb_tag']
 
 class AniDB:
-    def __init__(self, config):
+    def __init__(self, params, config):
         self.config = config
+
+        # Create a session so if we login we can continue to use the same session
+        self.anidb_session = requests.Session()
+
         self.urls = {
             "anime": "https://anidb.net/anime",
             "popular": "https://anidb.net/latest/anime/popular/?h=1",
-            "relation": "/relation/graph"
+            "relation": "/relation/graph",
+            "anidb_tag": "https://anidb.net/tag",
+            "login": "https://anidb.net/perl-bin/animedb.pl"
         }
 
+        if params and "username" in params and "password" in params:
+            result = str(self._login(params["username"], params["password"]).content)
+
+            # Login response does not use proper status codes so we have to check the content of the document
+            if "Wrong username/password" in result:
+                raise Failed("AniDB Error: Login failed")
+
     @retry(stop_max_attempt_number=6, wait_fixed=10000)
     def _request(self, url, language):
-        return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content)
+        return html.fromstring(self.anidb_session.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content)
+
+    def _login(self, username, password):
+        data = {
+            "show": "main",
+            "xuser": username,
+            "xpass": password,
+            "xdoautologin": "on"
+        }
+        return self.anidb_session.post(self.urls["login"], data, headers={"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 x64"})
 
     def _popular(self, language):
         response = self._request(self.urls["popular"], language)
@@ -47,12 +69,36 @@ class AniDB:
             return anidb_values
         raise Failed(f"AniDB Error: No valid AniDB IDs in {anidb_list}")
 
+    def _tag(self, tag, limit, language):
+        anidb_ids = []
+        next_page = True
+        current_url = self.urls["anidb_tag"] + "/" + str(tag)
+        while next_page:
+            logger.debug(f"Sending request to {current_url}")
+            response = self._request(current_url, language)
+            int_list = util.get_int_list(response.xpath("//td[@class='name main anime']/a/@href"), "AniDB ID")
+            anidb_ids.extend(int_list)
+            next_page_list = response.xpath("//li[@class='next']/a/@href")
+            logger.debug(f"next page list {next_page_list}")
+            if len(next_page_list) != 0 and len(anidb_ids) <= limit:
+                logger.debug(f"Loading next anidb page")
+                time.sleep(2)# Sleep as we are paging through anidb and don't want the ban hammer
+                current_url = "https://anidb.net" + next_page_list[0]
+            else:
+                logger.debug(f"Got to last page")
+                next_page = False
+        anidb_ids = anidb_ids[:limit]
+        return anidb_ids
+
     def get_items(self, method, data, language):
         pretty = util.pretty_names[method] if method in util.pretty_names else method
         anidb_ids = []
         if method == "anidb_popular":
             logger.info(f"Processing {pretty}: {data} Anime")
             anidb_ids.extend(self._popular(language)[:data])
+        elif method == "anidb_tag":
+            anidb_ids = self._tag(data["tag"], data["limit"], language)
+            logger.info(f"Processing {pretty}: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag ID: {data['tag']}")
         else:
             logger.info(f"Processing {pretty}: {data}")
             if method == "anidb_id":                            anidb_ids.append(data)

modules/config.py

@@ -108,6 +108,7 @@ class Config:
             if "omdb" in new_config:                        new_config["omdb"] = new_config.pop("omdb")
             if "trakt" in new_config:                       new_config["trakt"] = new_config.pop("trakt")
             if "mal" in new_config:                         new_config["mal"] = new_config.pop("mal")
+            if "anidb" in new_config:                       new_config["anidb"] = new_config.pop("anidb")
             yaml.round_trip_dump(new_config, open(self.config_path, "w", encoding="utf-8"), indent=ind, block_seq_indent=bsi)
             self.data = new_config
         except yaml.scanner.ScannerError as e:
@@ -270,9 +271,29 @@ class Config:
         else:
             logger.warning("mal attribute not found")
 
+        util.separator()
+
+        self.AniDB = None
+        anidb_username = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False)
+        anidb_password = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False)
+        if "anidb" in self.data and anidb_username and anidb_password:
+            logger.info("Connecting to AniDB...")
+            self.anidb = {}
+            try:
+                self.anidb["username"] = check_for_attribute(self.data, "username", parent="anidb", throw=True)
+                self.anidb["password"] = check_for_attribute(self.data, "password", parent="anidb", throw=True)
+                self.AniDB = AniDB(self.anidb, self)
+            except Failed as e:
+                logger.error(e)
+            logger.info(f"My Anime List Connection {'Failed' if self.MyAnimeList is None else 'Successful'}")
+        else:
+            logger.info("Using guest authentication for AniDB")
+            self.AniDB = AniDB(None, self)
+
+        util.separator()
+
         self.TVDb = TVDb(self)
         self.IMDb = IMDb(self)
-        self.AniDB = AniDB(self)
         self.Convert = Convert(self)
         self.AniList = AniList(self)
         self.Letterboxd = Letterboxd(self)