[28] add AniDB API

VERSION

@@ -1 +1 @@
-1.17.3-develop27
+1.17.3-develop28

docs/config/anidb.md

@@ -1,6 +1,10 @@
 # AniDB Attributes
 
-Configuring [AniDB](https://anidb.net/) is optional but can allow you to access mature content with AniDB Builders.
+Configuring [AniDB](https://anidb.net/) is optional but can unlock more features from the site
+
+Using `client` and `version` allows access to AniDB Library Operations.
+
+Using `username` and `password` allows you to access mature content with AniDB Builders.
 
 **All AniDB Builders still work without this, they will just not have mature content**
 
@@ -9,11 +13,46 @@ A `anidb` mapping is in the root of the config file.
 Below is a `anidb` mapping example and the full set of attributes:
 ```yaml
 anidb:
+  client: #######
+  version: 1
+  language: en
+  cache_expiration: 60
   username: ######
   password: ######
 ```
 
-| Attribute  | Allowed Values | Required |
+| Attribute          | Allowed Values                                                                                | Default | Required |
-|:-----------|:---------------|:--------:|
+|:-------------------|:----------------------------------------------------------------------------------------------|:-------:|:--------:|
-| `username` | AniDB Username | ✅  |
+| `client`           | AniDB Client Name                                                                             |   N/A   | ❌ |
-| `password` | AniDB Password | ✅  |
+| `version`          | AniDB Client Version                                                                          |   N/A   | ❌ |
+| `language`         | [ISO 639-1 Code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) of the User Language. |   en    | ❌ |
+| `cache_expiration` | Number of days before each cache mapping expires and has to be re-cached.                     |   60    | ❌ |
+| `username`         | AniDB Username                                                                                |   N/A   | ❌ |
+| `password`         | AniDB Password                                                                                |   N/A   | ❌ |
+
+* To get a Client Name and Client Version please follow the following steps.
+
+1. Login to [AniDB](https://anidb.net/)
+2. Go to you [API Client Page](https://anidb.net/software/add) and go to the `Add New Project` Tab.
+
+![AniDB Add Project](anidb-1.png)
+
+3. Fill in the Project Name with whatever name you want and then hit `+ Add Project`. The rest of the settings don't matter.
+4. After you've added the project you should end up on the Projects Page. If not go back to the [API Client Page](https://anidb.net/software/add) and click your projects name. 
+5. Once you're on the project page click `Add Client` in the top right.
+
+![AniDB Add Client](anidb-2.png)
+
+6. Come up with and enter a unique to AniDB Client Name, select `HTTP API` in the API Dropdown, and put `1` for Version.
+
+![AniDB Client Page](anidb-3.png)
+
+7. Put the Client Name and Client Version you just created in your config.yml as `client` and `version` respectively.
+
+```yaml
+anidb:
+  client: UniqueAniDBName
+  version: 1
+  language: en
+  cache_expiration: 60
+```

modules/anidb.py

@@ -1,4 +1,4 @@
-import time
+import json, time
 from datetime import datetime
 from modules import util
 from modules.util import Failed
@@ -7,6 +7,7 @@ logger = util.logger
 
 builders = ["anidb_id", "anidb_relation", "anidb_popular", "anidb_tag"]
 base_url = "https://anidb.net"
+api_url = "http://api.anidb.net:9001/httpapi"
 urls = {
     "anime": f"{base_url}/anime",
     "popular": f"{base_url}/latest/anime/popular/?h=1",
@@ -14,26 +15,37 @@ urls = {
     "tag": f"{base_url}/tag",
     "login": f"{base_url}/perl-bin/animedb.pl"
 }
-
 class AniDBObj:
-    def __init__(self, anidb, anidb_id, language):
+    def __init__(self, anidb, anidb_id, data):
-        self.anidb = anidb
+        self._anidb = anidb
         self.anidb_id = anidb_id
-        self.language = language
+        self._data = data
-        response = self.anidb._request(f"{urls['anime']}/{anidb_id}")
 
-        def parse_page(xpath, is_list=False, is_float=False, is_date=False, fail=False):
+        def _parse(attr, xpath, is_list=False, is_dict=False, is_float=False, is_date=False, fail=False):
-            parse_results = response.xpath(xpath)
             try:
+                if isinstance(data, dict):
+                    if is_list:
+                        return data[attr].split("|")
+                    elif is_dict:
+                        return json.loads(data[attr])
+                    elif is_float:
+                        return util.check_num(data[attr], is_int=False)
+                    elif is_date:
+                        return datetime.strptime(data[attr], "%Y-%m-%d")
+                    else:
+                        return data[attr]
+                parse_results = data.xpath(xpath)
                 if len(parse_results) > 0:
                     parse_results = [r.strip() for r in parse_results if len(r) > 0]
                 if parse_results:
                     if is_list:
                         return parse_results
+                    elif is_dict:
+                        return {ta.get("xml:lang"): ta.text_content() for ta in parse_results}
                     elif is_float:
                         return float(parse_results[0])
                     elif is_date:
-                        return datetime.strptime(parse_results[0], "%d.%m.%Y")
+                        return datetime.strptime(parse_results[0], "%Y-%m-%d")
                     else:
                         return parse_results[0]
             except (ValueError, TypeError):
@@ -42,26 +54,47 @@ class AniDBObj:
                 raise Failed(f"AniDB Error: No Anime Found for AniDB ID: {self.anidb_id}")
             elif is_list:
                 return []
-            elif is_float:
+            elif is_dict:
-                return 0
+                return {}
             else:
                 return None
 
-        self.official_title = parse_page(f"//th[text()='Main Title']/parent::tr/td/span/text()", fail=True)
+        self.main_title = _parse("main_title", "//anime/titles/title[@type='main']/text()", fail=True)
-        self.title = parse_page(f"//th[text()='Official Title']/parent::tr/td/span/span/span[text()='{self.language}']/parent::span/parent::span/parent::td/label/text()")
+        self.titles = _parse("titles", "//anime/titles/title[@type='official']", is_dict=True)
-        self.rating = parse_page(f"//th[text()='Rating']/parent::tr/td/span/a/span/text()", is_float=True)
+        self.official_title = self.titles[self._anidb.language] if self._anidb.language in self.titles else self.main_title
-        self.average = parse_page(f"//th[text()='Average']/parent::tr/td/span/a/span/text()", is_float=True)
+        self.rating = _parse("rating", "//anime/ratings/permanent/text()", is_float=True)
-        self.released = parse_page(f"//th[text()='Year']/parent::tr/td/span/text()", is_date=True)
+        self.average = _parse("average", "//anime/ratings/temporary/text()", is_float=True)
-        self.tags = [g.capitalize() for g in parse_page("//th/a[text()='Tags']/parent::th/parent::tr/td/span/a/span/text()", is_list=True)]
+        self.score = _parse("score", "//anime/ratings/review/text()", is_float=True)
-        self.description = response.xpath(f"string(//div[@itemprop='description'])")
+        self.released = _parse("released", "//anime/startdate/text()", is_date=True)
+        self.tags = _parse("tags", "//anime/tags/tag[@infobox='true']/name/text()", is_list=True)
 
 
 class AniDB:
-    def __init__(self, config, language):
+    def __init__(self, config, data):
         self.config = config
-        self.language = language
+        self.language = data["language"]
+        self.expiration = 60
+        self.client = None
+        self.version = None
         self.username = None
         self.password = None
+        self._delay = None
+
+    def authorize(self, client, version, expiration):
+        self.client = client
+        self.version = version
+        logger.secret(self.client)
+        self.expiration = expiration
+        try:
+            self.get_anime(69, ignore_cache=True)
+        except Failed:
+            self.client = None
+            self.version = None
+            raise
+
+    @property
+    def is_authorized(self):
+        return self.client is not None
 
     def login(self, username, password):
         self.username = username
@@ -72,12 +105,12 @@ class AniDB:
         if not self._request(urls["login"], data=data).xpath("//li[@class='sub-menu my']/@title"):
             raise Failed("AniDB Error: Login failed")
 
-    def _request(self, url, data=None):
+    def _request(self, url, params=None, data=None):
         logger.trace(f"URL: {url}")
         if data:
-            return self.config.post_html(url, data=data, headers=util.header(self.language))
+            return self.config.post_html(url, params=params, data=data, headers=util.header(self.language))
         else:
-            return self.config.get_html(url, headers=util.header(self.language))
+            return self.config.get_html(url, params=params, headers=util.header(self.language))
 
     def _popular(self):
         response = self._request(urls["popular"])
@@ -119,8 +152,28 @@ class AniDB:
             current_url = f"{base_url}{next_page_list[0]}"
         return anidb_ids[:limit]
 
-    def get_anime(self, anidb_id):
+    def get_anime(self, anidb_id, ignore_cache=False):
-        return AniDBObj(self, anidb_id, self.language)
+        expired = None
+        anidb_dict = None
+        if self.config.Cache and not ignore_cache:
+            anidb_dict, expired = self.config.Cache.query_anidb(anidb_id, self.expiration)
+        if expired or not anidb_dict:
+            time_check = time.time()
+            if self._delay is not None:
+                while time_check - self._delay < 2:
+                    time_check = time.time()
+            anidb_dict = self._request(api_url, params={
+                "client": self.client,
+                "clientver": self.version,
+                "protover": 1,
+                "request": "anime",
+                "aid": anidb_id
+            })
+            self._delay = time.time()
+        obj = AniDBObj(self, anidb_id, anidb_dict)
+        if self.config.Cache and not ignore_cache:
+            self.config.Cache.update_mdb(expired, anidb_id, obj, self.expiration)
+        return obj
 
     def get_anidb_ids(self, method, data):
         anidb_ids = []

modules/cache.py

@@ -118,6 +118,19 @@ class Cache:
                     certification TEXT,
                     expiration_date TEXT)"""
                 )
+                cursor.execute(
+                    """CREATE TABLE IF NOT EXISTS anidb_data (
+                    key INTEGER PRIMARY KEY,
+                    anidb_id INTEGER UNIQUE,
+                    main_title TEXT,
+                    titles TEXT,
+                    rating REAL,
+                    average REAL,
+                    score REAL,
+                    released TEXT,
+                    tags TEXT,
+                    expiration_date TEXT)"""
+                )
                 cursor.execute(
                     """CREATE TABLE IF NOT EXISTS tmdb_movie_data (
                     key INTEGER PRIMARY KEY,
@@ -471,6 +484,41 @@ class Cache:
                     expiration_date.strftime("%Y-%m-%d"), key_id
                 ))
 
+    def query_anidb(self, anidb_id, expiration):
+        anidb_dict = {}
+        expired = None
+        with sqlite3.connect(self.cache_path) as connection:
+            connection.row_factory = sqlite3.Row
+            with closing(connection.cursor()) as cursor:
+                cursor.execute("SELECT * FROM anidb_data WHERE anidb_id = ?", (anidb_id,))
+                row = cursor.fetchone()
+                if row:
+                    anidb_dict["main_title"] = row["main_title"]
+                    anidb_dict["titles"] = row["titles"] if row["titles"] else None
+                    anidb_dict["rating"] = row["rating"] if row["rating"] else None
+                    anidb_dict["average"] = row["average"] if row["average"] else None
+                    anidb_dict["score"] = row["score"] if row["score"] else None
+                    anidb_dict["released"] = row["released"] if row["released"] else None
+                    anidb_dict["tags"] = row["tags"] if row["tags"] else None
+                    datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d")
+                    time_between_insertion = datetime.now() - datetime_object
+                    expired = time_between_insertion.days > expiration
+        return anidb_dict, expired
+
+    def update_anidb(self, expired, anidb_id, anidb, expiration):
+        expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, expiration)))
+        with sqlite3.connect(self.cache_path) as connection:
+            connection.row_factory = sqlite3.Row
+            with closing(connection.cursor()) as cursor:
+                cursor.execute("INSERT OR IGNORE INTO anidb_data(anidb_id) VALUES(?)", (anidb_id,))
+                update_sql = "UPDATE anidb_data SET main_title = ?, titles = ?, rating = ?, average = ?, score = ?, " \
+                             "released = ?, tags = ?, expiration_date = ? WHERE anidb_id = ?"
+                cursor.execute(update_sql, (
+                    anidb.main_title, str(anidb.titles), anidb.rating, anidb.average, anidb.score,
+                    anidb.released.strftime("%Y-%m-%d") if anidb.released else None, "|".join(anidb.tags),
+                    expiration_date.strftime("%Y-%m-%d"), anidb_id
+                ))
+
     def query_tmdb_movie(self, tmdb_id, expiration):
         tmdb_dict = {}
         expired = None

modules/config.py

@@ -472,11 +472,16 @@ class ConfigFile:
             else:
                 logger.warning("mal attribute not found")
 
-            self.AniDB = AniDB(self, check_for_attribute(self.data, "language", parent="anidb", default="en"))
+            self.AniDB = AniDB(self, {"language": check_for_attribute(self.data, "language", parent="anidb", default="en")})
             if "anidb" in self.data:
                 logger.separator()
                 logger.info("Connecting to AniDB...")
                 try:
+                    self.AniDB.authorize(
+                        check_for_attribute(self.data, "client", parent="anidb", throw=True),
+                        check_for_attribute(self.data, "version", parent="anidb", var_type="int", throw=True),
+                        check_for_attribute(self.data, "cache_expiration", parent="anidb", var_type="int", default=60, int_min=1)
+                    )
                     self.AniDB.login(
                         check_for_attribute(self.data, "username", parent="anidb", throw=True),
                         check_for_attribute(self.data, "password", parent="anidb", throw=True)
@@ -750,6 +755,8 @@ class ConfigFile:
                         error_check(mass_key, "OMDb")
                     if params[mass_key] and params[mass_key].startswith("mdb") and not self.Mdblist.has_key:
                         error_check(mass_key, "MdbList")
+                    if params[mass_key] and params[mass_key].startswith("anidb") and not self.AniDB.is_authorized:
+                        error_check(mass_key, "AniDB")
                     if params[mass_key] and params[mass_key].startswith("trakt") and self.Trakt is None:
                         error_check(mass_key, "Trakt")