diff --git a/VERSION b/VERSION index 880de668..18fb59ea 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.17.3-develop27 +1.17.3-develop28 diff --git a/docs/config/anidb-1.png b/docs/config/anidb-1.png new file mode 100644 index 00000000..8ee0c11d Binary files /dev/null and b/docs/config/anidb-1.png differ diff --git a/docs/config/anidb-2.png b/docs/config/anidb-2.png new file mode 100644 index 00000000..2d9afd4e Binary files /dev/null and b/docs/config/anidb-2.png differ diff --git a/docs/config/anidb-3.png b/docs/config/anidb-3.png new file mode 100644 index 00000000..0d337930 Binary files /dev/null and b/docs/config/anidb-3.png differ diff --git a/docs/config/anidb.md b/docs/config/anidb.md index 90fc4f2e..5c82f42c 100644 --- a/docs/config/anidb.md +++ b/docs/config/anidb.md @@ -1,6 +1,10 @@ # AniDB Attributes -Configuring [AniDB](https://anidb.net/) is optional but can allow you to access mature content with AniDB Builders. +Configuring [AniDB](https://anidb.net/) is optional but can unlock more features from the site + +Using `client` and `version` allows access to AniDB Library Operations. + +Using `username` and `password` allows you to access mature content with AniDB Builders. **All AniDB Builders still work without this, they will just not have mature content** @@ -9,11 +13,46 @@ A `anidb` mapping is in the root of the config file. Below is a `anidb` mapping example and the full set of attributes: ```yaml anidb: + client: ####### + version: 1 + language: en + cache_expiration: 60 username: ###### password: ###### ``` -| Attribute | Allowed Values | Required | -|:-----------|:---------------|:--------:| -| `username` | AniDB Username | ✅ | -| `password` | AniDB Password | ✅ | +| Attribute | Allowed Values | Default | Required | +|:-------------------|:----------------------------------------------------------------------------------------------|:-------:|:--------:| +| `client` | AniDB Client Name | N/A | ❌ | +| `version` | AniDB Client Version | N/A | ❌ | +| `language` | [ISO 639-1 Code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) of the User Language. | en | ❌ | +| `cache_expiration` | Number of days before each cache mapping expires and has to be re-cached. | 60 | ❌ | +| `username` | AniDB Username | N/A | ❌ | +| `password` | AniDB Password | N/A | ❌ | + +* To get a Client Name and Client Version please follow the following steps. + +1. Login to [AniDB](https://anidb.net/) +2. Go to you [API Client Page](https://anidb.net/software/add) and go to the `Add New Project` Tab. + +![AniDB Add Project](anidb-1.png) + +3. Fill in the Project Name with whatever name you want and then hit `+ Add Project`. The rest of the settings don't matter. +4. After you've added the project you should end up on the Projects Page. If not go back to the [API Client Page](https://anidb.net/software/add) and click your projects name. +5. Once you're on the project page click `Add Client` in the top right. + +![AniDB Add Client](anidb-2.png) + +6. Come up with and enter a unique to AniDB Client Name, select `HTTP API` in the API Dropdown, and put `1` for Version. + +![AniDB Client Page](anidb-3.png) + +7. Put the Client Name and Client Version you just created in your config.yml as `client` and `version` respectively. + +```yaml +anidb: + client: UniqueAniDBName + version: 1 + language: en + cache_expiration: 60 +``` \ No newline at end of file diff --git a/modules/anidb.py b/modules/anidb.py index f62e88df..15c6f271 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -1,4 +1,4 @@ -import time +import json, time from datetime import datetime from modules import util from modules.util import Failed @@ -7,6 +7,7 @@ logger = util.logger builders = ["anidb_id", "anidb_relation", "anidb_popular", "anidb_tag"] base_url = "https://anidb.net" +api_url = "http://api.anidb.net:9001/httpapi" urls = { "anime": f"{base_url}/anime", "popular": f"{base_url}/latest/anime/popular/?h=1", @@ -14,26 +15,37 @@ urls = { "tag": f"{base_url}/tag", "login": f"{base_url}/perl-bin/animedb.pl" } - class AniDBObj: - def __init__(self, anidb, anidb_id, language): - self.anidb = anidb + def __init__(self, anidb, anidb_id, data): + self._anidb = anidb self.anidb_id = anidb_id - self.language = language - response = self.anidb._request(f"{urls['anime']}/{anidb_id}") + self._data = data - def parse_page(xpath, is_list=False, is_float=False, is_date=False, fail=False): - parse_results = response.xpath(xpath) + def _parse(attr, xpath, is_list=False, is_dict=False, is_float=False, is_date=False, fail=False): try: + if isinstance(data, dict): + if is_list: + return data[attr].split("|") + elif is_dict: + return json.loads(data[attr]) + elif is_float: + return util.check_num(data[attr], is_int=False) + elif is_date: + return datetime.strptime(data[attr], "%Y-%m-%d") + else: + return data[attr] + parse_results = data.xpath(xpath) if len(parse_results) > 0: parse_results = [r.strip() for r in parse_results if len(r) > 0] if parse_results: if is_list: return parse_results + elif is_dict: + return {ta.get("xml:lang"): ta.text_content() for ta in parse_results} elif is_float: return float(parse_results[0]) elif is_date: - return datetime.strptime(parse_results[0], "%d.%m.%Y") + return datetime.strptime(parse_results[0], "%Y-%m-%d") else: return parse_results[0] except (ValueError, TypeError): @@ -42,26 +54,47 @@ class AniDBObj: raise Failed(f"AniDB Error: No Anime Found for AniDB ID: {self.anidb_id}") elif is_list: return [] - elif is_float: - return 0 + elif is_dict: + return {} else: return None - self.official_title = parse_page(f"//th[text()='Main Title']/parent::tr/td/span/text()", fail=True) - self.title = parse_page(f"//th[text()='Official Title']/parent::tr/td/span/span/span[text()='{self.language}']/parent::span/parent::span/parent::td/label/text()") - self.rating = parse_page(f"//th[text()='Rating']/parent::tr/td/span/a/span/text()", is_float=True) - self.average = parse_page(f"//th[text()='Average']/parent::tr/td/span/a/span/text()", is_float=True) - self.released = parse_page(f"//th[text()='Year']/parent::tr/td/span/text()", is_date=True) - self.tags = [g.capitalize() for g in parse_page("//th/a[text()='Tags']/parent::th/parent::tr/td/span/a/span/text()", is_list=True)] - self.description = response.xpath(f"string(//div[@itemprop='description'])") + self.main_title = _parse("main_title", "//anime/titles/title[@type='main']/text()", fail=True) + self.titles = _parse("titles", "//anime/titles/title[@type='official']", is_dict=True) + self.official_title = self.titles[self._anidb.language] if self._anidb.language in self.titles else self.main_title + self.rating = _parse("rating", "//anime/ratings/permanent/text()", is_float=True) + self.average = _parse("average", "//anime/ratings/temporary/text()", is_float=True) + self.score = _parse("score", "//anime/ratings/review/text()", is_float=True) + self.released = _parse("released", "//anime/startdate/text()", is_date=True) + self.tags = _parse("tags", "//anime/tags/tag[@infobox='true']/name/text()", is_list=True) class AniDB: - def __init__(self, config, language): + def __init__(self, config, data): self.config = config - self.language = language + self.language = data["language"] + self.expiration = 60 + self.client = None + self.version = None self.username = None self.password = None + self._delay = None + + def authorize(self, client, version, expiration): + self.client = client + self.version = version + logger.secret(self.client) + self.expiration = expiration + try: + self.get_anime(69, ignore_cache=True) + except Failed: + self.client = None + self.version = None + raise + + @property + def is_authorized(self): + return self.client is not None def login(self, username, password): self.username = username @@ -72,12 +105,12 @@ class AniDB: if not self._request(urls["login"], data=data).xpath("//li[@class='sub-menu my']/@title"): raise Failed("AniDB Error: Login failed") - def _request(self, url, data=None): + def _request(self, url, params=None, data=None): logger.trace(f"URL: {url}") if data: - return self.config.post_html(url, data=data, headers=util.header(self.language)) + return self.config.post_html(url, params=params, data=data, headers=util.header(self.language)) else: - return self.config.get_html(url, headers=util.header(self.language)) + return self.config.get_html(url, params=params, headers=util.header(self.language)) def _popular(self): response = self._request(urls["popular"]) @@ -119,8 +152,28 @@ class AniDB: current_url = f"{base_url}{next_page_list[0]}" return anidb_ids[:limit] - def get_anime(self, anidb_id): - return AniDBObj(self, anidb_id, self.language) + def get_anime(self, anidb_id, ignore_cache=False): + expired = None + anidb_dict = None + if self.config.Cache and not ignore_cache: + anidb_dict, expired = self.config.Cache.query_anidb(anidb_id, self.expiration) + if expired or not anidb_dict: + time_check = time.time() + if self._delay is not None: + while time_check - self._delay < 2: + time_check = time.time() + anidb_dict = self._request(api_url, params={ + "client": self.client, + "clientver": self.version, + "protover": 1, + "request": "anime", + "aid": anidb_id + }) + self._delay = time.time() + obj = AniDBObj(self, anidb_id, anidb_dict) + if self.config.Cache and not ignore_cache: + self.config.Cache.update_mdb(expired, anidb_id, obj, self.expiration) + return obj def get_anidb_ids(self, method, data): anidb_ids = [] diff --git a/modules/cache.py b/modules/cache.py index 5f880fa8..eed5bde5 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -118,6 +118,19 @@ class Cache: certification TEXT, expiration_date TEXT)""" ) + cursor.execute( + """CREATE TABLE IF NOT EXISTS anidb_data ( + key INTEGER PRIMARY KEY, + anidb_id INTEGER UNIQUE, + main_title TEXT, + titles TEXT, + rating REAL, + average REAL, + score REAL, + released TEXT, + tags TEXT, + expiration_date TEXT)""" + ) cursor.execute( """CREATE TABLE IF NOT EXISTS tmdb_movie_data ( key INTEGER PRIMARY KEY, @@ -471,6 +484,41 @@ class Cache: expiration_date.strftime("%Y-%m-%d"), key_id )) + def query_anidb(self, anidb_id, expiration): + anidb_dict = {} + expired = None + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("SELECT * FROM anidb_data WHERE anidb_id = ?", (anidb_id,)) + row = cursor.fetchone() + if row: + anidb_dict["main_title"] = row["main_title"] + anidb_dict["titles"] = row["titles"] if row["titles"] else None + anidb_dict["rating"] = row["rating"] if row["rating"] else None + anidb_dict["average"] = row["average"] if row["average"] else None + anidb_dict["score"] = row["score"] if row["score"] else None + anidb_dict["released"] = row["released"] if row["released"] else None + anidb_dict["tags"] = row["tags"] if row["tags"] else None + datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d") + time_between_insertion = datetime.now() - datetime_object + expired = time_between_insertion.days > expiration + return anidb_dict, expired + + def update_anidb(self, expired, anidb_id, anidb, expiration): + expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, expiration))) + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("INSERT OR IGNORE INTO anidb_data(anidb_id) VALUES(?)", (anidb_id,)) + update_sql = "UPDATE anidb_data SET main_title = ?, titles = ?, rating = ?, average = ?, score = ?, " \ + "released = ?, tags = ?, expiration_date = ? WHERE anidb_id = ?" + cursor.execute(update_sql, ( + anidb.main_title, str(anidb.titles), anidb.rating, anidb.average, anidb.score, + anidb.released.strftime("%Y-%m-%d") if anidb.released else None, "|".join(anidb.tags), + expiration_date.strftime("%Y-%m-%d"), anidb_id + )) + def query_tmdb_movie(self, tmdb_id, expiration): tmdb_dict = {} expired = None diff --git a/modules/config.py b/modules/config.py index d01266f3..ada19014 100644 --- a/modules/config.py +++ b/modules/config.py @@ -472,11 +472,16 @@ class ConfigFile: else: logger.warning("mal attribute not found") - self.AniDB = AniDB(self, check_for_attribute(self.data, "language", parent="anidb", default="en")) + self.AniDB = AniDB(self, {"language": check_for_attribute(self.data, "language", parent="anidb", default="en")}) if "anidb" in self.data: logger.separator() logger.info("Connecting to AniDB...") try: + self.AniDB.authorize( + check_for_attribute(self.data, "client", parent="anidb", throw=True), + check_for_attribute(self.data, "version", parent="anidb", var_type="int", throw=True), + check_for_attribute(self.data, "cache_expiration", parent="anidb", var_type="int", default=60, int_min=1) + ) self.AniDB.login( check_for_attribute(self.data, "username", parent="anidb", throw=True), check_for_attribute(self.data, "password", parent="anidb", throw=True) @@ -750,6 +755,8 @@ class ConfigFile: error_check(mass_key, "OMDb") if params[mass_key] and params[mass_key].startswith("mdb") and not self.Mdblist.has_key: error_check(mass_key, "MdbList") + if params[mass_key] and params[mass_key].startswith("anidb") and not self.AniDB.is_authorized: + error_check(mass_key, "AniDB") if params[mass_key] and params[mass_key].startswith("trakt") and self.Trakt is None: error_check(mass_key, "Trakt")