|
|
@ -3,6 +3,7 @@ from datetime import datetime
|
|
|
|
from lxml.etree import ParserError
|
|
|
|
from lxml.etree import ParserError
|
|
|
|
from modules import util
|
|
|
|
from modules import util
|
|
|
|
from modules.util import Failed
|
|
|
|
from modules.util import Failed
|
|
|
|
|
|
|
|
from retrying import retry
|
|
|
|
|
|
|
|
|
|
|
|
logger = util.logger
|
|
|
|
logger = util.logger
|
|
|
|
|
|
|
|
|
|
|
@ -39,36 +40,20 @@ language_translation = {
|
|
|
|
"yo": "yor", "za": "zha", "zu": "zul"}
|
|
|
|
"yo": "yor", "za": "zha", "zu": "zul"}
|
|
|
|
|
|
|
|
|
|
|
|
class TVDbObj:
|
|
|
|
class TVDbObj:
|
|
|
|
def __init__(self, tvdb_url, language, is_movie, config):
|
|
|
|
def __init__(self, tvdb, tvdb_id, is_movie=False, ignore_cache=False):
|
|
|
|
self.tvdb_url = tvdb_url.strip()
|
|
|
|
self._tvdb = tvdb
|
|
|
|
self.language = language
|
|
|
|
self.tvdb_id = tvdb_id
|
|
|
|
self.is_movie = is_movie
|
|
|
|
self.is_movie = is_movie
|
|
|
|
self.config = config
|
|
|
|
self.ignore_cache = ignore_cache
|
|
|
|
if not self.is_movie and self.tvdb_url.startswith((urls["series"], urls["alt_series"], urls["series_id"])):
|
|
|
|
expired = None
|
|
|
|
self.media_type = "Series"
|
|
|
|
data = None
|
|
|
|
elif self.is_movie and self.tvdb_url.startswith((urls["movies"], urls["alt_movies"], urls["movie_id"])):
|
|
|
|
if self._tvdb.config.Cache and not ignore_cache:
|
|
|
|
self.media_type = "Movie"
|
|
|
|
data, expired = self._tvdb.config.Cache.query_tvdb(tvdb_id, is_movie, self._tvdb.expiration)
|
|
|
|
else:
|
|
|
|
if expired or not data:
|
|
|
|
raise Failed(f"TVDb Error: {self.tvdb_url} must begin with {urls['movies'] if self.is_movie else urls['series']}")
|
|
|
|
data = self._tvdb.get_request(f"{urls['movie_id' if is_movie else 'series_id']}{tvdb_id}")
|
|
|
|
|
|
|
|
|
|
|
|
if self.config.trace_mode:
|
|
|
|
|
|
|
|
logger.debug(f"URL: {tvdb_url}")
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
response = self.config.get_html(self.tvdb_url, headers=util.header(self.language))
|
|
|
|
|
|
|
|
except ParserError:
|
|
|
|
|
|
|
|
raise Failed(f"TVDb Error: Could not parse {self.tvdb_url}")
|
|
|
|
|
|
|
|
results = response.xpath(f"//*[text()='TheTVDB.com {self.media_type} ID']/parent::node()/span/text()")
|
|
|
|
|
|
|
|
if len(results) > 0:
|
|
|
|
|
|
|
|
self.id = int(results[0])
|
|
|
|
|
|
|
|
elif self.tvdb_url.startswith(urls["movie_id"]):
|
|
|
|
|
|
|
|
raise Failed(f"TVDb Error: Could not find a TVDb Movie using TVDb Movie ID: {self.tvdb_url[len(urls['movie_id']):]}")
|
|
|
|
|
|
|
|
elif self.tvdb_url.startswith(urls["series_id"]):
|
|
|
|
|
|
|
|
raise Failed(f"TVDb Error: Could not find a TVDb Series using TVDb Series ID: {self.tvdb_url[len(urls['series_id']):]}")
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise Failed(f"TVDb Error: Could not find a TVDb {self.media_type} ID at the URL {self.tvdb_url}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parse_page(xpath, is_list=False):
|
|
|
|
def parse_page(xpath, is_list=False):
|
|
|
|
parse_results = response.xpath(xpath)
|
|
|
|
parse_results = data.xpath(xpath)
|
|
|
|
if len(parse_results) > 0:
|
|
|
|
if len(parse_results) > 0:
|
|
|
|
parse_results = [r.strip() for r in parse_results if len(r) > 0]
|
|
|
|
parse_results = [r.strip() for r in parse_results if len(r) > 0]
|
|
|
|
return parse_results if is_list else parse_results[0] if len(parse_results) > 0 else None
|
|
|
|
return parse_results if is_list else parse_results[0] if len(parse_results) > 0 else None
|
|
|
@ -78,34 +63,91 @@ class TVDbObj:
|
|
|
|
place += f"@data-language='{lang}']" if lang else "not(@style='display:none')]"
|
|
|
|
place += f"@data-language='{lang}']" if lang else "not(@style='display:none')]"
|
|
|
|
return parse_page(f"{place}/@data-title"), parse_page(f"{place}/p/text()[normalize-space()]")
|
|
|
|
return parse_page(f"{place}/@data-title"), parse_page(f"{place}/p/text()[normalize-space()]")
|
|
|
|
|
|
|
|
|
|
|
|
self.title, self.summary = parse_title_summary(lang=self.language)
|
|
|
|
if isinstance(data, dict):
|
|
|
|
if not self.title and self.language in language_translation:
|
|
|
|
self.title = data["title"]
|
|
|
|
self.title, self.summary = parse_title_summary(lang=language_translation[self.language])
|
|
|
|
self.summary = data["summary"]
|
|
|
|
|
|
|
|
self.poster_url = data["poster_url"]
|
|
|
|
|
|
|
|
self.background_url = data["background_url"]
|
|
|
|
|
|
|
|
self.release_date = data["release_date"]
|
|
|
|
|
|
|
|
self.genres = data["genres"].split("|")
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
self.title, self.summary = parse_title_summary(lang=self._tvdb.language)
|
|
|
|
|
|
|
|
if not self.title and self._tvdb.language in language_translation:
|
|
|
|
|
|
|
|
self.title, self.summary = parse_title_summary(lang=language_translation[self._tvdb.language])
|
|
|
|
if not self.title:
|
|
|
|
if not self.title:
|
|
|
|
self.title, self.summary = parse_title_summary()
|
|
|
|
self.title, self.summary = parse_title_summary()
|
|
|
|
if not self.title:
|
|
|
|
if not self.title:
|
|
|
|
raise Failed(f"TVDb Error: Name not found from TVDb URL: {self.tvdb_url}")
|
|
|
|
raise Failed(f"TVDb Error: Name not found from TVDb ID: {self.tvdb_id}")
|
|
|
|
|
|
|
|
|
|
|
|
self.poster_path = parse_page("(//h2[@class='mt-4' and text()='Posters']/following::div/a/@href)[1]")
|
|
|
|
self.poster_url = parse_page("(//h2[@class='mt-4' and text()='Posters']/following::div/a/@href)[1]")
|
|
|
|
self.background_path = parse_page("(//h2[@class='mt-4' and text()='Backgrounds']/following::div/a/@href)[1]")
|
|
|
|
self.background_url = parse_page("(//h2[@class='mt-4' and text()='Backgrounds']/following::div/a/@href)[1]")
|
|
|
|
if self.is_movie:
|
|
|
|
if is_movie:
|
|
|
|
self.directors = parse_page("//strong[text()='Directors']/parent::li/span/a/text()[normalize-space()]", is_list=True)
|
|
|
|
|
|
|
|
self.writers = parse_page("//strong[text()='Writers']/parent::li/span/a/text()[normalize-space()]", is_list=True)
|
|
|
|
|
|
|
|
self.studios = parse_page("//strong[text()='Studio']/parent::li/span/a/text()[normalize-space()]", is_list=True)
|
|
|
|
|
|
|
|
released = parse_page("//strong[text()='Released']/parent::li/span/text()[normalize-space()]")
|
|
|
|
released = parse_page("//strong[text()='Released']/parent::li/span/text()[normalize-space()]")
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
self.networks = parse_page("//strong[text()='Networks']/parent::li/span/a/text()[normalize-space()]", is_list=True)
|
|
|
|
|
|
|
|
released = parse_page("//strong[text()='First Aired']/parent::li/span/text()[normalize-space()]")
|
|
|
|
released = parse_page("//strong[text()='First Aired']/parent::li/span/text()[normalize-space()]")
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
self.released = datetime.strptime(released, "%B %d, %Y") if released else released
|
|
|
|
self.release_date = datetime.strptime(released, "%B %d, %Y") if released else released
|
|
|
|
except ValueError:
|
|
|
|
except ValueError:
|
|
|
|
self.released = None
|
|
|
|
self.release_date = None
|
|
|
|
|
|
|
|
|
|
|
|
self.genres = parse_page("//strong[text()='Genres']/parent::li/span/a/text()[normalize-space()]", is_list=True)
|
|
|
|
self.genres = parse_page("//strong[text()='Genres']/parent::li/span/a/text()[normalize-space()]", is_list=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if self._tvdb.config.Cache and not ignore_cache:
|
|
|
|
|
|
|
|
self._tvdb.config.Cache.update_tvdb(expired, self, self._tvdb.expiration)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TVDb:
|
|
|
|
|
|
|
|
def __init__(self, config, tvdb_language, expiration):
|
|
|
|
|
|
|
|
self.config = config
|
|
|
|
|
|
|
|
self.language = tvdb_language
|
|
|
|
|
|
|
|
self.expiration = expiration
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_tvdb_obj(self, tvdb_url, is_movie=False):
|
|
|
|
|
|
|
|
tvdb_id, _, _ = self.get_id_from_url(tvdb_url, is_movie=is_movie)
|
|
|
|
|
|
|
|
return TVDbObj(self, tvdb_id, is_movie=is_movie)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_list_description(self, tvdb_url):
|
|
|
|
|
|
|
|
response = self.config.get_html(tvdb_url, headers=util.header(self.language))
|
|
|
|
|
|
|
|
description = response.xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()")
|
|
|
|
|
|
|
|
return description[0] if len(description) > 0 and len(description[0]) > 0 else ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed)
|
|
|
|
|
|
|
|
def get_request(self, tvdb_url):
|
|
|
|
|
|
|
|
return self.config.get_html(tvdb_url, headers=util.header(self.language))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_id_from_url(self, tvdb_url, is_movie=False, ignore_cache=False):
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
if not is_movie:
|
|
|
|
|
|
|
|
return int(tvdb_url), None, None
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
tvdb_url = f"{urls['movie_id']}{int(tvdb_url)}"
|
|
|
|
|
|
|
|
except ValueError:
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
tvdb_url = tvdb_url.strip()
|
|
|
|
|
|
|
|
if tvdb_url.startswith((urls["series"], urls["alt_series"], urls["series_id"])):
|
|
|
|
|
|
|
|
media_type = "Series"
|
|
|
|
|
|
|
|
elif tvdb_url.startswith((urls["movies"], urls["alt_movies"], urls["movie_id"])):
|
|
|
|
|
|
|
|
media_type = "Movie"
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise Failed(f"TVDb Error: {tvdb_url} must begin with {urls['movies']} or {urls['series']}")
|
|
|
|
|
|
|
|
expired = None
|
|
|
|
|
|
|
|
tvdb_id = None
|
|
|
|
|
|
|
|
if self.config.Cache and not ignore_cache:
|
|
|
|
|
|
|
|
tvdb_id, expired = self.config.Cache.query_tvdb_map(tvdb_url, self.expiration)
|
|
|
|
|
|
|
|
if tvdb_id and not expired and not is_movie:
|
|
|
|
|
|
|
|
return tvdb_id, None, None
|
|
|
|
|
|
|
|
if self.config.trace_mode:
|
|
|
|
|
|
|
|
logger.debug(f"URL: {tvdb_url}")
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
response = self.get_request(tvdb_url)
|
|
|
|
|
|
|
|
except ParserError:
|
|
|
|
|
|
|
|
raise Failed(f"TVDb Error: Could not parse {tvdb_url}")
|
|
|
|
|
|
|
|
results = response.xpath(f"//*[text()='TheTVDB.com {media_type} ID']/parent::node()/span/text()")
|
|
|
|
|
|
|
|
if len(results) > 0:
|
|
|
|
|
|
|
|
tvdb_id = int(results[0])
|
|
|
|
tmdb_id = None
|
|
|
|
tmdb_id = None
|
|
|
|
imdb_id = None
|
|
|
|
imdb_id = None
|
|
|
|
if self.is_movie:
|
|
|
|
if media_type == "Movie":
|
|
|
|
results = response.xpath("//*[text()='TheMovieDB.com']/@href")
|
|
|
|
results = response.xpath("//*[text()='TheMovieDB.com']/@href")
|
|
|
|
if len(results) > 0:
|
|
|
|
if len(results) > 0:
|
|
|
|
try:
|
|
|
|
try:
|
|
|
@ -119,36 +161,17 @@ class TVDbObj:
|
|
|
|
except Failed:
|
|
|
|
except Failed:
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
if tmdb_id is None and imdb_id is None:
|
|
|
|
if tmdb_id is None and imdb_id is None:
|
|
|
|
raise Failed(f"TVDb Error: No TMDb ID or IMDb ID found for {self.title}")
|
|
|
|
raise Failed(f"TVDb Error: No TMDb ID or IMDb ID found")
|
|
|
|
self.tmdb_id = tmdb_id
|
|
|
|
if self.config.Cache and not ignore_cache:
|
|
|
|
self.imdb_id = imdb_id
|
|
|
|
self.config.Cache.update_tvdb_map(expired, tvdb_url, tvdb_id, self.expiration)
|
|
|
|
|
|
|
|
return tvdb_id, tmdb_id, imdb_id
|
|
|
|
class TVDb:
|
|
|
|
elif tvdb_url.startswith(urls["movie_id"]):
|
|
|
|
def __init__(self, config, tvdb_language):
|
|
|
|
err_text = f"using TVDb Movie ID: {tvdb_url[len(urls['movie_id']):]}"
|
|
|
|
self.config = config
|
|
|
|
elif tvdb_url.startswith(urls["series_id"]):
|
|
|
|
self.tvdb_language = tvdb_language
|
|
|
|
err_text = f"using TVDb Series ID: {tvdb_url[len(urls['series_id']):]}"
|
|
|
|
|
|
|
|
else:
|
|
|
|
def get_item(self, tvdb_url, is_movie):
|
|
|
|
err_text = f"ID at the URL {tvdb_url}"
|
|
|
|
return self.get_movie(tvdb_url) if is_movie else self.get_series(tvdb_url)
|
|
|
|
raise Failed(f"TVDb Error: Could not find a TVDb {media_type} {err_text}")
|
|
|
|
|
|
|
|
|
|
|
|
def get_series(self, tvdb_url):
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
tvdb_url = f"{urls['series_id']}{int(tvdb_url)}"
|
|
|
|
|
|
|
|
except ValueError:
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
return TVDbObj(tvdb_url, self.tvdb_language, False, self.config)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_movie(self, tvdb_url):
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
tvdb_url = f"{urls['movie_id']}{int(tvdb_url)}"
|
|
|
|
|
|
|
|
except ValueError:
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
return TVDbObj(tvdb_url, self.tvdb_language, True, self.config)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_list_description(self, tvdb_url):
|
|
|
|
|
|
|
|
response = self.config.get_html(tvdb_url, headers=util.header(self.tvdb_language))
|
|
|
|
|
|
|
|
description = response.xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()")
|
|
|
|
|
|
|
|
return description[0] if len(description) > 0 and len(description[0]) > 0 else ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _ids_from_url(self, tvdb_url):
|
|
|
|
def _ids_from_url(self, tvdb_url):
|
|
|
|
ids = []
|
|
|
|
ids = []
|
|
|
@ -157,25 +180,27 @@ class TVDb:
|
|
|
|
logger.debug(f"URL: {tvdb_url}")
|
|
|
|
logger.debug(f"URL: {tvdb_url}")
|
|
|
|
if tvdb_url.startswith((urls["list"], urls["alt_list"])):
|
|
|
|
if tvdb_url.startswith((urls["list"], urls["alt_list"])):
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
response = self.config.get_html(tvdb_url, headers=util.header(self.tvdb_language))
|
|
|
|
response = self.config.get_html(tvdb_url, headers=util.header(self.language))
|
|
|
|
items = response.xpath("//div[@class='col-xs-12 col-sm-12 col-md-8 col-lg-8 col-md-pull-4']/div[@class='row']")
|
|
|
|
items = response.xpath("//div[@class='row']/div/div[@class='row']/div/h3/a")
|
|
|
|
for item in items:
|
|
|
|
for item in items:
|
|
|
|
title = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/text()")[0]
|
|
|
|
title = item.xpath("text()")[0]
|
|
|
|
item_url = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/@href")[0]
|
|
|
|
item_url = item.xpath("@href")[0]
|
|
|
|
if item_url.startswith("/series/"):
|
|
|
|
if item_url.startswith("/series/"):
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
ids.append((self.get_series(f"{base_url}{item_url}").id, "tvdb"))
|
|
|
|
tvdb_id, _, _ = self.get_id_from_url(f"{base_url}{item_url}")
|
|
|
|
|
|
|
|
if tvdb_id:
|
|
|
|
|
|
|
|
ids.append((tvdb_id, "tvdb"))
|
|
|
|
except Failed as e:
|
|
|
|
except Failed as e:
|
|
|
|
logger.error(f"{e} for series {title}")
|
|
|
|
logger.error(f"{e} for series {title}")
|
|
|
|
elif item_url.startswith("/movies/"):
|
|
|
|
elif item_url.startswith("/movies/"):
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
movie = self.get_movie(f"{base_url}{item_url}")
|
|
|
|
_, tmdb_id, imdb_id = self.get_id_from_url(f"{base_url}{item_url}")
|
|
|
|
if movie.tmdb_id:
|
|
|
|
if tmdb_id:
|
|
|
|
ids.append((movie.tmdb_id, "tmdb"))
|
|
|
|
ids.append((tmdb_id, "tmdb"))
|
|
|
|
elif movie.imdb_id:
|
|
|
|
elif imdb_id:
|
|
|
|
ids.append((movie.imdb_id, "imdb"))
|
|
|
|
ids.append((imdb_id, "imdb"))
|
|
|
|
except Failed as e:
|
|
|
|
except Failed as e:
|
|
|
|
logger.error(e)
|
|
|
|
logger.error(f"{e} for movie {title}")
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
logger.error(f"TVDb Error: Skipping Movie: {title}")
|
|
|
|
logger.error(f"TVDb Error: Skipping Movie: {title}")
|
|
|
|
time.sleep(2)
|
|
|
|
time.sleep(2)
|
|
|
@ -191,14 +216,26 @@ class TVDb:
|
|
|
|
def get_tvdb_ids(self, method, data):
|
|
|
|
def get_tvdb_ids(self, method, data):
|
|
|
|
if method == "tvdb_show":
|
|
|
|
if method == "tvdb_show":
|
|
|
|
logger.info(f"Processing TVDb Show: {data}")
|
|
|
|
logger.info(f"Processing TVDb Show: {data}")
|
|
|
|
return [(self.get_series(data).id, "tvdb")]
|
|
|
|
ids = []
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
tvdb_id, _, _ = self.get_id_from_url(data)
|
|
|
|
|
|
|
|
if tvdb_id:
|
|
|
|
|
|
|
|
ids.append((tvdb_id, "tvdb"))
|
|
|
|
|
|
|
|
except Failed as e:
|
|
|
|
|
|
|
|
logger.error(e)
|
|
|
|
|
|
|
|
return ids
|
|
|
|
elif method == "tvdb_movie":
|
|
|
|
elif method == "tvdb_movie":
|
|
|
|
logger.info(f"Processing TVDb Movie: {data}")
|
|
|
|
logger.info(f"Processing TVDb Movie: {data}")
|
|
|
|
movie = self.get_movie(data)
|
|
|
|
ids = []
|
|
|
|
if movie.tmdb_id:
|
|
|
|
try:
|
|
|
|
return [(movie.tmdb_id, "tmdb")]
|
|
|
|
_, tmdb_id, imdb_id = self.get_id_from_url(data)
|
|
|
|
elif movie.imdb_id:
|
|
|
|
if tmdb_id:
|
|
|
|
return [(movie.imdb_id, "imdb")]
|
|
|
|
ids.append((tmdb_id, "tmdb"))
|
|
|
|
|
|
|
|
elif imdb_id:
|
|
|
|
|
|
|
|
ids.append((imdb_id, "imdb"))
|
|
|
|
|
|
|
|
except Failed as e:
|
|
|
|
|
|
|
|
logger.error(e)
|
|
|
|
|
|
|
|
return ids
|
|
|
|
elif method == "tvdb_list":
|
|
|
|
elif method == "tvdb_list":
|
|
|
|
logger.info(f"Processing TVDb List: {data}")
|
|
|
|
logger.info(f"Processing TVDb List: {data}")
|
|
|
|
return self._ids_from_url(data)
|
|
|
|
return self._ids_from_url(data)
|
|
|
|