From 38fba77bcecf69e525e972cfaaa0c200a92ad6f4 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 2 Mar 2021 16:41:34 -0500 Subject: [PATCH 01/22] moved convert_from_imdb to config --- modules/anidb.py | 42 +++++----------------------------- modules/builder.py | 4 ++-- modules/config.py | 56 +++++++++++++++++++++++++++++++++++++++++++++- modules/imdb.py | 52 +++--------------------------------------- modules/tvdb.py | 31 +++---------------------- 5 files changed, 68 insertions(+), 117 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 5597b0e9..7e698d1a 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -62,7 +62,7 @@ class AniDBAPI: return anidb_values raise Failed(f"AniDB Error: No valid AniDB IDs in {anidb_list}") - def get_items(self, method, data, language, status_message=True): + def get_items(self, config, method, data, language, status_message=True): pretty = util.pretty_names[method] if method in util.pretty_names else method if status_message: logger.debug(f"Data: {data}") @@ -80,9 +80,10 @@ class AniDBAPI: movie_ids = [] for anidb_id in anime_ids: try: - tmdb_id = self.convert_from_imdb(self.convert_anidb_to_imdb(anidb_id)) - if tmdb_id: movie_ids.append(tmdb_id) - else: raise Failed + for imdb_id in self.convert_anidb_to_imdb(anidb_id): + tmdb_id, _ = config.convert_from_imdb(imdb_id, language) + if tmdb_id: movie_ids.append(tmdb_id) + else: raise Failed except Failed: try: show_ids.append(self.convert_anidb_to_tvdb(anidb_id)) except Failed: logger.error(f"AniDB Error: No TVDb ID or IMDb ID found for AniDB ID: {anidb_id}") @@ -91,36 +92,3 @@ class AniDBAPI: logger.debug(f"TMDb IDs Found: {movie_ids}") logger.debug(f"TVDb IDs Found: {show_ids}") return movie_ids, show_ids - - def convert_from_imdb(self, imdb_id): - output_tmdb_ids = [] - if not isinstance(imdb_id, list): - imdb_id = [imdb_id] - - for imdb in imdb_id: - expired = False - if self.Cache: - tmdb_id, tvdb_id = self.Cache.get_ids_from_imdb(imdb) - if not tmdb_id: - tmdb_id, expired = self.Cache.get_tmdb_from_imdb(imdb) - if expired: - tmdb_id = None - else: - tmdb_id = None - from_cache = tmdb_id is not None - - if not tmdb_id and self.TMDb: - try: tmdb_id = self.TMDb.convert_imdb_to_tmdb(imdb) - except Failed: pass - if not tmdb_id and self.Trakt: - try: tmdb_id = self.Trakt.convert_imdb_to_tmdb(imdb) - except Failed: pass - try: - if tmdb_id and not from_cache: self.TMDb.get_movie(tmdb_id) - except Failed: tmdb_id = None - if tmdb_id: output_tmdb_ids.append(tmdb_id) - if self.Cache and tmdb_id and expired is not False: - self.Cache.update_imdb("movie", expired, imdb, tmdb_id) - if len(output_tmdb_ids) == 0: raise Failed(f"AniDB Error: No TMDb ID found for IMDb: {imdb_id}") - elif len(output_tmdb_ids) == 1: return output_tmdb_ids[0] - else: return output_tmdb_ids diff --git a/modules/builder.py b/modules/builder.py index e4082b85..76ba01b5 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -644,10 +644,10 @@ class CollectionBuilder: elif "tautulli" in method: items = self.library.Tautulli.get_items(self.library, time_range=value["list_days"], stats_count=value["list_size"], list_type=value["list_type"], stats_count_buffer=value["list_buffer"]) items_found += len(items) - elif "anidb" in method: items_found += check_map(self.config.AniDB.get_items(method, value, self.library.Plex.language)) + elif "anidb" in method: items_found += check_map(self.config.AniDB.get_items(self.config, method, value, self.library.Plex.language)) elif "mal" in method: items_found += check_map(self.config.MyAnimeList.get_items(method, value)) elif "tvdb" in method: items_found += check_map(self.config.TVDb.get_items(method, value, self.library.Plex.language)) - elif "imdb" in method: items_found += check_map(self.config.IMDb.get_items(method, value, self.library.Plex.language)) + elif "imdb" in method: items_found += check_map(self.config.IMDb.get_items(self.config, method, value, self.library.Plex.language)) elif "tmdb" in method: items_found += check_map(self.config.TMDb.get_items(method, value, self.library.is_movie)) elif "trakt" in method: items_found += check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) else: logger.error(f"Collection Error: {method} method not supported") diff --git a/modules/config.py b/modules/config.py index ccedad5d..48f7a1b5 100644 --- a/modules/config.py +++ b/modules/config.py @@ -205,7 +205,7 @@ class Config: else: logger.warning("mal attribute not found") - self.TVDb = TVDbAPI(Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt) + self.TVDb = TVDbAPI(self, Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt) self.IMDb = IMDbAPI(Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt, TVDb=self.TVDb) if self.TMDb or self.Trakt else None self.AniDB = AniDBAPI(Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt) @@ -477,6 +477,60 @@ class Config: continue builder.run_collections_again(library, collection_obj, movie_map, show_map) + def convert_from_imdb(self, imdb_id, language): + update_tmdb = False + update_tvdb = False + if self.Cache: + tmdb_id, tvdb_id = self.Cache.get_ids_from_imdb(imdb_id) + update_tmdb = False + if not tmdb_id: + tmdb_id, update_tmdb = self.Cache.get_tmdb_from_imdb(imdb_id) + if update_tmdb: + tmdb_id = None + update_tvdb = False + if not tvdb_id: + tvdb_id, update_tvdb = self.Cache.get_tvdb_from_imdb(imdb_id) + if update_tvdb: + tvdb_id = None + else: + tmdb_id = None + tvdb_id = None + from_cache = tmdb_id is not None or tvdb_id is not None + + if not tmdb_id and not tvdb_id and self.TMDb: + try: + tmdb_id = self.TMDb.convert_imdb_to_tmdb(imdb_id) + except Failed: + pass + if not tmdb_id and not tvdb_id and self.TMDb: + try: + tvdb_id = self.TMDb.convert_imdb_to_tvdb(imdb_id) + except Failed: + pass + if not tmdb_id and not tvdb_id and self.Trakt: + try: + tmdb_id = self.Trakt.convert_imdb_to_tmdb(imdb_id) + except Failed: + pass + if not tmdb_id and not tvdb_id and self.Trakt: + try: + tvdb_id = self.Trakt.convert_imdb_to_tvdb(imdb_id) + except Failed: + pass + try: + if tmdb_id and not from_cache: self.TMDb.get_movie(tmdb_id) + except Failed: tmdb_id = None + try: + if tvdb_id and not from_cache: self.TVDb.get_series(language, tvdb_id=tvdb_id) + except Failed: tvdb_id = None + if not tmdb_id and not tvdb_id: raise Failed(f"IMDb Error: No TMDb ID or TVDb ID found for IMDb: {imdb_id}") + if self.Cache: + if tmdb_id and update_tmdb is not False: + self.Cache.update_imdb("movie", update_tmdb, imdb_id, tmdb_id) + if tvdb_id and update_tvdb is not False: + self.Cache.update_imdb("show", update_tvdb, imdb_id, tvdb_id) + return tmdb_id, tvdb_id + def map_guids(self, library): movie_map = {} show_map = {} diff --git a/modules/imdb.py b/modules/imdb.py index ba17d336..402e3d4d 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -52,7 +52,7 @@ class IMDbAPI: def send_request(self, url, header): return html.fromstring(requests.get(url, headers=header).content) - def get_items(self, method, data, language, status_message=True): + def get_items(self, config, method, data, language, status_message=True): pretty = util.pretty_names[method] if method in util.pretty_names else method if status_message: logger.debug(f"Data: {data}") @@ -61,7 +61,7 @@ class IMDbAPI: if method == "imdb_id": if status_message: logger.info(f"Processing {pretty}: {data}") - tmdb_id, tvdb_id = self.convert_from_imdb(data, language) + tmdb_id, tvdb_id = config.convert_from_imdb(data, language) if tmdb_id: movie_ids.append(tmdb_id) if tvdb_id: show_ids.append(tvdb_id) elif method == "imdb_list": @@ -74,7 +74,7 @@ class IMDbAPI: for i, imdb_id in enumerate(imdb_ids, 1): length = util.print_return(length, f"Converting IMDb ID {i}/{total_ids}") try: - tmdb_id, tvdb_id = self.convert_from_imdb(imdb_id, language) + tmdb_id, tvdb_id = config.convert_from_imdb(imdb_id, language) if tmdb_id: movie_ids.append(tmdb_id) if tvdb_id: show_ids.append(tvdb_id) except Failed as e: logger.warning(e) @@ -85,49 +85,3 @@ class IMDbAPI: logger.debug(f"TMDb IDs Found: {movie_ids}") logger.debug(f"TVDb IDs Found: {show_ids}") return movie_ids, show_ids - - def convert_from_imdb(self, imdb_id, language): - update_tmdb = False - update_tvdb = False - if self.Cache: - tmdb_id, tvdb_id = self.Cache.get_ids_from_imdb(imdb_id) - update_tmdb = False - if not tmdb_id: - tmdb_id, update_tmdb = self.Cache.get_tmdb_from_imdb(imdb_id) - if update_tmdb: - tmdb_id = None - update_tvdb = False - if not tvdb_id: - tvdb_id, update_tvdb = self.Cache.get_tvdb_from_imdb(imdb_id) - if update_tvdb: - tvdb_id = None - else: - tmdb_id = None - tvdb_id = None - from_cache = tmdb_id is not None or tvdb_id is not None - - if not tmdb_id and not tvdb_id and self.TMDb: - try: tmdb_id = self.TMDb.convert_imdb_to_tmdb(imdb_id) - except Failed: pass - if not tmdb_id and not tvdb_id and self.TMDb: - try: tvdb_id = self.TMDb.convert_imdb_to_tvdb(imdb_id) - except Failed: pass - if not tmdb_id and not tvdb_id and self.Trakt: - try: tmdb_id = self.Trakt.convert_imdb_to_tmdb(imdb_id) - except Failed: pass - if not tmdb_id and not tvdb_id and self.Trakt: - try: tvdb_id = self.Trakt.convert_imdb_to_tvdb(imdb_id) - except Failed: pass - try: - if tmdb_id and not from_cache: self.TMDb.get_movie(tmdb_id) - except Failed: tmdb_id = None - try: - if tvdb_id and not from_cache: self.TVDb.get_series(language, tvdb_id=tvdb_id) - except Failed: tvdb_id = None - if not tmdb_id and not tvdb_id: raise Failed(f"IMDb Error: No TMDb ID or TVDb ID found for IMDb: {imdb_id}") - if self.Cache: - if tmdb_id and update_tmdb is not False: - self.Cache.update_imdb("movie", update_tmdb, imdb_id, tmdb_id) - if tvdb_id and update_tvdb is not False: - self.Cache.update_imdb("show", update_tvdb, imdb_id, tvdb_id) - return tmdb_id, tvdb_id diff --git a/modules/tvdb.py b/modules/tvdb.py index b635d859..17a64ecc 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -45,7 +45,7 @@ class TVDbObj: if not tmdb_id: results = response.xpath("//*[text()='IMDB']/@href") if len(results) > 0: - try: tmdb_id = TVDb.convert_from_imdb(util.get_id_from_imdb_url(results[0])) + try: tmdb_id, _ = TVDb.config.convert_from_imdb(util.get_id_from_imdb_url(results[0])) except Failed as e: logger.error(e) self.tmdb_id = tmdb_id self.tvdb_url = tvdb_url @@ -54,7 +54,8 @@ class TVDbObj: self.TVDb = TVDb class TVDbAPI: - def __init__(self, Cache=None, TMDb=None, Trakt=None): + def __init__(self, config, Cache=None, TMDb=None, Trakt=None): + self.config = config self.Cache = Cache self.TMDb = TMDb self.Trakt = Trakt @@ -140,29 +141,3 @@ class TVDbAPI: logger.debug(f"TMDb IDs Found: {movie_ids}") logger.debug(f"TVDb IDs Found: {show_ids}") return movie_ids, show_ids - - def convert_from_imdb(self, imdb_id): - update = False - if self.Cache: - tmdb_id, tvdb_id = self.Cache.get_ids_from_imdb(imdb_id) - if not tmdb_id: - tmdb_id, update = self.Cache.get_tmdb_from_imdb(imdb_id) - if update: - tmdb_id = None - else: - tmdb_id = None - from_cache = tmdb_id is not None - - if not tmdb_id and self.TMDb: - try: tmdb_id = self.TMDb.convert_imdb_to_tmdb(imdb_id) - except Failed: pass - if not tmdb_id and self.Trakt: - try: tmdb_id = self.Trakt.convert_imdb_to_tmdb(imdb_id) - except Failed: pass - try: - if tmdb_id and not from_cache: self.TMDb.get_movie(tmdb_id) - except Failed: tmdb_id = None - if not tmdb_id: raise Failed(f"TVDb Error: No TMDb ID found for IMDb: {imdb_id}") - if self.Cache and tmdb_id and update is not False: - self.Cache.update_imdb("movie", update, imdb_id, tmdb_id) - return tmdb_id From 12c00ec114b507f5c04a653cd4d843abb060ee7d Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 3 Mar 2021 12:02:11 -0500 Subject: [PATCH 02/22] added support for the New TV Agent #77 --- modules/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/config.py b/modules/config.py index 48f7a1b5..bd21035f 100644 --- a/modules/config.py +++ b/modules/config.py @@ -15,6 +15,7 @@ from modules.trakttv import TraktAPI from modules.tvdb import TVDbAPI from modules.util import Failed from plexapi.exceptions import BadRequest +from plexapi.media import Guid from ruamel import yaml logger = logging.getLogger("Plex Meta Manager") @@ -575,11 +576,17 @@ class Config: item_type = guid.scheme.split(".")[-1] check_id = guid.netloc - if item_type == "plex" and library.is_movie: + if item_type == "plex" and check_id == "movie": for guid_tag in item.guids: url_parsed = requests.utils.urlparse(guid_tag.id) if url_parsed.scheme == "tmdb": tmdb_id = int(url_parsed.netloc) elif url_parsed.scheme == "imdb": imdb_id = url_parsed.netloc + elif item_type == "plex" and check_id == "show": + for guid_tag in item.findItems(item._data, Guid): + url_parsed = requests.utils.urlparse(guid_tag.id) + if url_parsed.scheme == "tvdb": tvdb_id = int(url_parsed.netloc) + elif url_parsed.scheme == "imdb": imdb_id = url_parsed.netloc + elif url_parsed.scheme == "tmdb": tmdb_id = int(url_parsed.netloc) elif item_type == "imdb": imdb_id = check_id elif item_type == "thetvdb": tvdb_id = int(check_id) elif item_type == "themoviedb": tmdb_id = int(check_id) From d101012772b343692d6e862b35b6800968d0b7f5 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 4 Mar 2021 01:16:37 -0500 Subject: [PATCH 03/22] Requirements Check #79 --- plex_meta_manager.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plex_meta_manager.py b/plex_meta_manager.py index f8475b68..accbb322 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -1,7 +1,12 @@ -import argparse, logging, os, re, schedule, sys, time +import argparse, logging, os, re, sys, time from datetime import datetime -from modules import tests, util -from modules.config import Config +try: + import schedule + from modules import tests, util + from modules.config import Config +except ModuleNotFoundError: + print("Error: Requirements are not installed") + sys.exit(0) parser = argparse.ArgumentParser() parser.add_argument("--my-tests", dest="tests", help=argparse.SUPPRESS, action="store_true", default=False) From 8ea4ab0950f7720f23e54b54fc685f6c7fefced7 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 4 Mar 2021 15:05:51 -0500 Subject: [PATCH 04/22] Add Letterboxd Support --- modules/builder.py | 3 +++ modules/config.py | 2 ++ modules/letterboxd.py | 54 +++++++++++++++++++++++++++++++++++++++++++ modules/util.py | 3 +++ 4 files changed, 62 insertions(+) create mode 100644 modules/letterboxd.py diff --git a/modules/builder.py b/modules/builder.py index 76ba01b5..42c49e90 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -327,6 +327,8 @@ class CollectionBuilder: list_count = 0 new_list.append({"url": imdb_url, "limit": list_count}) self.methods.append((method_name, new_list)) + elif method_name == "letterboxd_list": + self.methods.append((method_name, util.get_list(data[m], split=False))) elif method_name in util.dictionary_lists: if isinstance(data[m], dict): def get_int(parent, method, data_in, default_in, minimum=1, maximum=None): @@ -648,6 +650,7 @@ class CollectionBuilder: elif "mal" in method: items_found += check_map(self.config.MyAnimeList.get_items(method, value)) elif "tvdb" in method: items_found += check_map(self.config.TVDb.get_items(method, value, self.library.Plex.language)) elif "imdb" in method: items_found += check_map(self.config.IMDb.get_items(self.config, method, value, self.library.Plex.language)) + elif "letterboxd" in method: items_found += check_map(self.config.Letterboxd.get_items(method, value, self.library.Plex.language)) elif "tmdb" in method: items_found += check_map(self.config.TMDb.get_items(method, value, self.library.is_movie)) elif "trakt" in method: items_found += check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) else: logger.error(f"Collection Error: {method} method not supported") diff --git a/modules/config.py b/modules/config.py index bd21035f..6e0d9bc5 100644 --- a/modules/config.py +++ b/modules/config.py @@ -4,6 +4,7 @@ from modules.anidb import AniDBAPI from modules.builder import CollectionBuilder from modules.cache import Cache from modules.imdb import IMDbAPI +from modules.letterboxd import LetterboxdAPI from modules.mal import MyAnimeListAPI from modules.mal import MyAnimeListIDList from modules.plex import PlexAPI @@ -209,6 +210,7 @@ class Config: self.TVDb = TVDbAPI(self, Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt) self.IMDb = IMDbAPI(Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt, TVDb=self.TVDb) if self.TMDb or self.Trakt else None self.AniDB = AniDBAPI(Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt) + self.Letterboxd = LetterboxdAPI() util.separator() diff --git a/modules/letterboxd.py b/modules/letterboxd.py new file mode 100644 index 00000000..5671878d --- /dev/null +++ b/modules/letterboxd.py @@ -0,0 +1,54 @@ +import logging, math, re, requests +from lxml import html +from modules import util +from modules.util import Failed +from retrying import retry + +logger = logging.getLogger("Plex Meta Manager") + +class LetterboxdAPI: + def __init__(self): + self.url = "https://letterboxd.com" + + @retry(stop_max_attempt_number=6, wait_fixed=10000) + def send_request(self, url, header): + return html.fromstring(requests.get(url, headers=header).content) + + def parse_list_for_slugs(self, list_url, language): + response = self.send_request(list_url, header={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}) + slugs = response.xpath("//div[@class='poster film-poster really-lazy-load']/@data-film-slug") + next_url = response.xpath("//a[@class='next']/@href") + if len(next_url) > 0: + slugs.extend(self.parse_list_for_slugs(f"{self.url}{next_url[0]}", language)) + return slugs + + def get_tmdb_from_slug(self, slug, language): + return self.get_tmdb(f"{self.url}{slug}", language) + + def get_tmdb(self, letterboxd_url, language): + response = self.send_request(letterboxd_url, header={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}) + ids = response.xpath("//body/@data-tmdb-id") + if len(ids) > 0: + return int(ids[0]) + raise Failed(f"Letterboxd Error: TMDb ID not found at {letterboxd_url}") + + def get_items(self, method, data, language, status_message=True): + pretty = util.pretty_names[method] if method in util.pretty_names else method + movie_ids = [] + if status_message: + logger.info(f"Processing {pretty}: {data}") + slugs = self.parse_list_for_slugs(data, language) + total_slugs = len(slugs) + if total_slugs == 0: + raise Failed(f"Letterboxd Error: No List Items found in {data}") + length = 0 + for i, slug in enumerate(slugs, 1): + length = util.print_return(length, f"Finding TMDb ID {i}/{total_slugs}") + try: + movie_ids.append(self.get_tmdb(slug, language)) + except Failed as e: + logger.error(e) + util.print_end(length, f"Processed {total_slugs} TMDb IDs") + if status_message: + logger.debug(f"TMDb IDs Found: {movie_ids}") + return movie_ids, [] diff --git a/modules/util.py b/modules/util.py index 4ddfc1d2..53ec05b1 100644 --- a/modules/util.py +++ b/modules/util.py @@ -97,6 +97,7 @@ pretty_names = { "anidb_popular": "AniDB Popular", "imdb_list": "IMDb List", "imdb_id": "IMDb ID", + "letterboxd_list": "Letterboxd List", "mal_id": "MyAnimeList ID", "mal_all": "MyAnimeList All", "mal_airing": "MyAnimeList Airing", @@ -214,6 +215,7 @@ all_lists = [ "anidb_popular", "imdb_list", "imdb_id", + "letterboxd_list", "mal_id", "mal_all", "mal_airing", @@ -309,6 +311,7 @@ show_only_lists = [ "tvdb_show" ] movie_only_lists = [ + "letterboxd_list", "tmdb_collection", "tmdb_collection_details", "tmdb_movie", From acc7d8656127efc87563e1f7d68286a8f8c5ac29 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 5 Mar 2021 11:04:28 -0500 Subject: [PATCH 05/22] added mass_genre_update --- modules/builder.py | 10 ++-- modules/cache.py | 122 +++++++++++++++++++++++++++++++++++++-------- modules/config.py | 120 ++++++++++++++++++++++++++++++++++++++------ modules/omdb.py | 63 +++++++++++++++++++++++ modules/plex.py | 10 +--- 5 files changed, 276 insertions(+), 49 deletions(-) create mode 100644 modules/omdb.py diff --git a/modules/builder.py b/modules/builder.py index 42c49e90..1b1ceac0 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -878,17 +878,17 @@ class CollectionBuilder: elif "tmdb_show_details" in self.backgrounds: set_image("tmdb_show", self.backgrounds, is_background=True) else: logger.info("No background to update") - def run_collections_again(self, library, collection_obj, movie_map, show_map): + def run_collections_again(self, collection_obj, movie_map, show_map): collection_items = collection_obj.items() if isinstance(collection_obj, Collections) else [] name = collection_obj.title if isinstance(collection_obj, Collections) else collection_obj rating_keys = [movie_map[mm] for mm in self.missing_movies if mm in movie_map] - if library.is_show: + if self.library.is_show: rating_keys.extend([show_map[sm] for sm in self.missing_shows if sm in show_map]) if len(rating_keys) > 0: for rating_key in rating_keys: try: - current = library.fetchItem(int(rating_key)) + current = self.library.fetchItem(int(rating_key)) except (BadRequest, NotFound): logger.error(f"Plex Error: Item {rating_key} not found") continue @@ -897,7 +897,7 @@ class CollectionBuilder: else: current.addCollection(name) logger.info(f"{name} Collection | + | {current.title}") - logger.info(f"{len(rating_keys)} {'Movie' if library.is_movie else 'Show'}{'s' if len(rating_keys) > 1 else ''} Processed") + logger.info(f"{len(rating_keys)} {'Movie' if self.library.is_movie else 'Show'}{'s' if len(rating_keys) > 1 else ''} Processed") if len(self.missing_movies) > 0: logger.info("") @@ -913,7 +913,7 @@ class CollectionBuilder: logger.info("") logger.info(f"{len(self.missing_movies)} Movie{'s' if len(self.missing_movies) > 1 else ''} Missing") - if len(self.missing_shows) > 0 and library.is_show: + if len(self.missing_shows) > 0 and self.library.is_show: logger.info("") for missing_id in self.missing_shows: if missing_id not in show_map: diff --git a/modules/cache.py b/modules/cache.py index 2be4cd4f..ed795109 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -1,6 +1,7 @@ import logging, os, random, sqlite3 from contextlib import closing from datetime import datetime, timedelta +from modules.util import Failed logger = logging.getLogger("Plex Meta Manager") @@ -13,30 +14,45 @@ class Cache: cursor.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name='guids'") if cursor.fetchone()[0] == 0: logger.info(f"Initializing cache database at {cache}") - cursor.execute( - """CREATE TABLE IF NOT EXISTS guids ( - INTEGER PRIMARY KEY, - plex_guid TEXT, - tmdb_id TEXT, - imdb_id TEXT, - tvdb_id TEXT, - anidb_id TEXT, - mal_id TEXT, - expiration_date TEXT, - media_type TEXT)""" - ) - cursor.execute( - """CREATE TABLE IF NOT EXISTS imdb_map ( - INTEGER PRIMARY KEY, - imdb_id TEXT, - t_id TEXT, - expiration_date TEXT, - media_type TEXT)""" - ) else: logger.info(f"Using cache database at {cache}") + cursor.execute( + """CREATE TABLE IF NOT EXISTS guids ( + INTEGER PRIMARY KEY, + plex_guid TEXT UNIQUE, + tmdb_id TEXT, + imdb_id TEXT, + tvdb_id TEXT, + anidb_id TEXT, + mal_id TEXT, + expiration_date TEXT, + media_type TEXT)""" + ) + cursor.execute( + """CREATE TABLE IF NOT EXISTS imdb_map ( + INTEGER PRIMARY KEY, + imdb_id TEXT UNIQUE, + t_id TEXT, + expiration_date TEXT, + media_type TEXT)""" + ) + cursor.execute( + """CREATE TABLE IF NOT EXISTS omdb_data ( + INTEGER PRIMARY KEY, + imdb_id TEXT UNIQUE, + title TEXT, + year INTEGER, + content_rating TEXT, + genres TEXT, + imdb_rating REAL, + imdb_votes INTEGER, + metacritic_rating INTEGER, + type TEXT, + expiration_date TEXT)""" + ) self.expiration = expiration self.cache_path = cache + self.omdb_expiration = expiration def get_ids_from_imdb(self, imdb_id): tmdb_id, tmdb_expired = self.get_tmdb_id("movie", imdb_id=imdb_id) @@ -82,6 +98,40 @@ class Cache: expired = time_between_insertion.days > self.expiration return id_to_return, expired + def get_ids(self, media_type, plex_guid=None, tmdb_id=None, imdb_id=None, tvdb_id=None): + ids_to_return = {} + expired = None + if plex_guid: + key = plex_guid + key_type = "plex_guid" + elif tmdb_id: + key = tmdb_id + key_type = "tmdb_id" + elif imdb_id: + key = imdb_id + key_type = "imdb_id" + elif tvdb_id: + key = tvdb_id + key_type = "tvdb_id" + else: + raise Failed("ID Required") + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute(f"SELECT * FROM guids WHERE {key_type} = ? AND media_type = ?", (key, media_type)) + row = cursor.fetchone() + if row: + if row["plex_guid"]: ids_to_return["plex"] = row["plex_guid"] + if row["tmdb_id"]: ids_to_return["tmdb"] = int(row["tmdb_id"]) + if row["imdb_id"]: ids_to_return["imdb"] = row["imdb_id"] + if row["tvdb_id"]: ids_to_return["tvdb"] = int(row["tvdb_id"]) + if row["anidb_id"]: ids_to_return["anidb"] = int(row["anidb_id"]) + if row["mal_id"]: ids_to_return["mal"] = int(row["mal_id"]) + datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d") + time_between_insertion = datetime.now() - datetime_object + expired = time_between_insertion.days > self.expiration + return ids_to_return, expired + def update_guid(self, media_type, plex_guid, tmdb_id, imdb_id, tvdb_id, anidb_id, mal_id, expired): expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, self.expiration))) with sqlite3.connect(self.cache_path) as connection: @@ -126,3 +176,35 @@ class Cache: with closing(connection.cursor()) as cursor: cursor.execute("INSERT OR IGNORE INTO imdb_map(imdb_id) VALUES(?)", (imdb_id,)) cursor.execute("UPDATE imdb_map SET t_id = ?, expiration_date = ?, media_type = ? WHERE imdb_id = ?", (t_id, expiration_date.strftime("%Y-%m-%d"), media_type, imdb_id)) + + def query_omdb(self, imdb_id): + omdb_dict = {} + expired = None + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("SELECT * FROM omdb_data WHERE imdb_id = ?", (imdb_id,)) + row = cursor.fetchone() + if row: + omdb_dict["imdbID"] = row["imdb_id"] if row["imdb_id"] else None + omdb_dict["Title"] = row["title"] if row["title"] else None + omdb_dict["Year"] = row["year"] if row["year"] else None + omdb_dict["Rated"] = row["content_rating"] if row["content_rating"] else None + omdb_dict["Genre"] = row["genres"] if row["genres"] else None + omdb_dict["imdbRating"] = row["imdb_rating"] if row["imdb_rating"] else None + omdb_dict["imdbVotes"] = row["imdb_votes"] if row["imdb_votes"] else None + omdb_dict["Metascore"] = row["metacritic_rating"] if row["metacritic_rating"] else None + omdb_dict["Type"] = row["type"] if row["type"] else None + datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d") + time_between_insertion = datetime.now() - datetime_object + expired = time_between_insertion.days > self.omdb_expiration + return omdb_dict, expired + + def update_omdb(self, expired, omdb): + expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, self.omdb_expiration))) + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute("INSERT OR IGNORE INTO omdb_data(imdb_id) VALUES(?)", (omdb.imdb_id,)) + update_sql = "UPDATE omdb_data SET title = ?, year = ?, content_rating = ?, genres = ?, imdb_rating = ?, imdb_votes = ?, metacritic_rating = ?, type = ?, expiration_date = ? WHERE imdb_id = ?" + cursor.execute(update_sql, (omdb.title, omdb.year, omdb.content_rating, omdb.genres_str, omdb.imdb_rating, omdb.imdb_votes, omdb.metacritic_rating, omdb.type, expiration_date.strftime("%Y-%m-%d"), omdb.imdb_id)) diff --git a/modules/config.py b/modules/config.py index 6e0d9bc5..3e402b34 100644 --- a/modules/config.py +++ b/modules/config.py @@ -7,6 +7,7 @@ from modules.imdb import IMDbAPI from modules.letterboxd import LetterboxdAPI from modules.mal import MyAnimeListAPI from modules.mal import MyAnimeListIDList +from modules.omdb import OMDbAPI from modules.plex import PlexAPI from modules.radarr import RadarrAPI from modules.sonarr import SonarrAPI @@ -71,6 +72,7 @@ class Config: if "tautulli" in new_config: new_config["tautulli"] = new_config.pop("tautulli") if "radarr" in new_config: new_config["radarr"] = new_config.pop("radarr") if "sonarr" in new_config: new_config["sonarr"] = new_config.pop("sonarr") + if "omdb" in new_config: new_config["omdb"] = new_config.pop("omdb") if "trakt" in new_config: new_config["trakt"] = new_config.pop("trakt") if "mal" in new_config: new_config["mal"] = new_config.pop("mal") yaml.round_trip_dump(new_config, open(self.config_path, "w"), indent=ind, block_seq_indent=bsi) @@ -172,6 +174,23 @@ class Config: util.separator() + self.OMDb = None + if "omdb" in self.data: + logger.info("Connecting to OMDb...") + self.omdb = {} + try: + self.omdb["apikey"] = check_for_attribute(self.data, "apikey", parent="omdb", throw=True) + self.omdb["omdb_cache"] = check_for_attribute(self.data, "omdb_cache", parent="omdb", options=" true (Use a cache to store data)\n false (Do not use a cache to store data)", var_type="bool", default=True) + self.omdb["omdb_cache_expiration"] = check_for_attribute(self.data, "omdb_cache_expiration", parent="omdb", var_type="int", default=60) + self.OMDb = OMDbAPI(self.omdb, Cache=self.Cache) + except Failed as e: + logger.error(e) + logger.info(f"OMDb Connection {'Failed' if self.OMDb is None else 'Successful'}") + else: + logger.warning("omdb attribute not found") + + util.separator() + self.Trakt = None if "trakt" in self.data: logger.info("Connecting to Trakt...") @@ -263,12 +282,35 @@ class Config: if params["asset_directory"] is None: logger.warning("Config Warning: Assets will not be used asset_directory attribute must be set under config or under this specific Library") - params["sync_mode"] = check_for_attribute(libs[lib], "sync_mode", parent="settings", test_list=["append", "sync"], options=" append (Only Add Items to the Collection)\n sync (Add & Remove Items from the Collection)", default=self.general["sync_mode"], save=False) - params["show_unmanaged"] = check_for_attribute(libs[lib], "show_unmanaged", parent="settings", var_type="bool", default=self.general["show_unmanaged"], save=False) - params["show_filtered"] = check_for_attribute(libs[lib], "show_filtered", parent="settings", var_type="bool", default=self.general["show_filtered"], save=False) - params["show_missing"] = check_for_attribute(libs[lib], "show_missing", parent="settings", var_type="bool", default=self.general["show_missing"], save=False) - params["save_missing"] = check_for_attribute(libs[lib], "save_missing", parent="settings", var_type="bool", default=self.general["save_missing"], save=False) + if "settings" in libs[lib] and libs[lib]["settings"] and "sync_mode" in libs[lib]["settings"]: + params["sync_mode"] = check_for_attribute(libs[lib], "sync_mode", parent="settings", test_list=["append", "sync"], options=" append (Only Add Items to the Collection)\n sync (Add & Remove Items from the Collection)", default=self.general["sync_mode"], do_print=False, save=False) + else: + params["sync_mode"] = check_for_attribute(libs[lib], "sync_mode", test_list=["append", "sync"], options=" append (Only Add Items to the Collection)\n sync (Add & Remove Items from the Collection)", default=self.general["sync_mode"], do_print=False, save=False) + if "settings" in libs[lib] and libs[lib]["settings"] and "show_unmanaged" in libs[lib]["settings"]: + params["show_unmanaged"] = check_for_attribute(libs[lib], "show_unmanaged", parent="settings", var_type="bool", default=self.general["show_unmanaged"], do_print=False, save=False) + else: + params["show_unmanaged"] = check_for_attribute(libs[lib], "show_unmanaged", var_type="bool", default=self.general["show_unmanaged"], do_print=False, save=False) + + if "settings" in libs[lib] and libs[lib]["settings"] and "show_filtered" in libs[lib]["settings"]: + params["show_filtered"] = check_for_attribute(libs[lib], "show_filtered", parent="settings", var_type="bool", default=self.general["show_filtered"], do_print=False, save=False) + else: + params["show_filtered"] = check_for_attribute(libs[lib], "show_filtered", var_type="bool", default=self.general["show_filtered"], do_print=False, save=False) + + if "settings" in libs[lib] and libs[lib]["settings"] and "show_missing" in libs[lib]["settings"]: + params["show_missing"] = check_for_attribute(libs[lib], "show_missing", parent="settings", var_type="bool", default=self.general["show_missing"], do_print=False, save=False) + else: + params["show_missing"] = check_for_attribute(libs[lib], "show_missing", var_type="bool", default=self.general["show_missing"], do_print=False, save=False) + + if "settings" in libs[lib] and libs[lib]["settings"] and "save_missing" in libs[lib]["settings"]: + params["save_missing"] = check_for_attribute(libs[lib], "save_missing", parent="settings", var_type="bool", default=self.general["save_missing"], do_print=False, save=False) + else: + params["save_missing"] = check_for_attribute(libs[lib], "save_missing", var_type="bool", default=self.general["save_missing"], do_print=False, save=False) + + if "mass_genre_update" in libs[lib] and libs[lib]["mass_genre_update"]: + params["mass_genre_update"] = check_for_attribute(libs[lib], "mass_genre_update", test_list=["tmdb", "omdb"], options=" tmdb (Use TMDb Metadata)\n omdb (Use IMDb Metadata through OMDb)", default_is_none=True, save=False) + else: + params["mass_genre_update"] = None try: params["metadata_path"] = check_for_attribute(libs[lib], "metadata_path", var_type="path", default=os.path.join(default_dir, f"{lib}.yml"), throw=True) params["library_type"] = check_for_attribute(libs[lib], "library_type", test_list=["movie", "show"], options=" movie (For Movie Libraries)\n show (For Show Libraries)", throw=True) @@ -295,7 +337,7 @@ class Config: radarr_params["add"] = check_for_attribute(libs[lib], "add", parent="radarr", var_type="bool", default=self.general["radarr"]["add"], save=False) radarr_params["search"] = check_for_attribute(libs[lib], "search", parent="radarr", var_type="bool", default=self.general["radarr"]["search"], save=False) radarr_params["tag"] = check_for_attribute(libs[lib], "search", parent="radarr", var_type="lower_list", default=self.general["radarr"]["tag"], default_is_none=True, save=False) - library.add_Radarr(RadarrAPI(self.TMDb, radarr_params)) + library.Radarr = RadarrAPI(self.TMDb, radarr_params) except Failed as e: util.print_multiline(e) logger.info(f"{params['name']} library's Radarr Connection {'Failed' if library.Radarr is None else 'Successful'}") @@ -313,7 +355,7 @@ class Config: sonarr_params["search"] = check_for_attribute(libs[lib], "search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["search"], save=False) sonarr_params["season_folder"] = check_for_attribute(libs[lib], "season_folder", parent="sonarr", var_type="bool", default=self.general["sonarr"]["season_folder"], save=False) sonarr_params["tag"] = check_for_attribute(libs[lib], "search", parent="sonarr", var_type="lower_list", default=self.general["sonarr"]["tag"], default_is_none=True, save=False) - library.add_Sonarr(SonarrAPI(self.TVDb, sonarr_params, library.Plex.language)) + library.Sonarr = SonarrAPI(self.TVDb, sonarr_params, library.Plex.language) except Failed as e: util.print_multiline(e) logger.info(f"{params['name']} library's Sonarr Connection {'Failed' if library.Sonarr is None else 'Successful'}") @@ -324,7 +366,7 @@ class Config: try: tautulli_params["url"] = check_for_attribute(libs[lib], "url", parent="tautulli", default=self.general["tautulli"]["url"], req_default=True, save=False) tautulli_params["apikey"] = check_for_attribute(libs[lib], "apikey", parent="tautulli", default=self.general["tautulli"]["apikey"], req_default=True, save=False) - library.add_Tautulli(TautulliAPI(tautulli_params)) + library.Tautulli = TautulliAPI(tautulli_params) except Failed as e: util.print_multiline(e) logger.info(f"{params['name']} library's Tautulli Connection {'Failed' if library.Tautulli is None else 'Successful'}") @@ -345,16 +387,19 @@ class Config: os.environ["PLEXAPI_PLEXAPI_TIMEOUT"] = str(library.timeout) logger.info("") util.separator(f"{library.name} Library") - try: library.update_metadata(self.TMDb, test) - except Failed as e: logger.error(e) + logger.info("") + util.separator(f"Mapping {library.name} Library") + logger.info("") + movie_map, show_map = self.map_guids(library) + if not test: + if library.mass_genre_update: + self.mass_metadata(library) + try: library.update_metadata(self.TMDb, test) + except Failed as e: logger.error(e) logger.info("") util.separator(f"{library.name} Library {'Test ' if test else ''}Collections") collections = {c: library.collections[c] for c in util.get_list(requested_collections) if c in library.collections} if requested_collections else library.collections if collections: - logger.info("") - util.separator(f"Mapping {library.name} Library") - logger.info("") - movie_map, show_map = self.map_guids(library) for c in collections: if test and ("test" not in collections[c] or collections[c]["test"] is not True): no_template_test = True @@ -478,7 +523,7 @@ class Config: except Failed as e: util.print_multiline(e, error=True) continue - builder.run_collections_again(library, collection_obj, movie_map, show_map) + builder.run_collections_again(collection_obj, movie_map, show_map) def convert_from_imdb(self, imdb_id, language): update_tmdb = False @@ -534,6 +579,51 @@ class Config: self.Cache.update_imdb("show", update_tvdb, imdb_id, tvdb_id) return tmdb_id, tvdb_id + def mass_metadata(self, library): + length = 0 + logger.info("") + util.separator(f"Mass Editing {'Movie' if library.is_movie else 'Show'} Library: {library.name}") + logger.info("") + items = library.Plex.all() + for i, item in enumerate(items, 1): + length = util.print_return(length, f"Processing: {i}/{len(items)} {item.title}") + ids, expired = self.Cache.get_ids("movie" if library.is_movie else "show", plex_guid=item.guid) + if library.mass_genre_update: + if library.mass_genre_update == "tmdb": + if "tmdb" not in ids: + util.print_end(length, f"{item.title[:25]:<25} | No TMDb for Guid: {item.guid}") + continue + try: + tmdb_item = self.TMDb.get_movie(ids["tmdb"]) if library.is_movie else self.TMDb.get_show(ids["tmdb"]) + except Failed as e: + util.print_end(length, str(e)) + continue + new_genres = [genre.name for genre in tmdb_item.genres] + elif library.mass_genre_update == "omdb": + if self.OMDb.limit is True: + break + if "imdb" not in ids: + util.print_end(length, f"{item.title[:25]:<25} | No IMDb for Guid: {item.guid}") + continue + try: + omdb_item = self.OMDb.get_omdb(ids["imdb"]) + except Failed as e: + util.print_end(length, str(e)) + continue + new_genres = omdb_item.genres + else: + raise Failed + item_genres = [genre.tag for genre in item.genres] + display_str = "" + for genre in (g for g in item_genres if g not in new_genres): + item.removeGenre(genre) + display_str += f"{', ' if len(display_str) > 0 else ''}-{genre}" + for genre in (g for g in new_genres if g not in item_genres): + item.addGenre(genre) + display_str += f"{', ' if len(display_str) > 0 else ''}+{genre}" + if len(display_str) > 0: + util.print_end(length, f"{item.title[:25]:<25} | Genres | {display_str}") + def map_guids(self, library): movie_map = {} show_map = {} diff --git a/modules/omdb.py b/modules/omdb.py new file mode 100644 index 00000000..7470d594 --- /dev/null +++ b/modules/omdb.py @@ -0,0 +1,63 @@ +import logging, math, re, requests +from lxml import html +from modules import util +from modules.util import Failed +from retrying import retry + +logger = logging.getLogger("Plex Meta Manager") + +class OMDbObj: + def __init__(self, data): + self._data = data + self.title = data["Title"] + try: + self.year = int(data["Year"]) + except (ValueError, TypeError): + self.year = None + self.content_rating = data["Rated"] + self.genres = util.get_list(data["Genre"]) + self.genres_str = data["Genre"] + try: + self.imdb_rating = float(data["imdbRating"]) + except (ValueError, TypeError): + self.imdb_rating = None + try: + self.imdb_votes = int(str(data["imdbVotes"]).replace(',', '')) + except (ValueError, TypeError): + self.imdb_votes = None + try: + self.metacritic_rating = int(data["Metascore"]) + except (ValueError, TypeError): + self.metacritic_rating = None + self.imdb_id = data["imdbID"] + self.type = data["Type"] + +class OMDbAPI: + def __init__(self, params, Cache=None): + self.url = "http://www.omdbapi.com/" + self.apikey = params["apikey"] + self.cache = params["omdb_cache"] + self.cache_expiration = params["omdb_cache_expiration"] + self.limit = False + Cache.omdb_expiration = self.cache_expiration + self.Cache = Cache + self.get_omdb("tt0080684") + + #@retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) + def get_omdb(self, imdb_id): + expired = None + if self.cache and self.Cache: + omdb_dict, expired = self.Cache.query_omdb(imdb_id) + if omdb_dict and expired is False: + return OMDbObj(omdb_dict) + response = requests.get(self.url, params={"i": imdb_id, "apikey": self.apikey}) + if response.status_code < 400: + omdb = OMDbObj(response.json()) + if self.cache and self.Cache: + self.Cache.update_omdb(expired, omdb) + return omdb + else: + error = response.json()['Error'] + if error == "Request limit reached!": + self.limit = True + raise Failed(f"OMDb Error: {error}") diff --git a/modules/plex.py b/modules/plex.py index 88c10b89..bf8f10dc 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -60,20 +60,12 @@ class PlexAPI: self.show_filtered = params["show_filtered"] self.show_missing = params["show_missing"] self.save_missing = params["save_missing"] + self.mass_genre_update = params["mass_genre_update"] self.plex = params["plex"] self.timeout = params["plex"]["timeout"] self.missing = {} self.run_again = [] - def add_Radarr(self, Radarr): - self.Radarr = Radarr - - def add_Sonarr(self, Sonarr): - self.Sonarr = Sonarr - - def add_Tautulli(self, Tautulli): - self.Tautulli = Tautulli - @retry(stop_max_attempt_number=6, wait_fixed=10000) def search(self, title, libtype=None, year=None): if libtype is not None and year is not None: return self.Plex.search(title=title, year=year, libtype=libtype) From bc855df6ab4c16601ff012d41f27d3010115a66e Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 5 Mar 2021 11:05:06 -0500 Subject: [PATCH 06/22] fix for #82 --- modules/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/builder.py b/modules/builder.py index 1b1ceac0..31052b41 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -813,7 +813,7 @@ class CollectionBuilder: dirs = [folder for folder in os.listdir(path) if os.path.isdir(os.path.join(path, folder))] if len(dirs) > 0: for item in collection.items(): - folder = os.path.basename(os.path.dirname(item.locations[0])) + folder = os.path.basename(os.path.dirname(item.locations[0]) if self.library.is_movie else item.locations[0]) if folder in dirs: matches = glob.glob(os.path.join(path, folder, "poster.*")) poster_path = os.path.abspath(matches[0]) if len(matches) > 0 else None From 2528234a8849c90a1ce6e6f50321a4dcbb81dc52 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 5 Mar 2021 11:20:38 -0500 Subject: [PATCH 07/22] mass_genre_update update --- modules/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/config.py b/modules/config.py index 3e402b34..303bce32 100644 --- a/modules/config.py +++ b/modules/config.py @@ -180,8 +180,8 @@ class Config: self.omdb = {} try: self.omdb["apikey"] = check_for_attribute(self.data, "apikey", parent="omdb", throw=True) - self.omdb["omdb_cache"] = check_for_attribute(self.data, "omdb_cache", parent="omdb", options=" true (Use a cache to store data)\n false (Do not use a cache to store data)", var_type="bool", default=True) - self.omdb["omdb_cache_expiration"] = check_for_attribute(self.data, "omdb_cache_expiration", parent="omdb", var_type="int", default=60) + self.omdb["omdb_cache"] = check_for_attribute(self.data, "omdb_cache", parent="omdb", options=" true (Use a cache to store data)\n false (Do not use a cache to store data)", var_type="bool", default=self.general["cache"]) + self.omdb["omdb_cache_expiration"] = check_for_attribute(self.data, "omdb_cache_expiration", parent="omdb", var_type="int", default=self.general["cache_expiration"]) self.OMDb = OMDbAPI(self.omdb, Cache=self.Cache) except Failed as e: logger.error(e) @@ -311,6 +311,11 @@ class Config: params["mass_genre_update"] = check_for_attribute(libs[lib], "mass_genre_update", test_list=["tmdb", "omdb"], options=" tmdb (Use TMDb Metadata)\n omdb (Use IMDb Metadata through OMDb)", default_is_none=True, save=False) else: params["mass_genre_update"] = None + + if params["mass_genre_update"] == "omdb" and self.OMDb is None: + params["mass_genre_update"] = None + logger.error("Config Error: mass_genre_update cannot be omdb without a successful OMDb Connection") + try: params["metadata_path"] = check_for_attribute(libs[lib], "metadata_path", var_type="path", default=os.path.join(default_dir, f"{lib}.yml"), throw=True) params["library_type"] = check_for_attribute(libs[lib], "library_type", test_list=["movie", "show"], options=" movie (For Movie Libraries)\n show (For Show Libraries)", throw=True) From d56d1c6f546874c917640eacb2da89c3fd51ad0c Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 5 Mar 2021 13:40:17 -0500 Subject: [PATCH 08/22] fix for #77 --- modules/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/config.py b/modules/config.py index 303bce32..e48ab001 100644 --- a/modules/config.py +++ b/modules/config.py @@ -679,6 +679,7 @@ class Config: if url_parsed.scheme == "tmdb": tmdb_id = int(url_parsed.netloc) elif url_parsed.scheme == "imdb": imdb_id = url_parsed.netloc elif item_type == "plex" and check_id == "show": + item.reload() for guid_tag in item.findItems(item._data, Guid): url_parsed = requests.utils.urlparse(guid_tag.id) if url_parsed.scheme == "tvdb": tvdb_id = int(url_parsed.netloc) From fd0301e68a2176445844e762f26a04ecd1a05ca6 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 5 Mar 2021 14:19:22 -0500 Subject: [PATCH 09/22] fix for #84 --- modules/builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/builder.py b/modules/builder.py index 31052b41..9c3dbdfc 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -551,8 +551,10 @@ class CollectionBuilder: self.methods.append((method_name, util.get_list(data[m]))) elif method_name not in util.other_attributes: raise Failed(f"Collection Error: {method_name} attribute not supported") - else: + elif m in util.all_lists or m in util.method_alias or m in util.plex_searches: raise Failed(f"Collection Error: {m} attribute is blank") + else: + logger.warning(f"Collection Warning: {m} attribute is blank") self.sync = self.library.sync_mode == "sync" if "sync_mode" in data: From 4a8f825f7b5337cd316da76e428c10af226ae458 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 5 Mar 2021 15:33:24 -0500 Subject: [PATCH 10/22] clean up --- config/config.yml.template | 2 ++ modules/anidb.py | 10 ++++------ modules/builder.py | 4 ++-- modules/cache.py | 5 ++--- modules/config.py | 8 +++----- modules/imdb.py | 27 +++++++++++++-------------- modules/omdb.py | 9 +++------ modules/tvdb.py | 5 +---- 8 files changed, 30 insertions(+), 40 deletions(-) diff --git a/config/config.yml.template b/config/config.yml.template index bfec47e6..74bfefa2 100644 --- a/config/config.yml.template +++ b/config/config.yml.template @@ -43,6 +43,8 @@ sonarr: # Can be individually specified root_folder_path: "S:/TV Shows" add: false search: false +omdb: + apikey: ######## trakt: client_id: ################################################################ client_secret: ################################################################ diff --git a/modules/anidb.py b/modules/anidb.py index 7e698d1a..b6138fae 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -7,10 +7,8 @@ from retrying import retry logger = logging.getLogger("Plex Meta Manager") class AniDBAPI: - def __init__(self, Cache=None, TMDb=None, Trakt=None): - self.Cache = Cache - self.TMDb = TMDb - self.Trakt = Trakt + def __init__(self, config): + self.config = config self.urls = { "anime": "https://anidb.net/anime", "popular": "https://anidb.net/latest/anime/popular/?h=1", @@ -62,7 +60,7 @@ class AniDBAPI: return anidb_values raise Failed(f"AniDB Error: No valid AniDB IDs in {anidb_list}") - def get_items(self, config, method, data, language, status_message=True): + def get_items(self, method, data, language, status_message=True): pretty = util.pretty_names[method] if method in util.pretty_names else method if status_message: logger.debug(f"Data: {data}") @@ -81,7 +79,7 @@ class AniDBAPI: for anidb_id in anime_ids: try: for imdb_id in self.convert_anidb_to_imdb(anidb_id): - tmdb_id, _ = config.convert_from_imdb(imdb_id, language) + tmdb_id, _ = self.config.convert_from_imdb(imdb_id, language) if tmdb_id: movie_ids.append(tmdb_id) else: raise Failed except Failed: diff --git a/modules/builder.py b/modules/builder.py index 9c3dbdfc..3d26512b 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -648,10 +648,10 @@ class CollectionBuilder: elif "tautulli" in method: items = self.library.Tautulli.get_items(self.library, time_range=value["list_days"], stats_count=value["list_size"], list_type=value["list_type"], stats_count_buffer=value["list_buffer"]) items_found += len(items) - elif "anidb" in method: items_found += check_map(self.config.AniDB.get_items(self.config, method, value, self.library.Plex.language)) + elif "anidb" in method: items_found += check_map(self.config.AniDB.get_items(method, value, self.library.Plex.language)) elif "mal" in method: items_found += check_map(self.config.MyAnimeList.get_items(method, value)) elif "tvdb" in method: items_found += check_map(self.config.TVDb.get_items(method, value, self.library.Plex.language)) - elif "imdb" in method: items_found += check_map(self.config.IMDb.get_items(self.config, method, value, self.library.Plex.language)) + elif "imdb" in method: items_found += check_map(self.config.IMDb.get_items(method, value, self.library.Plex.language)) elif "letterboxd" in method: items_found += check_map(self.config.Letterboxd.get_items(method, value, self.library.Plex.language)) elif "tmdb" in method: items_found += check_map(self.config.TMDb.get_items(method, value, self.library.is_movie)) elif "trakt" in method: items_found += check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) diff --git a/modules/cache.py b/modules/cache.py index ed795109..d11f08f3 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -52,7 +52,6 @@ class Cache: ) self.expiration = expiration self.cache_path = cache - self.omdb_expiration = expiration def get_ids_from_imdb(self, imdb_id): tmdb_id, tmdb_expired = self.get_tmdb_id("movie", imdb_id=imdb_id) @@ -197,11 +196,11 @@ class Cache: omdb_dict["Type"] = row["type"] if row["type"] else None datetime_object = datetime.strptime(row["expiration_date"], "%Y-%m-%d") time_between_insertion = datetime.now() - datetime_object - expired = time_between_insertion.days > self.omdb_expiration + expired = time_between_insertion.days > self.expiration return omdb_dict, expired def update_omdb(self, expired, omdb): - expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, self.omdb_expiration))) + expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, self.expiration))) with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: diff --git a/modules/config.py b/modules/config.py index e48ab001..2598e323 100644 --- a/modules/config.py +++ b/modules/config.py @@ -180,8 +180,6 @@ class Config: self.omdb = {} try: self.omdb["apikey"] = check_for_attribute(self.data, "apikey", parent="omdb", throw=True) - self.omdb["omdb_cache"] = check_for_attribute(self.data, "omdb_cache", parent="omdb", options=" true (Use a cache to store data)\n false (Do not use a cache to store data)", var_type="bool", default=self.general["cache"]) - self.omdb["omdb_cache_expiration"] = check_for_attribute(self.data, "omdb_cache_expiration", parent="omdb", var_type="int", default=self.general["cache_expiration"]) self.OMDb = OMDbAPI(self.omdb, Cache=self.Cache) except Failed as e: logger.error(e) @@ -226,9 +224,9 @@ class Config: else: logger.warning("mal attribute not found") - self.TVDb = TVDbAPI(self, Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt) - self.IMDb = IMDbAPI(Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt, TVDb=self.TVDb) if self.TMDb or self.Trakt else None - self.AniDB = AniDBAPI(Cache=self.Cache, TMDb=self.TMDb, Trakt=self.Trakt) + self.TVDb = TVDbAPI(self) + self.IMDb = IMDbAPI(self) + self.AniDB = AniDBAPI(self) self.Letterboxd = LetterboxdAPI() util.separator() diff --git a/modules/imdb.py b/modules/imdb.py index 402e3d4d..48fadf6f 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -7,23 +7,22 @@ from retrying import retry logger = logging.getLogger("Plex Meta Manager") class IMDbAPI: - def __init__(self, Cache=None, TMDb=None, Trakt=None, TVDb=None): - if TMDb is None and Trakt is None: - raise Failed("IMDb Error: IMDb requires either TMDb or Trakt") - self.Cache = Cache - self.TMDb = TMDb - self.Trakt = Trakt - self.TVDb = TVDb + def __init__(self, config): + self.config = config + self.urls = { + "list": "https://www.imdb.com/list/ls", + "search": "https://www.imdb.com/search/title/?" + } def get_imdb_ids_from_url(self, imdb_url, language, limit): imdb_url = imdb_url.strip() - if not imdb_url.startswith("https://www.imdb.com/list/ls") and not imdb_url.startswith("https://www.imdb.com/search/title/?"): - raise Failed(f"IMDb Error: {imdb_url} must begin with either:\n| https://www.imdb.com/list/ls (For Lists)\n| https://www.imdb.com/search/title/? (For Searches)") + if not imdb_url.startswith(self.urls["list"]) and not imdb_url.startswith(self.urls["search"]): + raise Failed(f"IMDb Error: {imdb_url} must begin with either:\n| {self.urls['list']} (For Lists)\n| {self.urls['search']} (For Searches)") - if imdb_url.startswith("https://www.imdb.com/list/ls"): + if imdb_url.startswith(self.urls["list"]): try: list_id = re.search("(\\d+)", str(imdb_url)).group(1) except AttributeError: raise Failed(f"IMDb Error: Failed to parse List ID from {imdb_url}") - current_url = f"https://www.imdb.com/search/title/?lists=ls{list_id}" + current_url = f"{self.urls['search']}lists=ls{list_id}" else: current_url = imdb_url header = {"Accept-Language": language} @@ -52,7 +51,7 @@ class IMDbAPI: def send_request(self, url, header): return html.fromstring(requests.get(url, headers=header).content) - def get_items(self, config, method, data, language, status_message=True): + def get_items(self, method, data, language, status_message=True): pretty = util.pretty_names[method] if method in util.pretty_names else method if status_message: logger.debug(f"Data: {data}") @@ -61,7 +60,7 @@ class IMDbAPI: if method == "imdb_id": if status_message: logger.info(f"Processing {pretty}: {data}") - tmdb_id, tvdb_id = config.convert_from_imdb(data, language) + tmdb_id, tvdb_id = self.config.convert_from_imdb(data, language) if tmdb_id: movie_ids.append(tmdb_id) if tvdb_id: show_ids.append(tvdb_id) elif method == "imdb_list": @@ -74,7 +73,7 @@ class IMDbAPI: for i, imdb_id in enumerate(imdb_ids, 1): length = util.print_return(length, f"Converting IMDb ID {i}/{total_ids}") try: - tmdb_id, tvdb_id = config.convert_from_imdb(imdb_id, language) + tmdb_id, tvdb_id = self.config.convert_from_imdb(imdb_id, language) if tmdb_id: movie_ids.append(tmdb_id) if tvdb_id: show_ids.append(tvdb_id) except Failed as e: logger.warning(e) diff --git a/modules/omdb.py b/modules/omdb.py index 7470d594..ab42d66e 100644 --- a/modules/omdb.py +++ b/modules/omdb.py @@ -36,24 +36,21 @@ class OMDbAPI: def __init__(self, params, Cache=None): self.url = "http://www.omdbapi.com/" self.apikey = params["apikey"] - self.cache = params["omdb_cache"] - self.cache_expiration = params["omdb_cache_expiration"] self.limit = False - Cache.omdb_expiration = self.cache_expiration self.Cache = Cache self.get_omdb("tt0080684") - #@retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) + @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def get_omdb(self, imdb_id): expired = None - if self.cache and self.Cache: + if self.Cache: omdb_dict, expired = self.Cache.query_omdb(imdb_id) if omdb_dict and expired is False: return OMDbObj(omdb_dict) response = requests.get(self.url, params={"i": imdb_id, "apikey": self.apikey}) if response.status_code < 400: omdb = OMDbObj(response.json()) - if self.cache and self.Cache: + if self.Cache: self.Cache.update_omdb(expired, omdb) return omdb else: diff --git a/modules/tvdb.py b/modules/tvdb.py index 17a64ecc..23c1caa3 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -54,11 +54,8 @@ class TVDbObj: self.TVDb = TVDb class TVDbAPI: - def __init__(self, config, Cache=None, TMDb=None, Trakt=None): + def __init__(self, config): self.config = config - self.Cache = Cache - self.TMDb = TMDb - self.Trakt = Trakt self.site_url = "https://www.thetvdb.com" self.alt_site_url = "https://thetvdb.com" self.list_url = f"{self.site_url}/lists/" From ce67559a6a3256a721d33047e1910884cff95b03 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sat, 6 Mar 2021 11:44:56 -0500 Subject: [PATCH 11/22] fix for #89 --- modules/builder.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 3d26512b..5522ea1a 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -854,14 +854,14 @@ class CollectionBuilder: elif "tmdb_profile" in self.posters: set_image("tmdb_profile", self.posters) elif "asset_directory" in self.posters: set_image("asset_directory", self.posters) elif "tmdb_person" in self.posters: set_image("tmdb_person", self.posters) - elif "tmdb_collection_details" in self.posters: set_image("tmdb_collection", self.posters) + elif "tmdb_collection_details" in self.posters: set_image("tmdb_collection_details", self.posters) elif "tmdb_actor_details" in self.posters: set_image("tmdb_actor_details", self.posters) elif "tmdb_crew_details" in self.posters: set_image("tmdb_crew_details", self.posters) elif "tmdb_director_details" in self.posters: set_image("tmdb_director_details", self.posters) elif "tmdb_producer_details" in self.posters: set_image("tmdb_producer_details", self.posters) elif "tmdb_writer_details" in self.posters: set_image("tmdb_writer_details", self.posters) - elif "tmdb_movie_details" in self.posters: set_image("tmdb_movie", self.posters) - elif "tmdb_show_details" in self.posters: set_image("tmdb_show", self.posters) + elif "tmdb_movie_details" in self.posters: set_image("tmdb_movie_details", self.posters) + elif "tmdb_show_details" in self.posters: set_image("tmdb_show_details", self.posters) else: logger.info("No poster to update") logger.info("") @@ -875,9 +875,9 @@ class CollectionBuilder: elif "file_background" in self.backgrounds: set_image("file_poster", self.backgrounds, is_background=True) elif "tmdb_background" in self.backgrounds: set_image("tmdb_poster", self.backgrounds, is_background=True) elif "asset_directory" in self.backgrounds: set_image("asset_directory", self.backgrounds, is_background=True) - elif "tmdb_collection_details" in self.backgrounds: set_image("tmdb_collection", self.backgrounds, is_background=True) - elif "tmdb_movie_details" in self.backgrounds: set_image("tmdb_movie", self.backgrounds, is_background=True) - elif "tmdb_show_details" in self.backgrounds: set_image("tmdb_show", self.backgrounds, is_background=True) + elif "tmdb_collection_details" in self.backgrounds: set_image("tmdb_collection_details", self.backgrounds, is_background=True) + elif "tmdb_movie_details" in self.backgrounds: set_image("tmdb_movie_details", self.backgrounds, is_background=True) + elif "tmdb_show_details" in self.backgrounds: set_image("tmdb_show_details", self.backgrounds, is_background=True) else: logger.info("No background to update") def run_collections_again(self, collection_obj, movie_map, show_map): From 86d789b9303775ed1711697f973158cb7e8aafb5 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sat, 6 Mar 2021 11:54:43 -0500 Subject: [PATCH 12/22] fix for #90 --- modules/config.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/modules/config.py b/modules/config.py index 2598e323..c58658c1 100644 --- a/modules/config.py +++ b/modules/config.py @@ -396,7 +396,7 @@ class Config: movie_map, show_map = self.map_guids(library) if not test: if library.mass_genre_update: - self.mass_metadata(library) + self.mass_metadata(library, movie_map, show_map) try: library.update_metadata(self.TMDb, test) except Failed as e: logger.error(e) logger.info("") @@ -582,7 +582,7 @@ class Config: self.Cache.update_imdb("show", update_tvdb, imdb_id, tvdb_id) return tmdb_id, tvdb_id - def mass_metadata(self, library): + def mass_metadata(self, library, movie_map, show_map): length = 0 logger.info("") util.separator(f"Mass Editing {'Movie' if library.is_movie else 'Show'} Library: {library.name}") @@ -590,7 +590,20 @@ class Config: items = library.Plex.all() for i, item in enumerate(items, 1): length = util.print_return(length, f"Processing: {i}/{len(items)} {item.title}") - ids, expired = self.Cache.get_ids("movie" if library.is_movie else "show", plex_guid=item.guid) + ids = {} + if self.Cache: + ids, expired = self.Cache.get_ids("movie" if library.is_movie else "show", plex_guid=item.guid) + elif library.is_movie: + for tmdb in movie_map: + if movie_map[tmdb] == item.ratingKey: + ids["tmdb"] = tmdb + break + else: + for tvdb in show_map: + if show_map[tvdb] == item.ratingKey: + ids["tvdb"] = tvdb + break + if library.mass_genre_update: if library.mass_genre_update == "tmdb": if "tmdb" not in ids: @@ -777,7 +790,7 @@ class Config: elif id_name and api_name: error_message = f"Unable to convert {id_name} to {service_name} using {api_name}" elif id_name: error_message = f"Configure TMDb or Trakt to covert {id_name} to {service_name}" else: error_message = f"No ID to convert to {service_name}" - if self.Cache and (tmdb_id and library.is_movie) or ((tvdb_id or ((anidb_id or mal_id) and tmdb_id)) and library.is_show): + if self.Cache and ((tmdb_id and library.is_movie) or ((tvdb_id or ((anidb_id or mal_id) and tmdb_id)) and library.is_show)): if isinstance(tmdb_id, list): for i in range(len(tmdb_id)): util.print_end(length, f"Cache | {'^' if expired is True else '+'} | {item.guid:<46} | {tmdb_id[i] if tmdb_id[i] else 'None':<6} | {imdb_id[i] if imdb_id[i] else 'None':<10} | {tvdb_id if tvdb_id else 'None':<6} | {anidb_id if anidb_id else 'None':<5} | {mal_id if mal_id else 'None':<5} | {item.title}") From f218f2e8887ac52643d7bcf50f5ee42b57bfd7d9 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sat, 6 Mar 2021 18:30:53 -0500 Subject: [PATCH 13/22] fix for #91 --- modules/tvdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tvdb.py b/modules/tvdb.py index 23c1caa3..a4e6032c 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -45,7 +45,7 @@ class TVDbObj: if not tmdb_id: results = response.xpath("//*[text()='IMDB']/@href") if len(results) > 0: - try: tmdb_id, _ = TVDb.config.convert_from_imdb(util.get_id_from_imdb_url(results[0])) + try: tmdb_id, _ = TVDb.config.convert_from_imdb(util.get_id_from_imdb_url(results[0]), language) except Failed as e: logger.error(e) self.tmdb_id = tmdb_id self.tvdb_url = tvdb_url From 114408e4ce129194a168113fd057b647952d0183 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sat, 6 Mar 2021 23:12:56 -0500 Subject: [PATCH 14/22] fix for #92 --- modules/tmdb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/tmdb.py b/modules/tmdb.py index bdb635e8..cd71a214 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -123,7 +123,10 @@ class TMDbAPI: if credit.media_type == "movie": movie_ids.append(credit.id) elif credit.media_type == "tv": - show_ids.append(credit.id) + try: + show_ids.append(self.convert_tmdb_to_tvdb(credit.id)) + except Failed as e: + logger.warning(e) return movie_ids, show_ids def get_pagenation(self, method, amount, is_movie): From 1da0a47eabe4a94eae2f99b5bb9c784a4f5b79b1 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sun, 7 Mar 2021 00:53:50 -0500 Subject: [PATCH 15/22] Fix for #92 --- modules/tmdb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/tmdb.py b/modules/tmdb.py index cd71a214..076ae6d5 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -114,7 +114,10 @@ class TMDbAPI: if credit.media_type == "movie": movie_ids.append(credit.id) elif credit.media_type == "tv": - show_ids.append(credit.id) + try: + show_ids.append(self.convert_tmdb_to_tvdb(credit.id)) + except Failed as e: + logger.warning(e) for credit in actor_credits.crew: if crew or \ (director and credit.department == "Directing") or \ From e3ffced21939a8b747d1f55c39e6db0c2d69f2c1 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 8 Mar 2021 10:25:35 -0500 Subject: [PATCH 16/22] added trakt list descriptions --- modules/builder.py | 7 +++++++ modules/trakttv.py | 10 +++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 5522ea1a..9ba332d9 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -313,6 +313,12 @@ class CollectionBuilder: self.methods.append((method_name, config.AniDB.validate_anidb_list(util.get_int_list(data[m], "AniDB ID"), self.library.Plex.language))) elif method_name == "trakt_list": self.methods.append((method_name, config.Trakt.validate_trakt_list(util.get_list(data[m])))) + elif method_name == "trakt_list_details": + valid_list = config.Trakt.validate_trakt_list(util.get_list(data[m])) + item = config.Trakt.standard_list(valid_list[0]) + if hasattr(item, "description") and item.description: + self.summaries[method_name] = item.description + self.methods.append((method_name[:-8], valid_list)) elif method_name == "trakt_watchlist": self.methods.append((method_name, config.Trakt.validate_trakt_watchlist(util.get_list(data[m]), self.library.is_movie))) elif method_name == "imdb_list": @@ -747,6 +753,7 @@ class CollectionBuilder: elif "tmdb_biography" in self.summaries: summary = get_summary("tmdb_biography", self.summaries) elif "tmdb_person" in self.summaries: summary = get_summary("tmdb_person", self.summaries) elif "tmdb_collection_details" in self.summaries: summary = get_summary("tmdb_collection_details", self.summaries) + elif "trakt_list_details" in self.summaries: summary = get_summary("trakt_list_details", self.summaries) elif "tmdb_list_details" in self.summaries: summary = get_summary("tmdb_list_details", self.summaries) elif "tmdb_actor_details" in self.summaries: summary = get_summary("tmdb_actor_details", self.summaries) elif "tmdb_crew_details" in self.summaries: summary = get_summary("tmdb_crew_details", self.summaries) diff --git a/modules/trakttv.py b/modules/trakttv.py index c82d235c..2ffcb9ca 100644 --- a/modules/trakttv.py +++ b/modules/trakttv.py @@ -105,10 +105,10 @@ class TraktAPI: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def standard_list(self, data): - try: items = Trakt[requests.utils.urlparse(data).path].items() - except AttributeError: items = None - if items is None: raise Failed("Trakt Error: No List found") - else: return items + try: trakt_list = Trakt[requests.utils.urlparse(data).path].get() + except AttributeError: trakt_list = None + if trakt_list is None: raise Failed("Trakt Error: No List found") + else: return trakt_list def validate_trakt_list(self, values): trakt_values = [] @@ -145,7 +145,7 @@ class TraktAPI: logger.info(f"Processing {pretty}: {data} {media_type}{'' if data == 1 else 's'}") else: if method == "trakt_watchlist": trakt_items = self.watchlist(data, is_movie) - elif method == "trakt_list": trakt_items = self.standard_list(data) + elif method == "trakt_list": trakt_items = self.standard_list(data).items() else: raise Failed(f"Trakt Error: Method {method} not supported") if status_message: logger.info(f"Processing {pretty}: {data}") show_ids = [] From 5156d92b7c03218f9fa177fa9abefd8b1c0c772f Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 8 Mar 2021 12:02:40 -0500 Subject: [PATCH 17/22] added tvdb details methods #92 --- modules/builder.py | 38 ++++++++++++++++++++++++++++++++++++++ modules/tvdb.py | 10 ++++++++++ modules/util.py | 18 ++++++++++++++---- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 9ba332d9..554206e9 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -529,6 +529,38 @@ class CollectionBuilder: logger.warning(f"Collection Warning: {method_name} must be an integer greater then 0 defaulting to 20") list_count = 20 self.methods.append((method_name, [list_count])) + elif "tvdb" in method_name: + values = util.get_list(data[m]) + if method_name[-8:] == "_details": + if method_name == "tvdb_movie_details": + try: + item = config.TVDb.get_movie(self.library.Plex.language, tvdb_id=int(values[0])).id + except ValueError: + item = config.TVDb.get_movie(self.library.Plex.language, tvdb_url=values[0]).id + if hasattr(item, "description") and item.description: + self.summaries[method_name] = item.description + if hasattr(item, "background_path") and item.background_path: + self.backgrounds[method_name] = f"{config.TMDb.image_url}{item.background_path}" + if hasattr(item, "poster_path") and item.poster_path: + self.posters[method_name] = f"{config.TMDb.image_url}{item.poster_path}" + elif method_name == "tvdb_show_details": + try: + item = config.TVDb.get_series(self.library.Plex.language, tvdb_id=int(values[0])).id + except ValueError: + item = config.TVDb.get_series(self.library.Plex.language, tvdb_url=values[0]).id + if hasattr(item, "description") and item.description: + self.summaries[method_name] = item.description + if hasattr(item, "background_path") and item.background_path: + self.backgrounds[method_name] = f"{config.TMDb.image_url}{item.background_path}" + if hasattr(item, "poster_path") and item.poster_path: + self.posters[method_name] = f"{config.TMDb.image_url}{item.poster_path}" + elif method_name == "tvdb_list_details": + description = config.TVDb.get_list_description(self.library.Plex.language, values[0]) + if description and len(description) > 0: + self.summaries[method_name] = description + self.methods.append((method_name[:-8], values)) + else: + self.methods.append((method_name, values)) elif method_name in util.tmdb_lists: values = config.TMDb.validate_tmdb_list(util.get_int_list(data[m], f"TMDb {util.tmdb_type[method_name]} ID"), util.tmdb_type[method_name]) if method_name[-8:] == "_details": @@ -761,6 +793,8 @@ class CollectionBuilder: elif "tmdb_producer_details" in self.summaries: summary = get_summary("tmdb_producer_details", self.summaries) elif "tmdb_writer_details" in self.summaries: summary = get_summary("tmdb_writer_details", self.summaries) elif "tmdb_movie_details" in self.summaries: summary = get_summary("tmdb_movie_details", self.summaries) + elif "tvdb_movie_details" in self.summaries: summary = get_summary("tvdb_movie_details", self.summaries) + elif "tvdb_show_details" in self.summaries: summary = get_summary("tvdb_show_details", self.summaries) elif "tmdb_show_details" in self.summaries: summary = get_summary("tmdb_show_details", self.summaries) else: summary = None if summary: @@ -868,6 +902,8 @@ class CollectionBuilder: elif "tmdb_producer_details" in self.posters: set_image("tmdb_producer_details", self.posters) elif "tmdb_writer_details" in self.posters: set_image("tmdb_writer_details", self.posters) elif "tmdb_movie_details" in self.posters: set_image("tmdb_movie_details", self.posters) + elif "tvdb_movie_details" in self.posters: set_image("tvdb_movie_details", self.posters) + elif "tvdb_show_details" in self.posters: set_image("tvdb_show_details", self.posters) elif "tmdb_show_details" in self.posters: set_image("tmdb_show_details", self.posters) else: logger.info("No poster to update") @@ -884,6 +920,8 @@ class CollectionBuilder: elif "asset_directory" in self.backgrounds: set_image("asset_directory", self.backgrounds, is_background=True) elif "tmdb_collection_details" in self.backgrounds: set_image("tmdb_collection_details", self.backgrounds, is_background=True) elif "tmdb_movie_details" in self.backgrounds: set_image("tmdb_movie_details", self.backgrounds, is_background=True) + elif "tvdb_movie_details" in self.backgrounds: set_image("tvdb_movie_details", self.backgrounds, is_background=True) + elif "tvdb_show_details" in self.backgrounds: set_image("tvdb_show_details", self.backgrounds, is_background=True) elif "tmdb_show_details" in self.backgrounds: set_image("tmdb_show_details", self.backgrounds, is_background=True) else: logger.info("No background to update") diff --git a/modules/tvdb.py b/modules/tvdb.py index a4e6032c..d7b054d6 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -36,6 +36,12 @@ class TVDbObj: results = response.xpath("//div[@class='row hidden-xs hidden-sm']/div/img/@src") self.poster_path = results[0] if len(results) > 0 and len(results[0]) > 0 else None + results = response.xpath("(//h2[@class='mt-4' and text()='Backgrounds']/following::div/a/@href)[1]") + self.background_path = results[0] if len(results) > 0 and len(results[0]) > 0 else None + + results = response.xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()") + self.description = results[0] if len(results) > 0 and len(results[0]) > 0 else None + tmdb_id = None if is_movie: results = response.xpath("//*[text()='TheMovieDB.com']/@href") @@ -81,6 +87,10 @@ class TVDbAPI: tvdb_url = f"{self.movie_id_url}{tvdb_id}" return TVDbObj(tvdb_url, language, True, self) + def get_list_description(self, language, tvdb_url): + description = self.send_request(tvdb_url, language).xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()") + return description[0] if len(description) > 0 and len(description[0]) > 0 else "" + def get_tvdb_ids_from_url(self, tvdb_url, language): show_ids = [] movie_ids = [] diff --git a/modules/util.py b/modules/util.py index 53ec05b1..11e6ded4 100644 --- a/modules/util.py +++ b/modules/util.py @@ -145,11 +145,15 @@ pretty_names = { "tmdb_writer": "TMDb Writer", "tmdb_writer_details": "TMDb Writer", "trakt_list": "Trakt List", + "trakt_list_details": "Trakt List", "trakt_trending": "Trakt Trending", "trakt_watchlist": "Trakt Watchlist", "tvdb_list": "TVDb List", + "tvdb_list_details": "TVDb List", "tvdb_movie": "TVDb Movie", - "tvdb_show": "TVDb Show" + "tvdb_movie_details": "TVDb Movie", + "tvdb_show": "TVDb Show", + "tvdb_show_details": "TVDb Show" } mal_ranked_name = { "mal_all": "all", @@ -261,11 +265,15 @@ all_lists = [ "tmdb_writer", "tmdb_writer_details", "trakt_list", + "trakt_list_details", "trakt_trending", "trakt_watchlist", "tvdb_list", + "tvdb_list_details", "tvdb_movie", - "tvdb_show" + "tvdb_movie_details", + "tvdb_show", + "tvdb_show_details" ] collectionless_lists = [ "sort_title", "content_rating", @@ -308,7 +316,8 @@ show_only_lists = [ "tmdb_network", "tmdb_show", "tmdb_show_details", - "tvdb_show" + "tvdb_show", + "tvdb_show_details" ] movie_only_lists = [ "letterboxd_list", @@ -317,7 +326,8 @@ movie_only_lists = [ "tmdb_movie", "tmdb_movie_details", "tmdb_now_playing", - "tvdb_movie" + "tvdb_movie", + "tvdb_movie_details" ] movie_only_searches = [ "actor", "actor.not", From 17be256282c48f944ed6ca40f91aacec99d570fb Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 8 Mar 2021 14:53:05 -0500 Subject: [PATCH 18/22] added more collection details --- README.md | 5 +++-- modules/builder.py | 35 ++++++++++++++++++++--------------- modules/config.py | 2 +- modules/sonarr.py | 2 +- modules/tvdb.py | 37 +++++++++++++++++++------------------ modules/util.py | 6 +++--- 6 files changed, 47 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 0d02f4b1..9cc43a5e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ The original concept for Plex Meta Manager is [Plex Auto Collections](https://gi The script can update many metadata fields for movies, shows, collections, seasons, and episodes and can act as a backup if your plex DB goes down. It can even update metadata the plex UI can't like Season Names. If the time is put into the metadata configuration file you can have a way to recreate your library and all its metadata changes with the click of a button. -The script is designed to work with most Metadata agents including the new Plex Movie Agent, [Hama Anime Agent](https://github.com/ZeroQI/Hama.bundle), and [MyAnimeList Anime Agent](https://github.com/Fribb/MyAnimeList.bundle). +The script is designed to work with most Metadata agents including the new Plex Movie Agent, New Plex TV Agent, [Hama Anime Agent](https://github.com/ZeroQI/Hama.bundle), and [MyAnimeList Anime Agent](https://github.com/Fribb/MyAnimeList.bundle). + +[![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/donate?business=JTK3CVKF3ZHP2&item_name=Plex+Meta+Manager¤cy_code=USD) ## Getting Started @@ -20,4 +22,3 @@ The script is designed to work with most Metadata agents including the new Plex * To see user submitted Metadata configuration files and you could even add your own go to the [Plex Meta Manager Configs](https://github.com/meisnate12/Plex-Meta-Manager-Configs) * Pull Request are welcome but please submit them to the develop branch * If you wish to contribute to the Wiki please fork and send a pull request on the [Plex Meta Manager Wiki Repository](https://github.com/meisnate12/Plex-Meta-Manager-Wiki) -* [Buy Me a Pizza](https://www.buymeacoffee.com/meisnate12) diff --git a/modules/builder.py b/modules/builder.py index 554206e9..aaee5a0c 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -240,6 +240,12 @@ class CollectionBuilder: self.summaries[method_name] = config.TMDb.get_list(util.regex_first_int(data[m], "TMDb List ID")).description elif method_name == "tmdb_biography": self.summaries[method_name] = config.TMDb.get_person(util.regex_first_int(data[m], "TMDb Person ID")).biography + elif method_name == "tvdb_summary": + self.summaries[method_name] = config.TVDb.get_movie_or_show(data[m], self.library.Plex.language, self.library.is_movie).summary + elif method_name == "tvdb_description": + self.summaries[method_name] = config.TVDb.get_list_description(data[m], self.library.Plex.language) + elif method_name == "trakt_description": + self.summaries[method_name] = config.Trakt.standard_list(config.Trakt.validate_trakt_list(util.get_list(data[m]))[0]).description elif method_name == "collection_mode": if data[m] in ["default", "hide", "hide_items", "show_items", "hideItems", "showItems"]: if data[m] == "hide_items": self.details[method_name] = "hideItems" @@ -258,6 +264,8 @@ class CollectionBuilder: self.posters[method_name] = f"{config.TMDb.image_url}{config.TMDb.get_movie_show_or_collection(util.regex_first_int(data[m], 'TMDb ID'), self.library.is_movie).poster_path}" elif method_name == "tmdb_profile": self.posters[method_name] = f"{config.TMDb.image_url}{config.TMDb.get_person(util.regex_first_int(data[m], 'TMDb Person ID')).profile_path}" + elif method_name == "tvdb_poster": + self.posters[method_name] = f"{config.TVDb.get_movie_or_series(data[m], self.library.Plex.language, self.library.is_movie).poster_path}" elif method_name == "file_poster": if os.path.exists(data[m]): self.posters[method_name] = os.path.abspath(data[m]) else: raise Failed(f"Collection Error: Poster Path Does Not Exist: {os.path.abspath(data[m])}") @@ -265,6 +273,8 @@ class CollectionBuilder: self.backgrounds[method_name] = data[m] elif method_name == "tmdb_background": self.backgrounds[method_name] = f"{config.TMDb.image_url}{config.TMDb.get_movie_show_or_collection(util.regex_first_int(data[m], 'TMDb ID'), self.library.is_movie).poster_path}" + elif method_name == "tvdb_background": + self.posters[method_name] = f"{config.TVDb.get_movie_or_series(data[m], self.library.Plex.language, self.library.is_movie).background_path}" elif method_name == "file_background": if os.path.exists(data[m]): self.backgrounds[method_name] = os.path.abspath(data[m]) else: raise Failed(f"Collection Error: Background Path Does Not Exist: {os.path.abspath(data[m])}") @@ -533,10 +543,7 @@ class CollectionBuilder: values = util.get_list(data[m]) if method_name[-8:] == "_details": if method_name == "tvdb_movie_details": - try: - item = config.TVDb.get_movie(self.library.Plex.language, tvdb_id=int(values[0])).id - except ValueError: - item = config.TVDb.get_movie(self.library.Plex.language, tvdb_url=values[0]).id + item = config.TVDb.get_movie(self.library.Plex.language, values[0]) if hasattr(item, "description") and item.description: self.summaries[method_name] = item.description if hasattr(item, "background_path") and item.background_path: @@ -544,10 +551,7 @@ class CollectionBuilder: if hasattr(item, "poster_path") and item.poster_path: self.posters[method_name] = f"{config.TMDb.image_url}{item.poster_path}" elif method_name == "tvdb_show_details": - try: - item = config.TVDb.get_series(self.library.Plex.language, tvdb_id=int(values[0])).id - except ValueError: - item = config.TVDb.get_series(self.library.Plex.language, tvdb_url=values[0]).id + item = config.TVDb.get_series(self.library.Plex.language, values[0]) if hasattr(item, "description") and item.description: self.summaries[method_name] = item.description if hasattr(item, "background_path") and item.background_path: @@ -555,9 +559,7 @@ class CollectionBuilder: if hasattr(item, "poster_path") and item.poster_path: self.posters[method_name] = f"{config.TMDb.image_url}{item.poster_path}" elif method_name == "tvdb_list_details": - description = config.TVDb.get_list_description(self.library.Plex.language, values[0]) - if description and len(description) > 0: - self.summaries[method_name] = description + self.summaries[method_name] = config.TVDb.get_list_description(values[0], self.library.Plex.language) self.methods.append((method_name[:-8], values)) else: self.methods.append((method_name, values)) @@ -737,7 +739,7 @@ class CollectionBuilder: missing_shows_with_names = [] for missing_id in missing_shows: try: - title = str(self.config.TVDb.get_series(self.library.Plex.language, tvdb_id=missing_id).title.encode("ascii", "replace").decode()) + title = str(self.config.TVDb.get_series(self.library.Plex.language, missing_id).title.encode("ascii", "replace").decode()) except Failed as e: logger.error(e) continue @@ -782,6 +784,7 @@ class CollectionBuilder: if "summary" in self.summaries: summary = get_summary("summary", self.summaries) elif "tmdb_description" in self.summaries: summary = get_summary("tmdb_description", self.summaries) elif "tmdb_summary" in self.summaries: summary = get_summary("tmdb_summary", self.summaries) + elif "tvdb_summary" in self.summaries: summary = get_summary("tvdb_summary", self.summaries) elif "tmdb_biography" in self.summaries: summary = get_summary("tmdb_biography", self.summaries) elif "tmdb_person" in self.summaries: summary = get_summary("tmdb_person", self.summaries) elif "tmdb_collection_details" in self.summaries: summary = get_summary("tmdb_collection_details", self.summaries) @@ -893,6 +896,7 @@ class CollectionBuilder: elif "file_poster" in self.posters: set_image("file_poster", self.posters) elif "tmdb_poster" in self.posters: set_image("tmdb_poster", self.posters) elif "tmdb_profile" in self.posters: set_image("tmdb_profile", self.posters) + elif "tvdb_poster" in self.posters: set_image("tvdb_poster", self.posters) elif "asset_directory" in self.posters: set_image("asset_directory", self.posters) elif "tmdb_person" in self.posters: set_image("tmdb_person", self.posters) elif "tmdb_collection_details" in self.posters: set_image("tmdb_collection_details", self.posters) @@ -915,8 +919,9 @@ class CollectionBuilder: logger.info(f"Method: {b} Background: {self.backgrounds[b]}") if "url_background" in self.backgrounds: set_image("url_background", self.backgrounds, is_background=True) - elif "file_background" in self.backgrounds: set_image("file_poster", self.backgrounds, is_background=True) - elif "tmdb_background" in self.backgrounds: set_image("tmdb_poster", self.backgrounds, is_background=True) + elif "file_background" in self.backgrounds: set_image("file_background", self.backgrounds, is_background=True) + elif "tmdb_background" in self.backgrounds: set_image("tmdb_background", self.backgrounds, is_background=True) + elif "tvdb_background" in self.backgrounds: set_image("tvdb_background", self.backgrounds, is_background=True) elif "asset_directory" in self.backgrounds: set_image("asset_directory", self.backgrounds, is_background=True) elif "tmdb_collection_details" in self.backgrounds: set_image("tmdb_collection_details", self.backgrounds, is_background=True) elif "tmdb_movie_details" in self.backgrounds: set_image("tmdb_movie_details", self.backgrounds, is_background=True) @@ -965,7 +970,7 @@ class CollectionBuilder: for missing_id in self.missing_shows: if missing_id not in show_map: try: - title = str(self.config.TVDb.get_series(self.library.Plex.language, tvdb_id=missing_id).title.encode("ascii", "replace").decode()) + title = str(self.config.TVDb.get_series(self.library.Plex.language, missing_id).title.encode("ascii", "replace").decode()) except Failed as e: logger.error(e) continue diff --git a/modules/config.py b/modules/config.py index c58658c1..ad438961 100644 --- a/modules/config.py +++ b/modules/config.py @@ -572,7 +572,7 @@ class Config: if tmdb_id and not from_cache: self.TMDb.get_movie(tmdb_id) except Failed: tmdb_id = None try: - if tvdb_id and not from_cache: self.TVDb.get_series(language, tvdb_id=tvdb_id) + if tvdb_id and not from_cache: self.TVDb.get_series(language, tvdb_id) except Failed: tvdb_id = None if not tmdb_id and not tvdb_id: raise Failed(f"IMDb Error: No TMDb ID or TVDb ID found for IMDb: {imdb_id}") if self.Cache: diff --git a/modules/sonarr.py b/modules/sonarr.py index a2c29bd3..a13c5c27 100644 --- a/modules/sonarr.py +++ b/modules/sonarr.py @@ -57,7 +57,7 @@ class SonarrAPI: tag_nums.append(tag_cache[label]) for tvdb_id in tvdb_ids: try: - show = self.tvdb.get_series(self.language, tvdb_id=tvdb_id) + show = self.tvdb.get_series(self.language, tvdb_id) except Failed as e: logger.error(e) continue diff --git a/modules/tvdb.py b/modules/tvdb.py index d7b054d6..4d7a5b8a 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -40,7 +40,7 @@ class TVDbObj: self.background_path = results[0] if len(results) > 0 and len(results[0]) > 0 else None results = response.xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()") - self.description = results[0] if len(results) > 0 and len(results[0]) > 0 else None + self.summary = results[0] if len(results) > 0 and len(results[0]) > 0 else None tmdb_id = None if is_movie: @@ -73,21 +73,24 @@ class TVDbAPI: self.series_id_url = f"{self.site_url}/dereferrer/series/" self.movie_id_url = f"{self.site_url}/dereferrer/movie/" - def get_series(self, language, tvdb_url=None, tvdb_id=None): - if not tvdb_url and not tvdb_id: - raise Failed("TVDB Error: get_series requires either tvdb_url or tvdb_id") - elif not tvdb_url and tvdb_id: - tvdb_url = f"{self.series_id_url}{tvdb_id}" + def get_movie_or_series(self, language, tvdb_url, is_movie): + return self.get_movie(language, tvdb_url) if is_movie else self.get_series(language, tvdb_url) + + def get_series(self, language, tvdb_url): + try: + tvdb_url = f"{self.series_id_url}{int(tvdb_url)}" + except ValueError: + pass return TVDbObj(tvdb_url, language, False, self) - def get_movie(self, language, tvdb_url=None, tvdb_id=None): - if not tvdb_url and not tvdb_id: - raise Failed("TVDB Error: get_movie requires either tvdb_url or tvdb_id") - elif not tvdb_url and tvdb_id: - tvdb_url = f"{self.movie_id_url}{tvdb_id}" + def get_movie(self, language, tvdb_url): + try: + tvdb_url = f"{self.movie_id_url}{int(tvdb_url)}" + except ValueError: + pass return TVDbObj(tvdb_url, language, True, self) - def get_list_description(self, language, tvdb_url): + def get_list_description(self, tvdb_url, language): description = self.send_request(tvdb_url, language).xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()") return description[0] if len(description) > 0 and len(description[0]) > 0 else "" @@ -102,11 +105,11 @@ class TVDbAPI: title = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/text()")[0] item_url = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/@href")[0] if item_url.startswith("/series/"): - try: show_ids.append(self.get_series(language, tvdb_url=f"{self.site_url}{item_url}").id) + try: show_ids.append(self.get_series(language, f"{self.site_url}{item_url}").id) except Failed as e: logger.error(f"{e} for series {title}") elif item_url.startswith("/movies/"): try: - tmdb_id = self.get_movie(language, tvdb_url=f"{self.site_url}{item_url}").tmdb_id + tmdb_id = self.get_movie(language, f"{self.site_url}{item_url}").tmdb_id if tmdb_id: movie_ids.append(tmdb_id) else: raise Failed(f"TVDb Error: TMDb ID not found from TVDb URL: {tvdb_url}") except Failed as e: @@ -133,11 +136,9 @@ class TVDbAPI: if status_message: logger.info(f"Processing {pretty}: {data}") if method == "tvdb_show": - try: show_ids.append(self.get_series(language, tvdb_id=int(data)).id) - except ValueError: show_ids.append(self.get_series(language, tvdb_url=data).id) + show_ids.append(self.get_series(language, data).id) elif method == "tvdb_movie": - try: movie_ids.append(self.get_movie(language, tvdb_id=int(data)).id) - except ValueError: movie_ids.append(self.get_movie(language, tvdb_url=data).id) + movie_ids.append(self.get_movie(language, data).id) elif method == "tvdb_list": tmdb_ids, tvdb_ids = self.get_tvdb_ids_from_url(data, language) movie_ids.extend(tmdb_ids) diff --git a/modules/util.py b/modules/util.py index 11e6ded4..35667dea 100644 --- a/modules/util.py +++ b/modules/util.py @@ -453,10 +453,10 @@ boolean_details = [ ] all_details = [ "sort_title", "content_rating", - "summary", "tmdb_summary", "tmdb_description", "tmdb_biography", + "summary", "tmdb_summary", "tmdb_description", "tmdb_biography", "tvdb_summary", "tvdb_description", "trakt_description", "collection_mode", "collection_order", - "url_poster", "tmdb_poster", "tmdb_profile", "file_poster", - "url_background", "file_background", + "url_poster", "tmdb_poster", "tmdb_profile", "tvdb_poster", "file_poster", + "url_background", "tmdb_background", "tvdb_background", "file_background", "name_mapping", "add_to_arr", "arr_tag", "label", "show_filtered", "show_missing", "save_missing" ] From dfcf5ed6247dc15cb040eef11a3d353d33ed6afa Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 8 Mar 2021 15:33:11 -0500 Subject: [PATCH 19/22] asset directory can handle Season posters #82 --- modules/builder.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/builder.py b/modules/builder.py index aaee5a0c..7ef19cae 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -873,6 +873,13 @@ class CollectionBuilder: logger.info(f"Detail: asset_directory updated {item.title}'s background to [file] {background_path}") if poster_path is None and background_path is None: logger.warning(f"No Files Found: {os.path.join(path, folder)}") + if self.library.is_show: + for season in item.seasons(): + matches = glob.glob(os.path.join(path, folder, f"Season{'0' if season.seasonNumber < 10 else ''}{season.seasonNumber}.*")) + if len(matches) > 0: + season_path = os.path.abspath(matches[0]) + season.uploadPoster(filepath=season_path) + logger.info(f"Detail: asset_directory updated {item.title} Season {season.seasonNumber}'s poster to [file] {season_path}") else: logger.warning(f"No Folder: {os.path.join(path, folder)}") From 716d5d483753ae7c767383f24bef21787ce976f6 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 8 Mar 2021 16:09:53 -0500 Subject: [PATCH 20/22] added `title` search #88 --- modules/builder.py | 44 ++++++++++++++++++++++++++++++++------------ modules/util.py | 1 + 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 7ef19cae..06ca9f9a 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -304,6 +304,8 @@ class CollectionBuilder: else: final_values.append(value) self.methods.append(("plex_search", [[(method_name, final_values)]])) + elif method_name == "title": + self.methods.append(("plex_search", [[(method_name, data[m])]])) elif method_name in util.plex_searches: self.methods.append(("plex_search", [[(method_name, util.get_list(data[m]))]])) elif method_name == "plex_all": @@ -420,6 +422,9 @@ class CollectionBuilder: if len(years) > 0: used.append(util.remove_not(search)) searches.append((search, util.get_int_list(data[m][s], util.remove_not(search)))) + elif search == "title": + used.append(util.remove_not(search)) + searches.append((search, data[m][s])) elif search in util.plex_searches: used.append(util.remove_not(search)) searches.append((search, util.get_list(data[m][s]))) @@ -644,18 +649,33 @@ class CollectionBuilder: items_found += len(items) elif method == "plex_search": search_terms = {} - for i, attr_pair in enumerate(value): - search_list = attr_pair[1] - final_method = attr_pair[0][:-4] + "!" if attr_pair[0][-4:] == ".not" else attr_pair[0] - if self.library.is_show: - final_method = "show." + final_method - search_terms[final_method] = search_list - ors = "" - for o, param in enumerate(attr_pair[1]): - or_des = " OR " if o > 0 else f"{attr_pair[0]}(" - ors += f"{or_des}{param}" - logger.info(f"\t\t AND {ors})" if i > 0 else f"Processing {pretty}: {ors})") - items = self.library.Plex.search(**search_terms) + title_search = None + has_processed = False + for search_method, search_data in value: + if search_method == "title": + title_search = search_data + logger.info(f"Processing {pretty}: title({title_search})") + has_processed = True + + for search_method, search_list in value: + if search_method != "title": + final_method = search_method[:-4] + "!" if search_method[-4:] == ".not" else search_method + if self.library.is_show: + final_method = "show." + final_method + search_terms[final_method] = search_list + ors = "" + for o, param in enumerate(search_list): + or_des = " OR " if o > 0 else f"{search_method}(" + ors += f"{or_des}{param}" + if title_search or has_processed: + logger.info(f"\t\t AND {ors})") + else: + logger.info(f"Processing {pretty}: {ors})") + has_processed = True + if title_search: + items = self.library.Plex.search(title_search, **search_terms) + else: + items = self.library.Plex.search(**search_terms) items_found += len(items) elif method == "plex_collectionless": good_collections = [] diff --git a/modules/util.py b/modules/util.py index 35667dea..0f659df0 100644 --- a/modules/util.py +++ b/modules/util.py @@ -309,6 +309,7 @@ plex_searches = [ "genre", #"genre.not", "producer", #"producer.not", "studio", #"studio.not", + "title", "writer", #"writer.not" "year" #"year.not", ] From 1ebaf445766ae4abe85985b19ada7bf27af21ed1 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 8 Mar 2021 16:54:19 -0500 Subject: [PATCH 21/22] added letterboxd list descriptions --- modules/builder.py | 7 +++++++ modules/letterboxd.py | 12 ++++++++---- modules/util.py | 5 ++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 06ca9f9a..b1061add 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -246,6 +246,8 @@ class CollectionBuilder: self.summaries[method_name] = config.TVDb.get_list_description(data[m], self.library.Plex.language) elif method_name == "trakt_description": self.summaries[method_name] = config.Trakt.standard_list(config.Trakt.validate_trakt_list(util.get_list(data[m]))[0]).description + elif method_name == "letterboxd_description": + self.summaries[method_name] = config.Letterboxd.get_list_description(data[m], self.library.Plex.language) elif method_name == "collection_mode": if data[m] in ["default", "hide", "hide_items", "show_items", "hideItems", "showItems"]: if data[m] == "hide_items": self.details[method_name] = "hideItems" @@ -347,6 +349,10 @@ class CollectionBuilder: self.methods.append((method_name, new_list)) elif method_name == "letterboxd_list": self.methods.append((method_name, util.get_list(data[m], split=False))) + elif method_name == "letterboxd_list_details": + values = util.get_list(data[m], split=False) + self.summaries[method_name] = config.Letterboxd.get_list_description(values[0], self.library.Plex.language) + self.methods.append((method_name[:-8], values)) elif method_name in util.dictionary_lists: if isinstance(data[m], dict): def get_int(parent, method, data_in, default_in, minimum=1, maximum=None): @@ -803,6 +809,7 @@ class CollectionBuilder: return summaries[summary_method] if "summary" in self.summaries: summary = get_summary("summary", self.summaries) elif "tmdb_description" in self.summaries: summary = get_summary("tmdb_description", self.summaries) + elif "letterboxd_description" in self.summaries: summary = get_summary("letterboxd_description", self.summaries) elif "tmdb_summary" in self.summaries: summary = get_summary("tmdb_summary", self.summaries) elif "tvdb_summary" in self.summaries: summary = get_summary("tvdb_summary", self.summaries) elif "tmdb_biography" in self.summaries: summary = get_summary("tmdb_biography", self.summaries) diff --git a/modules/letterboxd.py b/modules/letterboxd.py index 5671878d..16b817ef 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -11,11 +11,15 @@ class LetterboxdAPI: self.url = "https://letterboxd.com" @retry(stop_max_attempt_number=6, wait_fixed=10000) - def send_request(self, url, header): - return html.fromstring(requests.get(url, headers=header).content) + def send_request(self, url, language): + return html.fromstring(requests.get(url, header={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + + def get_list_description(self, list_url, language): + descriptions = self.send_request(list_url, language).xpath("//meta[@property='og:description']/@content") + return descriptions[0] if len(descriptions) > 0 and len(descriptions[0]) > 0 else None def parse_list_for_slugs(self, list_url, language): - response = self.send_request(list_url, header={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}) + response = self.send_request(list_url, language) slugs = response.xpath("//div[@class='poster film-poster really-lazy-load']/@data-film-slug") next_url = response.xpath("//a[@class='next']/@href") if len(next_url) > 0: @@ -26,7 +30,7 @@ class LetterboxdAPI: return self.get_tmdb(f"{self.url}{slug}", language) def get_tmdb(self, letterboxd_url, language): - response = self.send_request(letterboxd_url, header={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}) + response = self.send_request(letterboxd_url, language) ids = response.xpath("//body/@data-tmdb-id") if len(ids) > 0: return int(ids[0]) diff --git a/modules/util.py b/modules/util.py index 0f659df0..9fdcb953 100644 --- a/modules/util.py +++ b/modules/util.py @@ -98,6 +98,7 @@ pretty_names = { "imdb_list": "IMDb List", "imdb_id": "IMDb ID", "letterboxd_list": "Letterboxd List", + "letterboxd_list_details": "Letterboxd List", "mal_id": "MyAnimeList ID", "mal_all": "MyAnimeList All", "mal_airing": "MyAnimeList Airing", @@ -220,6 +221,7 @@ all_lists = [ "imdb_list", "imdb_id", "letterboxd_list", + "letterboxd_list_details", "mal_id", "mal_all", "mal_airing", @@ -322,6 +324,7 @@ show_only_lists = [ ] movie_only_lists = [ "letterboxd_list", + "letterboxd_list_details", "tmdb_collection", "tmdb_collection_details", "tmdb_movie", @@ -454,7 +457,7 @@ boolean_details = [ ] all_details = [ "sort_title", "content_rating", - "summary", "tmdb_summary", "tmdb_description", "tmdb_biography", "tvdb_summary", "tvdb_description", "trakt_description", + "summary", "tmdb_summary", "tmdb_description", "tmdb_biography", "tvdb_summary", "tvdb_description", "trakt_description", "letterboxd_description", "collection_mode", "collection_order", "url_poster", "tmdb_poster", "tmdb_profile", "tvdb_poster", "file_poster", "url_background", "tmdb_background", "tvdb_background", "file_background", From f1966a0ef3d979ebac236a96bbebb0bd55df898a Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 8 Mar 2021 18:20:26 -0500 Subject: [PATCH 22/22] v1.4.0 --- README.md | 2 +- plex_meta_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9cc43a5e..5608bfbf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Plex Meta Manager -#### Version 1.3.0 +#### Version 1.4.0 The original concept for Plex Meta Manager is [Plex Auto Collections](https://github.com/mza921/Plex-Auto-Collections), but this is rewritten from the ground up to be able to include a scheduler, metadata edits, multiple libraries, and logging. Plex Meta Manager is a Python 3 script that can be continuously run using YAML configuration files to update on a schedule the metadata of the movies, shows, and collections in your libraries as well as automatically build collections based on various methods all detailed in the wiki. Some collection examples that the script can automatically build and update daily include Plex Based Searches like actor, genre, or studio collections or Collections based on TMDb, IMDb, Trakt, TVDb, AniDB, or MyAnimeList lists and various other services. diff --git a/plex_meta_manager.py b/plex_meta_manager.py index accbb322..522e56d2 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -65,7 +65,7 @@ logger.info(util.get_centered_text("| |_) | |/ _ \\ \\/ / | |\\/| |/ _ \\ __/ _` logger.info(util.get_centered_text("| __/| | __/> < | | | | __/ || (_| | | | | | (_| | | | | (_| | (_| | __/ | ")) logger.info(util.get_centered_text("|_| |_|\\___/_/\\_\\ |_| |_|\\___|\\__\\__,_| |_| |_|\\__,_|_| |_|\\__,_|\\__, |\\___|_| ")) logger.info(util.get_centered_text(" |___/ ")) -logger.info(util.get_centered_text(" Version: 1.3.0 ")) +logger.info(util.get_centered_text(" Version: 1.4.0 ")) util.separator() if args.tests: