From 71cf71798e8d67fc08201fbf54a56bd16f15c9a6 Mon Sep 17 00:00:00 2001 From: Critical-Impact Date: Sun, 11 Jul 2021 18:12:52 +1000 Subject: [PATCH 01/95] AniDB Authentication and Tag Builder --- config/config.yml.template | 3 ++ modules/anidb.py | 58 ++++++++++++++++++++++++++++++++++---- modules/config.py | 23 ++++++++++++++- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/config/config.yml.template b/config/config.yml.template index 9ee8c3de..9a2bb7d6 100644 --- a/config/config.yml.template +++ b/config/config.yml.template @@ -89,3 +89,6 @@ mal: token_type: expires_in: refresh_token: +anidb: + username: ###### - optional + password: ###### \ No newline at end of file diff --git a/modules/anidb.py b/modules/anidb.py index d9147f84..6f05424b 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -1,4 +1,4 @@ -import logging, requests +import logging, requests,time from lxml import html from modules import util from modules.util import Failed @@ -6,20 +6,42 @@ from retrying import retry logger = logging.getLogger("Plex Meta Manager") -builders = ["anidb_id", "anidb_relation", "anidb_popular"] +builders = ["anidb_id", "anidb_relation", "anidb_popular", 'anidb_tag'] class AniDB: - def __init__(self, config): + def __init__(self, params, config): self.config = config + + # Create a session so if we login we can continue to use the same session + self.anidb_session = requests.Session() + self.urls = { "anime": "https://anidb.net/anime", "popular": "https://anidb.net/latest/anime/popular/?h=1", - "relation": "/relation/graph" + "relation": "/relation/graph", + "anidb_tag": "https://anidb.net/tag", + "login": "https://anidb.net/perl-bin/animedb.pl" } + if params and "username" in params and "password" in params: + result = str(self._login(params["username"], params["password"]).content) + + # Login response does not use proper status codes so we have to check the content of the document + if "Wrong username/password" in result: + raise Failed("AniDB Error: Login failed") + @retry(stop_max_attempt_number=6, wait_fixed=10000) def _request(self, url, language): - return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + return html.fromstring(self.anidb_session.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + + def _login(self, username, password): + data = { + "show": "main", + "xuser": username, + "xpass": password, + "xdoautologin": "on" + } + return self.anidb_session.post(self.urls["login"], data, headers={"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 x64"}) def _popular(self, language): response = self._request(self.urls["popular"], language) @@ -47,12 +69,36 @@ class AniDB: return anidb_values raise Failed(f"AniDB Error: No valid AniDB IDs in {anidb_list}") + def _tag(self, tag, limit, language): + anidb_ids = [] + next_page = True + current_url = self.urls["anidb_tag"] + "/" + str(tag) + while next_page: + logger.debug(f"Sending request to {current_url}") + response = self._request(current_url, language) + int_list = util.get_int_list(response.xpath("//td[@class='name main anime']/a/@href"), "AniDB ID") + anidb_ids.extend(int_list) + next_page_list = response.xpath("//li[@class='next']/a/@href") + logger.debug(f"next page list {next_page_list}") + if len(next_page_list) != 0 and len(anidb_ids) <= limit: + logger.debug(f"Loading next anidb page") + time.sleep(2)# Sleep as we are paging through anidb and don't want the ban hammer + current_url = "https://anidb.net" + next_page_list[0] + else: + logger.debug(f"Got to last page") + next_page = False + anidb_ids = anidb_ids[:limit] + return anidb_ids + def get_items(self, method, data, language): pretty = util.pretty_names[method] if method in util.pretty_names else method anidb_ids = [] if method == "anidb_popular": logger.info(f"Processing {pretty}: {data} Anime") anidb_ids.extend(self._popular(language)[:data]) + elif method == "anidb_tag": + anidb_ids = self._tag(data["tag"], data["limit"], language) + logger.info(f"Processing {pretty}: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag ID: {data['tag']}") else: logger.info(f"Processing {pretty}: {data}") if method == "anidb_id": anidb_ids.append(data) @@ -63,4 +109,4 @@ class AniDB: logger.debug(f"{len(anidb_ids)} AniDB IDs Found: {anidb_ids}") logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids + return movie_ids, show_ids \ No newline at end of file diff --git a/modules/config.py b/modules/config.py index b097d5ad..f3394a25 100644 --- a/modules/config.py +++ b/modules/config.py @@ -108,6 +108,7 @@ class Config: if "omdb" in new_config: new_config["omdb"] = new_config.pop("omdb") if "trakt" in new_config: new_config["trakt"] = new_config.pop("trakt") if "mal" in new_config: new_config["mal"] = new_config.pop("mal") + if "anidb" in new_config: new_config["anidb"] = new_config.pop("anidb") yaml.round_trip_dump(new_config, open(self.config_path, "w", encoding="utf-8"), indent=ind, block_seq_indent=bsi) self.data = new_config except yaml.scanner.ScannerError as e: @@ -270,9 +271,29 @@ class Config: else: logger.warning("mal attribute not found") + util.separator() + + self.AniDB = None + anidb_username = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False) + anidb_password = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False) + if "anidb" in self.data and anidb_username and anidb_password: + logger.info("Connecting to AniDB...") + self.anidb = {} + try: + self.anidb["username"] = check_for_attribute(self.data, "username", parent="anidb", throw=True) + self.anidb["password"] = check_for_attribute(self.data, "password", parent="anidb", throw=True) + self.AniDB = AniDB(self.anidb, self) + except Failed as e: + logger.error(e) + logger.info(f"My Anime List Connection {'Failed' if self.MyAnimeList is None else 'Successful'}") + else: + logger.info("Using guest authentication for AniDB") + self.AniDB = AniDB(None, self) + + util.separator() + self.TVDb = TVDb(self) self.IMDb = IMDb(self) - self.AniDB = AniDB(self) self.Convert = Convert(self) self.AniList = AniList(self) self.Letterboxd = Letterboxd(self) From 4ed4a62c8818ca987db456294c09c6069a1f07f4 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 12 Jul 2021 15:06:13 -0400 Subject: [PATCH 02/95] extra assets printout --- modules/plex.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/plex.py b/modules/plex.py index 316201de..6a375740 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -771,6 +771,7 @@ class Plex: def update_item_from_assets(self, item, overlay=None): name = os.path.basename(os.path.dirname(item.locations[0]) if self.is_movie else item.locations[0]) found_one = False + uploaded = False for ad in self.asset_directory: poster = None background = None @@ -797,6 +798,7 @@ class Plex: if len(matches) > 0: background = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_poster=False, is_url=False) if poster or background: + uploaded = True self.upload_images(item, poster=poster, background=background, overlay=overlay) if self.is_show: for season in self.query(item.seasons): @@ -821,6 +823,8 @@ class Plex: self.upload_images(item, overlay=overlay) elif not found_one: logger.error(f"Asset Warning: No asset folder found called '{name}'") + elif not uploaded: + logger.error(f"Asset Warning: No poster or background found in an assets folder") def find_collection_assets(self, item, name=None): if name is None: From 1ea405bebbbad63e9a0993d525b6b639f39d423e Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 12 Jul 2021 15:24:31 -0400 Subject: [PATCH 03/95] more debug prints --- modules/plex.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/plex.py b/modules/plex.py index 6a375740..9ae4c7dc 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -770,7 +770,7 @@ class Plex: def update_item_from_assets(self, item, overlay=None): name = os.path.basename(os.path.dirname(item.locations[0]) if self.is_movie else item.locations[0]) - found_one = False + found_folder = False uploaded = False for ad in self.asset_directory: poster = None @@ -785,20 +785,21 @@ class Plex: item_dir = os.path.abspath(matches[0]) if item_dir is None: continue - found_one = True + found_folder = True poster_filter = os.path.join(item_dir, "poster.*") background_filter = os.path.join(item_dir, "background.*") else: poster_filter = os.path.join(ad, f"{name}.*") background_filter = os.path.join(ad, f"{name}_background.*") + logger.debug(f"Poster Filter: {poster_filter}") matches = glob.glob(poster_filter) if len(matches) > 0: poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_url=False) + logger.debug(f"Background Filter: {background_filter}") matches = glob.glob(background_filter) if len(matches) > 0: background = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_poster=False, is_url=False) if poster or background: - uploaded = True self.upload_images(item, poster=poster, background=background, overlay=overlay) if self.is_show: for season in self.query(item.seasons): @@ -819,12 +820,12 @@ class Plex: if len(matches) > 0: episode_poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title} {episode.seasonEpisode.upper()}'s ", is_url=False) self.upload_images(episode, poster=episode_poster) - if not found_one and overlay: + if not found_folder and overlay: self.upload_images(item, overlay=overlay) - elif not found_one: + elif self.asset_folders and not found_folder: logger.error(f"Asset Warning: No asset folder found called '{name}'") - elif not uploaded: - logger.error(f"Asset Warning: No poster or background found in an assets folder") + elif not poster and not background: + logger.error(f"Asset Warning: No poster or background found in an assets folder for {name}") def find_collection_assets(self, item, name=None): if name is None: From cd0265d8f4cdf9c5eddcda92608103941d28bf28 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 12 Jul 2021 16:05:35 -0400 Subject: [PATCH 04/95] more debugs --- modules/plex.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/plex.py b/modules/plex.py index 9ae4c7dc..09449598 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -769,6 +769,12 @@ class Plex: return updated def update_item_from_assets(self, item, overlay=None): + logger.debug(item.locations) + logger.debug(item.locations[0]) + self.reload(item) + logger.debug(item.locations) + logger.debug(item.locations[0]) + name = os.path.basename(os.path.dirname(item.locations[0]) if self.is_movie else item.locations[0]) found_folder = False uploaded = False From 7d2bd7025d14dde13e9a06c409af2b2b33756af2 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 12 Jul 2021 16:21:07 -0400 Subject: [PATCH 05/95] more debug --- modules/plex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/plex.py b/modules/plex.py index 09449598..290d9b16 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -771,11 +771,11 @@ class Plex: def update_item_from_assets(self, item, overlay=None): logger.debug(item.locations) logger.debug(item.locations[0]) - self.reload(item) - logger.debug(item.locations) - logger.debug(item.locations[0]) + logger.debug(os.path.dirname(item.locations[0])) + logger.debug(os.path.basename(os.path.dirname(item.locations[0]))) name = os.path.basename(os.path.dirname(item.locations[0]) if self.is_movie else item.locations[0]) + logger.debug(name) found_folder = False uploaded = False for ad in self.asset_directory: From 556653a4bae9a75b7068789e0b48bc499e7477b1 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 12 Jul 2021 16:35:35 -0400 Subject: [PATCH 06/95] cast to string --- modules/plex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/plex.py b/modules/plex.py index 290d9b16..dd28ce76 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -771,10 +771,10 @@ class Plex: def update_item_from_assets(self, item, overlay=None): logger.debug(item.locations) logger.debug(item.locations[0]) - logger.debug(os.path.dirname(item.locations[0])) - logger.debug(os.path.basename(os.path.dirname(item.locations[0]))) + logger.debug(os.path.dirname(str(item.locations[0]))) + logger.debug(os.path.basename(os.path.dirname(str(item.locations[0])))) - name = os.path.basename(os.path.dirname(item.locations[0]) if self.is_movie else item.locations[0]) + name = os.path.basename(os.path.dirname(str(item.locations[0])) if self.is_movie else str(item.locations[0])) logger.debug(name) found_folder = False uploaded = False From 3d6c1c66ff3c7ad59ae4569ea2054fbc64de2a6e Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 12 Jul 2021 16:39:08 -0400 Subject: [PATCH 07/95] more debug --- modules/plex.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/plex.py b/modules/plex.py index dd28ce76..93fb76ed 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -773,6 +773,9 @@ class Plex: logger.debug(item.locations[0]) logger.debug(os.path.dirname(str(item.locations[0]))) logger.debug(os.path.basename(os.path.dirname(str(item.locations[0])))) + logger.debug(os.path.dirname(os.path.abspath(item.locations[0]))) + logger.debug(os.path.basename(os.path.dirname(os.path.abspath(item.locations[0])))) + name = os.path.basename(os.path.dirname(str(item.locations[0])) if self.is_movie else str(item.locations[0])) logger.debug(name) From 65361d0a370e7611d0d1e7f42f172e4024c06e0c Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 13 Jul 2021 14:00:09 -0400 Subject: [PATCH 08/95] fixed true tag issue --- modules/config.py | 4 ++-- modules/plex.py | 17 +++-------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/modules/config.py b/modules/config.py index b097d5ad..889bb8a5 100644 --- a/modules/config.py +++ b/modules/config.py @@ -476,7 +476,7 @@ class Config: radarr_params["monitor"] = check_for_attribute(lib, "monitor", parent="radarr", var_type="bool", default=self.general["radarr"]["monitor"], save=False) radarr_params["availability"] = check_for_attribute(lib, "availability", parent="radarr", test_list=radarr_availabilities, default=self.general["radarr"]["availability"], save=False) radarr_params["quality_profile"] = check_for_attribute(lib, "quality_profile", parent="radarr", default=self.general["radarr"]["quality_profile"], req_default=True, save=False) - radarr_params["tag"] = check_for_attribute(lib, "search", parent="radarr", var_type="lower_list", default=self.general["radarr"]["tag"], default_is_none=True, save=False) + radarr_params["tag"] = check_for_attribute(lib, "tag", parent="radarr", var_type="lower_list", default=self.general["radarr"]["tag"], default_is_none=True, save=False) radarr_params["search"] = check_for_attribute(lib, "search", parent="radarr", var_type="bool", default=self.general["radarr"]["search"], save=False) library.Radarr = Radarr(radarr_params) except Failed as e: @@ -505,7 +505,7 @@ class Config: sonarr_params["language_profile"] = check_for_attribute(lib, "language_profile", parent="sonarr", default_is_none=True, save=False) sonarr_params["series_type"] = check_for_attribute(lib, "series_type", parent="sonarr", test_list=sonarr_series_types, default=self.general["sonarr"]["series_type"], save=False) sonarr_params["season_folder"] = check_for_attribute(lib, "season_folder", parent="sonarr", var_type="bool", default=self.general["sonarr"]["season_folder"], save=False) - sonarr_params["tag"] = check_for_attribute(lib, "search", parent="sonarr", var_type="lower_list", default=self.general["sonarr"]["tag"], default_is_none=True, save=False) + sonarr_params["tag"] = check_for_attribute(lib, "tag", parent="sonarr", var_type="lower_list", default=self.general["sonarr"]["tag"], default_is_none=True, save=False) sonarr_params["search"] = check_for_attribute(lib, "search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["search"], save=False) sonarr_params["cutoff_search"] = check_for_attribute(lib, "cutoff_search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["cutoff_search"], save=False) library.Sonarr = Sonarr(sonarr_params) diff --git a/modules/plex.py b/modules/plex.py index 93fb76ed..eff47a7f 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -769,18 +769,9 @@ class Plex: return updated def update_item_from_assets(self, item, overlay=None): - logger.debug(item.locations) - logger.debug(item.locations[0]) - logger.debug(os.path.dirname(str(item.locations[0]))) - logger.debug(os.path.basename(os.path.dirname(str(item.locations[0])))) - logger.debug(os.path.dirname(os.path.abspath(item.locations[0]))) - logger.debug(os.path.basename(os.path.dirname(os.path.abspath(item.locations[0])))) - - name = os.path.basename(os.path.dirname(str(item.locations[0])) if self.is_movie else str(item.locations[0])) logger.debug(name) found_folder = False - uploaded = False for ad in self.asset_directory: poster = None background = None @@ -800,11 +791,9 @@ class Plex: else: poster_filter = os.path.join(ad, f"{name}.*") background_filter = os.path.join(ad, f"{name}_background.*") - logger.debug(f"Poster Filter: {poster_filter}") matches = glob.glob(poster_filter) if len(matches) > 0: poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_url=False) - logger.debug(f"Background Filter: {background_filter}") matches = glob.glob(background_filter) if len(matches) > 0: background = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_poster=False, is_url=False) @@ -829,12 +818,12 @@ class Plex: if len(matches) > 0: episode_poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title} {episode.seasonEpisode.upper()}'s ", is_url=False) self.upload_images(episode, poster=episode_poster) - if not found_folder and overlay: + if not poster and overlay: self.upload_images(item, overlay=overlay) - elif self.asset_folders and not found_folder: + if not overlay and self.asset_folders and not found_folder: logger.error(f"Asset Warning: No asset folder found called '{name}'") elif not poster and not background: - logger.error(f"Asset Warning: No poster or background found in an assets folder for {name}") + logger.error(f"Asset Warning: No poster or background found in an assets folder for '{name}'") def find_collection_assets(self, item, name=None): if name is None: From 8f7f35c413abe4327ba18a1f41871c3a174fb936 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 13 Jul 2021 14:32:46 -0400 Subject: [PATCH 09/95] #335 fix episode number --- modules/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/meta.py b/modules/meta.py index c102001f..e75f9769 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -366,7 +366,7 @@ class Metadata: if edit_tags("writer", episode, episode_dict, episode_methods): updated = True set_images(episode, episode_dict, episode_methods) - logger.info(f"Episode S{episode_id}E{season_id} of {mapping_name} Details Update {'Complete' if updated else 'Not Needed'}") + logger.info(f"Episode S{season_id}E{episode_id} of {mapping_name} Details Update {'Complete' if updated else 'Not Needed'}") else: logger.error(f"Metadata Error: episode {episode_str} invalid must have S##E## format") else: From 21097c97fe1cf60b98b9429a08f59b2ea9ced3b8 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 13 Jul 2021 14:51:04 -0400 Subject: [PATCH 10/95] metadata attributes should work better --- modules/meta.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/modules/meta.py b/modules/meta.py index e75f9769..5bf08ee8 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -81,19 +81,21 @@ class Metadata: edits = {} advance_edits = {} - def add_edit(name, current, group, alias, key=None, value=None, var_type="str"): + def add_edit(name, current_item, group, alias, key=None, value=None, var_type="str"): if value or name in alias: if value or group[alias[name]]: if key is None: key = name if value is None: value = group[alias[name]] try: + current = str(getattr(current_item, key, "")) if var_type == "date": final_value = util.check_date(value, name, return_string=True, plex_date=True) + current = current[:-9] elif var_type == "float": final_value = util.check_number(value, name, number_type="float", minimum=0, maximum=10) else: final_value = value - if str(current) != str(final_value): + if current != str(final_value): edits[f"{key}.value"] = final_value edits[f"{key}.locked"] = 1 logger.info(f"Detail: {name} updated to {final_value}") @@ -242,16 +244,16 @@ class Metadata: genres = [genre.name for genre in tmdb_item.genres] edits = {} - add_edit("title", item.title, meta, methods, value=title) - add_edit("sort_title", item.titleSort, meta, methods, key="titleSort") - add_edit("originally_available", str(item.originallyAvailableAt)[:-9], meta, methods, key="originallyAvailableAt", value=originally_available, var_type="date") - add_edit("critic_rating", item.rating, meta, methods, value=rating, key="rating", var_type="float") - add_edit("audience_rating", item.audienceRating, meta, methods, key="audienceRating", var_type="float") - add_edit("content_rating", item.contentRating, meta, methods, key="contentRating") - add_edit("original_title", item.originalTitle, meta, methods, key="originalTitle", value=original_title) - add_edit("studio", item.studio, meta, methods, value=studio) - add_edit("tagline", item.tagline, meta, methods, value=tagline) - add_edit("summary", item.summary, meta, methods, value=summary) + add_edit("title", item, meta, methods, value=title) + add_edit("sort_title", item, meta, methods, key="titleSort") + add_edit("originally_available", item, meta, methods, key="originallyAvailableAt", value=originally_available, var_type="date") + add_edit("critic_rating", item, meta, methods, value=rating, key="rating", var_type="float") + add_edit("audience_rating", item, meta, methods, key="audienceRating", var_type="float") + add_edit("content_rating", item, meta, methods, key="contentRating") + add_edit("original_title", item, meta, methods, key="originalTitle", value=original_title) + add_edit("studio", item, meta, methods, value=studio) + add_edit("tagline", item, meta, methods, value=tagline) + add_edit("summary", item, meta, methods, value=summary) if self.library.edit_item(item, mapping_name, item_type, edits): updated = True @@ -306,8 +308,8 @@ class Metadata: logger.error("Metadata Error: sub attribute must be True or False") edits = {} - add_edit("title", season.title, season_dict, season_methods, value=title) - add_edit("summary", season.summary, season_dict, season_methods) + add_edit("title", season, season_dict, season_methods, value=title) + add_edit("summary", season, season_dict, season_methods) if self.library.edit_item(season, season_id, "Season", edits): updated = True set_images(season, season_dict, season_methods) @@ -352,13 +354,11 @@ class Metadata: else: logger.error("Metadata Error: sub attribute must be True or False") edits = {} - add_edit("title", episode.title, episode_dict, episode_methods, value=title) - add_edit("sort_title", episode.titleSort, episode_dict, episode_methods, - key="titleSort") - add_edit("rating", episode.rating, episode_dict, episode_methods) - add_edit("originally_available", str(episode.originallyAvailableAt)[:-9], - episode_dict, episode_methods, key="originallyAvailableAt") - add_edit("summary", episode.summary, episode_dict, episode_methods) + add_edit("title", episode, episode_dict, episode_methods, value=title) + add_edit("sort_title", episode, episode_dict, episode_methods, key="titleSort") + add_edit("rating", episode, episode_dict, episode_methods, var_type="float") + add_edit("originally_available", episode, episode_dict, episode_methods, key="originallyAvailableAt", var_type="date") + add_edit("summary", episode, episode_dict, episode_methods) if self.library.edit_item(episode, f"{season_id} Episode: {episode_id}", "Season", edits): updated = True if edit_tags("director", episode, episode_dict, episode_methods): From 97a01a64968288ed2bc431d0d8dea4402503155e Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 13 Jul 2021 16:53:49 -0400 Subject: [PATCH 11/95] change up anidb login --- modules/anidb.py | 43 +++++++++++++++---------------------------- modules/builder.py | 12 ++++++++++++ modules/config.py | 16 +++++++--------- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 6f05424b..3a62700b 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -1,4 +1,4 @@ -import logging, requests,time +import logging, requests, time from lxml import html from modules import util from modules.util import Failed @@ -6,15 +6,12 @@ from retrying import retry logger = logging.getLogger("Plex Meta Manager") -builders = ["anidb_id", "anidb_relation", "anidb_popular", 'anidb_tag'] +builders = ["anidb_id", "anidb_relation", "anidb_popular", "anidb_tag"] class AniDB: def __init__(self, params, config): self.config = config - # Create a session so if we login we can continue to use the same session - self.anidb_session = requests.Session() - self.urls = { "anime": "https://anidb.net/anime", "popular": "https://anidb.net/latest/anime/popular/?h=1", @@ -22,18 +19,15 @@ class AniDB: "anidb_tag": "https://anidb.net/tag", "login": "https://anidb.net/perl-bin/animedb.pl" } - - if params and "username" in params and "password" in params: - result = str(self._login(params["username"], params["password"]).content) - - # Login response does not use proper status codes so we have to check the content of the document - if "Wrong username/password" in result: + if params: + if not self._login(params["username"], params["password"]).xpath("//li[@class='sub-menu my']/@title"): raise Failed("AniDB Error: Login failed") @retry(stop_max_attempt_number=6, wait_fixed=10000) def _request(self, url, language): - return html.fromstring(self.anidb_session.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + return html.fromstring(self.config.session.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + @retry(stop_max_attempt_number=6, wait_fixed=10000) def _login(self, username, password): data = { "show": "main", @@ -41,7 +35,7 @@ class AniDB: "xpass": password, "xdoautologin": "on" } - return self.anidb_session.post(self.urls["login"], data, headers={"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 x64"}) + return html.fromstring(self.config.session.post(self.urls["login"], data, headers={"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 x64"}).content) def _popular(self, language): response = self._request(self.urls["popular"], language) @@ -71,24 +65,17 @@ class AniDB: def _tag(self, tag, limit, language): anidb_ids = [] - next_page = True - current_url = self.urls["anidb_tag"] + "/" + str(tag) - while next_page: - logger.debug(f"Sending request to {current_url}") + current_url = f"{self.urls['anidb_tag']}/{tag}" + while True: response = self._request(current_url, language) int_list = util.get_int_list(response.xpath("//td[@class='name main anime']/a/@href"), "AniDB ID") anidb_ids.extend(int_list) next_page_list = response.xpath("//li[@class='next']/a/@href") - logger.debug(f"next page list {next_page_list}") - if len(next_page_list) != 0 and len(anidb_ids) <= limit: - logger.debug(f"Loading next anidb page") - time.sleep(2)# Sleep as we are paging through anidb and don't want the ban hammer - current_url = "https://anidb.net" + next_page_list[0] - else: - logger.debug(f"Got to last page") - next_page = False - anidb_ids = anidb_ids[:limit] - return anidb_ids + if len(anidb_ids) >= limit or len(next_page_list) == 0: + break + time.sleep(2) + current_url = f"https://anidb.net{next_page_list[0]}" + return anidb_ids[:limit] def get_items(self, method, data, language): pretty = util.pretty_names[method] if method in util.pretty_names else method @@ -109,4 +96,4 @@ class AniDB: logger.debug(f"{len(anidb_ids)} AniDB IDs Found: {anidb_ids}") logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids \ No newline at end of file + return movie_ids, show_ids diff --git a/modules/builder.py b/modules/builder.py index 375bd39a..f530e91b 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -61,6 +61,7 @@ modifier_alias = {".greater": ".gt", ".less": ".lt"} all_builders = anidb.builders + anilist.builders + icheckmovies.builders + imdb.builders + letterboxd.builders + mal.builders + plex.builders + tautulli.builders + tmdb.builders + trakttv.builders + tvdb.builders dictionary_builders = [ "filters", + "anidb_tag", "anilist_genre", "anilist_season", "anilist_tag", @@ -950,6 +951,17 @@ class CollectionBuilder: new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 100, maximum=1000) self.methods.append((method_name, [new_dictionary])) + elif method_name == "anidb_tag": + new_dictionary = {} + dict_methods = {dm.lower(): dm for dm in dict_data} + if "tag" not in dict_methods: + raise Failed("Collection Error: anidb_tag tag attribute is required") + elif not dict_data[dict_methods["tag"]]: + raise Failed("Collection Error: anidb_tag tag attribute is blank") + else: + new_dictionary["tag"] = util.regex_first_int(dict_data[dict_methods["username"]], "AniDB Tag ID") + new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 0, minimum=0) + self.methods.append((method_name, [new_dictionary])) elif "anilist" in method_name: new_dictionary = {"sort_by": "score"} dict_methods = {dm.lower(): dm for dm in dict_data} diff --git a/modules/config.py b/modules/config.py index 392bfaa2..fcfd0997 100644 --- a/modules/config.py +++ b/modules/config.py @@ -1,4 +1,4 @@ -import logging, os +import logging, os, requests from datetime import datetime from modules import util from modules.anidb import AniDB @@ -188,6 +188,8 @@ class Config: util.print_multiline(options) return default + self.session = requests.Session() + self.general = {} self.general["cache"] = check_for_attribute(self.data, "cache", parent="settings", var_type="bool", default=True) self.general["cache_expiration"] = check_for_attribute(self.data, "cache_expiration", parent="settings", var_type="int", default=60) @@ -274,9 +276,8 @@ class Config: util.separator() self.AniDB = None - anidb_username = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False) - anidb_password = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False) - if "anidb" in self.data and anidb_username and anidb_password: + if "anidb" in self.data: + util.separator() logger.info("Connecting to AniDB...") self.anidb = {} try: @@ -285,13 +286,10 @@ class Config: self.AniDB = AniDB(self.anidb, self) except Failed as e: logger.error(e) - logger.info(f"My Anime List Connection {'Failed' if self.MyAnimeList is None else 'Successful'}") - else: - logger.info("Using guest authentication for AniDB") + logger.info(f"My Anime List Connection {'Failed Continuing as Guest ' if self.MyAnimeList is None else 'Successful'}") + if self.AniDB is None: self.AniDB = AniDB(None, self) - util.separator() - self.TVDb = TVDb(self) self.IMDb = IMDb(self) self.Convert = Convert(self) From 2f15564e6254e09759dd453004e8079f93410c74 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 14 Jul 2021 10:47:20 -0400 Subject: [PATCH 12/95] #306 Added a Session --- modules/anidb.py | 59 +++++++++++++-------------- modules/anilist.py | 6 +-- modules/builder.py | 25 ++++++------ modules/config.py | 37 +++++++++++++---- modules/convert.py | 19 +++------ modules/icheckmovies.py | 20 ++++------ modules/imdb.py | 46 ++++++++++----------- modules/letterboxd.py | 22 +++++------ modules/mal.py | 42 +++++++++----------- modules/meta.py | 9 +++-- modules/omdb.py | 21 +++++----- modules/plex.py | 18 ++++----- modules/radarr.py | 4 +- modules/sonarr.py | 3 +- modules/tautulli.py | 9 ++--- modules/trakttv.py | 6 +-- modules/tvdb.py | 88 ++++++++++++++++++++--------------------- modules/util.py | 3 ++ 18 files changed, 214 insertions(+), 223 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 3a62700b..647495e8 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -1,52 +1,48 @@ -import logging, requests, time -from lxml import html +import logging, time from modules import util from modules.util import Failed -from retrying import retry logger = logging.getLogger("Plex Meta Manager") builders = ["anidb_id", "anidb_relation", "anidb_popular", "anidb_tag"] +base_url = "https://anidb.net" +urls = { + "anime": f"{base_url}/anime", + "popular": f"{base_url}/latest/anime/popular/?h=1", + "relation": "/relation/graph", + "tag": f"{base_url}/tag", + "login": f"{base_url}/perl-bin/animedb.pl" +} class AniDB: - def __init__(self, params, config): + def __init__(self, config, params): self.config = config - - self.urls = { - "anime": "https://anidb.net/anime", - "popular": "https://anidb.net/latest/anime/popular/?h=1", - "relation": "/relation/graph", - "anidb_tag": "https://anidb.net/tag", - "login": "https://anidb.net/perl-bin/animedb.pl" - } + self.username = params["username"] if params else None + self.password = params["password"] if params else None if params: - if not self._login(params["username"], params["password"]).xpath("//li[@class='sub-menu my']/@title"): + if not self._login(self.username, self.password).xpath("//li[@class='sub-menu my']/@title"): raise Failed("AniDB Error: Login failed") - @retry(stop_max_attempt_number=6, wait_fixed=10000) - def _request(self, url, language): - return html.fromstring(self.config.session.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + def _request(self, url, language=None, postData=None): + if postData: + return self.config.post_html(url, postData, headers=util.header(language)) + else: + return self.config.get_html(url, headers=util.header(language)) - @retry(stop_max_attempt_number=6, wait_fixed=10000) def _login(self, username, password): - data = { - "show": "main", - "xuser": username, - "xpass": password, - "xdoautologin": "on" - } - return html.fromstring(self.config.session.post(self.urls["login"], data, headers={"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 x64"}).content) + data = {"show": "main", "xuser": username, "xpass": password, "xdoautologin": "on"} + return self._request(urls["login"], postData=data) def _popular(self, language): - response = self._request(self.urls["popular"], language) + response = self._request(urls["popular"], language=language) return util.get_int_list(response.xpath("//td[@class='name anime']/a/@href"), "AniDB ID") def _relations(self, anidb_id, language): - response = self._request(f"{self.urls['anime']}/{anidb_id}{self.urls['relation']}", language) + response = self._request(f"{urls['anime']}/{anidb_id}{urls['relation']}", language=language) return util.get_int_list(response.xpath("//area/@href"), "AniDB ID") def _validate(self, anidb_id, language): - response = self._request(f"{self.urls['anime']}/{anidb_id}", language) + response = self._request(f"{urls['anime']}/{anidb_id}", language=language) ids = response.xpath(f"//*[text()='a{anidb_id}']/text()") if len(ids) > 0: return util.regex_first_int(ids[0], "AniDB ID") @@ -65,16 +61,15 @@ class AniDB: def _tag(self, tag, limit, language): anidb_ids = [] - current_url = f"{self.urls['anidb_tag']}/{tag}" + current_url = f"{urls['tag']}/{tag}" while True: - response = self._request(current_url, language) - int_list = util.get_int_list(response.xpath("//td[@class='name main anime']/a/@href"), "AniDB ID") - anidb_ids.extend(int_list) + response = self._request(current_url, language=language) + anidb_ids.extend(util.get_int_list(response.xpath("//td[@class='name main anime']/a/@href"), "AniDB ID")) next_page_list = response.xpath("//li[@class='next']/a/@href") if len(anidb_ids) >= limit or len(next_page_list) == 0: break time.sleep(2) - current_url = f"https://anidb.net{next_page_list[0]}" + current_url = f"{base_url}{next_page_list[0]}" return anidb_ids[:limit] def get_items(self, method, data, language): diff --git a/modules/anilist.py b/modules/anilist.py index 34959081..8df509e1 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -1,4 +1,4 @@ -import logging, requests, time +import logging, time from modules import util from modules.util import Failed from retrying import retry @@ -19,13 +19,13 @@ pretty_names = { "score": "Average Score", "popular": "Popularity" } +base_url = "https://graphql.anilist.co" tag_query = "query{MediaTagCollection {name}}" genre_query = "query{GenreCollection}" class AniList: def __init__(self, config): self.config = config - self.url = "https://graphql.anilist.co" self.tags = {} self.genres = {} self.tags = {t["name"].lower(): t["name"] for t in self._request(tag_query, {})["data"]["MediaTagCollection"]} @@ -33,7 +33,7 @@ class AniList: @retry(stop_max_attempt_number=2, retry_on_exception=util.retry_if_not_failed) def _request(self, query, variables): - response = requests.post(self.url, json={"query": query, "variables": variables}) + response = self.config.post(base_url, json={"query": query, "variables": variables}) json_obj = response.json() if "errors" in json_obj: if json_obj['errors'][0]['message'] == "Too Many Requests.": diff --git a/modules/builder.py b/modules/builder.py index f530e91b..1d8d8204 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -2,7 +2,7 @@ import logging, os, re from datetime import datetime, timedelta from modules import anidb, anilist, icheckmovies, imdb, letterboxd, mal, plex, radarr, sonarr, tautulli, tmdb, trakttv, tvdb, util from modules.util import Failed, ImageData -from PIL import Image, UnidentifiedImageError +from PIL import Image from plexapi.exceptions import BadRequest, NotFound from plexapi.video import Movie, Show from urllib.parse import quote @@ -1238,7 +1238,7 @@ class CollectionBuilder: indent = f"\n{' ' * level}" conjunction = f"{'and' if is_all else 'or'}=1&" for _key, _data in filter_dict.items(): - attr, modifier, final = self._split(_key) + attr, modifier, final_attr = self._split(_key) def build_url_arg(arg, mod=None, arg_s=None, mod_s=None): arg_key = plex.search_translation[attr] if attr in plex.search_translation else attr @@ -1254,15 +1254,15 @@ class CollectionBuilder: display_line = f"{indent}{param_s} {mod_s} {arg_s}" return f"{arg_key}{mod}={arg}&", display_line - if final not in plex.searches and not final.startswith(("any", "all")): - raise Failed(f"Collection Error: {final} is not a valid {method} attribute") - elif final in plex.movie_only_searches and self.library.is_show: - raise Failed(f"Collection Error: {final} {method} attribute only works for movie libraries") - elif final in plex.show_only_searches and self.library.is_movie: - raise Failed(f"Collection Error: {final} {method} attribute only works for show libraries") + if final_attr not in plex.searches and not final_attr.startswith(("any", "all")): + raise Failed(f"Collection Error: {final_attr} is not a valid {method} attribute") + elif final_attr in plex.movie_only_searches and self.library.is_show: + raise Failed(f"Collection Error: {final_attr} {method} attribute only works for movie libraries") + elif final_attr in plex.show_only_searches and self.library.is_movie: + raise Failed(f"Collection Error: {final_attr} {method} attribute only works for show libraries") elif _data is None: - raise Failed(f"Collection Error: {final} {method} attribute is blank") - elif final.startswith(("any", "all")): + raise Failed(f"Collection Error: {final_attr} {method} attribute is blank") + elif final_attr.startswith(("any", "all")): dicts = util.get_list(_data) results = "" display_add = "" @@ -1274,7 +1274,7 @@ class CollectionBuilder: display_add += inside_display results += f"{conjunction if len(results) > 0 else ''}push=1&{inside_filter}pop=1&" else: - validation = self.validate_attribute(attr, modifier, final, _data, validate, pairs=True) + validation = self.validate_attribute(attr, modifier, final_attr, _data, validate, pairs=True) if validation is None: continue elif attr in plex.date_attributes and modifier in ["", ".not"]: @@ -1436,7 +1436,6 @@ class CollectionBuilder: def add_to_collection(self): name, collection_items = self.library.get_collection_name_and_items(self.obj if self.obj else self.name, self.smart_label_collection) total = len(self.rating_keys) - max_length = len(str(total)) for i, item in enumerate(self.rating_keys, 1): try: current = self.fetch_item(item) @@ -1772,7 +1771,7 @@ class CollectionBuilder: continue og_image = os.path.join(overlay_folder, f"{rating_key}.png") if os.path.exists(og_image): - self.library._upload_file_poster(item, og_image) + self.library.upload_file_poster(item, og_image) os.remove(og_image) self.config.Cache.update_image_map(item.ratingKey, self.library.original_mapping_name, "poster", "", "", "") diff --git a/modules/config.py b/modules/config.py index fcfd0997..5979e18d 100644 --- a/modules/config.py +++ b/modules/config.py @@ -1,5 +1,6 @@ import logging, os, requests from datetime import datetime +from lxml import html from modules import util from modules.anidb import AniDB from modules.anilist import AniList @@ -18,6 +19,7 @@ from modules.tmdb import TMDb from modules.trakttv import Trakt from modules.tvdb import TVDb from modules.util import Failed +from retrying import retry from ruamel import yaml logger = logging.getLogger("Plex Meta Manager") @@ -230,7 +232,7 @@ class Config: self.omdb = {} try: self.omdb["apikey"] = check_for_attribute(self.data, "apikey", parent="omdb", throw=True) - self.OMDb = OMDb(self.omdb, Cache=self.Cache) + self.OMDb = OMDb(self, self.omdb) except Failed as e: logger.error(e) logger.info(f"OMDb Connection {'Failed' if self.OMDb is None else 'Successful'}") @@ -248,7 +250,7 @@ class Config: self.trakt["client_secret"] = check_for_attribute(self.data, "client_secret", parent="trakt", throw=True) self.trakt["config_path"] = self.config_path authorization = self.data["trakt"]["authorization"] if "authorization" in self.data["trakt"] and self.data["trakt"]["authorization"] else None - self.Trakt = Trakt(self.trakt, authorization) + self.Trakt = Trakt(self, self.trakt, authorization) except Failed as e: logger.error(e) logger.info(f"Trakt Connection {'Failed' if self.Trakt is None else 'Successful'}") @@ -266,7 +268,7 @@ class Config: self.mal["client_secret"] = check_for_attribute(self.data, "client_secret", parent="mal", throw=True) self.mal["config_path"] = self.config_path authorization = self.data["mal"]["authorization"] if "authorization" in self.data["mal"] and self.data["mal"]["authorization"] else None - self.MyAnimeList = MyAnimeList(self.mal, self, authorization) + self.MyAnimeList = MyAnimeList(self, self.mal, authorization) except Failed as e: logger.error(e) logger.info(f"My Anime List Connection {'Failed' if self.MyAnimeList is None else 'Successful'}") @@ -283,12 +285,12 @@ class Config: try: self.anidb["username"] = check_for_attribute(self.data, "username", parent="anidb", throw=True) self.anidb["password"] = check_for_attribute(self.data, "password", parent="anidb", throw=True) - self.AniDB = AniDB(self.anidb, self) + self.AniDB = AniDB(self, self.anidb) except Failed as e: logger.error(e) logger.info(f"My Anime List Connection {'Failed Continuing as Guest ' if self.MyAnimeList is None else 'Successful'}") if self.AniDB is None: - self.AniDB = AniDB(None, self) + self.AniDB = AniDB(self, None) self.TVDb = TVDb(self) self.IMDb = IMDb(self) @@ -497,7 +499,7 @@ class Config: radarr_params["quality_profile"] = check_for_attribute(lib, "quality_profile", parent="radarr", default=self.general["radarr"]["quality_profile"], req_default=True, save=False) radarr_params["tag"] = check_for_attribute(lib, "tag", parent="radarr", var_type="lower_list", default=self.general["radarr"]["tag"], default_is_none=True, save=False) radarr_params["search"] = check_for_attribute(lib, "search", parent="radarr", var_type="bool", default=self.general["radarr"]["search"], save=False) - library.Radarr = Radarr(radarr_params) + library.Radarr = Radarr(self, radarr_params) except Failed as e: util.print_stacktrace() util.print_multiline(e, error=True) @@ -527,7 +529,7 @@ class Config: sonarr_params["tag"] = check_for_attribute(lib, "tag", parent="sonarr", var_type="lower_list", default=self.general["sonarr"]["tag"], default_is_none=True, save=False) sonarr_params["search"] = check_for_attribute(lib, "search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["search"], save=False) sonarr_params["cutoff_search"] = check_for_attribute(lib, "cutoff_search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["cutoff_search"], save=False) - library.Sonarr = Sonarr(sonarr_params) + library.Sonarr = Sonarr(self, sonarr_params) except Failed as e: util.print_stacktrace() util.print_multiline(e, error=True) @@ -544,7 +546,7 @@ class Config: try: tautulli_params["url"] = check_for_attribute(lib, "url", parent="tautulli", var_type="url", default=self.general["tautulli"]["url"], req_default=True, save=False) tautulli_params["apikey"] = check_for_attribute(lib, "apikey", parent="tautulli", default=self.general["tautulli"]["apikey"], req_default=True, save=False) - library.Tautulli = Tautulli(tautulli_params) + library.Tautulli = Tautulli(self, tautulli_params) except Failed as e: util.print_stacktrace() util.print_multiline(e, error=True) @@ -563,3 +565,22 @@ class Config: util.separator() + def get_html(self, url, headers=None): + return html.fromstring(self.get(url, headers=headers).content) + + def get_json(self, url, headers=None): + return self.get(url, headers=headers).json() + + @retry(stop_max_attempt_number=6, wait_fixed=10000) + def get(self, url, headers=None, params=None): + return self.session.get(url, headers=headers, params=params) + + def post_html(self, url, data=None, json=None, headers=None): + return html.fromstring(self.post(url, data=data, json=json, headers=headers).content) + + def post_json(self, url, data=None, json=None, headers=None): + return self.post(url, data=data, json=json, headers=headers).json() + + @retry(stop_max_attempt_number=6, wait_fixed=10000) + def post(self, url, data=None, json=None, headers=None): + return self.session.post(url, data=data, json=json, headers=headers) diff --git a/modules/convert.py b/modules/convert.py index 66408258..ac0ad97d 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -1,22 +1,17 @@ import logging, re, requests -from lxml import html from modules import util from modules.util import Failed from plexapi.exceptions import BadRequest -from retrying import retry logger = logging.getLogger("Plex Meta Manager") +arms_url = "https://relations.yuna.moe/api/ids" +anidb_url = "https://raw.githubusercontent.com/Anime-Lists/anime-lists/master/anime-list-master.xml" + class Convert: def __init__(self, config): self.config = config - self.arms_url = "https://relations.yuna.moe/api/ids" - self.anidb_url = "https://raw.githubusercontent.com/Anime-Lists/anime-lists/master/anime-list-master.xml" - self.AniDBIDs = self._get_anidb() - - @retry(stop_max_attempt_number=6, wait_fixed=10000) - def _get_anidb(self): - return html.fromstring(requests.get(self.anidb_url).content) + self.AniDBIDs = self.config.get_html(anidb_url) def _anidb(self, input_id, to_id, fail=False): ids = self.AniDBIDs.xpath(f"//anime[contains(@anidbid, '{input_id}')]/@{to_id}") @@ -33,10 +28,6 @@ class Convert: raise Failed(fail_text) return [] if to_id == "imdbid" else None - @retry(stop_max_attempt_number=6, wait_fixed=10000) - def _request(self, ids): - return requests.post(self.arms_url, json=ids).json() - def _arms_ids(self, anilist_ids=None, anidb_ids=None, mal_ids=None): all_ids = [] def collect_ids(ids, id_name): @@ -68,7 +59,7 @@ class Convert: if len(unconverted_ids) > 0: unconverted_id_sets.append(unconverted_ids) for unconverted_id_set in unconverted_id_sets: - for anime_ids in self._request(unconverted_id_set): + for anime_ids in self.config.post_json(arms_url, json=unconverted_id_set): if anime_ids: if self.config.Cache: self.config.Cache.update_anime_map(False, anime_ids) diff --git a/modules/icheckmovies.py b/modules/icheckmovies.py index ebba4051..117cbf1c 100644 --- a/modules/icheckmovies.py +++ b/modules/icheckmovies.py @@ -1,35 +1,31 @@ -import logging, requests -from lxml import html +import logging from modules import util from modules.util import Failed -from retrying import retry logger = logging.getLogger("Plex Meta Manager") builders = ["icheckmovies_list", "icheckmovies_list_details"] +base_url = "https://www.icheckmovies.com/lists/" class ICheckMovies: def __init__(self, config): self.config = config - self.list_url = "https://www.icheckmovies.com/lists/" - @retry(stop_max_attempt_number=6, wait_fixed=10000) - def _request(self, url, language): - return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) + def _request(self, url, language, xpath): + return self.config.get_html(url, headers=util.header(language)).xpath(xpath) def _parse_list(self, list_url, language): - response = self._request(list_url, language) - imdb_urls = response.xpath("//a[@class='optionIcon optionIMDB external']/@href") + imdb_urls = self._request(list_url, language, "//a[@class='optionIcon optionIMDB external']/@href") return [t[t.find("/tt") + 1:-1] for t in imdb_urls] def get_list_description(self, list_url, language): - descriptions = self._request(list_url, language).xpath("//div[@class='span-19 last']/p/em/text()") + descriptions = self._request(list_url, language, "//div[@class='span-19 last']/p/em/text()") return descriptions[0] if len(descriptions) > 0 and len(descriptions[0]) > 0 else None def validate_icheckmovies_list(self, list_url, language): list_url = list_url.strip() - if not list_url.startswith(self.list_url): - raise Failed(f"ICheckMovies Error: {list_url} must begin with: {self.list_url}") + if not list_url.startswith(base_url): + raise Failed(f"ICheckMovies Error: {list_url} must begin with: {base_url}") if len(self._parse_list(list_url, language)) > 0: return list_url raise Failed(f"ICheckMovies Error: {list_url} failed to parse") diff --git a/modules/imdb.py b/modules/imdb.py index e50257c4..e544a038 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -1,45 +1,44 @@ -import logging, math, re, requests -from lxml import html +import logging, math, re, time from modules import util from modules.util import Failed -from retrying import retry logger = logging.getLogger("Plex Meta Manager") builders = ["imdb_list", "imdb_id"] +base_url = "https://www.imdb.com" +urls = { + "list": f"{base_url}/list/ls", + "search": f"{base_url}/search/title/?", + "keyword": f"{base_url}/search/keyword/?" +} class IMDb: def __init__(self, config): self.config = config - self.urls = { - "list": "https://www.imdb.com/list/ls", - "search": "https://www.imdb.com/search/title/?", - "keyword": "https://www.imdb.com/search/keyword/?" - } def validate_imdb_url(self, imdb_url, language): imdb_url = imdb_url.strip() - if not imdb_url.startswith(self.urls["list"]) and not imdb_url.startswith(self.urls["search"]) and not imdb_url.startswith(self.urls["keyword"]): - raise Failed(f"IMDb Error: {imdb_url} must begin with either:\n{self.urls['list']} (For Lists)\n{self.urls['search']} (For Searches)\n{self.urls['keyword']} (For Keyword Searches)") + if not imdb_url.startswith(urls["list"]) and not imdb_url.startswith(urls["search"]) and not imdb_url.startswith(urls["keyword"]): + raise Failed(f"IMDb Error: {imdb_url} must begin with either:\n{urls['list']} (For Lists)\n{urls['search']} (For Searches)\n{urls['keyword']} (For Keyword Searches)") total, _ = self._total(self._fix_url(imdb_url), language) if total > 0: return imdb_url raise Failed(f"IMDb Error: {imdb_url} failed to parse") def _fix_url(self, imdb_url): - if imdb_url.startswith(self.urls["list"]): + if imdb_url.startswith(urls["list"]): try: list_id = re.search("(\\d+)", str(imdb_url)).group(1) except AttributeError: raise Failed(f"IMDb Error: Failed to parse List ID from {imdb_url}") - return f"{self.urls['search']}lists=ls{list_id}" + return f"{urls['search']}lists=ls{list_id}" elif imdb_url.endswith("/"): return imdb_url[:-1] else: return imdb_url def _total(self, imdb_url, language): - header = {"Accept-Language": language} - if imdb_url.startswith(self.urls["keyword"]): - results = self._request(imdb_url, header).xpath("//div[@class='desc']/text()") + headers = util.header(language) + if imdb_url.startswith(urls["keyword"]): + results = self.config.get_html(imdb_url, headers=headers).xpath("//div[@class='desc']/text()") total = None for result in results: if "title" in result: @@ -52,7 +51,7 @@ class IMDb: raise Failed(f"IMDb Error: No Results at URL: {imdb_url}") return total, 50 else: - try: results = self._request(imdb_url, header).xpath("//div[@class='desc']/span/text()")[0].replace(",", "") + try: results = self.config.get_html(imdb_url, headers=headers).xpath("//div[@class='desc']/span/text()")[0].replace(",", "") except IndexError: raise Failed(f"IMDb Error: Failed to parse URL: {imdb_url}") try: total = int(re.findall("(\\d+) title", results)[0]) except IndexError: raise Failed(f"IMDb Error: No Results at URL: {imdb_url}") @@ -61,7 +60,7 @@ class IMDb: def _ids_from_url(self, imdb_url, language, limit): current_url = self._fix_url(imdb_url) total, item_count = self._total(current_url, language) - header = {"Accept-Language": language} + headers = util.header(language) imdb_ids = [] if "&start=" in current_url: current_url = re.sub("&start=\\d+", "", current_url) if "&count=" in current_url: current_url = re.sub("&count=\\d+", "", current_url) @@ -74,22 +73,19 @@ class IMDb: for i in range(1, num_of_pages + 1): start_num = (i - 1) * item_count + 1 util.print_return(f"Parsing Page {i}/{num_of_pages} {start_num}-{limit if i == num_of_pages else i * item_count}") - if imdb_url.startswith(self.urls["keyword"]): - response = self._request(f"{current_url}&page={i}", header) + if imdb_url.startswith(urls["keyword"]): + response = self.config.get_html(f"{current_url}&page={i}", headers=headers) else: - response = self._request(f"{current_url}&count={remainder if i == num_of_pages else item_count}&start={start_num}", header) - if imdb_url.startswith(self.urls["keyword"]) and i == num_of_pages: + response = self.config.get_html(f"{current_url}&count={remainder if i == num_of_pages else item_count}&start={start_num}", headers=headers) + if imdb_url.startswith(urls["keyword"]) and i == num_of_pages: imdb_ids.extend(response.xpath("//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst")[:remainder]) else: imdb_ids.extend(response.xpath("//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst")) + time.sleep(2) util.print_end() if imdb_ids: return imdb_ids else: raise Failed(f"IMDb Error: No IMDb IDs Found at {imdb_url}") - @retry(stop_max_attempt_number=6, wait_fixed=10000) - def _request(self, url, header): - return html.fromstring(requests.get(url, headers=header).content) - def get_items(self, method, data, language, is_movie): pretty = util.pretty_names[method] if method in util.pretty_names else method show_ids = [] diff --git a/modules/letterboxd.py b/modules/letterboxd.py index a88a9d23..a53535cf 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -1,24 +1,18 @@ -import logging, requests -from lxml import html +import logging, time from modules import util from modules.util import Failed -from retrying import retry logger = logging.getLogger("Plex Meta Manager") builders = ["letterboxd_list", "letterboxd_list_details"] +base_url = "https://letterboxd.com" class Letterboxd: def __init__(self, config): self.config = config - self.url = "https://letterboxd.com" - - @retry(stop_max_attempt_number=6, wait_fixed=10000) - def _request(self, url, language): - return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) def _parse_list(self, list_url, language): - response = self._request(list_url, language) + response = self.config.get_html(list_url, headers=util.header(language)) letterboxd_ids = response.xpath("//div[@class='poster film-poster really-lazy-load']/@data-film-id") items = [] for letterboxd_id in letterboxd_ids: @@ -26,11 +20,12 @@ class Letterboxd: items.append((letterboxd_id, slugs[0])) next_url = response.xpath("//a[@class='next']/@href") if len(next_url) > 0: - items.extend(self._parse_list(f"{self.url}{next_url[0]}", language)) + time.sleep(2) + items.extend(self._parse_list(f"{base_url}{next_url[0]}", language)) return items def _tmdb(self, letterboxd_url, language): - response = self._request(letterboxd_url, language) + response = self.config.get_html(letterboxd_url, headers=util.header(language)) ids = response.xpath("//a[@data-track-action='TMDb']/@href") if len(ids) > 0 and ids[0]: if "themoviedb.org/movie" in ids[0]: @@ -39,7 +34,8 @@ class Letterboxd: raise Failed(f"Letterboxd Error: TMDb Movie ID not found at {letterboxd_url}") def get_list_description(self, list_url, language): - descriptions = self._request(list_url, language).xpath("//meta[@property='og:description']/@content") + response = self.config.get_html(list_url, headers=util.header(language)) + descriptions = response.xpath("//meta[@property='og:description']/@content") return descriptions[0] if len(descriptions) > 0 and len(descriptions[0]) > 0 else None def get_items(self, method, data, language): @@ -58,7 +54,7 @@ class Letterboxd: tmdb_id, expired = self.config.Cache.query_letterboxd_map(letterboxd_id) if not tmdb_id or expired is not False: try: - tmdb_id = self._tmdb(f"{self.url}{slug}", language) + tmdb_id = self._tmdb(f"{base_url}{slug}", language) except Failed as e: logger.error(e) continue diff --git a/modules/mal.py b/modules/mal.py index 25d797ad..dc3ada82 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -1,7 +1,6 @@ -import logging, re, requests, secrets, webbrowser +import logging, re, secrets, webbrowser from modules import util from modules.util import Failed, TimeoutExpired -from retrying import retry from ruamel import yaml logger = logging.getLogger("Plex Meta Manager") @@ -71,18 +70,17 @@ userlist_status = [ "dropped", "plan_to_watch" ] - +urls = { + "oauth_token": "https://myanimelist.net/v1/oauth2/token", + "oauth_authorize": "https://myanimelist.net/v1/oauth2/authorize", + "ranking": "https://api.myanimelist.net/v2/anime/ranking", + "season": "https://api.myanimelist.net/v2/anime/season", + "suggestions": "https://api.myanimelist.net/v2/anime/suggestions", + "user": "https://api.myanimelist.net/v2/users" +} class MyAnimeList: - def __init__(self, params, config, authorization=None): + def __init__(self, config, params, authorization=None): self.config = config - self.urls = { - "oauth_token": "https://myanimelist.net/v1/oauth2/token", - "oauth_authorize": "https://myanimelist.net/v1/oauth2/authorize", - "ranking": "https://api.myanimelist.net/v2/anime/ranking", - "season": "https://api.myanimelist.net/v2/anime/season", - "suggestions": "https://api.myanimelist.net/v2/anime/suggestions", - "user": "https://api.myanimelist.net/v2/users" - } self.client_id = params["client_id"] self.client_secret = params["client_secret"] self.config_path = params["config_path"] @@ -93,7 +91,7 @@ class MyAnimeList: def _authorization(self): code_verifier = secrets.token_urlsafe(100)[:128] - url = f"{self.urls['oauth_authorize']}?response_type=code&client_id={self.client_id}&code_challenge={code_verifier}" + url = f"{urls['oauth_authorize']}?response_type=code&client_id={self.client_id}&code_challenge={code_verifier}" logger.info("") logger.info(f"Navigate to: {url}") logger.info("") @@ -122,7 +120,7 @@ class MyAnimeList: def _check(self, authorization): try: - self._request(self.urls["suggestions"], authorization=authorization) + self._request(urls["suggestions"], authorization=authorization) return True except Failed as e: logger.debug(e) @@ -158,14 +156,12 @@ class MyAnimeList: return True return False - @retry(stop_max_attempt_number=6, wait_fixed=10000) def _oauth(self, data): - return requests.post(self.urls["oauth_token"], data).json() + return self.config.post_json(urls["oauth_token"], data=data) - @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def _request(self, url, authorization=None): new_authorization = authorization if authorization else self.authorization - response = requests.get(url, headers={"Authorization": f"Bearer {new_authorization['access_token']}"}).json() + response = self.config.get_json(url, headers={"Authorization": f"Bearer {new_authorization['access_token']}"}) if "error" in response: raise Failed(f"MyAnimeList Error: {response['error']}") else: return response @@ -174,23 +170,23 @@ class MyAnimeList: return [d["node"]["id"] for d in data["data"]] if "data" in data else [] def _username(self): - return self._request(f"{self.urls['user']}/@me")["name"] + return self._request(f"{urls['user']}/@me")["name"] def _ranked(self, ranking_type, limit): - url = f"{self.urls['ranking']}?ranking_type={ranking_type}&limit={limit}" + url = f"{urls['ranking']}?ranking_type={ranking_type}&limit={limit}" return self._parse_request(url) def _season(self, season, year, sort_by, limit): - url = f"{self.urls['season']}/{year}/{season}?sort={sort_by}&limit={limit}" + url = f"{urls['season']}/{year}/{season}?sort={sort_by}&limit={limit}" return self._parse_request(url) def _suggestions(self, limit): - url = f"{self.urls['suggestions']}?limit={limit}" + url = f"{urls['suggestions']}?limit={limit}" return self._parse_request(url) def _userlist(self, username, status, sort_by, limit): final_status = "" if status == "all" else f"status={status}&" - url = f"{self.urls['user']}/{username}/animelist?{final_status}sort={sort_by}&limit={limit}" + url = f"{urls['user']}/{username}/animelist?{final_status}sort={sort_by}&limit={limit}" return self._parse_request(url) def get_items(self, method, data): diff --git a/modules/meta.py b/modules/meta.py index 5bf08ee8..309f6e74 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -1,4 +1,4 @@ -import logging, os, re, requests +import logging, os, re from datetime import datetime from modules import plex, util from modules.util import Failed, ImageData @@ -7,13 +7,14 @@ from ruamel import yaml logger = logging.getLogger("Plex Meta Manager") +github_base = "https://raw.githubusercontent.com/meisnate12/Plex-Meta-Manager-Configs/master/" + class Metadata: def __init__(self, config, library, file_type, path): self.config = config self.library = library self.type = file_type self.path = path - self.github_base = "https://raw.githubusercontent.com/meisnate12/Plex-Meta-Manager-Configs/master/" logger.info("") logger.info(f"Loading Metadata {file_type}: {path}") def get_dict(attribute, attr_data, check_list=None): @@ -37,8 +38,8 @@ class Metadata: return None try: if file_type in ["URL", "Git"]: - content_path = path if file_type == "URL" else f"{self.github_base}{path}.yml" - response = requests.get(content_path) + content_path = path if file_type == "URL" else f"{github_base}{path}.yml" + response = self.config.get(content_path) if response.status_code >= 400: raise Failed(f"URL Error: No file found at {content_path}") content = response.content diff --git a/modules/omdb.py b/modules/omdb.py index bcf513fe..45051c92 100644 --- a/modules/omdb.py +++ b/modules/omdb.py @@ -1,10 +1,11 @@ -import logging, requests +import logging from modules import util from modules.util import Failed -from retrying import retry logger = logging.getLogger("Plex Meta Manager") +base_url = "http://www.omdbapi.com/" + class OMDbObj: def __init__(self, imdb_id, data): self._imdb_id = imdb_id @@ -35,25 +36,23 @@ class OMDbObj: self.type = data["Type"] class OMDb: - def __init__(self, params, Cache=None): - self.url = "http://www.omdbapi.com/" + def __init__(self, config, params): + self.config = config self.apikey = params["apikey"] self.limit = False - self.Cache = Cache self.get_omdb("tt0080684") - @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def get_omdb(self, imdb_id): expired = None - if self.Cache: - omdb_dict, expired = self.Cache.query_omdb(imdb_id) + if self.config.Cache: + omdb_dict, expired = self.config.Cache.query_omdb(imdb_id) if omdb_dict and expired is False: return OMDbObj(imdb_id, omdb_dict) - response = requests.get(self.url, params={"i": imdb_id, "apikey": self.apikey}) + response = self.config.get(base_url, params={"i": imdb_id, "apikey": self.apikey}) if response.status_code < 400: omdb = OMDbObj(imdb_id, response.json()) - if self.Cache: - self.Cache.update_omdb(expired, omdb) + if self.config.Cache: + self.config.Cache.update_omdb(expired, omdb) return omdb else: error = response.json()['Error'] diff --git a/modules/plex.py b/modules/plex.py index eff47a7f..1fd5dbce 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -255,8 +255,12 @@ sort_types = { class Plex: def __init__(self, config, params): self.config = config + self.plex = params["plex"] + self.url = params["plex"]["url"] + self.token = params["plex"]["token"] + self.timeout = params["plex"]["timeout"] try: - self.PlexServer = PlexServer(params["plex"]["url"], params["plex"]["token"], timeout=params["plex"]["timeout"]) + self.PlexServer = PlexServer(baseurl=self.url, token=self.token, session=self.config.session, timeout=self.timeout) except Unauthorized: raise Failed("Plex Error: Plex token is invalid") except ValueError as e: @@ -322,10 +326,6 @@ class Plex: self.radarr_add_all = params["radarr_add_all"] self.sonarr_add_all = params["sonarr_add_all"] self.mass_update = self.mass_genre_update or self.mass_audience_rating_update or self.mass_critic_rating_update or self.split_duplicates or self.radarr_add_all or self.sonarr_add_all - self.plex = params["plex"] - self.url = params["plex"]["url"] - self.token = params["plex"]["token"] - self.timeout = params["plex"]["timeout"] self.clean_bundles = params["plex"]["clean_bundles"] self.empty_trash = params["plex"]["empty_trash"] self.optimize = params["plex"]["optimize"] @@ -427,7 +427,7 @@ class Plex: self.reload(item) @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) - def _upload_file_poster(self, item, image): + def upload_file_poster(self, item, image): item.uploadPoster(filepath=image) self.reload(item) @@ -470,7 +470,7 @@ class Plex: new_poster = new_poster.resize(overlay_image.size, Image.ANTIALIAS) new_poster.paste(overlay_image, (0, 0), overlay_image) new_poster.save(temp_image) - self._upload_file_poster(item, temp_image) + self.upload_file_poster(item, temp_image) poster_uploaded = True logger.info(f"Detail: Overlay: {overlay_name} applied to {item.title}") @@ -772,9 +772,9 @@ class Plex: name = os.path.basename(os.path.dirname(str(item.locations[0])) if self.is_movie else str(item.locations[0])) logger.debug(name) found_folder = False + poster = None + background = None for ad in self.asset_directory: - poster = None - background = None item_dir = None if self.asset_folders: if os.path.isdir(os.path.join(ad, name)): diff --git a/modules/radarr.py b/modules/radarr.py index 86ab67de..5c2cd069 100644 --- a/modules/radarr.py +++ b/modules/radarr.py @@ -19,7 +19,8 @@ apply_tags_translation = { } class Radarr: - def __init__(self, params): + def __init__(self, config, params): + self.config = config self.url = params["url"] self.token = params["token"] try: @@ -83,4 +84,3 @@ class Radarr: logger.info("") for tmdb_id in not_exists: logger.info(f"TMDb ID Not in Radarr | {tmdb_id}") - diff --git a/modules/sonarr.py b/modules/sonarr.py index 150e32dd..eee671af 100644 --- a/modules/sonarr.py +++ b/modules/sonarr.py @@ -24,7 +24,8 @@ apply_tags_translation = { } class Sonarr: - def __init__(self, params): + def __init__(self, config, params): + self.config = config self.url = params["url"] self.token = params["token"] try: diff --git a/modules/tautulli.py b/modules/tautulli.py index bc0f9dfd..c00e6335 100644 --- a/modules/tautulli.py +++ b/modules/tautulli.py @@ -1,15 +1,15 @@ -import logging, requests +import logging from modules import util from modules.util import Failed from plexapi.exceptions import BadRequest, NotFound -from retrying import retry logger = logging.getLogger("Plex Meta Manager") builders = ["tautulli_popular", "tautulli_watched"] class Tautulli: - def __init__(self, params): + def __init__(self, config, params): + self.config = config self.url = params["url"] self.apikey = params["apikey"] try: @@ -62,7 +62,6 @@ class Tautulli: if section_id: return section_id else: raise Failed(f"Tautulli Error: No Library named {library_name} in the response") - @retry(stop_max_attempt_number=6, wait_fixed=10000) def _request(self, url): logger.debug(f"Tautulli URL: {url.replace(self.apikey, '###############')}") - return requests.get(url).json() + return self.config.get_json(url) diff --git a/modules/trakttv.py b/modules/trakttv.py index 35de3cf9..52f52161 100644 --- a/modules/trakttv.py +++ b/modules/trakttv.py @@ -24,7 +24,8 @@ builders = [ ] class Trakt: - def __init__(self, params, authorization=None): + def __init__(self, config, params, authorization=None): + self.config = config self.base_url = "https://api.trakt.tv" self.redirect_uri = "urn:ietf:wg:oauth:2.0:oob" self.aliases = { @@ -118,9 +119,8 @@ class Trakt: if trakt_list is None: raise Failed("Trakt Error: No List found") else: return trakt_list - @retry(stop_max_attempt_number=6, wait_fixed=10000) def _request(self, url): - return requests.get(url, headers={"Content-Type": "application/json", "trakt-api-version": "2", "trakt-api-key": self.client_id}).json() + return self.config.get_json(url, headers={"Content-Type": "application/json", "trakt-api-version": "2", "trakt-api-key": self.client_id}) def _collection(self, username, is_movie): items = self._request(f"{self.base_url}/users/{username}/collection/{'movies' if is_movie else 'shows'}") diff --git a/modules/tvdb.py b/modules/tvdb.py index 94016831..d338642e 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -1,8 +1,6 @@ -import logging, requests -from lxml import html +import logging, requests, time from modules import util from modules.util import Failed -from retrying import retry logger = logging.getLogger("Plex Meta Manager") @@ -14,33 +12,48 @@ builders = [ "tvdb_show", "tvdb_show_details" ] +base_url = "https://www.thetvdb.com" +alt_url = "https://thetvdb.com" +urls = { + "list": f"{base_url}/lists/", + "alt_list": f"{alt_url}/lists/", + "series": f"{base_url}/series/", + "alt_series": f"{alt_url}/series/", + "movies": f"{base_url}/movies/", + "alt_movies": f"{alt_url}/movies/", + "series_id": f"{base_url}/dereferrer/series/", + "movie_id": f"{base_url}/dereferrer/movie/" +} class TVDbObj: - def __init__(self, tvdb_url, language, is_movie, TVDb): - tvdb_url = tvdb_url.strip() - if not is_movie and tvdb_url.startswith((TVDb.series_url, TVDb.alt_series_url, TVDb.series_id_url)): + def __init__(self, tvdb_url, language, is_movie, config): + self.tvdb_url = tvdb_url.strip() + self.language = language + self.is_movie = is_movie + self.config = config + if not self.is_movie and self.tvdb_url.startswith((urls["series"], urls["alt_series"], urls["series_id"])): self.media_type = "Series" - elif is_movie and tvdb_url.startswith((TVDb.movies_url, TVDb.alt_movies_url, TVDb.movie_id_url)): + elif self.is_movie and self.tvdb_url.startswith((urls["movies"], urls["alt_movies"], urls["movie_id"])): self.media_type = "Movie" else: - raise Failed(f"TVDb Error: {tvdb_url} must begin with {TVDb.movies_url if is_movie else TVDb.series_url}") + raise Failed(f"TVDb Error: {self.tvdb_url} must begin with {urls['movies'] if self.is_movie else urls['series']}") - response = TVDb._request(tvdb_url, language) + response = self.config.get_html(self.tvdb_url, headers=util.header(self.language)) results = response.xpath(f"//*[text()='TheTVDB.com {self.media_type} ID']/parent::node()/span/text()") if len(results) > 0: self.id = int(results[0]) - elif tvdb_url.startswith(TVDb.movie_id_url): - raise Failed(f"TVDb Error: Could not find a TVDb Movie using TVDb Movie ID: {tvdb_url[len(TVDb.movie_id_url):]}") - elif tvdb_url.startswith(TVDb.series_id_url): - raise Failed(f"TVDb Error: Could not find a TVDb Series using TVDb Series ID: {tvdb_url[len(TVDb.series_id_url):]}") + elif self.tvdb_url.startswith(urls["movie_id"]): + raise Failed(f"TVDb Error: Could not find a TVDb Movie using TVDb Movie ID: {self.tvdb_url[len(urls['movie_id']):]}") + elif self.tvdb_url.startswith(urls["series_id"]): + raise Failed(f"TVDb Error: Could not find a TVDb Series using TVDb Series ID: {self.tvdb_url[len(urls['series_id']):]}") else: - raise Failed(f"TVDb Error: Could not find a TVDb {self.media_type} ID at the URL {tvdb_url}") + raise Failed(f"TVDb Error: Could not find a TVDb {self.media_type} ID at the URL {self.tvdb_url}") results = response.xpath("//div[@class='change_translation_text' and @data-language='eng']/@data-title") if len(results) > 0 and len(results[0]) > 0: self.title = results[0] else: - raise Failed(f"TVDb Error: Name not found from TVDb URL: {tvdb_url}") + raise Failed(f"TVDb Error: Name not found from TVDb URL: {self.tvdb_url}") results = response.xpath("//div[@class='row hidden-xs hidden-sm']/div/img/@src") self.poster_path = results[0] if len(results) > 0 and len(results[0]) > 0 else None @@ -52,7 +65,7 @@ class TVDbObj: self.summary = results[0] if len(results) > 0 and len(results[0]) > 0 else None tmdb_id = None - if is_movie: + if self.is_movie: results = response.xpath("//*[text()='TheMovieDB.com']/@href") if len(results) > 0: try: @@ -63,70 +76,58 @@ class TVDbObj: results = response.xpath("//*[text()='IMDB']/@href") if len(results) > 0: try: - tmdb_id = TVDb.config.Convert.imdb_to_tmdb(util.get_id_from_imdb_url(results[0]), fail=True) + tmdb_id = self.config.Convert.imdb_to_tmdb(util.get_id_from_imdb_url(results[0]), fail=True) except Failed: pass if tmdb_id is None: raise Failed(f"TVDB Error: No TMDb ID found for {self.title}") self.tmdb_id = tmdb_id - self.tvdb_url = tvdb_url - self.language = language - self.is_movie = is_movie - self.TVDb = TVDb class TVDb: def __init__(self, config): self.config = config - self.site_url = "https://www.thetvdb.com" - self.alt_site_url = "https://thetvdb.com" - self.list_url = f"{self.site_url}/lists/" - self.alt_list_url = f"{self.alt_site_url}/lists/" - self.series_url = f"{self.site_url}/series/" - self.alt_series_url = f"{self.alt_site_url}/series/" - self.movies_url = f"{self.site_url}/movies/" - self.alt_movies_url = f"{self.alt_site_url}/movies/" - self.series_id_url = f"{self.site_url}/dereferrer/series/" - self.movie_id_url = f"{self.site_url}/dereferrer/movie/" def get_movie_or_series(self, language, tvdb_url, is_movie): return self.get_movie(language, tvdb_url) if is_movie else self.get_series(language, tvdb_url) def get_series(self, language, tvdb_url): try: - tvdb_url = f"{self.series_id_url}{int(tvdb_url)}" + tvdb_url = f"{urls['series_id']}{int(tvdb_url)}" except ValueError: pass - return TVDbObj(tvdb_url, language, False, self) + return TVDbObj(tvdb_url, language, False, self.config) def get_movie(self, language, tvdb_url): try: - tvdb_url = f"{self.movie_id_url}{int(tvdb_url)}" + tvdb_url = f"{urls['movie_id']}{int(tvdb_url)}" except ValueError: pass - return TVDbObj(tvdb_url, language, True, self) + return TVDbObj(tvdb_url, language, True, self.config) def get_list_description(self, tvdb_url, language): - description = self._request(tvdb_url, language).xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()") + response = self.config.get_html(tvdb_url, headers=util.header(language)) + description = response.xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()") return description[0] if len(description) > 0 and len(description[0]) > 0 else "" def _ids_from_url(self, tvdb_url, language): show_ids = [] movie_ids = [] tvdb_url = tvdb_url.strip() - if tvdb_url.startswith((self.list_url, self.alt_list_url)): + if tvdb_url.startswith((urls["list"], urls["alt_list"])): try: - items = self._request(tvdb_url, language).xpath("//div[@class='col-xs-12 col-sm-12 col-md-8 col-lg-8 col-md-pull-4']/div[@class='row']") + response = self.config.get_html(tvdb_url, headers=util.header(language)) + items = response.xpath("//div[@class='col-xs-12 col-sm-12 col-md-8 col-lg-8 col-md-pull-4']/div[@class='row']") for item in items: title = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/text()")[0] item_url = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/@href")[0] if item_url.startswith("/series/"): try: - show_ids.append(self.get_series(language, f"{self.site_url}{item_url}").id) + show_ids.append(self.get_series(language, f"{base_url}{item_url}").id) except Failed as e: logger.error(f"{e} for series {title}") elif item_url.startswith("/movies/"): try: - tmdb_id = self.get_movie(language, f"{self.site_url}{item_url}").tmdb_id + tmdb_id = self.get_movie(language, f"{base_url}{item_url}").tmdb_id if tmdb_id: movie_ids.append(tmdb_id) else: @@ -135,6 +136,7 @@ class TVDb: logger.error(f"{e} for series {title}") else: logger.error(f"TVDb Error: Skipping Movie: {title}") + time.sleep(2) if len(show_ids) > 0 or len(movie_ids) > 0: return movie_ids, show_ids raise Failed(f"TVDb Error: No TVDb IDs found at {tvdb_url}") @@ -142,11 +144,7 @@ class TVDb: util.print_stacktrace() raise Failed(f"TVDb Error: URL Lookup Failed for {tvdb_url}") else: - raise Failed(f"TVDb Error: {tvdb_url} must begin with {self.list_url}") - - @retry(stop_max_attempt_number=6, wait_fixed=10000) - def _request(self, url, language): - return html.fromstring(requests.get(url, headers={"Accept-Language": language}).content) + raise Failed(f"TVDb Error: {tvdb_url} must begin with {urls['list']}") def get_items(self, method, data, language): pretty = util.pretty_names[method] if method in util.pretty_names else method diff --git a/modules/util.py b/modules/util.py index cc058f18..76f74bee 100644 --- a/modules/util.py +++ b/modules/util.py @@ -280,6 +280,9 @@ def logger_input(prompt, timeout=60): elif hasattr(signal, "SIGALRM"): return unix_input(prompt, timeout) else: raise SystemError("Input Timeout not supported on this system") +def header(language="en-US,en;q=0.5"): + return {"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"} + def alarm_handler(signum, frame): raise TimeoutExpired From f9e736649ec12371cf475fb235a586d110062fc6 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 15 Jul 2021 13:42:28 -0400 Subject: [PATCH 13/95] removed trakt.py --- modules/builder.py | 16 ++- modules/config.py | 2 +- modules/mal.py | 1 + modules/{trakttv.py => trakt.py} | 165 +++++++++++++++---------------- requirements.txt | 4 - 5 files changed, 90 insertions(+), 98 deletions(-) rename modules/{trakttv.py => trakt.py} (50%) diff --git a/modules/builder.py b/modules/builder.py index 1d8d8204..0dae742b 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1,6 +1,6 @@ import logging, os, re from datetime import datetime, timedelta -from modules import anidb, anilist, icheckmovies, imdb, letterboxd, mal, plex, radarr, sonarr, tautulli, tmdb, trakttv, tvdb, util +from modules import anidb, anilist, icheckmovies, imdb, letterboxd, mal, plex, radarr, sonarr, tautulli, tmdb, trakt, tvdb, util from modules.util import Failed, ImageData from PIL import Image from plexapi.exceptions import BadRequest, NotFound @@ -58,7 +58,7 @@ filter_translation = { "writer": "writers" } modifier_alias = {".greater": ".gt", ".less": ".lt"} -all_builders = anidb.builders + anilist.builders + icheckmovies.builders + imdb.builders + letterboxd.builders + mal.builders + plex.builders + tautulli.builders + tmdb.builders + trakttv.builders + tvdb.builders +all_builders = anidb.builders + anilist.builders + icheckmovies.builders + imdb.builders + letterboxd.builders + mal.builders + plex.builders + tautulli.builders + tmdb.builders + trakt.builders + tvdb.builders dictionary_builders = [ "filters", "anidb_tag", @@ -570,7 +570,7 @@ class CollectionBuilder: elif method_name == "tvdb_description": self.summaries[method_name] = config.TVDb.get_list_description(method_data, self.library.Plex.language) elif method_name == "trakt_description": - self.summaries[method_name] = config.Trakt.standard_list(config.Trakt.validate_trakt(util.get_list(method_data))[0]).description + self.summaries[method_name] = config.Trakt.list_description(config.Trakt.validate_trakt(util.get_list(method_data), self.library.is_movie)[0]) elif method_name == "letterboxd_description": self.summaries[method_name] = config.Letterboxd.get_list_description(method_data, self.library.Plex.language) elif method_name == "icheckmovies_description": @@ -719,15 +719,13 @@ class CollectionBuilder: elif method_name in ["anilist_id", "anilist_relations", "anilist_studio"]: self.methods.append((method_name, config.AniList.validate_anilist_ids(util.get_int_list(method_data, "AniList ID"), studio=method_name == "anilist_studio"))) elif method_name == "trakt_list": - self.methods.append((method_name, config.Trakt.validate_trakt(util.get_list(method_data)))) + self.methods.append((method_name, config.Trakt.validate_trakt(util.get_list(method_data), self.library.is_movie))) elif method_name == "trakt_list_details": - valid_list = config.Trakt.validate_trakt(util.get_list(method_data)) - item = config.Trakt.standard_list(valid_list[0]) - if hasattr(item, "description") and item.description: - self.summaries[method_name] = item.description + valid_list = config.Trakt.validate_trakt(util.get_list(method_data), self.library.is_movie) + self.summaries[method_name] = config.Trakt.list_description(valid_list[0]) self.methods.append((method_name[:-8], valid_list)) elif method_name in ["trakt_watchlist", "trakt_collection"]: - self.methods.append((method_name, config.Trakt.validate_trakt(util.get_list(method_data), trakt_type=method_name[6:], is_movie=self.library.is_movie))) + self.methods.append((method_name, config.Trakt.validate_trakt(util.get_list(method_data), self.library.is_movie, trakt_type=method_name[6:]))) elif method_name == "imdb_list": new_list = [] for imdb_list in util.get_list(method_data, split=False): diff --git a/modules/config.py b/modules/config.py index 5979e18d..764bc0f2 100644 --- a/modules/config.py +++ b/modules/config.py @@ -16,7 +16,7 @@ from modules.radarr import Radarr from modules.sonarr import Sonarr from modules.tautulli import Tautulli from modules.tmdb import TMDb -from modules.trakttv import Trakt +from modules.trakt import Trakt from modules.tvdb import TVDb from modules.util import Failed from retrying import retry diff --git a/modules/mal.py b/modules/mal.py index dc3ada82..a27a6aea 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -78,6 +78,7 @@ urls = { "suggestions": "https://api.myanimelist.net/v2/anime/suggestions", "user": "https://api.myanimelist.net/v2/users" } + class MyAnimeList: def __init__(self, config, params, authorization=None): self.config = config diff --git a/modules/trakttv.py b/modules/trakt.py similarity index 50% rename from modules/trakttv.py rename to modules/trakt.py index 52f52161..ce04ddd0 100644 --- a/modules/trakttv.py +++ b/modules/trakt.py @@ -1,16 +1,13 @@ import logging, requests, webbrowser from modules import util from modules.util import Failed, TimeoutExpired -from retrying import retry from ruamel import yaml -from trakt import Trakt as TraktAPI -from trakt.objects.episode import Episode -from trakt.objects.movie import Movie -from trakt.objects.season import Season -from trakt.objects.show import Show logger = logging.getLogger("Plex Meta Manager") +redirect_uri = "urn:ietf:wg:oauth:2.0:oob" +redirect_uri_encoded = redirect_uri.replace(":", "%3A") +base_url = "https://api.trakt.tv" builders = [ "trakt_collected", "trakt_collection", @@ -26,49 +23,59 @@ builders = [ class Trakt: def __init__(self, config, params, authorization=None): self.config = config - self.base_url = "https://api.trakt.tv" - self.redirect_uri = "urn:ietf:wg:oauth:2.0:oob" - self.aliases = { - "trakt_trending": "Trakt Trending", - "trakt_watchlist": "Trakt Watchlist", - "trakt_list": "Trakt List" - } self.client_id = params["client_id"] self.client_secret = params["client_secret"] self.config_path = params["config_path"] self.authorization = authorization - TraktAPI.configuration.defaults.client(self.client_id, self.client_secret) if not self._save(self.authorization): if not self._refresh(): self._authorization() def _authorization(self): - url = TraktAPI["oauth"].authorize_url(self.redirect_uri) + url = f"https://trakt.tv/oauth/authorize?response_type=code&client_id={self.client_id}&redirect_uri={redirect_uri_encoded}" logger.info(f"Navigate to: {url}") logger.info("If you get an OAuth error your client_id or client_secret is invalid") webbrowser.open(url, new=2) try: pin = util.logger_input("Trakt pin (case insensitive)", timeout=300).strip() except TimeoutExpired: raise Failed("Input Timeout: Trakt pin required.") if not pin: raise Failed("Trakt Error: No input Trakt pin required.") - new_authorization = TraktAPI["oauth"].token(pin, self.redirect_uri) - if not new_authorization: + json = { + "code": pin, + "client_id": self.client_id, + "client_secret": self.client_secret, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code" + } + response = self.config.post(f"{base_url}/oauth/token", json=json, headers={"Content-Type": "application/json"}) + if response.status_code != 200: raise Failed("Trakt Error: Invalid trakt pin. If you're sure you typed it in correctly your client_id or client_secret may be invalid") - if not self._save(new_authorization): + elif not self._save(response.json()): raise Failed("Trakt Error: New Authorization Failed") - def _check(self, authorization): - try: - with TraktAPI.configuration.oauth.from_response(authorization, refresh=True): - if TraktAPI["users/settings"].get(): - return True - except ValueError: pass - return False + def _check(self, authorization=None): + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.authorization['access_token'] if authorization is None else authorization['access_token']}", + "trakt-api-version": "2", + "trakt-api-key": self.client_id + } + response = self.config.get(f"{base_url}/users/settings", headers=headers) + return response.status_code == 200 def _refresh(self): if self.authorization and "refresh_token" in self.authorization and self.authorization["refresh_token"]: logger.info("Refreshing Access Token...") - refreshed_authorization = TraktAPI["oauth"].token_refresh(self.authorization["refresh_token"], self.redirect_uri) - return self._save(refreshed_authorization) + json = { + "refresh_token": self.authorization["refresh_token"], + "client_id": self.client_id, + "client_secret": self.client_secret, + "redirect_uri": redirect_uri, + "grant_type": "refresh_token" + } + response = self.config.post(f"{base_url}/oauth/token", json=json, headers={"Content-Type": "application/json"}) + if response.status_code != 200: + return False + return self._save(response.json()) return False def _save(self, authorization): @@ -86,100 +93,90 @@ class Trakt: } logger.info(f"Saving authorization information to {self.config_path}") yaml.round_trip_dump(config, open(self.config_path, "w"), indent=ind, block_seq_indent=bsi) - self.authorization = authorization - TraktAPI.configuration.defaults.oauth.from_response(self.authorization) + self.authorization = authorization return True return False - @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) + def _request(self, url): + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.authorization['access_token']}", + "trakt-api-version": "2", + "trakt-api-key": self.client_id + } + response = self.config.get(url, headers=headers) + if response.status_code == 200: + return response.json() + else: + raise Failed(f"({response.status_code}) {response.reason}") + def convert(self, external_id, from_source, to_source, media_type): - lookup = TraktAPI["search"].lookup(external_id, from_source, media_type) - if lookup: - lookup = lookup[0] if isinstance(lookup, list) else lookup - if lookup.get_key(to_source): - return lookup.get_key(to_source) if to_source == "imdb" else int(lookup.get_key(to_source)) + path = f"/search/{from_source}/{external_id}" + if from_source in ["tmdb", "tvdb"]: + path = f"{path}?type={media_type}" + lookup = self._request(f"{base_url}{path}") + if lookup and media_type in lookup[0] and to_source in lookup[0][media_type]["ids"]: + return lookup[0][media_type]["ids"][to_source] raise Failed(f"Trakt Error: No {to_source.upper().replace('B', 'b')} ID found for {from_source.upper().replace('B', 'b')} ID: {external_id}") - def collection(self, data, is_movie): - return self._user_list("collection", data, is_movie) - - def _watchlist(self, data, is_movie): - return self._user_list("watchlist", data, is_movie) + def list_description(self, data): + try: + return self._request(f"{base_url}{requests.utils.urlparse(data).path}")["description"] + except Failed: + raise Failed(f"Trakt Error: List {data} not found") - @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def _user_list(self, list_type, data, is_movie): - items = TraktAPI[f"users/{data}/{list_type}"].movies() if is_movie else TraktAPI[f"users/{data}/{list_type}"].shows() - if items is None: raise Failed("Trakt Error: No List found") - else: return [i for i in items] - - @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) - def standard_list(self, data): - try: trakt_list = TraktAPI[requests.utils.urlparse(data).path].get() - except AttributeError: trakt_list = None - if trakt_list is None: raise Failed("Trakt Error: No List found") - else: return trakt_list - - def _request(self, url): - return self.config.get_json(url, headers={"Content-Type": "application/json", "trakt-api-version": "2", "trakt-api-key": self.client_id}) - - def _collection(self, username, is_movie): - items = self._request(f"{self.base_url}/users/{username}/collection/{'movies' if is_movie else 'shows'}") + path = f"{requests.utils.urlparse(data).path}/items" if list_type == "list" else f"/users/{data}/{list_type}" + try: + items = self._request(f"{base_url}{path}/{'movies' if is_movie else 'shows'}") + except Failed: + raise Failed(f"Trakt Error: {'List' if list_type == 'list' else 'User'} {data} not found") + if len(items) == 0: + if list_type == "list": + raise Failed(f"Trakt Error: List {data} is empty") + else: + raise Failed(f"Trakt Error: {data}'s {list_type.capitalize()} is empty") if is_movie: return [item["movie"]["ids"]["tmdb"] for item in items], [] else: return [], [item["show"]["ids"]["tvdb"] for item in items] def _pagenation(self, pagenation, amount, is_movie): - items = self._request(f"{self.base_url}/{'movies' if is_movie else 'shows'}/{pagenation}?limit={amount}") + items = self._request(f"{base_url}/{'movies' if is_movie else 'shows'}/{pagenation}?limit={amount}") if pagenation == "popular" and is_movie: return [item["ids"]["tmdb"] for item in items], [] elif pagenation == "popular": return [], [item["ids"]["tvdb"] for item in items] elif is_movie: return [item["movie"]["ids"]["tmdb"] for item in items], [] else: return [], [item["show"]["ids"]["tvdb"] for item in items] - def validate_trakt(self, values, trakt_type=None, is_movie=None): + def validate_trakt(self, values, is_movie, trakt_type="list"): trakt_values = [] for value in values: try: - if trakt_type == "watchlist" and is_movie is not None: - self._watchlist(value, is_movie) - elif trakt_type == "collection" and is_movie is not None: - self._collection(value, is_movie) - else: - self.standard_list(value) + self._user_list(trakt_type, value, is_movie) trakt_values.append(value) except Failed as e: logger.error(e) if len(trakt_values) == 0: - if trakt_type == "watchlist" and is_movie is not None: + if trakt_type == "watchlist": raise Failed(f"Trakt Error: No valid Trakt Watchlists in {values}") - elif trakt_type == "collection" and is_movie is not None: + elif trakt_type == "collection": raise Failed(f"Trakt Error: No valid Trakt Collections in {values}") else: raise Failed(f"Trakt Error: No valid Trakt Lists in {values}") return trakt_values def get_items(self, method, data, is_movie): - pretty = self.aliases[method] if method in self.aliases else method + pretty = util.pretty_names[method] if method in util.pretty_names else method media_type = "Movie" if is_movie else "Show" if method in ["trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected"]: movie_ids, show_ids = self._pagenation(method[6:], data, is_movie) logger.info(f"Processing {pretty}: {data} {media_type}{'' if data == 1 else 's'}") - elif method == "trakt_collection": - movie_ids, show_ids = self._collection(data, is_movie) + elif method in ["trakt_collection", "trakt_watchlist"]: + movie_ids, show_ids = self._user_list(method[6:], data, is_movie) logger.info(f"Processing {pretty} {media_type}s for {data}") - else: - show_ids = [] - movie_ids = [] - if method == "trakt_watchlist": trakt_items = self._watchlist(data, is_movie) - elif method == "trakt_list": trakt_items = self.standard_list(data).items() - else: raise Failed(f"Trakt Error: Method {method} not supported") + elif method == "trakt_list": + movie_ids, show_ids = self._user_list(method[6:], data, is_movie) logger.info(f"Processing {pretty}: {data}") - for trakt_item in trakt_items: - if isinstance(trakt_item, Movie): - movie_ids.append(int(trakt_item.get_key("tmdb"))) - elif isinstance(trakt_item, Show) and trakt_item.pk[1] not in show_ids: - show_ids.append(int(trakt_item.pk[1])) - elif (isinstance(trakt_item, (Season, Episode))) and trakt_item.show.pk[1] not in show_ids: - show_ids.append(int(trakt_item.show.pk[1])) - logger.debug(f"Trakt {media_type} Found: {trakt_items}") + else: + raise Failed(f"Trakt Error: Method {method} not supported") logger.debug("") logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") diff --git a/requirements.txt b/requirements.txt index 2d778224..b78713c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,6 @@ -# Remove -# Less common, pinned PlexAPI==4.6.1 tmdbv3api==1.7.5 -trakt.py==4.3.0 arrapi==1.0.2 -# More common, flexible lxml requests>=2.4.2 ruamel.yaml From d25568d67ace95766789240e3eff149b5d85bc76 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 15 Jul 2021 18:08:47 -0400 Subject: [PATCH 14/95] tmdb and arr sessions requirements update --- modules/radarr.py | 2 +- modules/sonarr.py | 2 +- modules/tmdb.py | 2 +- requirements.txt | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/radarr.py b/modules/radarr.py index 5c2cd069..1a135aaf 100644 --- a/modules/radarr.py +++ b/modules/radarr.py @@ -24,7 +24,7 @@ class Radarr: self.url = params["url"] self.token = params["token"] try: - self.api = RadarrAPI(self.url, self.token) + self.api = RadarrAPI(self.url, self.token, session=self.config.session) except ArrException as e: raise Failed(e) self.add = params["add"] diff --git a/modules/sonarr.py b/modules/sonarr.py index eee671af..55782b84 100644 --- a/modules/sonarr.py +++ b/modules/sonarr.py @@ -29,7 +29,7 @@ class Sonarr: self.url = params["url"] self.token = params["token"] try: - self.api = SonarrAPI(self.url, self.token) + self.api = SonarrAPI(self.url, self.token, session=self.config.session) except ArrException as e: raise Failed(e) self.add = params["add"] diff --git a/modules/tmdb.py b/modules/tmdb.py index 5179e91a..5ecfc579 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -111,7 +111,7 @@ discover_tv_sort = [ class TMDb: def __init__(self, config, params): self.config = config - self.TMDb = tmdbv3api.TMDb() + self.TMDb = tmdbv3api.TMDb(session=self.config.session) self.TMDb.api_key = params["apikey"] self.TMDb.language = params["language"] response = tmdbv3api.Configuration().info() diff --git a/requirements.txt b/requirements.txt index b78713c5..1273e1b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PlexAPI==4.6.1 -tmdbv3api==1.7.5 -arrapi==1.0.2 +tmdbv3api==1.7.6 +arrapi==1.1.0 lxml requests>=2.4.2 ruamel.yaml @@ -8,3 +8,4 @@ schedule retrying pathvalidate pillow +python-slugify \ No newline at end of file From 7b2d5579e04946858f615a3d1b885442101eca7c Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 16 Jul 2021 09:55:46 -0400 Subject: [PATCH 15/95] upped arrapi requirements to 1.1.1 --- modules/config.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/config.py b/modules/config.py index 764bc0f2..ae68493a 100644 --- a/modules/config.py +++ b/modules/config.py @@ -186,7 +186,7 @@ class Config: raise Failed(f"Config Error: {message}") if do_print: util.print_multiline(f"Config Warning: {message}") - if attribute in data and data[attribute] and test_list is not None and data[attribute] not in test_list: + if data and attribute in data and data[attribute] and test_list is not None and data[attribute] not in test_list: util.print_multiline(options) return default diff --git a/requirements.txt b/requirements.txt index 1273e1b0..0bda40ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PlexAPI==4.6.1 tmdbv3api==1.7.6 -arrapi==1.1.0 +arrapi==1.1.1 lxml requests>=2.4.2 ruamel.yaml From f422f428e772d1a3274bb8e2bc4db6ca71e881d7 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 19 Jul 2021 16:12:29 -0400 Subject: [PATCH 16/95] fix letterboxd IDs --- modules/letterboxd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/letterboxd.py b/modules/letterboxd.py index a53535cf..a00184e8 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -13,7 +13,7 @@ class Letterboxd: def _parse_list(self, list_url, language): response = self.config.get_html(list_url, headers=util.header(language)) - letterboxd_ids = response.xpath("//div[@class='poster film-poster really-lazy-load']/@data-film-id") + letterboxd_ids = response.xpath("//li[@class='poster-container']/div/@data-film-id") items = [] for letterboxd_id in letterboxd_ids: slugs = response.xpath(f"//div[@data-film-id='{letterboxd_id}']/@data-film-slug") From 88369c3802b0ff892b6f98b1e07c6a2e1b8fd348 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 20 Jul 2021 08:42:58 -0400 Subject: [PATCH 17/95] hopefully fixed letterboxed IDs --- modules/letterboxd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/letterboxd.py b/modules/letterboxd.py index a00184e8..3b829811 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -13,7 +13,7 @@ class Letterboxd: def _parse_list(self, list_url, language): response = self.config.get_html(list_url, headers=util.header(language)) - letterboxd_ids = response.xpath("//li[@class='poster-container']/div/@data-film-id") + letterboxd_ids = response.xpath("//li[contains(@class, 'poster-container')]/div/@data-film-id") items = [] for letterboxd_id in letterboxd_ids: slugs = response.xpath(f"//div[@data-film-id='{letterboxd_id}']/@data-film-slug") From 8dbb0763dbbae14e64d9946d39de9ddc5a70bd37 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 20 Jul 2021 10:38:08 -0400 Subject: [PATCH 18/95] add overlay debugs --- modules/plex.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/plex.py b/modules/plex.py index 1fd5dbce..a3962583 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -416,6 +416,8 @@ class Plex: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def _upload_image(self, item, image): + logger.debug(item) + logger.debug(image) if image.is_poster and image.is_url: item.uploadPoster(url=image.location) elif image.is_poster: @@ -428,6 +430,8 @@ class Plex: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def upload_file_poster(self, item, image): + logger.debug(item) + logger.debug(image) item.uploadPoster(filepath=image) self.reload(item) @@ -438,8 +442,12 @@ class Plex: image = None if self.config.Cache: image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, self.original_mapping_name, "poster") + logger.debug(poster.compare) + logger.debug(image_compare) if str(poster.compare) != str(image_compare): image = None + logger.debug(image) + logger.debug(item.thumb) if image is None or image != item.thumb: self._upload_image(item, poster) poster_uploaded = True @@ -455,8 +463,11 @@ class Plex: overlay_name, overlay_folder, overlay_image, temp_image = overlay image_overlay = None if self.config.Cache: - image, _, image_overlay = self.config.Cache.query_image_map(item.ratingKey, self.original_mapping_name, "poster") + _, _, image_overlay = self.config.Cache.query_image_map(item.ratingKey, self.original_mapping_name, "poster") if poster_uploaded or not image_overlay or image_overlay != overlay_name: + logger.debug(poster_uploaded) + logger.debug(image_overlay) + logger.debug(overlay_name) response = requests.get(item.posterUrl) if response.status_code >= 400: raise Failed(f"Overlay Error: Overlay Failed for {item.title}") From 486e85970079a6d8454927bc3b9fc7ce27400c18 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 20 Jul 2021 10:55:07 -0400 Subject: [PATCH 19/95] add more overlay debugs --- modules/builder.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/builder.py b/modules/builder.py index 0dae742b..5e8d34cc 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1732,9 +1732,12 @@ class CollectionBuilder: temp_image = os.path.join(overlay_folder, f"temp.png") overlay = (overlay_name, overlay_folder, overlay_image, temp_image) + logger.debug(rating_keys) + tmdb_ids = [] tvdb_ids = [] for item in items: + logger.debug(item.ratingKey) if int(item.ratingKey) in rating_keys: rating_keys.remove(int(item.ratingKey)) if self.details["item_assets"] or overlay is not None: @@ -1761,6 +1764,8 @@ class CollectionBuilder: if len(tvdb_ids) > 0: self.library.Sonarr.edit_tags(tvdb_ids, self.item_details["item_sonarr_tag"], self.item_details["apply_tags"]) + logger.debug(rating_keys) + for rating_key in rating_keys: try: item = self.fetch_item(rating_key) From 371cb567e73faf0e5571929d4595f5745e05f558 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 20 Jul 2021 16:52:07 -0400 Subject: [PATCH 20/95] fixed overlay --- modules/builder.py | 9 +---- modules/cache.py | 94 ++++++++++++++++++++++++++++++++-------------- modules/plex.py | 22 +++-------- 3 files changed, 74 insertions(+), 51 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 5e8d34cc..964759c9 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1726,18 +1726,15 @@ class CollectionBuilder: if "item_overlay" in self.item_details: overlay_name = self.item_details["item_overlay"] if self.config.Cache: - rating_keys = self.config.Cache.query_image_map_overlay(self.library.original_mapping_name, "poster", overlay_name) + rating_keys = self.config.Cache.query_image_map_overlay(self.library.image_table_name, overlay_name) overlay_folder = os.path.join(self.config.default_dir, "overlays", overlay_name) overlay_image = Image.open(os.path.join(overlay_folder, "overlay.png")) temp_image = os.path.join(overlay_folder, f"temp.png") overlay = (overlay_name, overlay_folder, overlay_image, temp_image) - logger.debug(rating_keys) - tmdb_ids = [] tvdb_ids = [] for item in items: - logger.debug(item.ratingKey) if int(item.ratingKey) in rating_keys: rating_keys.remove(int(item.ratingKey)) if self.details["item_assets"] or overlay is not None: @@ -1764,8 +1761,6 @@ class CollectionBuilder: if len(tvdb_ids) > 0: self.library.Sonarr.edit_tags(tvdb_ids, self.item_details["item_sonarr_tag"], self.item_details["apply_tags"]) - logger.debug(rating_keys) - for rating_key in rating_keys: try: item = self.fetch_item(rating_key) @@ -1776,7 +1771,7 @@ class CollectionBuilder: if os.path.exists(og_image): self.library.upload_file_poster(item, og_image) os.remove(og_image) - self.config.Cache.update_image_map(item.ratingKey, self.library.original_mapping_name, "poster", "", "", "") + self.config.Cache.update_image_map(item.ratingKey, self.library.image_table_name, "", "", "") def update_details(self): if not self.obj and self.smart_url: diff --git a/modules/cache.py b/modules/cache.py index 95c004ce..ecf1865c 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -6,22 +6,23 @@ logger = logging.getLogger("Plex Meta Manager") class Cache: def __init__(self, config_path, expiration): - cache = f"{os.path.splitext(config_path)[0]}.cache" - with sqlite3.connect(cache) as connection: + self.cache_path = f"{os.path.splitext(config_path)[0]}.cache" + self.expiration = expiration + with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: cursor.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name='guid_map'") if cursor.fetchone()[0] == 0: - logger.info(f"Initializing cache database at {cache}") + logger.info(f"Initializing cache database at {self.cache_path}") else: - logger.info(f"Using cache database at {cache}") + logger.info(f"Using cache database at {self.cache_path}") cursor.execute("DROP TABLE IF EXISTS guids") cursor.execute("DROP TABLE IF EXISTS imdb_to_tvdb_map") cursor.execute("DROP TABLE IF EXISTS tmdb_to_tvdb_map") cursor.execute("DROP TABLE IF EXISTS imdb_map") cursor.execute( """CREATE TABLE IF NOT EXISTS guid_map ( - INTEGER PRIMARY KEY, + key INTEGER PRIMARY KEY, plex_guid TEXT UNIQUE, t_id TEXT, media_type TEXT, @@ -29,7 +30,7 @@ class Cache: ) cursor.execute( """CREATE TABLE IF NOT EXISTS imdb_to_tmdb_map ( - INTEGER PRIMARY KEY, + key INTEGER PRIMARY KEY, imdb_id TEXT UNIQUE, tmdb_id TEXT, media_type TEXT, @@ -37,28 +38,28 @@ class Cache: ) cursor.execute( """CREATE TABLE IF NOT EXISTS imdb_to_tvdb_map2 ( - INTEGER PRIMARY KEY, + key INTEGER PRIMARY KEY, imdb_id TEXT UNIQUE, tvdb_id TEXT, expiration_date TEXT)""" ) cursor.execute( """CREATE TABLE IF NOT EXISTS tmdb_to_tvdb_map2 ( - INTEGER PRIMARY KEY, + key INTEGER PRIMARY KEY, tmdb_id TEXT UNIQUE, tvdb_id TEXT, expiration_date TEXT)""" ) cursor.execute( """CREATE TABLE IF NOT EXISTS letterboxd_map ( - INTEGER PRIMARY KEY, + key INTEGER PRIMARY KEY, letterboxd_id TEXT UNIQUE, tmdb_id TEXT, expiration_date TEXT)""" ) cursor.execute( """CREATE TABLE IF NOT EXISTS omdb_data ( - INTEGER PRIMARY KEY, + key INTEGER PRIMARY KEY, imdb_id TEXT UNIQUE, title TEXT, year INTEGER, @@ -72,7 +73,7 @@ class Cache: ) cursor.execute( """CREATE TABLE IF NOT EXISTS anime_map ( - INTEGER PRIMARY KEY, + key INTEGER PRIMARY KEY, anidb TEXT UNIQUE, anilist TEXT, myanimelist TEXT, @@ -80,17 +81,21 @@ class Cache: expiration_date TEXT)""" ) cursor.execute( - """CREATE TABLE IF NOT EXISTS image_map ( - INTEGER PRIMARY KEY, - rating_key TEXT, - library TEXT, - type TEXT, - overlay TEXT, - compare TEXT, - location TEXT)""" + """CREATE TABLE IF NOT EXISTS image_maps ( + key INTEGER PRIMARY KEY, + library TEXT UNIQUE)""" ) - self.expiration = expiration - self.cache_path = cache + cursor.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name='image_map'") + if cursor.fetchone()[0] > 0: + cursor.execute(f"SELECT DISTINCT library FROM image_map") + for library in cursor.fetchall(): + table_name = self.get_image_table_name(library["library"]) + cursor.execute(f"SELECT DISTINCT * FROM image_map WHERE library='{library['library']}'") + for row in cursor.fetchall(): + if row["type"] == "poster": + final_table = table_name if row["type"] == "poster" else f"{table_name}_backgrounds" + self.update_image_map(row["rating_key"], final_table, row["location"], row["compare"], row["overlay"]) + cursor.execute("DROP TABLE IF EXISTS image_map") def query_guid_map(self, plex_guid): id_to_return = None @@ -233,30 +238,63 @@ class Cache: cursor.execute("INSERT OR IGNORE INTO anime_map(anidb) VALUES(?)", (anime_ids["anidb"],)) cursor.execute("UPDATE anime_map SET anilist = ?, myanimelist = ?, kitsu = ?, expiration_date = ? WHERE anidb = ?", (anime_ids["anidb"], anime_ids["myanimelist"], anime_ids["kitsu"], expiration_date.strftime("%Y-%m-%d"), anime_ids["anidb"])) - def query_image_map_overlay(self, library, image_type, overlay): + def get_image_table_name(self, library): + table_name = None + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute(f"SELECT * FROM image_maps WHERE library = ?", (library,)) + row = cursor.fetchone() + if row and row["key"]: + table_name = f"image_map_{row['key']}" + else: + cursor.execute("INSERT OR IGNORE INTO image_maps(library) VALUES(?)", (library,)) + cursor.execute(f"SELECT * FROM image_maps WHERE library = ?", (library,)) + row = cursor.fetchone() + if row and row["key"]: + table_name = f"image_map_{row['key']}" + cursor.execute( + f"""CREATE TABLE IF NOT EXISTS {table_name} ( + key INTEGER PRIMARY KEY, + rating_key TEXT UNIQUE, + overlay TEXT, + compare TEXT, + location TEXT)""" + ) + cursor.execute( + f"""CREATE TABLE IF NOT EXISTS {table_name}_backgrounds ( + key INTEGER PRIMARY KEY, + rating_key TEXT UNIQUE, + overlay TEXT, + compare TEXT, + location TEXT)""" + ) + return table_name + + def query_image_map_overlay(self, table_name, overlay): rks = [] with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: - cursor.execute(f"SELECT * FROM image_map WHERE overlay = ? AND library = ? AND type = ?", (overlay, library, image_type)) + cursor.execute(f"SELECT * FROM {table_name} WHERE overlay = ?", (overlay,)) rows = cursor.fetchall() for row in rows: rks.append(int(row["rating_key"])) return rks - def query_image_map(self, rating_key, library, image_type): + def query_image_map(self, rating_key, table_name): with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: - cursor.execute(f"SELECT * FROM image_map WHERE rating_key = ? AND library = ? AND type = ?", (rating_key, library, image_type)) + cursor.execute(f"SELECT * FROM {table_name} WHERE rating_key = ?", (rating_key,)) row = cursor.fetchone() if row and row["location"]: return row["location"], row["compare"], row["overlay"] return None, None, None - def update_image_map(self, rating_key, library, image_type, location, compare, overlay): + def update_image_map(self, rating_key, table_name, location, compare, overlay): with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: - cursor.execute("INSERT OR IGNORE INTO image_map(rating_key, library, type) VALUES(?, ?, ?)", (rating_key, library, image_type)) - cursor.execute("UPDATE image_map SET location = ?, compare = ?, overlay = ? WHERE rating_key = ? AND library = ? AND type = ?", (location, compare, overlay, rating_key, library, image_type)) + cursor.execute(f"INSERT OR IGNORE INTO {table_name}(rating_key) VALUES(?)", (rating_key,)) + cursor.execute(f"UPDATE {table_name} SET location = ?, compare = ?, overlay = ? WHERE rating_key = ?", (location, compare, overlay, rating_key)) diff --git a/modules/plex.py b/modules/plex.py index a3962583..511602c9 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -309,6 +309,7 @@ class Plex: self.mapping_name, output = util.validate_filename(self.original_mapping_name) if output: logger.info(output) + self.image_table_name = self.config.Cache.get_image_table_name(self.original_mapping_name) if self.config.Cache else None self.missing_path = os.path.join(params["default_dir"], f"{self.name}_missing.yml") self.metadata_path = params["metadata_path"] self.asset_directory = params["asset_directory"] @@ -416,8 +417,6 @@ class Plex: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def _upload_image(self, item, image): - logger.debug(item) - logger.debug(image) if image.is_poster and image.is_url: item.uploadPoster(url=image.location) elif image.is_poster: @@ -430,8 +429,6 @@ class Plex: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def upload_file_poster(self, item, image): - logger.debug(item) - logger.debug(image) item.uploadPoster(filepath=image) self.reload(item) @@ -441,13 +438,9 @@ class Plex: try: image = None if self.config.Cache: - image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, self.original_mapping_name, "poster") - logger.debug(poster.compare) - logger.debug(image_compare) + image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) if str(poster.compare) != str(image_compare): image = None - logger.debug(image) - logger.debug(item.thumb) if image is None or image != item.thumb: self._upload_image(item, poster) poster_uploaded = True @@ -463,11 +456,8 @@ class Plex: overlay_name, overlay_folder, overlay_image, temp_image = overlay image_overlay = None if self.config.Cache: - _, _, image_overlay = self.config.Cache.query_image_map(item.ratingKey, self.original_mapping_name, "poster") + _, _, image_overlay = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) if poster_uploaded or not image_overlay or image_overlay != overlay_name: - logger.debug(poster_uploaded) - logger.debug(image_overlay) - logger.debug(overlay_name) response = requests.get(item.posterUrl) if response.status_code >= 400: raise Failed(f"Overlay Error: Overlay Failed for {item.title}") @@ -490,7 +480,7 @@ class Plex: try: image = None if self.config.Cache: - image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, self.original_mapping_name, "background") + image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, f"{self.image_table_name}_background") if str(background.compare) != str(image_compare): image = None if image is None or image != item.art: @@ -505,9 +495,9 @@ class Plex: if self.config.Cache: if poster_uploaded: - self.config.Cache.update_image_map(item.ratingKey, self.original_mapping_name, "poster", item.thumb, poster.compare if poster else "", overlay_name) + self.config.Cache.update_image_map(item.ratingKey, self.image_table_name, item.thumb, poster.compare if poster else "", overlay_name) if background_uploaded: - self.config.Cache.update_image_map(item.ratingKey, self.original_mapping_name, "background", item.art, background.compare, "") + self.config.Cache.update_image_map(item.ratingKey, f"{self.image_table_name}_backgrounds", item.art, background.compare, "") @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def get_search_choices(self, search_name, title=True): From f7d1992fc2ec44664e7135eac6cede25281c18b6 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 21 Jul 2021 10:28:12 -0400 Subject: [PATCH 21/95] fix date error --- modules/builder.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 964759c9..b8239a26 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1468,9 +1468,12 @@ class CollectionBuilder: if (modifier == "" and (current_data is None or current_data < threshold_date)) \ or (modifier == ".not" and current_data and current_data >= threshold_date): return False - elif (modifier == ".before" and (current_data is None or current_data >= filter_data)) \ - or (modifier == ".after" and (current_data is None or current_data <= filter_data)): - return False + elif modifier in [".before", ".after"]: + if current_data is None: + return False + filter_date = datetime.strptime(str(filter_data), "%m/%d/%Y") + if (modifier == ".before" and current_data >= filter_data) or (modifier == ".after" and current_data <= filter_data): + return False elif filter_attr in ["release", "added", "last_played"] and modifier == ".regex": jailbreak = False current_data = getattr(current, filter_actual) From 31f6129c7ff10d436cf187ac9c57e329959935b3 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 21 Jul 2021 13:40:05 -0400 Subject: [PATCH 22/95] prep for sort --- modules/anidb.py | 3 +- modules/anilist.py | 1 + modules/builder.py | 149 ++++++++++++++++++---------------------- modules/icheckmovies.py | 17 +++-- modules/imdb.py | 21 +++++- modules/letterboxd.py | 11 +++ modules/tmdb.py | 8 ++- modules/trakt.py | 3 +- 8 files changed, 119 insertions(+), 94 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 647495e8..7cd299f4 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -48,7 +48,8 @@ class AniDB: return util.regex_first_int(ids[0], "AniDB ID") raise Failed(f"AniDB Error: AniDB ID: {anidb_id} not found") - def validate_anidb_list(self, anidb_list, language): + def validate_anidb_ids(self, anidb_ids, language): + anidb_list = util.get_int_list(anidb_ids, "AniDB ID") anidb_values = [] for anidb_id in anidb_list: try: diff --git a/modules/anilist.py b/modules/anilist.py index 8df509e1..5a31cf02 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -205,6 +205,7 @@ class AniList: raise Failed(f"AniList Error: Tag: {tag} does not exist") def validate_anilist_ids(self, anilist_ids, studio=False): + anilist_id_list = util.get_int_list(anilist_ids, "AniList ID") anilist_values = [] for anilist_id in anilist_ids: if studio: query = "query ($id: Int) {Studio(id: $id) {name}}" diff --git a/modules/builder.py b/modules/builder.py index b8239a26..55a08603 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -534,8 +534,7 @@ class CollectionBuilder: continue logger.info("") logger.info(f"Validating Method: {method_key}") - if "trakt" in method_key.lower() and not config.Trakt: raise Failed(f"Collection Error: {method_key} requires Trakt todo be configured") - elif "imdb" in method_key.lower() and not config.IMDb: raise Failed(f"Collection Error: {method_key} requires TMDb or Trakt to be configured") + if "trakt" in method_key.lower() and not config.Trakt: raise Failed(f"Collection Error: {method_key} requires Trakt to be configured") elif "radarr" in method_key.lower() and not self.library.Radarr: raise Failed(f"Collection Error: {method_key} requires Radarr to be configured") elif "sonarr" in method_key.lower() and not self.library.Sonarr: raise Failed(f"Collection Error: {method_key} requires Sonarr to be configured") elif "tautulli" in method_key.lower() and not self.library.Tautulli: raise Failed(f"Collection Error: {method_key} requires Tautulli to be configured") @@ -570,7 +569,7 @@ class CollectionBuilder: elif method_name == "tvdb_description": self.summaries[method_name] = config.TVDb.get_list_description(method_data, self.library.Plex.language) elif method_name == "trakt_description": - self.summaries[method_name] = config.Trakt.list_description(config.Trakt.validate_trakt(util.get_list(method_data), self.library.is_movie)[0]) + self.summaries[method_name] = config.Trakt.list_description(config.Trakt.validate_trakt(method_data, self.library.is_movie)[0]) elif method_name == "letterboxd_description": self.summaries[method_name] = config.Letterboxd.get_list_description(method_data, self.library.Plex.language) elif method_name == "icheckmovies_description": @@ -702,65 +701,55 @@ class CollectionBuilder: elif method_name == "sonarr_tag": self.sonarr_options["tag"] = util.get_list(method_data) elif method_final in plex.searches: - self.methods.append(("plex_search", [self.build_filter("plex_search", {"any": {method_name: method_data}})])) + self.methods.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) elif method_name == "plex_all": - self.methods.append((method_name, [""])) + self.methods.append((method_name, True)) elif method_name == "anidb_popular": list_count = util.regex_first_int(method_data, "List Size", default=40) if 1 <= list_count <= 30: - self.methods.append((method_name, [list_count])) + self.methods.append((method_name, list_count)) else: logger.warning("Collection Error: anidb_popular must be an integer between 1 and 30 defaulting to 30") - self.methods.append((method_name, [30])) + self.methods.append((method_name, 30)) elif method_name == "mal_id": - self.methods.append((method_name, util.get_int_list(method_data, "MyAnimeList ID"))) + for mal_id in util.get_int_list(method_data, "MyAnimeList ID"): + self.methods.append((method_name, mal_id)) elif method_name in ["anidb_id", "anidb_relation"]: - self.methods.append((method_name, config.AniDB.validate_anidb_list(util.get_int_list(method_data, "AniDB ID"), self.library.Plex.language))) + for anidb_id in config.AniDB.validate_anidb_ids(method_data, self.library.Plex.language): + self.methods.append((method_name, anidb_id)) elif method_name in ["anilist_id", "anilist_relations", "anilist_studio"]: - self.methods.append((method_name, config.AniList.validate_anilist_ids(util.get_int_list(method_data, "AniList ID"), studio=method_name == "anilist_studio"))) + for anilist_id in config.AniList.validate_anilist_ids(method_data, studio=method_name == "anilist_studio"): + self.methods.append((method_name, anilist_id)) elif method_name == "trakt_list": - self.methods.append((method_name, config.Trakt.validate_trakt(util.get_list(method_data), self.library.is_movie))) + for trakt_list in config.Trakt.validate_trakt(method_data, self.library.is_movie): + self.methods.append((method_name, trakt_list)) elif method_name == "trakt_list_details": - valid_list = config.Trakt.validate_trakt(util.get_list(method_data), self.library.is_movie) - self.summaries[method_name] = config.Trakt.list_description(valid_list[0]) - self.methods.append((method_name[:-8], valid_list)) + trakt_lists = config.Trakt.validate_trakt(method_data, self.library.is_movie) + self.summaries[method_name] = config.Trakt.list_description(trakt_lists[0]) + for trakt_list in trakt_lists: + self.methods.append((method_name[:-8], trakt_list)) elif method_name in ["trakt_watchlist", "trakt_collection"]: - self.methods.append((method_name, config.Trakt.validate_trakt(util.get_list(method_data), self.library.is_movie, trakt_type=method_name[6:]))) + for trakt_list in config.Trakt.validate_trakt(method_data, self.library.is_movie, trakt_type=method_name[6:]): + self.methods.append((method_name, trakt_list)) elif method_name == "imdb_list": - new_list = [] - for imdb_list in util.get_list(method_data, split=False): - if isinstance(imdb_list, dict): - dict_methods = {dm.lower(): dm for dm in imdb_list} - if "url" in dict_methods and imdb_list[dict_methods["url"]]: - imdb_url = config.IMDb.validate_imdb_url(imdb_list[dict_methods["url"]], self.library.Plex.language) - else: - raise Failed("Collection Error: imdb_list attribute url is required") - if "limit" in dict_methods and imdb_list[dict_methods["limit"]]: - list_count = util.regex_first_int(imdb_list[dict_methods["limit"]], "List Limit", default=0) - else: - list_count = 0 - else: - imdb_url = config.IMDb.validate_imdb_url(str(imdb_list), self.library.Plex.language) - list_count = 0 - new_list.append({"url": imdb_url, "limit": list_count}) - self.methods.append((method_name, new_list)) + for imdb_dict in self.config.IMDb.validate_imdb_lists(method_data, self.library.Plex.language): + self.methods.append((method_name, imdb_dict)) elif method_name == "icheckmovies_list": - valid_lists = [] - for icheckmovies_list in util.get_list(method_data, split=False): - valid_lists.append(config.ICheckMovies.validate_icheckmovies_list(icheckmovies_list, self.library.Plex.language)) - self.methods.append((method_name, valid_lists)) + for icheckmovies_list in self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.library.Plex.language): + self.methods.append((method_name, icheckmovies_list)) elif method_name == "icheckmovies_list_details": - valid_lists = [] - for icheckmovies_list in util.get_list(method_data, split=False): - valid_lists.append(config.ICheckMovies.validate_icheckmovies_list(icheckmovies_list, self.library.Plex.language)) - self.methods.append((method_name[:-8], valid_lists)) - self.summaries[method_name] = config.ICheckMovies.get_list_description(method_data, self.library.Plex.language) + icheckmovies_lists = self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.library.Plex.language) + for icheckmovies_list in icheckmovies_lists: + self.methods.append((method_name[:-8], icheckmovies_list)) + self.summaries[method_name] = self.config.ICheckMovies.get_list_description(icheckmovies_lists[0], self.library.Plex.language) elif method_name == "letterboxd_list": - self.methods.append((method_name, util.get_list(method_data, split=False))) + for letterboxd_list in self.config.Letterboxd.validate_letterboxd_lists(method_data, self.library.Plex.language): + self.methods.append((method_name, letterboxd_list)) elif method_name == "letterboxd_list_details": - values = util.get_list(method_data, split=False) - self.summaries[method_name] = config.Letterboxd.get_list_description(values[0], self.library.Plex.language) - self.methods.append((method_name[:-8], values)) + letterboxd_lists = self.config.Letterboxd.validate_letterboxd_lists(method_data, self.library.Plex.language) + for letterboxd_list in letterboxd_lists: + self.methods.append((method_name[:-8], letterboxd_list)) + self.summaries[method_name] = config.Letterboxd.get_list_description(letterboxd_lists[0], self.library.Plex.language) elif method_name in dictionary_builders: for dict_data in util.get_list(method_data): if isinstance(dict_data, dict): @@ -817,9 +806,9 @@ class CollectionBuilder: exact_list.append(self.name) new_dictionary["exclude_prefix"] = prefix_list new_dictionary["exclude"] = exact_list - self.methods.append((method_name, [new_dictionary])) + self.methods.append((method_name, new_dictionary)) elif method_name == "plex_search": - self.methods.append((method_name, [self.build_filter("plex_search", dict_data)])) + self.methods.append((method_name, self.build_filter("plex_search", dict_data))) elif method_name == "tmdb_discover": new_dictionary = {"limit": 100} for discover_name, discover_data in dict_data.items(): @@ -874,7 +863,7 @@ class CollectionBuilder: else: raise Failed(f"Collection Error: {method_name} parameter {discover_final} is blank") if len(new_dictionary) > 1: - self.methods.append((method_name, [new_dictionary])) + self.methods.append((method_name, new_dictionary)) else: raise Failed(f"Collection Error: {method_name} had no valid fields") elif "tautulli" in method_name: @@ -889,7 +878,7 @@ class CollectionBuilder: new_dictionary["list_days"] = get_int(method_name, "list_days", dict_data, dict_methods, 30) new_dictionary["list_size"] = get_int(method_name, "list_size", dict_data, dict_methods, 10) new_dictionary["list_buffer"] = get_int(method_name, "list_buffer", dict_data, dict_methods, 20) - self.methods.append((method_name, [new_dictionary])) + self.methods.append((method_name, new_dictionary)) elif method_name == "mal_season": new_dictionary = {"sort_by": "anime_num_list_users"} dict_methods = {dm.lower(): dm for dm in dict_data} @@ -918,7 +907,7 @@ class CollectionBuilder: new_dictionary["year"] = get_int(method_name, "year", dict_data, dict_methods, self.current_time.year, minimum=1917, maximum=self.current_time.year + 1) new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 100, maximum=500) - self.methods.append((method_name, [new_dictionary])) + self.methods.append((method_name, new_dictionary)) elif method_name == "mal_userlist": new_dictionary = {"status": "all", "sort_by": "list_score"} dict_methods = {dm.lower(): dm for dm in dict_data} @@ -948,7 +937,7 @@ class CollectionBuilder: new_dictionary["sort_by"] = mal.userlist_sort[dict_data[dict_methods["sort_by"]]] new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 100, maximum=1000) - self.methods.append((method_name, [new_dictionary])) + self.methods.append((method_name, new_dictionary)) elif method_name == "anidb_tag": new_dictionary = {} dict_methods = {dm.lower(): dm for dm in dict_data} @@ -959,7 +948,7 @@ class CollectionBuilder: else: new_dictionary["tag"] = util.regex_first_int(dict_data[dict_methods["username"]], "AniDB Tag ID") new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 0, minimum=0) - self.methods.append((method_name, [new_dictionary])) + self.methods.append((method_name, new_dictionary)) elif "anilist" in method_name: new_dictionary = {"sort_by": "score"} dict_methods = {dm.lower(): dm for dm in dict_data} @@ -1005,7 +994,7 @@ class CollectionBuilder: new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 0, maximum=500) - self.methods.append((method_name, [new_dictionary])) + self.methods.append((method_name, new_dictionary)) else: raise Failed(f"Collection Error: {method_name} attribute is not a dictionary: {dict_data}") elif method_name in numbered_builders: @@ -1013,7 +1002,7 @@ class CollectionBuilder: if list_count < 1: logger.warning(f"Collection Warning: {method_name} must be an integer greater then 0 defaulting to 10") list_count = 10 - self.methods.append((method_name, [list_count])) + self.methods.append((method_name, list_count)) elif "tvdb" in method_name: values = util.get_list(method_data) if method_name[-8:] == "_details": @@ -1035,11 +1024,10 @@ class CollectionBuilder: self.posters[method_name] = f"{config.TMDb.image_url}{item.poster_path}" elif method_name == "tvdb_list_details": self.summaries[method_name] = config.TVDb.get_list_description(values[0], self.library.Plex.language) - self.methods.append((method_name[:-8], values)) - else: - self.methods.append((method_name, values)) + for value in values: + self.methods.append((method_name[:-8] if method_name[-8:] == "_details" else method_name, value)) elif method_name in tmdb.builders: - values = config.TMDb.validate_tmdb_list(util.get_int_list(method_data, f"TMDb {tmdb.type_map[method_name]} ID"), tmdb.type_map[method_name]) + values = config.TMDb.validate_tmdb_ids(method_data, method_name) if method_name[-8:] == "_details": if method_name in ["tmdb_collection_details", "tmdb_movie_details", "tmdb_show_details"]: item = config.TMDb.get_movie_show_or_collection(values[0], self.library.is_movie) @@ -1059,11 +1047,11 @@ class CollectionBuilder: item = config.TMDb.get_list(values[0]) if hasattr(item, "description") and item.description: self.summaries[method_name] = item.description - self.methods.append((method_name[:-8], values)) - else: - self.methods.append((method_name, values)) + for value in values: + self.methods.append((method_name[:-8] if method_name[-8:] == "_details" else method_name, value)) elif method_name in all_builders: - self.methods.append((method_name, util.get_list(method_data))) + for value in util.get_list(method_data): + self.methods.append((method_name, value)) elif method_name not in ignored_details: raise Failed(f"Collection Error: {method_name} attribute not supported") elif method_key.lower() in all_builders or method_key.lower() in method_alias or method_key.lower() in plex.searches: @@ -1156,23 +1144,22 @@ class CollectionBuilder: elif show_id not in self.missing_shows: self.missing_shows.append(show_id) return items_found_inside - for method, values in self.methods: - for value in values: - logger.debug("") - logger.debug(f"Builder: {method}: {value}") - logger.info("") - if "plex" in method: add_rating_keys(self.library.get_items(method, value)) - elif "tautulli" in method: add_rating_keys(self.library.Tautulli.get_items(self.library, value)) - elif "anidb" in method: check_map(self.config.AniDB.get_items(method, value, self.library.Plex.language)) - elif "anilist" in method: check_map(self.config.AniList.get_items(method, value)) - elif "mal" in method: check_map(self.config.MyAnimeList.get_items(method, value)) - elif "tvdb" in method: check_map(self.config.TVDb.get_items(method, value, self.library.Plex.language)) - elif "imdb" in method: check_map(self.config.IMDb.get_items(method, value, self.library.Plex.language, self.library.is_movie)) - elif "icheckmovies" in method: check_map(self.config.ICheckMovies.get_items(method, value, self.library.Plex.language)) - elif "letterboxd" in method: check_map(self.config.Letterboxd.get_items(method, value, self.library.Plex.language)) - elif "tmdb" in method: check_map(self.config.TMDb.get_items(method, value, self.library.is_movie)) - elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) - else: logger.error(f"Collection Error: {method} method not supported") + for method, value in self.methods: + logger.debug("") + logger.debug(f"Builder: {method}: {value}") + logger.info("") + if "plex" in method: add_rating_keys(self.library.get_items(method, value)) + elif "tautulli" in method: add_rating_keys(self.library.Tautulli.get_items(self.library, value)) + elif "anidb" in method: check_map(self.config.AniDB.get_items(method, value, self.library.Plex.language)) + elif "anilist" in method: check_map(self.config.AniList.get_items(method, value)) + elif "mal" in method: check_map(self.config.MyAnimeList.get_items(method, value)) + elif "tvdb" in method: check_map(self.config.TVDb.get_items(method, value, self.library.Plex.language)) + elif "imdb" in method: check_map(self.config.IMDb.get_items(method, value, self.library.Plex.language, self.library.is_movie)) + elif "icheckmovies" in method: check_map(self.config.ICheckMovies.get_items(method, value, self.library.Plex.language)) + elif "letterboxd" in method: check_map(self.config.Letterboxd.get_items(method, value, self.library.Plex.language)) + elif "tmdb" in method: check_map(self.config.TMDb.get_items(method, value, self.library.is_movie)) + elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) + else: logger.error(f"Collection Error: {method} method not supported") def build_filter(self, method, plex_filter, smart=False): if smart: @@ -1389,9 +1376,9 @@ class CollectionBuilder: else: logger.error(error) return valid_list - elif attribute in ["year", "episode_year"] and modifier in [".gt", ".gte", ".lt", ".lte"]:# + elif attribute in ["year", "episode_year"] and modifier in [".gt", ".gte", ".lt", ".lte"]: return util.check_year(data, self.current_year, final) - elif attribute in plex.date_attributes and modifier in [".before", ".after"]:# + elif attribute in plex.date_attributes and modifier in [".before", ".after"]: return util.check_date(data, final, return_string=True, plex_date=True) elif attribute in plex.number_attributes and modifier in ["", ".not", ".gt", ".gte", ".lt", ".lte"]: return util.check_number(data, final, minimum=1) diff --git a/modules/icheckmovies.py b/modules/icheckmovies.py index 117cbf1c..dc13e201 100644 --- a/modules/icheckmovies.py +++ b/modules/icheckmovies.py @@ -22,13 +22,16 @@ class ICheckMovies: descriptions = self._request(list_url, language, "//div[@class='span-19 last']/p/em/text()") return descriptions[0] if len(descriptions) > 0 and len(descriptions[0]) > 0 else None - def validate_icheckmovies_list(self, list_url, language): - list_url = list_url.strip() - if not list_url.startswith(base_url): - raise Failed(f"ICheckMovies Error: {list_url} must begin with: {base_url}") - if len(self._parse_list(list_url, language)) > 0: - return list_url - raise Failed(f"ICheckMovies Error: {list_url} failed to parse") + def validate_icheckmovies_lists(self, icheckmovies_lists, language): + valid_lists = [] + for icheckmovies_list in util.get_list(icheckmovies_lists, split=False): + list_url = icheckmovies_list.strip() + if not list_url.startswith(base_url): + raise Failed(f"ICheckMovies Error: {list_url} must begin with: {base_url}") + if len(self._parse_list(list_url, language)) > 0: + valid_lists.append(list_url) + raise Failed(f"ICheckMovies Error: {list_url} failed to parse") + return valid_lists def get_items(self, method, data, language): pretty = util.pretty_names[method] if method in util.pretty_names else method diff --git a/modules/imdb.py b/modules/imdb.py index e544a038..bfab31e6 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -16,7 +16,7 @@ class IMDb: def __init__(self, config): self.config = config - def validate_imdb_url(self, imdb_url, language): + def _validate_url(self, imdb_url, language): imdb_url = imdb_url.strip() if not imdb_url.startswith(urls["list"]) and not imdb_url.startswith(urls["search"]) and not imdb_url.startswith(urls["keyword"]): raise Failed(f"IMDb Error: {imdb_url} must begin with either:\n{urls['list']} (For Lists)\n{urls['search']} (For Searches)\n{urls['keyword']} (For Keyword Searches)") @@ -25,6 +25,25 @@ class IMDb: return imdb_url raise Failed(f"IMDb Error: {imdb_url} failed to parse") + def validate_imdb_lists(self, imdb_lists, language): + valid_lists = [] + for imdb_list in util.get_list(imdb_lists, split=False): + if isinstance(imdb_list, dict): + dict_methods = {dm.lower(): dm for dm in imdb_list} + if "url" in dict_methods and imdb_list[dict_methods["url"]]: + imdb_url = self._validate_url(imdb_list[dict_methods["url"]], language) + else: + raise Failed("Collection Error: imdb_list attribute url is required") + if "limit" in dict_methods and imdb_list[dict_methods["limit"]]: + list_count = util.regex_first_int(imdb_list[dict_methods["limit"]], "List Limit", default=0) + else: + list_count = 0 + else: + imdb_url = self._validate_url(str(imdb_list), language) + list_count = 0 + valid_lists.append({"url": imdb_url, "limit": list_count}) + return valid_lists + def _fix_url(self, imdb_url): if imdb_url.startswith(urls["list"]): try: list_id = re.search("(\\d+)", str(imdb_url)).group(1) diff --git a/modules/letterboxd.py b/modules/letterboxd.py index 3b829811..8f6ec302 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -38,6 +38,17 @@ class Letterboxd: descriptions = response.xpath("//meta[@property='og:description']/@content") return descriptions[0] if len(descriptions) > 0 and len(descriptions[0]) > 0 else None + def validate_letterboxd_lists(self, letterboxd_lists, language): + valid_lists = [] + for letterboxd_list in util.get_list(letterboxd_lists, split=False): + list_url = letterboxd_list.strip() + if not list_url.startswith(base_url): + raise Failed(f"Letterboxd Error: {list_url} must begin with: {base_url}") + if len(self._parse_list(list_url, language)) > 0: + valid_lists.append(list_url) + raise Failed(f"Letterboxd Error: {list_url} failed to parse") + return valid_lists + def get_items(self, method, data, language): pretty = util.pretty_names[method] if method in util.pretty_names else method movie_ids = [] diff --git a/modules/tmdb.py b/modules/tmdb.py index 5ecfc579..3881a58e 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -273,15 +273,17 @@ class TMDb: if count == amount: break return ids, amount - def validate_tmdb_list(self, tmdb_list, tmdb_type): + def validate_tmdb_ids(self, tmdb_ids, tmdb_method): + tmdb_list = util.get_int_list(tmdb_ids, f"TMDb {type_map[tmdb_method]} ID") tmdb_values = [] for tmdb_id in tmdb_list: - try: tmdb_values.append(self.validate_tmdb(tmdb_id, tmdb_type)) + try: tmdb_values.append(self.validate_tmdb(tmdb_id, tmdb_method)) except Failed as e: logger.error(e) if len(tmdb_values) == 0: raise Failed(f"TMDb Error: No valid TMDb IDs in {tmdb_list}") return tmdb_values - def validate_tmdb(self, tmdb_id, tmdb_type): + def validate_tmdb(self, tmdb_id, tmdb_method): + tmdb_type = type_map[tmdb_method] if tmdb_type == "Movie": self.get_movie(tmdb_id) elif tmdb_type == "Show": self.get_show(tmdb_id) elif tmdb_type == "Collection": self.get_collection(tmdb_id) diff --git a/modules/trakt.py b/modules/trakt.py index ce04ddd0..60221049 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -146,7 +146,8 @@ class Trakt: elif is_movie: return [item["movie"]["ids"]["tmdb"] for item in items], [] else: return [], [item["show"]["ids"]["tvdb"] for item in items] - def validate_trakt(self, values, is_movie, trakt_type="list"): + def validate_trakt(self, trakt_lists, is_movie, trakt_type="list"): + values = util.get_list(trakt_lists) trakt_values = [] for value in values: try: From 18897852d3c51675ec0b45f41a66ced87b7383eb Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 21 Jul 2021 15:25:29 -0400 Subject: [PATCH 23/95] made date consistent --- modules/builder.py | 12 ++++++------ modules/meta.py | 2 +- modules/tmdb.py | 3 +-- modules/util.py | 8 ++++---- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 55a08603..24403caf 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -844,7 +844,7 @@ class CollectionBuilder: if discover_data is True: new_dictionary[discover_final] = discover_data elif discover_final in tmdb.discover_dates: - new_dictionary[discover_final] = util.check_date(discover_data, f"{method_name} attribute {discover_final}", return_string=True) + new_dictionary[discover_final] = util.validate_date(discover_data, f"{method_name} attribute {discover_final}", return_as="%m/%d/%Y") elif discover_final in ["primary_release_year", "year", "first_air_date_year"]: new_dictionary[discover_final] = util.check_number(discover_data, f"{method_name} attribute {discover_final}", minimum=1800, maximum=self.current_year + 1) elif discover_final in ["vote_count.gte", "vote_count.lte", "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte"]: @@ -1379,7 +1379,7 @@ class CollectionBuilder: elif attribute in ["year", "episode_year"] and modifier in [".gt", ".gte", ".lt", ".lte"]: return util.check_year(data, self.current_year, final) elif attribute in plex.date_attributes and modifier in [".before", ".after"]: - return util.check_date(data, final, return_string=True, plex_date=True) + return util.validate_date(data, final, return_as="%Y-%m-%d") elif attribute in plex.number_attributes and modifier in ["", ".not", ".gt", ".gte", ".lt", ".lte"]: return util.check_number(data, final, minimum=1) elif attribute in plex.float_attributes and modifier in [".gt", ".gte", ".lt", ".lte"]: @@ -1458,8 +1458,8 @@ class CollectionBuilder: elif modifier in [".before", ".after"]: if current_data is None: return False - filter_date = datetime.strptime(str(filter_data), "%m/%d/%Y") - if (modifier == ".before" and current_data >= filter_data) or (modifier == ".after" and current_data <= filter_data): + filter_date = util.validate_date(filter_data) + if (modifier == ".before" and current_data >= filter_date) or (modifier == ".after" and current_data <= filter_date): return False elif filter_attr in ["release", "added", "last_played"] and modifier == ".regex": jailbreak = False @@ -1618,7 +1618,7 @@ class CollectionBuilder: or (filter_method == "tmdb_vote_count.lte" and movie.vote_count > filter_data): match = False break - current_title = f"{movie.title} ({util.check_date(movie.release_date, 'test', plex_date=True).year})" if movie.release_date else movie.title + current_title = f"{movie.title} ({util.validate_date(movie.release_date, 'test').year})" if movie.release_date else movie.title if match: missing_movies_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: @@ -1958,7 +1958,7 @@ class CollectionBuilder: logger.error(e) continue if self.details["show_missing"] is True: - current_title = f"{movie.title} ({util.check_date(movie.release_date, 'test', plex_date=True).year})" if movie.release_date else movie.title + current_title = f"{movie.title} ({util.validate_date(movie.release_date, 'test').year})" if movie.release_date else movie.title logger.info(f"{name} Collection | ? | {current_title} (TMDb: {missing_id})") logger.info("") logger.info(f"{len(self.run_again_movies)} Movie{'s' if len(self.run_again_movies) > 1 else ''} Missing") diff --git a/modules/meta.py b/modules/meta.py index 309f6e74..f4805e56 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -90,7 +90,7 @@ class Metadata: try: current = str(getattr(current_item, key, "")) if var_type == "date": - final_value = util.check_date(value, name, return_string=True, plex_date=True) + final_value = util.validate_date(value, name, return_as="%Y-%m-%d") current = current[:-9] elif var_type == "float": final_value = util.check_number(value, name, number_type="float", minimum=0, maximum=10) diff --git a/modules/tmdb.py b/modules/tmdb.py index 3881a58e..ea0278c3 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -1,5 +1,4 @@ import logging, tmdbv3api -from datetime import datetime from modules import util from modules.util import Failed from retrying import retry @@ -254,7 +253,7 @@ class TMDb: count = 0 for date_attr in discover_dates: if date_attr in attrs: - attrs[date_attr] = datetime.strftime(datetime.strptime(attrs[date_attr], "%m/%d/%Y"), "%Y-%m-%d") + attrs[date_attr] = util.validate_date(attrs[date_attr], return_as="%Y-%m-%d") self.Discover.discover_movies(attrs) if is_movie else self.Discover.discover_tv_shows(attrs) total_pages = int(self.TMDb.total_pages) total_results = int(self.TMDb.total_results) diff --git a/modules/util.py b/modules/util.py index 76f74bee..984638a1 100644 --- a/modules/util.py +++ b/modules/util.py @@ -270,10 +270,10 @@ def check_number(value, method, number_type="int", minimum=None, maximum=None): else: return num_value -def check_date(date_text, method, return_string=False, plex_date=False): - try: date_obg = datetime.strptime(str(date_text), "%Y-%m-%d" if plex_date else "%m/%d/%Y") - except ValueError: raise Failed(f"Collection Error: {method}: {date_text} must match pattern {'YYYY-MM-DD e.g. 2020-12-25' if plex_date else 'MM/DD/YYYY e.g. 12/25/2020'}") - return str(date_text) if return_string else date_obg +def validate_date(date_text, method, return_as=None): + try: date_obg = datetime.strptime(str(date_text), "%Y-%m-%d" if "-" in str(date_text) else "%m/%d/%Y") + except ValueError: raise Failed(f"Collection Error: {method}: {date_text} must match pattern YYYY-MM-DD (e.g. 2020-12-25) or MM/DD/YYYY (e.g. 12/25/2020)") + return datetime.strftime(date_obg, return_as) if return_as else date_obg def logger_input(prompt, timeout=60): if windows: return windows_input(prompt, timeout) From dab8600e842a28ebc4b83ee750cabda26516fd87 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 21 Jul 2021 15:50:28 -0400 Subject: [PATCH 24/95] fixed method error --- modules/builder.py | 2 +- modules/tmdb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 24403caf..991ea965 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1458,7 +1458,7 @@ class CollectionBuilder: elif modifier in [".before", ".after"]: if current_data is None: return False - filter_date = util.validate_date(filter_data) + filter_date = util.validate_date(filter_data, filter_final) if (modifier == ".before" and current_data >= filter_date) or (modifier == ".after" and current_data <= filter_date): return False elif filter_attr in ["release", "added", "last_played"] and modifier == ".regex": diff --git a/modules/tmdb.py b/modules/tmdb.py index ea0278c3..a3065403 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -253,7 +253,7 @@ class TMDb: count = 0 for date_attr in discover_dates: if date_attr in attrs: - attrs[date_attr] = util.validate_date(attrs[date_attr], return_as="%Y-%m-%d") + attrs[date_attr] = util.validate_date(attrs[date_attr], f"tmdb_discover attribute {date_attr}", return_as="%Y-%m-%d") self.Discover.discover_movies(attrs) if is_movie else self.Discover.discover_tv_shows(attrs) total_pages = int(self.TMDb.total_pages) total_results = int(self.TMDb.total_results) From c3a8d788f3e2e6f23a77834d26daf543127f5037 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 21 Jul 2021 16:59:27 -0400 Subject: [PATCH 25/95] organizing --- modules/builder.py | 403 +++++++++++++++++++++++++-------------------- modules/plex.py | 12 ++ 2 files changed, 232 insertions(+), 183 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 991ea965..d465107e 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -163,10 +163,18 @@ ignored_details = [ "tmdb_person", "build_collection" ] +details = [ + "collection_mode", "collection_order", "label" +] + boolean_details + string_details collectionless_details = [ "collection_order", "plex_collectionless", "label", "label_sync_mode", "test" ] + poster_details + background_details + summary_details + string_details +item_details = [ + "item_label", "item_radarr_tag", "item_sonarr_tag", "item_overlay" +] + plex.item_advance_keys +radarr_details = ["radarr_add", "radarr_folder", "radarr_monitor", "radarr_search", "radarr_availability", "radarr_quality", "radarr_tag"] +sonarr_details = ["sonarr_add", "sonarr_folder", "sonarr_monitor", "sonarr_language", "sonarr_series", "sonarr_quality", "sonarr_season", "sonarr_search", "sonarr_cutoff_search", "sonarr_tag"] all_filters = [ "actor", "actor.not", "audio_language", "audio_language.not", @@ -216,6 +224,7 @@ class CollectionBuilder: self.metadata = metadata self.name = name self.data = data + self.language = self.library.Plex.language self.details = { "show_filtered": self.library.show_filtered, "show_missing": self.library.show_missing, @@ -381,7 +390,7 @@ class CollectionBuilder: try: if 0 <= int(param) <= 23: self.schedule += f"\nScheduled to run only on the {util.make_ordinal(param)} hour" - if config.run_hour == int(param): + if self.config.run_hour == int(param): skip_collection = False else: raise ValueError @@ -467,12 +476,12 @@ class CollectionBuilder: logger.debug(f"Value: {self.data[methods['tmdb_person']]}") valid_names = [] for tmdb_id in util.get_int_list(self.data[methods["tmdb_person"]], "TMDb Person ID"): - person = config.TMDb.get_person(tmdb_id) + person = self.config.TMDb.get_person(tmdb_id) valid_names.append(person.name) if hasattr(person, "biography") and person.biography: self.summaries["tmdb_person"] = person.biography if hasattr(person, "profile_path") and person.profile_path: - self.posters["tmdb_person"] = f"{config.TMDb.image_url}{person.profile_path}" + self.posters["tmdb_person"] = f"{self.config.TMDb.image_url}{person.profile_path}" if len(valid_names) > 0: self.details["tmdb_person"] = valid_names else: @@ -534,11 +543,11 @@ class CollectionBuilder: continue logger.info("") logger.info(f"Validating Method: {method_key}") - if "trakt" in method_key.lower() and not config.Trakt: raise Failed(f"Collection Error: {method_key} requires Trakt to be configured") + if "trakt" in method_key.lower() and not self.config.Trakt: raise Failed(f"Collection Error: {method_key} requires Trakt to be configured") elif "radarr" in method_key.lower() and not self.library.Radarr: raise Failed(f"Collection Error: {method_key} requires Radarr to be configured") elif "sonarr" in method_key.lower() and not self.library.Sonarr: raise Failed(f"Collection Error: {method_key} requires Sonarr to be configured") elif "tautulli" in method_key.lower() and not self.library.Tautulli: raise Failed(f"Collection Error: {method_key} requires Tautulli to be configured") - elif "mal" in method_key.lower() and not config.MyAnimeList: raise Failed(f"Collection Error: {method_key} requires MyAnimeList to be configured") + elif "mal" in method_key.lower() and not self.config.MyAnimeList: raise Failed(f"Collection Error: {method_key} requires MyAnimeList to be configured") elif method_data is not None: logger.debug(f"Value: {method_data}") method_name, method_mod, method_final = self._split(method_key) @@ -556,150 +565,22 @@ class CollectionBuilder: raise Failed(f"Collection Error: {method_name} attribute does not work for Collectionless collection") elif self.smart_url and (method_name in all_builders or method_name in smart_url_collection_invalid): raise Failed(f"Collection Error: {method_name} builder not allowed when using smart_filter") - elif method_name == "summary": - self.summaries[method_name] = method_data - elif method_name == "tmdb_summary": - self.summaries[method_name] = config.TMDb.get_movie_show_or_collection(util.regex_first_int(method_data, "TMDb ID"), self.library.is_movie).overview - elif method_name == "tmdb_description": - self.summaries[method_name] = config.TMDb.get_list(util.regex_first_int(method_data, "TMDb List ID")).description - elif method_name == "tmdb_biography": - self.summaries[method_name] = config.TMDb.get_person(util.regex_first_int(method_data, "TMDb Person ID")).biography - elif method_name == "tvdb_summary": - self.summaries[method_name] = config.TVDb.get_movie_or_show(method_data, self.library.Plex.language, self.library.is_movie).summary - elif method_name == "tvdb_description": - self.summaries[method_name] = config.TVDb.get_list_description(method_data, self.library.Plex.language) - elif method_name == "trakt_description": - self.summaries[method_name] = config.Trakt.list_description(config.Trakt.validate_trakt(method_data, self.library.is_movie)[0]) - elif method_name == "letterboxd_description": - self.summaries[method_name] = config.Letterboxd.get_list_description(method_data, self.library.Plex.language) - elif method_name == "icheckmovies_description": - self.summaries[method_name] = config.ICheckMovies.get_list_description(method_data, self.library.Plex.language) - elif method_name == "collection_mode": - if str(method_data).lower() == "default": - self.details[method_name] = "default" - elif str(method_data).lower() == "hide": - self.details[method_name] = "hide" - elif str(method_data).lower() in ["hide_items", "hideitems"]: - self.details[method_name] = "hideItems" - elif str(method_data).lower() in ["show_items", "showitems"]: - self.details[method_name] = "showItems" - else: - raise Failed(f"Collection Error: {method_data} collection_mode invalid\n\tdefault (Library default)\n\thide (Hide Collection)\n\thide_items (Hide Items in this Collection)\n\tshow_items (Show this Collection and its Items)") - elif method_name == "collection_order": - if str(method_data).lower() == "release": - self.details[method_name] = "release" - elif str(method_data).lower() == "alpha": - self.details[method_name] = "alpha" - else: - raise Failed(f"Collection Error: {method_data} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)") - elif method_name == "url_poster": - self.posters[method_name] = method_data - elif method_name == "tmdb_poster": - self.posters[method_name] = f"{config.TMDb.image_url}{config.TMDb.get_movie_show_or_collection(util.regex_first_int(method_data, 'TMDb ID'), self.library.is_movie).poster_path}" - elif method_name == "tmdb_profile": - self.posters[method_name] = f"{config.TMDb.image_url}{config.TMDb.get_person(util.regex_first_int(method_data, 'TMDb Person ID')).profile_path}" - elif method_name == "tvdb_poster": - self.posters[method_name] = f"{config.TVDb.get_movie_or_series(method_data, self.library.Plex.language, self.library.is_movie).poster_path}" - elif method_name == "file_poster": - if os.path.exists(method_data): - self.posters[method_name] = os.path.abspath(method_data) - else: - raise Failed(f"Collection Error: Poster Path Does Not Exist: {os.path.abspath(method_data)}") - elif method_name == "url_background": - self.backgrounds[method_name] = method_data - elif method_name == "tmdb_background": - self.backgrounds[method_name] = f"{config.TMDb.image_url}{config.TMDb.get_movie_show_or_collection(util.regex_first_int(method_data, 'TMDb ID'), self.library.is_movie).poster_path}" - elif method_name == "tvdb_background": - self.posters[method_name] = f"{config.TVDb.get_movie_or_series(method_data, self.library.Plex.language, self.library.is_movie).background_path}" - elif method_name == "file_background": - if os.path.exists(method_data): self.backgrounds[method_name] = os.path.abspath(method_data) - else: raise Failed(f"Collection Error: Background Path Does Not Exist: {os.path.abspath(method_data)}") - elif method_name == "label": - if "label" in methods and "label.sync" in methods: - raise Failed("Collection Error: Cannot use label and label.sync together") - if "label.remove" in methods and "label.sync" in methods: - raise Failed("Collection Error: Cannot use label.remove and label.sync together") - if method_final == "label" and "label_sync_mode" in methods and self.data[methods["label_sync_mode"]] == "sync": - self.details["label.sync"] = util.get_list(method_data) - else: - self.details[method_final] = util.get_list(method_data) - elif method_name == "item_label": - if "item_label" in methods and "item_label.sync" in methods: - raise Failed(f"Collection Error: Cannot use item_label and item_label.sync together") - if "item_label.remove" in methods and "item_label.sync" in methods: - raise Failed(f"Collection Error: Cannot use item_label.remove and item_label.sync together") - self.item_details[method_final] = util.get_list(method_data) - elif method_name in ["item_radarr_tag", "item_sonarr_tag"]: - if method_name in methods and f"{method_name}.sync" in methods: - raise Failed(f"Collection Error: Cannot use {method_name} and {method_name}.sync together") - if f"{method_name}.remove" in methods and f"{method_name}.sync" in methods: - raise Failed(f"Collection Error: Cannot use {method_name}.remove and {method_name}.sync together") - if method_name in methods and f"{method_name}.remove" in methods: - raise Failed(f"Collection Error: Cannot use {method_name} and {method_name}.remove together") - self.item_details[method_name] = util.get_list(method_data) - self.item_details["apply_tags"] = method_mod[1:] if method_mod else "" - elif method_name == "item_overlay": - overlay = os.path.join(config.default_dir, "overlays", method_data, "overlay.png") - if not os.path.exists(overlay): - raise Failed(f"Collection Error: {method_data} overlay image not found at {overlay}") - if method_data in self.library.overlays: - raise Failed("Each Overlay can only be used once per Library") - self.library.overlays.append(method_data) - self.item_details[method_name] = method_data - elif method_name in plex.item_advance_keys: - key, options = plex.item_advance_keys[method_name] - if method_name in advance_new_agent and self.library.agent not in plex.new_plex_agents: - logger.error(f"Metadata Error: {method_name} attribute only works for with the New Plex Movie Agent and New Plex TV Agent") - elif method_name in advance_show and not self.library.is_show: - logger.error(f"Metadata Error: {method_name} attribute only works for show libraries") - elif str(method_data).lower() not in options: - logger.error(f"Metadata Error: {method_data} {method_name} attribute invalid") - else: - self.item_details[method_name] = str(method_data).lower() - elif method_name in boolean_details: - self.details[method_name] = util.get_bool(method_name, method_data) - elif method_name in string_details: - self.details[method_name] = str(method_data) - elif method_name == "radarr_add": - self.add_to_radarr = util.get_bool(method_name, method_data) - elif method_name == "radarr_folder": - self.radarr_options["folder"] = method_data - elif method_name in ["radarr_monitor", "radarr_search"]: - self.radarr_options[method_name[7:]] = util.get_bool(method_name, method_data) - elif method_name == "radarr_availability": - if str(method_data).lower() in radarr.availability_translation: - self.radarr_options["availability"] = str(method_data).lower() - else: - raise Failed(f"Collection Error: {method_name} attribute must be either announced, cinemas, released or db") - elif method_name == "radarr_quality": - self.library.Radarr.get_profile_id(method_data) - self.radarr_options["quality"] = method_data - elif method_name == "radarr_tag": - self.radarr_options["tag"] = util.get_list(method_data) - elif method_name == "sonarr_add": - self.add_to_sonarr = util.get_bool(method_name, method_data) - elif method_name == "sonarr_folder": - self.sonarr_options["folder"] = method_data - elif method_name == "sonarr_monitor": - if str(method_data).lower() in sonarr.monitor_translation: - self.sonarr_options["monitor"] = str(method_data).lower() - else: - raise Failed(f"Collection Error: {method_name} attribute must be either all, future, missing, existing, pilot, first, latest or none") - elif method_name == "sonarr_quality": - self.library.Sonarr.get_profile_id(method_data, "quality_profile") - self.sonarr_options["quality"] = method_data - elif method_name == "sonarr_language": - self.library.Sonarr.get_profile_id(method_data, "language_profile") - self.sonarr_options["language"] = method_data - elif method_name == "sonarr_series": - if str(method_data).lower() in sonarr.series_type: - self.sonarr_options["series"] = str(method_data).lower() - else: - raise Failed(f"Collection Error: {method_name} attribute must be either standard, daily, or anime") - elif method_name in ["sonarr_season", "sonarr_search", "sonarr_cutoff_search"]: - self.sonarr_options[method_name[7:]] = util.get_bool(method_name, method_data) - elif method_name == "sonarr_tag": - self.sonarr_options["tag"] = util.get_list(method_data) + elif method_name in summary_details: + self._summary(method_name, method_data) + elif method_name in poster_details: + self._poster(method_name, method_data) + elif method_name in background_details: + self._background(method_name, method_data) + elif method_name in details: + self._details(method_name, method_data, method_final, methods) + elif method_name in details: + self._details(method_name, method_data, method_final, methods) + elif method_name in item_details: + self._item_details(method_name, method_data, method_mod, method_final, methods) + elif method_name in radarr_details: + self._radarr(method_name, method_data) + elif method_name in sonarr_details: + self._sonarr(method_name, method_data) elif method_final in plex.searches: self.methods.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) elif method_name == "plex_all": @@ -715,41 +596,41 @@ class CollectionBuilder: for mal_id in util.get_int_list(method_data, "MyAnimeList ID"): self.methods.append((method_name, mal_id)) elif method_name in ["anidb_id", "anidb_relation"]: - for anidb_id in config.AniDB.validate_anidb_ids(method_data, self.library.Plex.language): + for anidb_id in self.config.AniDB.validate_anidb_ids(method_data, self.language): self.methods.append((method_name, anidb_id)) elif method_name in ["anilist_id", "anilist_relations", "anilist_studio"]: - for anilist_id in config.AniList.validate_anilist_ids(method_data, studio=method_name == "anilist_studio"): + for anilist_id in self.config.AniList.validate_anilist_ids(method_data, studio=method_name == "anilist_studio"): self.methods.append((method_name, anilist_id)) elif method_name == "trakt_list": - for trakt_list in config.Trakt.validate_trakt(method_data, self.library.is_movie): + for trakt_list in self.config.Trakt.validate_trakt(method_data, self.library.is_movie): self.methods.append((method_name, trakt_list)) elif method_name == "trakt_list_details": - trakt_lists = config.Trakt.validate_trakt(method_data, self.library.is_movie) - self.summaries[method_name] = config.Trakt.list_description(trakt_lists[0]) + trakt_lists = self.config.Trakt.validate_trakt(method_data, self.library.is_movie) + self.summaries[method_name] = self.config.Trakt.list_description(trakt_lists[0]) for trakt_list in trakt_lists: self.methods.append((method_name[:-8], trakt_list)) elif method_name in ["trakt_watchlist", "trakt_collection"]: - for trakt_list in config.Trakt.validate_trakt(method_data, self.library.is_movie, trakt_type=method_name[6:]): + for trakt_list in self.config.Trakt.validate_trakt(method_data, self.library.is_movie, trakt_type=method_name[6:]): self.methods.append((method_name, trakt_list)) elif method_name == "imdb_list": - for imdb_dict in self.config.IMDb.validate_imdb_lists(method_data, self.library.Plex.language): + for imdb_dict in self.config.IMDb.validate_imdb_lists(method_data, self.language): self.methods.append((method_name, imdb_dict)) elif method_name == "icheckmovies_list": - for icheckmovies_list in self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.library.Plex.language): + for icheckmovies_list in self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.language): self.methods.append((method_name, icheckmovies_list)) elif method_name == "icheckmovies_list_details": - icheckmovies_lists = self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.library.Plex.language) + icheckmovies_lists = self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.language) for icheckmovies_list in icheckmovies_lists: self.methods.append((method_name[:-8], icheckmovies_list)) - self.summaries[method_name] = self.config.ICheckMovies.get_list_description(icheckmovies_lists[0], self.library.Plex.language) + self.summaries[method_name] = self.config.ICheckMovies.get_list_description(icheckmovies_lists[0], self.language) elif method_name == "letterboxd_list": - for letterboxd_list in self.config.Letterboxd.validate_letterboxd_lists(method_data, self.library.Plex.language): + for letterboxd_list in self.config.Letterboxd.validate_letterboxd_lists(method_data, self.language): self.methods.append((method_name, letterboxd_list)) elif method_name == "letterboxd_list_details": - letterboxd_lists = self.config.Letterboxd.validate_letterboxd_lists(method_data, self.library.Plex.language) + letterboxd_lists = self.config.Letterboxd.validate_letterboxd_lists(method_data, self.language) for letterboxd_list in letterboxd_lists: self.methods.append((method_name[:-8], letterboxd_list)) - self.summaries[method_name] = config.Letterboxd.get_list_description(letterboxd_lists[0], self.library.Plex.language) + self.summaries[method_name] = self.config.Letterboxd.get_list_description(letterboxd_lists[0], self.language) elif method_name in dictionary_builders: for dict_data in util.get_list(method_data): if isinstance(dict_data, dict): @@ -1007,44 +888,44 @@ class CollectionBuilder: values = util.get_list(method_data) if method_name[-8:] == "_details": if method_name == "tvdb_movie_details": - item = config.TVDb.get_movie(self.library.Plex.language, values[0]) + item = self.config.TVDb.get_movie(self.language, values[0]) if hasattr(item, "description") and item.description: self.summaries[method_name] = item.description if hasattr(item, "background_path") and item.background_path: - self.backgrounds[method_name] = f"{config.TMDb.image_url}{item.background_path}" + self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.background_path}" if hasattr(item, "poster_path") and item.poster_path: - self.posters[method_name] = f"{config.TMDb.image_url}{item.poster_path}" + self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" elif method_name == "tvdb_show_details": - item = config.TVDb.get_series(self.library.Plex.language, values[0]) + item = self.config.TVDb.get_series(self.language, values[0]) if hasattr(item, "description") and item.description: self.summaries[method_name] = item.description if hasattr(item, "background_path") and item.background_path: - self.backgrounds[method_name] = f"{config.TMDb.image_url}{item.background_path}" + self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.background_path}" if hasattr(item, "poster_path") and item.poster_path: - self.posters[method_name] = f"{config.TMDb.image_url}{item.poster_path}" + self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" elif method_name == "tvdb_list_details": - self.summaries[method_name] = config.TVDb.get_list_description(values[0], self.library.Plex.language) + self.summaries[method_name] = self.config.TVDb.get_list_description(values[0], self.language) for value in values: self.methods.append((method_name[:-8] if method_name[-8:] == "_details" else method_name, value)) elif method_name in tmdb.builders: - values = config.TMDb.validate_tmdb_ids(method_data, method_name) + values = self.config.TMDb.validate_tmdb_ids(method_data, method_name) if method_name[-8:] == "_details": if method_name in ["tmdb_collection_details", "tmdb_movie_details", "tmdb_show_details"]: - item = config.TMDb.get_movie_show_or_collection(values[0], self.library.is_movie) + item = self.config.TMDb.get_movie_show_or_collection(values[0], self.library.is_movie) if hasattr(item, "overview") and item.overview: self.summaries[method_name] = item.overview if hasattr(item, "backdrop_path") and item.backdrop_path: - self.backgrounds[method_name] = f"{config.TMDb.image_url}{item.backdrop_path}" + self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.backdrop_path}" if hasattr(item, "poster_path") and item.poster_path: - self.posters[method_name] = f"{config.TMDb.image_url}{item.poster_path}" + self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" elif method_name in ["tmdb_actor_details", "tmdb_crew_details", "tmdb_director_details", "tmdb_producer_details", "tmdb_writer_details"]: - item = config.TMDb.get_person(values[0]) + item = self.config.TMDb.get_person(values[0]) if hasattr(item, "biography") and item.biography: self.summaries[method_name] = item.biography if hasattr(item, "profile_path") and item.profile_path: - self.posters[method_name] = f"{config.TMDb.image_url}{item.profile_path}" + self.posters[method_name] = f"{self.config.TMDb.image_url}{item.profile_path}" else: - item = config.TMDb.get_list(values[0]) + item = self.config.TMDb.get_list(values[0]) if hasattr(item, "description") and item.description: self.summaries[method_name] = item.description for value in values: @@ -1096,6 +977,162 @@ class CollectionBuilder: logger.info("") logger.info("Validation Successful") + def _summary(self, method_name, method_data): + if method_name == "summary": + self.summaries[method_name] = method_data + elif method_name == "tmdb_summary": + self.summaries[method_name] = self.config.TMDb.get_movie_show_or_collection(util.regex_first_int(method_data, "TMDb ID"), self.library.is_movie).overview + elif method_name == "tmdb_description": + self.summaries[method_name] = self.config.TMDb.get_list(util.regex_first_int(method_data, "TMDb List ID")).description + elif method_name == "tmdb_biography": + self.summaries[method_name] = self.config.TMDb.get_person(util.regex_first_int(method_data, "TMDb Person ID")).biography + elif method_name == "tvdb_summary": + self.summaries[method_name] = self.config.TVDb.get_movie_or_show(method_data, self.language, self.library.is_movie).summary + elif method_name == "tvdb_description": + self.summaries[method_name] = self.config.TVDb.get_list_description(method_data, self.language) + elif method_name == "trakt_description": + self.summaries[method_name] = self.config.Trakt.list_description(self.config.Trakt.validate_trakt(method_data, self.library.is_movie)[0]) + elif method_name == "letterboxd_description": + self.summaries[method_name] = self.config.Letterboxd.get_list_description(method_data, self.language) + elif method_name == "icheckmovies_description": + self.summaries[method_name] = self.config.ICheckMovies.get_list_description(method_data, self.language) + + def _poster(self, method_name, method_data): + if method_name == "url_poster": + self.posters[method_name] = method_data + elif method_name == "tmdb_poster": + url_slug = self.config.TMDb.get_movie_show_or_collection(util.regex_first_int(method_data, 'TMDb ID'), self.library.is_movie).poster_path + self.posters[method_name] = f"{self.config.TMDb.image_url}{url_slug}" + elif method_name == "tmdb_profile": + url_slug = self.config.TMDb.get_person(util.regex_first_int(method_data, 'TMDb Person ID')).profile_path + self.posters[method_name] = f"{self.config.TMDb.image_url}{url_slug}" + elif method_name == "tvdb_poster": + self.posters[method_name] = f"{self.config.TVDb.get_movie_or_series(method_data, self.language, self.library.is_movie).poster_path}" + elif method_name == "file_poster": + if os.path.exists(method_data): + self.posters[method_name] = os.path.abspath(method_data) + else: + raise Failed(f"Collection Error: Poster Path Does Not Exist: {os.path.abspath(method_data)}") + + def _background(self, method_name, method_data): + if method_name == "url_background": + self.backgrounds[method_name] = method_data + elif method_name == "tmdb_background": + url_slug = self.config.TMDb.get_movie_show_or_collection(util.regex_first_int(method_data, 'TMDb ID'), self.library.is_movie).poster_path + self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{url_slug}" + elif method_name == "tvdb_background": + self.posters[method_name] = f"{self.config.TVDb.get_movie_or_series(method_data, self.language, self.library.is_movie).background_path}" + elif method_name == "file_background": + if os.path.exists(method_data): + self.backgrounds[method_name] = os.path.abspath(method_data) + else: + raise Failed(f"Collection Error: Background Path Does Not Exist: {os.path.abspath(method_data)}") + + def _details(self, method_name, method_data, method_final, methods): + if method_name == "collection_mode": + if str(method_data).lower() in plex.collection_mode_options: + self.details[method_name] = plex.collection_mode_options[str(method_data).lower()] + else: + raise Failed(f"Collection Error: {method_data} collection_mode invalid\n\tdefault (Library default)\n\thide (Hide Collection)\n\thide_items (Hide Items in this Collection)\n\tshow_items (Show this Collection and its Items)") + elif method_name == "collection_order": + if str(method_data).lower() in plex.collection_order_options: + self.details[method_name] = plex.collection_order_options[str(method_data).lower()] + else: + raise Failed(f"Collection Error: {method_data} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)") + elif method_name == "label": + if "label" in methods and "label.sync" in methods: + raise Failed("Collection Error: Cannot use label and label.sync together") + if "label.remove" in methods and "label.sync" in methods: + raise Failed("Collection Error: Cannot use label.remove and label.sync together") + if method_final == "label" and "label_sync_mode" in methods and self.data[methods["label_sync_mode"]] == "sync": + self.details["label.sync"] = util.get_list(method_data) + else: + self.details[method_final] = util.get_list(method_data) + elif method_name in boolean_details: + self.details[method_name] = util.get_bool(method_name, method_data) + elif method_name in string_details: + self.details[method_name] = str(method_data) + + def _item_details(self, method_name, method_data, method_mod, method_final, methods): + if method_name == "item_label": + if "item_label" in methods and "item_label.sync" in methods: + raise Failed(f"Collection Error: Cannot use item_label and item_label.sync together") + if "item_label.remove" in methods and "item_label.sync" in methods: + raise Failed(f"Collection Error: Cannot use item_label.remove and item_label.sync together") + self.item_details[method_final] = util.get_list(method_data) + elif method_name in ["item_radarr_tag", "item_sonarr_tag"]: + if method_name in methods and f"{method_name}.sync" in methods: + raise Failed(f"Collection Error: Cannot use {method_name} and {method_name}.sync together") + if f"{method_name}.remove" in methods and f"{method_name}.sync" in methods: + raise Failed(f"Collection Error: Cannot use {method_name}.remove and {method_name}.sync together") + if method_name in methods and f"{method_name}.remove" in methods: + raise Failed(f"Collection Error: Cannot use {method_name} and {method_name}.remove together") + self.item_details[method_name] = util.get_list(method_data) + self.item_details["apply_tags"] = method_mod[1:] if method_mod else "" + elif method_name == "item_overlay": + overlay = os.path.join(self.config.default_dir, "overlays", method_data, "overlay.png") + if not os.path.exists(overlay): + raise Failed(f"Collection Error: {method_data} overlay image not found at {overlay}") + if method_data in self.library.overlays: + raise Failed("Each Overlay can only be used once per Library") + self.library.overlays.append(method_data) + self.item_details[method_name] = method_data + elif method_name in plex.item_advance_keys: + key, options = plex.item_advance_keys[method_name] + if method_name in advance_new_agent and self.library.agent not in plex.new_plex_agents: + logger.error( + f"Metadata Error: {method_name} attribute only works for with the New Plex Movie Agent and New Plex TV Agent") + elif method_name in advance_show and not self.library.is_show: + logger.error(f"Metadata Error: {method_name} attribute only works for show libraries") + elif str(method_data).lower() not in options: + logger.error(f"Metadata Error: {method_data} {method_name} attribute invalid") + else: + self.item_details[method_name] = str(method_data).lower() + + def _radarr(self, method_name, method_data): + if method_name == "radarr_add": + self.add_to_radarr = util.get_bool(method_name, method_data) + elif method_name == "radarr_folder": + self.radarr_options["folder"] = method_data + elif method_name in ["radarr_monitor", "radarr_search"]: + self.radarr_options[method_name[7:]] = util.get_bool(method_name, method_data) + elif method_name == "radarr_availability": + if str(method_data).lower() in radarr.availability_translation: + self.radarr_options["availability"] = str(method_data).lower() + else: + raise Failed(f"Collection Error: {method_name} attribute must be either announced, cinemas, released or db") + elif method_name == "radarr_quality": + self.library.Radarr.get_profile_id(method_data) + self.radarr_options["quality"] = method_data + elif method_name == "radarr_tag": + self.radarr_options["tag"] = util.get_list(method_data) + + def _sonarr(self, method_name, method_data): + if method_name == "sonarr_add": + self.add_to_sonarr = util.get_bool(method_name, method_data) + elif method_name == "sonarr_folder": + self.sonarr_options["folder"] = method_data + elif method_name == "sonarr_monitor": + if str(method_data).lower() in sonarr.monitor_translation: + self.sonarr_options["monitor"] = str(method_data).lower() + else: + raise Failed(f"Collection Error: {method_name} attribute must be either all, future, missing, existing, pilot, first, latest or none") + elif method_name == "sonarr_quality": + self.library.Sonarr.get_profile_id(method_data, "quality_profile") + self.sonarr_options["quality"] = method_data + elif method_name == "sonarr_language": + self.library.Sonarr.get_profile_id(method_data, "language_profile") + self.sonarr_options["language"] = method_data + elif method_name == "sonarr_series": + if str(method_data).lower() in sonarr.series_type: + self.sonarr_options["series"] = str(method_data).lower() + else: + raise Failed(f"Collection Error: {method_name} attribute must be either standard, daily, or anime") + elif method_name in ["sonarr_season", "sonarr_search", "sonarr_cutoff_search"]: + self.sonarr_options[method_name[7:]] = util.get_bool(method_name, method_data) + elif method_name == "sonarr_tag": + self.sonarr_options["tag"] = util.get_list(method_data) + def collect_rating_keys(self): filtered_keys = {} name = self.obj.title if self.obj else self.name @@ -1150,13 +1187,13 @@ class CollectionBuilder: logger.info("") if "plex" in method: add_rating_keys(self.library.get_items(method, value)) elif "tautulli" in method: add_rating_keys(self.library.Tautulli.get_items(self.library, value)) - elif "anidb" in method: check_map(self.config.AniDB.get_items(method, value, self.library.Plex.language)) + elif "anidb" in method: check_map(self.config.AniDB.get_items(method, value, self.language)) elif "anilist" in method: check_map(self.config.AniList.get_items(method, value)) elif "mal" in method: check_map(self.config.MyAnimeList.get_items(method, value)) - elif "tvdb" in method: check_map(self.config.TVDb.get_items(method, value, self.library.Plex.language)) - elif "imdb" in method: check_map(self.config.IMDb.get_items(method, value, self.library.Plex.language, self.library.is_movie)) - elif "icheckmovies" in method: check_map(self.config.ICheckMovies.get_items(method, value, self.library.Plex.language)) - elif "letterboxd" in method: check_map(self.config.Letterboxd.get_items(method, value, self.library.Plex.language)) + elif "tvdb" in method: check_map(self.config.TVDb.get_items(method, value, self.language)) + elif "imdb" in method: check_map(self.config.IMDb.get_items(method, value, self.language, self.library.is_movie)) + elif "icheckmovies" in method: check_map(self.config.ICheckMovies.get_items(method, value, self.language)) + elif "letterboxd" in method: check_map(self.config.Letterboxd.get_items(method, value, self.language)) elif "tmdb" in method: check_map(self.config.TMDb.get_items(method, value, self.library.is_movie)) elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) else: logger.error(f"Collection Error: {method} method not supported") @@ -1642,7 +1679,7 @@ class CollectionBuilder: missing_shows_with_names = [] for missing_id in self.missing_shows: try: - title = str(self.config.TVDb.get_series(self.library.Plex.language, missing_id).title.encode("ascii", "replace").decode()) + title = str(self.config.TVDb.get_series(self.language, missing_id).title.encode("ascii", "replace").decode()) except Failed as e: logger.error(e) continue @@ -1968,7 +2005,7 @@ class CollectionBuilder: for missing_id in self.run_again_shows: if missing_id not in self.library.show_map: try: - title = str(self.config.TVDb.get_series(self.library.Plex.language, missing_id).title.encode("ascii", "replace").decode()) + title = str(self.config.TVDb.get_series(self.language, missing_id).title.encode("ascii", "replace").decode()) except Failed as e: logger.error(e) continue diff --git a/modules/plex.py b/modules/plex.py index 511602c9..cf23b896 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -57,6 +57,18 @@ plex_languages = ["default", "ar-SA", "ca-ES", "cs-CZ", "da-DK", "de-DE", "el-GR metadata_language_options = {lang.lower(): lang for lang in plex_languages} metadata_language_options["default"] = None use_original_title_options = {"default": -1, "no": 0, "yes": 1} +collection_mode_options = { + "default": "default", + "hide": "hide", + "hide_items": "hideItems", + "hideitems": "hideItems", + "show_items": "showItems", + "showitems": "showItems" +} +collection_order_options = { + "release": "release", + "alpha": "alpha" +} collection_mode_keys = {-1: "default", 0: "hide", 1: "hideItems", 2: "showItems"} collection_order_keys = {0: "release", 1: "alpha", 2: "custom"} item_advance_keys = { From 10b0cf8a6d33cf7658158c45b72dc2c339d875be Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 22 Jul 2021 08:56:58 -0400 Subject: [PATCH 26/95] fixed dict and table errors --- modules/builder.py | 6 ++++-- modules/plex.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index d465107e..a13f8431 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -171,8 +171,10 @@ collectionless_details = [ "label", "label_sync_mode", "test" ] + poster_details + background_details + summary_details + string_details item_details = [ - "item_label", "item_radarr_tag", "item_sonarr_tag", "item_overlay" -] + plex.item_advance_keys + "item_label", "item_radarr_tag", "item_sonarr_tag", "item_overlay", + "item_episode_sorting", "item_keep_episodes", "item_delete_episodes", "item_season_display", + "item_episode_ordering", "item_metadata_language", "item_use_original_title" +] radarr_details = ["radarr_add", "radarr_folder", "radarr_monitor", "radarr_search", "radarr_availability", "radarr_quality", "radarr_tag"] sonarr_details = ["sonarr_add", "sonarr_folder", "sonarr_monitor", "sonarr_language", "sonarr_series", "sonarr_quality", "sonarr_season", "sonarr_search", "sonarr_cutoff_search", "sonarr_tag"] all_filters = [ diff --git a/modules/plex.py b/modules/plex.py index cf23b896..3cc0ccb4 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -492,7 +492,7 @@ class Plex: try: image = None if self.config.Cache: - image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, f"{self.image_table_name}_background") + image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, f"{self.image_table_name}_backgrounds") if str(background.compare) != str(image_compare): image = None if image is None or image != item.art: From 992c5444d9791f884aec30ca44c7049d3efd79da Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 22 Jul 2021 13:44:34 -0400 Subject: [PATCH 27/95] added dev version --- plex_meta_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plex_meta_manager.py b/plex_meta_manager.py index a04d16c6..cd477c86 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -105,7 +105,7 @@ def start(config_path, is_test=False, time_scheduled=None, requested_collections logger.info(util.centered("| __/| | __/> < | | | | __/ || (_| | | | | | (_| | | | | (_| | (_| | __/ | ")) logger.info(util.centered("|_| |_|\\___/_/\\_\\ |_| |_|\\___|\\__\\__,_| |_| |_|\\__,_|_| |_|\\__,_|\\__, |\\___|_| ")) logger.info(util.centered(" |___/ ")) - logger.info(util.centered(" Version: 1.11.3 ")) + logger.info(util.centered(" Version: 1.11.3-beta1 ")) if time_scheduled: start_type = f"{time_scheduled} " elif is_test: start_type = "Test " elif requested_collections: start_type = "Collections " From 527bc7cf723e9e7f3c354452205be3b3c56d8965 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 22 Jul 2021 13:57:23 -0400 Subject: [PATCH 28/95] letterboxd fix --- modules/letterboxd.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/letterboxd.py b/modules/letterboxd.py index 8f6ec302..d936546f 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -44,9 +44,10 @@ class Letterboxd: list_url = letterboxd_list.strip() if not list_url.startswith(base_url): raise Failed(f"Letterboxd Error: {list_url} must begin with: {base_url}") - if len(self._parse_list(list_url, language)) > 0: + elif len(self._parse_list(list_url, language)) > 0: valid_lists.append(list_url) - raise Failed(f"Letterboxd Error: {list_url} failed to parse") + else: + raise Failed(f"Letterboxd Error: {list_url} failed to parse") return valid_lists def get_items(self, method, data, language): From fcb52d3b9c4522d89a7dcb5708331e323646a2be Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 22 Jul 2021 17:00:45 -0400 Subject: [PATCH 29/95] more organizing --- modules/builder.py | 139 ++++++++++++++++++++------------------------- modules/util.py | 48 ++++++++++++++++ 2 files changed, 109 insertions(+), 78 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index a13f8431..4a5ec58f 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -583,26 +583,24 @@ class CollectionBuilder: self._radarr(method_name, method_data) elif method_name in sonarr_details: self._sonarr(method_name, method_data) + elif method_name in anidb.builders: + self._anidb(method_name, method_data) + elif method_name in anilist.builders: + self._anilist(method_name, method_data) + elif method_name in icheckmovies.builders: + self._icheckmovies(method_name, method_data) + + + + + elif method_final in plex.searches: self.methods.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) elif method_name == "plex_all": self.methods.append((method_name, True)) - elif method_name == "anidb_popular": - list_count = util.regex_first_int(method_data, "List Size", default=40) - if 1 <= list_count <= 30: - self.methods.append((method_name, list_count)) - else: - logger.warning("Collection Error: anidb_popular must be an integer between 1 and 30 defaulting to 30") - self.methods.append((method_name, 30)) elif method_name == "mal_id": for mal_id in util.get_int_list(method_data, "MyAnimeList ID"): self.methods.append((method_name, mal_id)) - elif method_name in ["anidb_id", "anidb_relation"]: - for anidb_id in self.config.AniDB.validate_anidb_ids(method_data, self.language): - self.methods.append((method_name, anidb_id)) - elif method_name in ["anilist_id", "anilist_relations", "anilist_studio"]: - for anilist_id in self.config.AniList.validate_anilist_ids(method_data, studio=method_name == "anilist_studio"): - self.methods.append((method_name, anilist_id)) elif method_name == "trakt_list": for trakt_list in self.config.Trakt.validate_trakt(method_data, self.library.is_movie): self.methods.append((method_name, trakt_list)) @@ -617,14 +615,6 @@ class CollectionBuilder: elif method_name == "imdb_list": for imdb_dict in self.config.IMDb.validate_imdb_lists(method_data, self.language): self.methods.append((method_name, imdb_dict)) - elif method_name == "icheckmovies_list": - for icheckmovies_list in self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.language): - self.methods.append((method_name, icheckmovies_list)) - elif method_name == "icheckmovies_list_details": - icheckmovies_lists = self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.language) - for icheckmovies_list in icheckmovies_lists: - self.methods.append((method_name[:-8], icheckmovies_list)) - self.summaries[method_name] = self.config.ICheckMovies.get_list_description(icheckmovies_lists[0], self.language) elif method_name == "letterboxd_list": for letterboxd_list in self.config.Letterboxd.validate_letterboxd_lists(method_data, self.language): self.methods.append((method_name, letterboxd_list)) @@ -820,63 +810,6 @@ class CollectionBuilder: new_dictionary["sort_by"] = mal.userlist_sort[dict_data[dict_methods["sort_by"]]] new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 100, maximum=1000) - self.methods.append((method_name, new_dictionary)) - elif method_name == "anidb_tag": - new_dictionary = {} - dict_methods = {dm.lower(): dm for dm in dict_data} - if "tag" not in dict_methods: - raise Failed("Collection Error: anidb_tag tag attribute is required") - elif not dict_data[dict_methods["tag"]]: - raise Failed("Collection Error: anidb_tag tag attribute is blank") - else: - new_dictionary["tag"] = util.regex_first_int(dict_data[dict_methods["username"]], "AniDB Tag ID") - new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 0, minimum=0) - self.methods.append((method_name, new_dictionary)) - elif "anilist" in method_name: - new_dictionary = {"sort_by": "score"} - dict_methods = {dm.lower(): dm for dm in dict_data} - if method_name == "anilist_season": - if self.current_time.month in [12, 1, 2]: new_dictionary["season"] = "winter" - elif self.current_time.month in [3, 4, 5]: new_dictionary["season"] = "spring" - elif self.current_time.month in [6, 7, 8]: new_dictionary["season"] = "summer" - elif self.current_time.month in [9, 10, 11]: new_dictionary["season"] = "fall" - - if "season" not in dict_methods: - logger.warning(f"Collection Warning: anilist_season season attribute not found using the current season: {new_dictionary['season']} as default") - elif not dict_data[dict_methods["season"]]: - logger.warning(f"Collection Warning: anilist_season season attribute is blank using the current season: {new_dictionary['season']} as default") - elif dict_data[dict_methods["season"]] not in util.pretty_seasons: - logger.warning(f"Collection Warning: anilist_season season attribute {dict_data[dict_methods['season']]} invalid must be either 'winter', 'spring', 'summer' or 'fall' using the current season: {new_dictionary['season']} as default") - else: - new_dictionary["season"] = dict_data[dict_methods["season"]] - - new_dictionary["year"] = get_int(method_name, "year", dict_data, dict_methods, self.current_time.year, minimum=1917, maximum=self.current_time.year + 1) - elif method_name == "anilist_genre": - if "genre" not in dict_methods: - raise Failed(f"Collection Warning: anilist_genre genre attribute not found") - elif not dict_data[dict_methods["genre"]]: - raise Failed(f"Collection Warning: anilist_genre genre attribute is blank") - else: - new_dictionary["genre"] = self.config.AniList.validate_genre(dict_data[dict_methods["genre"]]) - elif method_name == "anilist_tag": - if "tag" not in dict_methods: - raise Failed(f"Collection Warning: anilist_tag tag attribute not found") - elif not dict_data[dict_methods["tag"]]: - raise Failed(f"Collection Warning: anilist_tag tag attribute is blank") - else: - new_dictionary["tag"] = self.config.AniList.validate_tag(dict_data[dict_methods["tag"]]) - - if "sort_by" not in dict_methods: - logger.warning(f"Collection Warning: {method_name} sort_by attribute not found using score as default") - elif not dict_data[dict_methods["sort_by"]]: - logger.warning(f"Collection Warning: {method_name} sort_by attribute is blank using score as default") - elif str(dict_data[dict_methods["sort_by"]]).lower() not in ["score", "popular"]: - logger.warning(f"Collection Warning: {method_name} sort_by attribute {dict_data[dict_methods['sort_by']]} invalid must be either 'score' or 'popular' using score as default") - else: - new_dictionary["sort_by"] = dict_data[dict_methods["sort_by"]] - - new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 0, maximum=500) - self.methods.append((method_name, new_dictionary)) else: raise Failed(f"Collection Error: {method_name} attribute is not a dictionary: {dict_data}") @@ -1135,6 +1068,56 @@ class CollectionBuilder: elif method_name == "sonarr_tag": self.sonarr_options["tag"] = util.get_list(method_data) + def _anidb(self, method_name, method_data): + if method_name == "anidb_popular": + self.methods.append((method_name, util.parse_int(method_name, method_data, 30, maximum=30))) + elif method_name in ["anidb_id", "anidb_relation"]: + for anidb_id in self.config.AniDB.validate_anidb_ids(method_data, self.language): + self.methods.append((method_name, anidb_id)) + elif method_name == "anidb_tag": + for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + new_dictionary = {} + if "tag" not in dict_methods: + raise Failed("Collection Error: anidb_tag tag attribute is required") + elif not dict_data[dict_methods["tag"]]: + raise Failed("Collection Error: anidb_tag tag attribute is blank") + else: + new_dictionary["tag"] = util.regex_first_int(dict_data[dict_methods["username"]], "AniDB Tag ID") + new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 0, minimum=0) + self.methods.append((method_name, new_dictionary)) + + def _anilist(self, method_name, method_data): + if method_name in ["anilist_id", "anilist_relations", "anilist_studio"]: + for anilist_id in self.config.AniList.validate_anilist_ids(method_data, studio=method_name == "anilist_studio"): + self.methods.append((method_name, anilist_id)) + elif method_name in ["anilist_popular", "anilist_top_rated"]: + self.methods.append((method_name, util.parse_int(method_name, method_data, 10))) + elif method_name in ["anilist_season", "anilist_genre", "anilist_tag"]: + for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + new_dictionary = {} + if method_name == "anilist_season": + if self.current_time.month in [12, 1, 2]: new_dictionary["season"] = "winter" + elif self.current_time.month in [3, 4, 5]: new_dictionary["season"] = "spring" + elif self.current_time.month in [6, 7, 8]: new_dictionary["season"] = "summer" + elif self.current_time.month in [9, 10, 11]: new_dictionary["season"] = "fall" + new_dictionary["season"] = util.parse_from_dict(method_name, "season", dict_data, dict_methods, default=new_dictionary["season"], options=["winter", "spring", "summer", "fall"]) + new_dictionary["year"] = util.parse_int_from_dict(method_name, "year", dict_data, dict_methods, self.current_time.year, minimum=1917, maximum=self.current_time.year + 1) + elif method_name == "anilist_genre": + new_dictionary["genre"] = self.config.AniList.validate_genre(util.parse_from_dict(method_name, "genre", dict_data, dict_methods)) + elif method_name == "anilist_tag": + new_dictionary["tag"] = self.config.AniList.validate_tag(util.parse_from_dict(method_name, "tag", dict_data, dict_methods)) + new_dictionary["sort_by"] = util.parse_from_dict(method_name, "sort_by", dict_data, dict_methods, default="score", options=["score", "popular"]) + new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 0, maximum=500) + self.methods.append((method_name, new_dictionary)) + + def _icheckmovies(self, method_name, method_data): + if method_name.startswith("icheckmovies_list"): + icheckmovies_lists = self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.language) + for icheckmovies_list in icheckmovies_lists: + self.methods.append(("icheckmovies_list", icheckmovies_list)) + if method_name.endswith("_details"): + self.summaries[method_name] = self.config.ICheckMovies.get_list_description(icheckmovies_lists[0], self.language) + def collect_rating_keys(self): filtered_keys = {} name = self.obj.title if self.obj else self.name diff --git a/modules/util.py b/modules/util.py index 984638a1..4f32ae42 100644 --- a/modules/util.py +++ b/modules/util.py @@ -438,3 +438,51 @@ def is_locked(filepath): if file_object: file_object.close() return locked + +def validate_dict_list(method_name, data): + final_list = [] + for dict_data in get_list(data): + if isinstance(dict_data, dict): + final_list.append((dict_data, {dm.lower(): dm for dm in dict_data})) + else: + raise Failed(f"Collection Error: {method_name} attribute is not a dictionary: {dict_data}") + return final_list + +def parse_int(method, data, default=10, minimum=1, maximum=None): + list_count = regex_first_int(data, "List Size", default=default) + if maximum is None and list_count < minimum: + logger.warning(f"Collection Warning: {method} must an integer >= {minimum} using {default} as default") + elif maximum is not None and (list_count < minimum or list_count > maximum): + logger.warning(f"Collection Warning: {method} must an integer between {minimum} and {maximum} using {default} as default") + else: + return list_count + return default + +def parse_from_dict(parent, method, data, methods, default=None, options=None): + message = "" + if method not in methods: + message = f"Collection Warning: {parent} {method} attribute not found" + elif data[methods[method]] is None: + message = f"Collection Warning: {parent} {method} attribute is blank" + elif options is not None and str(data[methods[method]]).lower() not in options: + message = f"Collection Warning: {parent} {method} attribute {data[methods[method]]} must be in {options}" + else: + return data[methods[method]] + if default is None: + raise Failed(message) + else: + logger.warning(f"{message} using {default} as default") + return default + +def parse_int_from_dict(parent, method, data, methods, default, minimum=1, maximum=None): + if method not in methods: + logger.warning(f"Collection Warning: {parent} {method} attribute not found using {default} as default") + elif not data[methods[method]]: + logger.warning(f"Collection Warning: {parent} {methods[method]} attribute is blank using {default} as default") + elif maximum is None and (not isinstance(data[methods[method]], int) or data[methods[method]] < minimum): + logger.warning(f"Collection Warning: {parent} {methods[method]} attribute {data[methods[method]]} must an integer >= {minimum} using {default} as default") + elif maximum is not None and (not isinstance(data[methods[method]], int) or data[methods[method]] < minimum or data[methods[method]] > maximum): + logger.warning(f"Collection Warning: {parent} {methods[method]} attribute {data[methods[method]]} must an integer between {minimum} and {maximum} using {default} as default") + else: + return data[methods[method]] + return default From a2fad0a170bb3b12b8bcfde2f4d349db9856b727 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 23 Jul 2021 14:45:49 -0400 Subject: [PATCH 30/95] finish organizing --- modules/builder.py | 602 +++++++++++++++++++-------------------------- modules/mal.py | 6 +- modules/tvdb.py | 2 +- modules/util.py | 26 +- 4 files changed, 278 insertions(+), 358 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 4a5ec58f..bf85d610 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -118,8 +118,8 @@ numbered_builders = [ "trakt_watched", "trakt_collected" ] -smart_collection_invalid = ["collection_order"] -smart_url_collection_invalid = [ +smart_invalid = ["collection_order"] +smart_url_invalid = [ "run_again", "sync_mode", "show_filtered", "show_missing", "save_missing", "smart_label", "radarr_add", "radarr_folder", "radarr_monitor", "radarr_availability", "radarr_quality", "radarr_tag", "radarr_search", @@ -163,18 +163,12 @@ ignored_details = [ "tmdb_person", "build_collection" ] -details = [ - "collection_mode", "collection_order", "label" -] + boolean_details + string_details +details = ["collection_mode", "collection_order", "label"] + boolean_details + string_details collectionless_details = [ "collection_order", "plex_collectionless", "label", "label_sync_mode", "test" ] + poster_details + background_details + summary_details + string_details -item_details = [ - "item_label", "item_radarr_tag", "item_sonarr_tag", "item_overlay", - "item_episode_sorting", "item_keep_episodes", "item_delete_episodes", "item_season_display", - "item_episode_ordering", "item_metadata_language", "item_use_original_title" -] +item_details = ["item_label", "item_radarr_tag", "item_sonarr_tag", "item_overlay"] + list(plex.item_advance_keys.keys()) radarr_details = ["radarr_add", "radarr_folder", "radarr_monitor", "radarr_search", "radarr_availability", "radarr_quality", "radarr_tag"] sonarr_details = ["sonarr_add", "sonarr_folder", "sonarr_monitor", "sonarr_language", "sonarr_series", "sonarr_quality", "sonarr_season", "sonarr_search", "sonarr_cutoff_search", "sonarr_tag"] all_filters = [ @@ -541,339 +535,46 @@ class CollectionBuilder: self.smart = self.smart_url or self.smart_label_collection for method_key, method_data in self.data.items(): - if method_key.lower() in ignored_details: + method_name, method_mod, method_final = self._split(method_key) + if method_name in ignored_details: continue - logger.info("") - logger.info(f"Validating Method: {method_key}") - if "trakt" in method_key.lower() and not self.config.Trakt: raise Failed(f"Collection Error: {method_key} requires Trakt to be configured") - elif "radarr" in method_key.lower() and not self.library.Radarr: raise Failed(f"Collection Error: {method_key} requires Radarr to be configured") - elif "sonarr" in method_key.lower() and not self.library.Sonarr: raise Failed(f"Collection Error: {method_key} requires Sonarr to be configured") - elif "tautulli" in method_key.lower() and not self.library.Tautulli: raise Failed(f"Collection Error: {method_key} requires Tautulli to be configured") - elif "mal" in method_key.lower() and not self.config.MyAnimeList: raise Failed(f"Collection Error: {method_key} requires MyAnimeList to be configured") - elif method_data is not None: - logger.debug(f"Value: {method_data}") - method_name, method_mod, method_final = self._split(method_key) - if method_name in show_only_builders and self.library.is_movie: - raise Failed(f"Collection Error: {method_name} attribute only works for show libraries") - elif method_name in movie_only_builders and self.library.is_show: - raise Failed(f"Collection Error: {method_name} attribute only works for movie libraries") - elif method_name in plex.movie_only_searches and self.library.is_show: - raise Failed(f"Collection Error: {method_name} plex search only works for movie libraries") - elif method_name in plex.show_only_searches and self.library.is_movie: - raise Failed(f"Collection Error: {method_name} plex search only works for show libraries") - elif method_name in smart_collection_invalid and self.smart: - raise Failed(f"Collection Error: {method_name} attribute only works with normal collections") - elif method_name not in collectionless_details and self.collectionless: - raise Failed(f"Collection Error: {method_name} attribute does not work for Collectionless collection") - elif self.smart_url and (method_name in all_builders or method_name in smart_url_collection_invalid): - raise Failed(f"Collection Error: {method_name} builder not allowed when using smart_filter") - elif method_name in summary_details: - self._summary(method_name, method_data) - elif method_name in poster_details: - self._poster(method_name, method_data) - elif method_name in background_details: - self._background(method_name, method_data) - elif method_name in details: - self._details(method_name, method_data, method_final, methods) - elif method_name in details: - self._details(method_name, method_data, method_final, methods) - elif method_name in item_details: - self._item_details(method_name, method_data, method_mod, method_final, methods) - elif method_name in radarr_details: - self._radarr(method_name, method_data) - elif method_name in sonarr_details: - self._sonarr(method_name, method_data) - elif method_name in anidb.builders: - self._anidb(method_name, method_data) - elif method_name in anilist.builders: - self._anilist(method_name, method_data) - elif method_name in icheckmovies.builders: - self._icheckmovies(method_name, method_data) - - - - - - elif method_final in plex.searches: - self.methods.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) - elif method_name == "plex_all": - self.methods.append((method_name, True)) - elif method_name == "mal_id": - for mal_id in util.get_int_list(method_data, "MyAnimeList ID"): - self.methods.append((method_name, mal_id)) - elif method_name == "trakt_list": - for trakt_list in self.config.Trakt.validate_trakt(method_data, self.library.is_movie): - self.methods.append((method_name, trakt_list)) - elif method_name == "trakt_list_details": - trakt_lists = self.config.Trakt.validate_trakt(method_data, self.library.is_movie) - self.summaries[method_name] = self.config.Trakt.list_description(trakt_lists[0]) - for trakt_list in trakt_lists: - self.methods.append((method_name[:-8], trakt_list)) - elif method_name in ["trakt_watchlist", "trakt_collection"]: - for trakt_list in self.config.Trakt.validate_trakt(method_data, self.library.is_movie, trakt_type=method_name[6:]): - self.methods.append((method_name, trakt_list)) - elif method_name == "imdb_list": - for imdb_dict in self.config.IMDb.validate_imdb_lists(method_data, self.language): - self.methods.append((method_name, imdb_dict)) - elif method_name == "letterboxd_list": - for letterboxd_list in self.config.Letterboxd.validate_letterboxd_lists(method_data, self.language): - self.methods.append((method_name, letterboxd_list)) - elif method_name == "letterboxd_list_details": - letterboxd_lists = self.config.Letterboxd.validate_letterboxd_lists(method_data, self.language) - for letterboxd_list in letterboxd_lists: - self.methods.append((method_name[:-8], letterboxd_list)) - self.summaries[method_name] = self.config.Letterboxd.get_list_description(letterboxd_lists[0], self.language) - elif method_name in dictionary_builders: - for dict_data in util.get_list(method_data): - if isinstance(dict_data, dict): - def get_int(parent, int_method, data_in, methods_in, default_in, minimum=1, maximum=None): - if int_method not in methods_in: - logger.warning(f"Collection Warning: {parent} {int_method} attribute not found using {default_in} as default") - elif not data_in[methods_in[int_method]]: - logger.warning(f"Collection Warning: {parent} {methods_in[int_method]} attribute is blank using {default_in} as default") - elif isinstance(data_in[methods_in[int_method]], int) and data_in[methods_in[int_method]] >= minimum: - if maximum is None or data_in[methods_in[int_method]] <= maximum: - return data_in[methods_in[int_method]] - else: - logger.warning(f"Collection Warning: {parent} {methods_in[int_method]} attribute {data_in[methods_in[int_method]]} invalid must an integer <= {maximum} using {default_in} as default") - else: - logger.warning(f"Collection Warning: {parent} {methods_in[int_method]} attribute {data_in[methods_in[int_method]]} invalid must an integer >= {minimum} using {default_in} as default") - return default_in - if method_name == "filters": - validate = True - if "validate" in dict_data: - if dict_data["validate"] is None: - raise Failed("Collection Error: validate filter attribute is blank") - if not isinstance(dict_data["validate"], bool): - raise Failed("Collection Error: validate filter attribute must be either true or false") - validate = dict_data["validate"] - for filter_method, filter_data in dict_data.items(): - filter_attr, modifier, filter_final = self._split(filter_method) - if filter_final not in all_filters: - raise Failed(f"Collection Error: {filter_final} is not a valid filter attribute") - elif filter_final in movie_only_filters and self.library.is_show: - raise Failed(f"Collection Error: {filter_final} filter attribute only works for movie libraries") - elif filter_final in show_only_filters and self.library.is_movie: - raise Failed(f"Collection Error: {filter_final} filter attribute only works for show libraries") - elif filter_final is None: - raise Failed(f"Collection Error: {filter_final} filter attribute is blank") - else: - self.filters.append((filter_final, self.validate_attribute(filter_attr, modifier, f"{filter_final} filter", filter_data, validate))) - elif method_name == "plex_collectionless": - new_dictionary = {} - dict_methods = {dm.lower(): dm for dm in dict_data} - prefix_list = [] - if "exclude_prefix" in dict_methods and dict_data[dict_methods["exclude_prefix"]]: - if isinstance(dict_data[dict_methods["exclude_prefix"]], list): - prefix_list.extend([exclude for exclude in dict_data[dict_methods["exclude_prefix"]] if exclude]) - else: - prefix_list.append(str(dict_data[dict_methods["exclude_prefix"]])) - exact_list = [] - if "exclude" in dict_methods and dict_data[dict_methods["exclude"]]: - if isinstance(dict_data[dict_methods["exclude"]], list): - exact_list.extend([exclude for exclude in dict_data[dict_methods["exclude"]] if exclude]) - else: - exact_list.append(str(dict_data[dict_methods["exclude"]])) - if len(prefix_list) == 0 and len(exact_list) == 0: - raise Failed("Collection Error: you must have at least one exclusion") - exact_list.append(self.name) - new_dictionary["exclude_prefix"] = prefix_list - new_dictionary["exclude"] = exact_list - self.methods.append((method_name, new_dictionary)) - elif method_name == "plex_search": - self.methods.append((method_name, self.build_filter("plex_search", dict_data))) - elif method_name == "tmdb_discover": - new_dictionary = {"limit": 100} - for discover_name, discover_data in dict_data.items(): - discover_final = discover_name.lower() - if discover_data: - if (self.library.is_movie and discover_final in tmdb.discover_movie) or (self.library.is_show and discover_final in tmdb.discover_tv): - if discover_final == "language": - if re.compile("([a-z]{2})-([A-Z]{2})").match(str(discover_data)): - new_dictionary[discover_final] = str(discover_data) - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} must match pattern ([a-z]{{2}})-([A-Z]{{2}}) e.g. en-US") - elif discover_final == "region": - if re.compile("^[A-Z]{2}$").match(str(discover_data)): - new_dictionary[discover_final] = str(discover_data) - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} must match pattern ^[A-Z]{{2}}$ e.g. US") - elif discover_final == "sort_by": - if (self.library.is_movie and discover_data in tmdb.discover_movie_sort) or (self.library.is_show and discover_data in tmdb.discover_tv_sort): - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} is invalid") - elif discover_final == "certification_country": - if "certification" in dict_data or "certification.lte" in dict_data or "certification.gte" in dict_data: - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be used with either certification, certification.lte, or certification.gte") - elif discover_final in ["certification", "certification.lte", "certification.gte"]: - if "certification_country" in dict_data: - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be used with certification_country") - elif discover_final in ["include_adult", "include_null_first_air_dates", "screened_theatrically"]: - if discover_data is True: - new_dictionary[discover_final] = discover_data - elif discover_final in tmdb.discover_dates: - new_dictionary[discover_final] = util.validate_date(discover_data, f"{method_name} attribute {discover_final}", return_as="%m/%d/%Y") - elif discover_final in ["primary_release_year", "year", "first_air_date_year"]: - new_dictionary[discover_final] = util.check_number(discover_data, f"{method_name} attribute {discover_final}", minimum=1800, maximum=self.current_year + 1) - elif discover_final in ["vote_count.gte", "vote_count.lte", "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte"]: - new_dictionary[discover_final] = util.check_number(discover_data, f"{method_name} attribute {discover_final}", minimum=1) - elif discover_final in ["with_cast", "with_crew", "with_people", "with_companies", "with_networks", "with_genres", "without_genres", "with_keywords", "without_keywords", "with_original_language", "timezone"]: - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final} not supported") - elif discover_final == "limit": - if isinstance(discover_data, int) and discover_data > 0: - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be a valid number greater then 0") - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final} not supported") - else: - raise Failed(f"Collection Error: {method_name} parameter {discover_final} is blank") - if len(new_dictionary) > 1: - self.methods.append((method_name, new_dictionary)) - else: - raise Failed(f"Collection Error: {method_name} had no valid fields") - elif "tautulli" in method_name: - new_dictionary = {} - if method_name == "tautulli_popular": - new_dictionary["list_type"] = "popular" - elif method_name == "tautulli_watched": - new_dictionary["list_type"] = "watched" - else: - raise Failed(f"Collection Error: {method_name} attribute not supported") - dict_methods = {dm.lower(): dm for dm in dict_data} - new_dictionary["list_days"] = get_int(method_name, "list_days", dict_data, dict_methods, 30) - new_dictionary["list_size"] = get_int(method_name, "list_size", dict_data, dict_methods, 10) - new_dictionary["list_buffer"] = get_int(method_name, "list_buffer", dict_data, dict_methods, 20) - self.methods.append((method_name, new_dictionary)) - elif method_name == "mal_season": - new_dictionary = {"sort_by": "anime_num_list_users"} - dict_methods = {dm.lower(): dm for dm in dict_data} - if "sort_by" not in dict_methods: - logger.warning("Collection Warning: mal_season sort_by attribute not found using members as default") - elif not dict_data[dict_methods["sort_by"]]: - logger.warning("Collection Warning: mal_season sort_by attribute is blank using members as default") - elif dict_data[dict_methods["sort_by"]] not in mal.season_sort: - logger.warning(f"Collection Warning: mal_season sort_by attribute {dict_data[dict_methods['sort_by']]} invalid must be either 'members' or 'score' using members as default") - else: - new_dictionary["sort_by"] = mal.season_sort[dict_data[dict_methods["sort_by"]]] - - if self.current_time.month in [1, 2, 3]: new_dictionary["season"] = "winter" - elif self.current_time.month in [4, 5, 6]: new_dictionary["season"] = "spring" - elif self.current_time.month in [7, 8, 9]: new_dictionary["season"] = "summer" - elif self.current_time.month in [10, 11, 12]: new_dictionary["season"] = "fall" - - if "season" not in dict_methods: - logger.warning(f"Collection Warning: mal_season season attribute not found using the current season: {new_dictionary['season']} as default") - elif not dict_data[dict_methods["season"]]: - logger.warning(f"Collection Warning: mal_season season attribute is blank using the current season: {new_dictionary['season']} as default") - elif dict_data[dict_methods["season"]] not in util.pretty_seasons: - logger.warning(f"Collection Warning: mal_season season attribute {dict_data[dict_methods['season']]} invalid must be either 'winter', 'spring', 'summer' or 'fall' using the current season: {new_dictionary['season']} as default") - else: - new_dictionary["season"] = dict_data[dict_methods["season"]] - - new_dictionary["year"] = get_int(method_name, "year", dict_data, dict_methods, self.current_time.year, minimum=1917, maximum=self.current_time.year + 1) - new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 100, maximum=500) - self.methods.append((method_name, new_dictionary)) - elif method_name == "mal_userlist": - new_dictionary = {"status": "all", "sort_by": "list_score"} - dict_methods = {dm.lower(): dm for dm in dict_data} - if "username" not in dict_methods: - raise Failed("Collection Error: mal_userlist username attribute is required") - elif not dict_data[dict_methods["username"]]: - raise Failed("Collection Error: mal_userlist username attribute is blank") - else: - new_dictionary["username"] = dict_data[dict_methods["username"]] - - if "status" not in dict_methods: - logger.warning("Collection Warning: mal_season status attribute not found using all as default") - elif not dict_data[dict_methods["status"]]: - logger.warning("Collection Warning: mal_season status attribute is blank using all as default") - elif dict_data[dict_methods["status"]] not in mal.userlist_status: - logger.warning(f"Collection Warning: mal_season status attribute {dict_data[dict_methods['status']]} invalid must be either 'all', 'watching', 'completed', 'on_hold', 'dropped' or 'plan_to_watch' using all as default") - else: - new_dictionary["status"] = mal.userlist_status[dict_data[dict_methods["status"]]] - - if "sort_by" not in dict_methods: - logger.warning("Collection Warning: mal_season sort_by attribute not found using score as default") - elif not dict_data[dict_methods["sort_by"]]: - logger.warning("Collection Warning: mal_season sort_by attribute is blank using score as default") - elif dict_data[dict_methods["sort_by"]] not in mal.userlist_sort: - logger.warning(f"Collection Warning: mal_season sort_by attribute {dict_data[dict_methods['sort_by']]} invalid must be either 'score', 'last_updated', 'title' or 'start_date' using score as default") - else: - new_dictionary["sort_by"] = mal.userlist_sort[dict_data[dict_methods["sort_by"]]] - - new_dictionary["limit"] = get_int(method_name, "limit", dict_data, dict_methods, 100, maximum=1000) - self.methods.append((method_name, new_dictionary)) - else: - raise Failed(f"Collection Error: {method_name} attribute is not a dictionary: {dict_data}") - elif method_name in numbered_builders: - list_count = util.regex_first_int(method_data, "List Size", default=10) - if list_count < 1: - logger.warning(f"Collection Warning: {method_name} must be an integer greater then 0 defaulting to 10") - list_count = 10 - self.methods.append((method_name, list_count)) - elif "tvdb" in method_name: - values = util.get_list(method_data) - if method_name[-8:] == "_details": - if method_name == "tvdb_movie_details": - item = self.config.TVDb.get_movie(self.language, values[0]) - if hasattr(item, "description") and item.description: - self.summaries[method_name] = item.description - if hasattr(item, "background_path") and item.background_path: - self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.background_path}" - if hasattr(item, "poster_path") and item.poster_path: - self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" - elif method_name == "tvdb_show_details": - item = self.config.TVDb.get_series(self.language, values[0]) - if hasattr(item, "description") and item.description: - self.summaries[method_name] = item.description - if hasattr(item, "background_path") and item.background_path: - self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.background_path}" - if hasattr(item, "poster_path") and item.poster_path: - self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" - elif method_name == "tvdb_list_details": - self.summaries[method_name] = self.config.TVDb.get_list_description(values[0], self.language) - for value in values: - self.methods.append((method_name[:-8] if method_name[-8:] == "_details" else method_name, value)) - elif method_name in tmdb.builders: - values = self.config.TMDb.validate_tmdb_ids(method_data, method_name) - if method_name[-8:] == "_details": - if method_name in ["tmdb_collection_details", "tmdb_movie_details", "tmdb_show_details"]: - item = self.config.TMDb.get_movie_show_or_collection(values[0], self.library.is_movie) - if hasattr(item, "overview") and item.overview: - self.summaries[method_name] = item.overview - if hasattr(item, "backdrop_path") and item.backdrop_path: - self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.backdrop_path}" - if hasattr(item, "poster_path") and item.poster_path: - self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" - elif method_name in ["tmdb_actor_details", "tmdb_crew_details", "tmdb_director_details", "tmdb_producer_details", "tmdb_writer_details"]: - item = self.config.TMDb.get_person(values[0]) - if hasattr(item, "biography") and item.biography: - self.summaries[method_name] = item.biography - if hasattr(item, "profile_path") and item.profile_path: - self.posters[method_name] = f"{self.config.TMDb.image_url}{item.profile_path}" - else: - item = self.config.TMDb.get_list(values[0]) - if hasattr(item, "description") and item.description: - self.summaries[method_name] = item.description - for value in values: - self.methods.append((method_name[:-8] if method_name[-8:] == "_details" else method_name, value)) - elif method_name in all_builders: - for value in util.get_list(method_data): - self.methods.append((method_name, value)) - elif method_name not in ignored_details: - raise Failed(f"Collection Error: {method_name} attribute not supported") - elif method_key.lower() in all_builders or method_key.lower() in method_alias or method_key.lower() in plex.searches: - raise Failed(f"Collection Error: {method_key} attribute is blank") - else: - logger.warning(f"Collection Warning: {method_key} attribute is blank") + logger.debug("") + logger.debug(f"Validating Method: {method_key}") + logger.debug(f"Value: {method_data}") + if method_data is None and method_name in all_builders + plex.searches: raise Failed(f"Collection Error: {method_final} attribute is blank") + elif method_data is None: logger.warning(f"Collection Warning: {method_final} attribute is blank") + elif not self.config.Trakt and "trakt" in method_name: raise Failed(f"Collection Error: {method_final} requires Trakt to be configured") + elif not self.library.Radarr and "radarr" in method_name: raise Failed(f"Collection Error: {method_final} requires Radarr to be configured") + elif not self.library.Sonarr and "sonarr" in method_name: raise Failed(f"Collection Error: {method_final} requires Sonarr to be configured") + elif not self.library.Tautulli and "tautulli" in method_name: raise Failed(f"Collection Error: {method_final} requires Tautulli to be configured") + elif not self.config.MyAnimeList and "mal" in method_name: raise Failed(f"Collection Error: {method_final} requires MyAnimeList to be configured") + elif self.library.is_movie and method_name in show_only_builders: raise Failed(f"Collection Error: {method_final} attribute only works for show libraries") + elif self.library.is_show and method_name in movie_only_builders: raise Failed(f"Collection Error: {method_final} attribute only works for movie libraries") + elif self.library.is_show and method_name in plex.movie_only_searches: raise Failed(f"Collection Error: {method_final} plex search only works for movie libraries") + elif self.library.is_movie and method_name in plex.show_only_searches: raise Failed(f"Collection Error: {method_final} plex search only works for show libraries") + elif self.smart and method_name in smart_invalid: raise Failed(f"Collection Error: {method_final} attribute only works with normal collections") + elif self.collectionless and method_name not in collectionless_details: raise Failed(f"Collection Error: {method_final} attribute does not work for Collectionless collection") + elif self.smart_url and method_name in all_builders + smart_url_invalid: raise Failed(f"Collection Error: {method_final} builder not allowed when using smart_filter") + elif method_name in summary_details: self._summary(method_name, method_data) + elif method_name in poster_details: self._poster(method_name, method_data) + elif method_name in background_details: self._background(method_name, method_data) + elif method_name in details: self._details(method_name, method_data, method_final, methods) + elif method_name in item_details: self._item_details(method_name, method_data, method_mod, method_final, methods) + elif method_name in radarr_details: self._radarr(method_name, method_data) + elif method_name in sonarr_details: self._sonarr(method_name, method_data) + elif method_name in anidb.builders: self._anidb(method_name, method_data) + elif method_name in anilist.builders: self._anilist(method_name, method_data) + elif method_name in icheckmovies.builders: self._icheckmovies(method_name, method_data) + elif method_name in letterboxd.builders: self._letterboxd(method_name, method_data) + elif method_name in imdb.builders: self._imdb(method_name, method_data) + elif method_name in mal.builders: self._mal(method_name, method_data) + elif method_name in plex.builders or method_final in plex.searches: self._plex(method_name, method_data) + elif method_name in tautulli.builders: self._tautulli(method_name, method_data) + elif method_name in tmdb.builders: self._tmdb(method_name, method_data) + elif method_name in trakt.builders: self._trakt(method_name, method_data) + elif method_name in tvdb.builders: self._tvdb(method_name, method_data) + elif method_name == "filters": self._filters(method_name, method_data) + else: raise Failed(f"Collection Error: {method_final} attribute not supported") if self.add_to_radarr is None: self.add_to_radarr = self.library.Radarr.add if self.library.Radarr else False @@ -942,7 +643,7 @@ class CollectionBuilder: url_slug = self.config.TMDb.get_person(util.regex_first_int(method_data, 'TMDb Person ID')).profile_path self.posters[method_name] = f"{self.config.TMDb.image_url}{url_slug}" elif method_name == "tvdb_poster": - self.posters[method_name] = f"{self.config.TVDb.get_movie_or_series(method_data, self.language, self.library.is_movie).poster_path}" + self.posters[method_name] = f"{self.config.TVDb.get_item(method_data, self.language, self.library.is_movie).poster_path}" elif method_name == "file_poster": if os.path.exists(method_data): self.posters[method_name] = os.path.abspath(method_data) @@ -956,7 +657,7 @@ class CollectionBuilder: url_slug = self.config.TMDb.get_movie_show_or_collection(util.regex_first_int(method_data, 'TMDb ID'), self.library.is_movie).poster_path self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{url_slug}" elif method_name == "tvdb_background": - self.posters[method_name] = f"{self.config.TVDb.get_movie_or_series(method_data, self.language, self.library.is_movie).background_path}" + self.posters[method_name] = f"{self.config.TVDb.get_item(method_data, self.language, self.library.is_movie).background_path}" elif method_name == "file_background": if os.path.exists(method_data): self.backgrounds[method_name] = os.path.abspath(method_data) @@ -1070,7 +771,7 @@ class CollectionBuilder: def _anidb(self, method_name, method_data): if method_name == "anidb_popular": - self.methods.append((method_name, util.parse_int(method_name, method_data, 30, maximum=30))) + self.methods.append((method_name, util.parse_int(method_name, method_data, default=30, maximum=30))) elif method_name in ["anidb_id", "anidb_relation"]: for anidb_id in self.config.AniDB.validate_anidb_ids(method_data, self.language): self.methods.append((method_name, anidb_id)) @@ -1091,7 +792,7 @@ class CollectionBuilder: for anilist_id in self.config.AniList.validate_anilist_ids(method_data, studio=method_name == "anilist_studio"): self.methods.append((method_name, anilist_id)) elif method_name in ["anilist_popular", "anilist_top_rated"]: - self.methods.append((method_name, util.parse_int(method_name, method_data, 10))) + self.methods.append((method_name, util.parse_int(method_name, method_data))) elif method_name in ["anilist_season", "anilist_genre", "anilist_tag"]: for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): new_dictionary = {} @@ -1118,6 +819,215 @@ class CollectionBuilder: if method_name.endswith("_details"): self.summaries[method_name] = self.config.ICheckMovies.get_list_description(icheckmovies_lists[0], self.language) + def _imdb(self, method_name, method_data): + if method_name == "imdb_id": + for value in util.get_list(method_data): + if str(value).startswith("tt"): + self.methods.append((method_name, value)) + else: + raise Failed(f"Collection Error: imdb_id {value} must begin with tt") + elif method_name == "imdb_list": + for imdb_dict in self.config.IMDb.validate_imdb_lists(method_data, self.language): + self.methods.append((method_name, imdb_dict)) + + def _letterboxd(self, method_name, method_data): + if method_name.startswith("letterboxd_list"): + letterboxd_lists = self.config.Letterboxd.validate_letterboxd_lists(method_data, self.language) + for letterboxd_list in letterboxd_lists: + self.methods.append(("letterboxd_list", letterboxd_list)) + if method_name.endswith("_details"): + self.summaries[method_name] = self.config.Letterboxd.get_list_description(letterboxd_lists[0], self.language) + + def _mal(self, method_name, method_data): + if method_name == "mal_id": + for mal_id in util.get_int_list(method_data, "MyAnimeList ID"): + self.methods.append((method_name, mal_id)) + elif method_name in ["mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_ova", "mal_movie", "mal_special", "mal_popular", "mal_favorite", "mal_suggested"]: + self.methods.append((method_name, util.parse_int(method_name, method_data))) + elif method_name in ["mal_season", "mal_userlist"]: + for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + new_dictionary = {} + if method_name == "mal_season": + if self.current_time.month in [1, 2, 3]: new_dictionary["season"] = "winter" + elif self.current_time.month in [4, 5, 6]: new_dictionary["season"] = "spring" + elif self.current_time.month in [7, 8, 9]: new_dictionary["season"] = "summer" + elif self.current_time.month in [10, 11, 12]: new_dictionary["season"] = "fall" + new_dictionary["season"] = util.parse_from_dict(method_name, "season", dict_data, dict_methods, default=new_dictionary["season"], options=["winter", "spring", "summer", "fall"]) + new_dictionary["sort_by"] = util.parse_from_dict(method_name, "sort_by", dict_data, dict_methods, default="members", options=mal.season_sort_options, translation=mal.season_sort_translation) + new_dictionary["year"] = util.parse_int_from_dict(method_name, "year", dict_data, dict_methods, self.current_time.year, minimum=1917, maximum=self.current_time.year + 1) + new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 100, maximum=500) + elif method_name == "mal_userlist": + new_dictionary["username"] = util.parse_from_dict(method_name, "username", dict_data, dict_methods) + new_dictionary["status"] = util.parse_from_dict(method_name, "status", dict_data, dict_methods, default="all", options=mal.userlist_status) + new_dictionary["sort_by"] = util.parse_from_dict(method_name, "sort_by", dict_data, dict_methods, default="score", options=mal.userlist_sort_options, translation=mal.userlist_sort_translation) + new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 100, maximum=1000) + self.methods.append((method_name, new_dictionary)) + + def _plex(self, method_name, method_data): + if method_name == "plex_all": + self.methods.append((method_name, True)) + elif method_name in ["plex_search", "plex_collectionless"]: + for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + new_dictionary = {} + if method_name == "plex_search": + new_dictionary = self.build_filter("plex_search", dict_data) + elif method_name == "plex_collectionless": + prefix_list = util.parse_list("exclude_prefix", dict_data, dict_methods) + exact_list = util.parse_list("exclude", dict_data, dict_methods) + if len(prefix_list) == 0 and len(exact_list) == 0: + raise Failed("Collection Error: you must have at least one exclusion") + exact_list.append(self.name) + new_dictionary["exclude_prefix"] = prefix_list + new_dictionary["exclude"] = exact_list + self.methods.append((method_name, new_dictionary)) + else: + self.methods.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) + + def _tautulli(self, method_name, method_data): + for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + self.methods.append((method_name, { + "list_type": "popular" if method_name == "tautulli_popular" else "watched", + "list_days": util.parse_int_from_dict(method_name, "list_days", dict_data, dict_methods, 30), + "list_size": util.parse_int_from_dict(method_name, "list_size", dict_data, dict_methods, 10), + "list_buffer": util.parse_int_from_dict(method_name, "list_buffer", dict_data, dict_methods, 20) + })) + + def _tmdb(self, method_name, method_data): + if method_name == "tmdb_discover": + for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + new_dictionary = {"limit": 100} + for discover_name, discover_data in dict_data.items(): + discover_final = discover_name.lower() + if discover_data: + if (self.library.is_movie and discover_final in tmdb.discover_movie) or (self.library.is_show and discover_final in tmdb.discover_tv): + if discover_final == "language": + if re.compile("([a-z]{2})-([A-Z]{2})").match(str(discover_data)): + new_dictionary[discover_final] = str(discover_data) + else: + raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} must match pattern ([a-z]{{2}})-([A-Z]{{2}}) e.g. en-US") + elif discover_final == "region": + if re.compile("^[A-Z]{2}$").match(str(discover_data)): + new_dictionary[discover_final] = str(discover_data) + else: + raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} must match pattern ^[A-Z]{{2}}$ e.g. US") + elif discover_final == "sort_by": + if (self.library.is_movie and discover_data in tmdb.discover_movie_sort) or (self.library.is_show and discover_data in tmdb.discover_tv_sort): + new_dictionary[discover_final] = discover_data + else: + raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} is invalid") + elif discover_final == "certification_country": + if "certification" in dict_data or "certification.lte" in dict_data or "certification.gte" in dict_data: + new_dictionary[discover_final] = discover_data + else: + raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be used with either certification, certification.lte, or certification.gte") + elif discover_final in ["certification", "certification.lte", "certification.gte"]: + if "certification_country" in dict_data: + new_dictionary[discover_final] = discover_data + else: + raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be used with certification_country") + elif discover_final in ["include_adult", "include_null_first_air_dates", "screened_theatrically"]: + if discover_data is True: + new_dictionary[discover_final] = discover_data + elif discover_final in tmdb.discover_dates: + new_dictionary[discover_final] = util.validate_date(discover_data, f"{method_name} attribute {discover_final}", return_as="%m/%d/%Y") + elif discover_final in ["primary_release_year", "year", "first_air_date_year"]: + new_dictionary[discover_final] = util.check_number(discover_data, f"{method_name} attribute {discover_final}", minimum=1800, maximum=self.current_year + 1) + elif discover_final in ["vote_count.gte", "vote_count.lte", "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte"]: + new_dictionary[discover_final] = util.check_number(discover_data, f"{method_name} attribute {discover_final}", minimum=1) + elif discover_final in ["with_cast", "with_crew", "with_people", "with_companies", "with_networks", "with_genres", "without_genres", "with_keywords", "without_keywords", "with_original_language", "timezone"]: + new_dictionary[discover_final] = discover_data + else: + raise Failed(f"Collection Error: {method_name} attribute {discover_final} not supported") + elif discover_final == "limit": + if isinstance(discover_data, int) and discover_data > 0: + new_dictionary[discover_final] = discover_data + else: + raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be a valid number greater then 0") + else: + raise Failed(f"Collection Error: {method_name} attribute {discover_final} not supported") + else: + raise Failed(f"Collection Error: {method_name} parameter {discover_final} is blank") + if len(new_dictionary) > 1: + self.methods.append((method_name, new_dictionary)) + else: + raise Failed(f"Collection Error: {method_name} had no valid fields") + elif method_name in ["tmdb_popular", "tmdb_top_rated", "tmdb_now_playing", "tmdb_trending_daily", "tmdb_trending_weekly"]: + self.methods.append((method_name, util.parse_int(method_name, method_data))) + else: + values = self.config.TMDb.validate_tmdb_ids(method_data, method_name) + if method_name.endswith("_details"): + if method_name.beginswith(("tmdb_collection", "tmdb_movie", "tmdb_show")): + item = self.config.TMDb.get_movie_show_or_collection(values[0], self.library.is_movie) + if hasattr(item, "overview") and item.overview: + self.summaries[method_name] = item.overview + if hasattr(item, "backdrop_path") and item.backdrop_path: + self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.backdrop_path}" + if hasattr(item, "poster_path") and item.poster_path: + self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" + elif method_name.beginswith(("tmdb_actor", "tmdb_crew", "tmdb_director", "tmdb_producer", "tmdb_writer")): + item = self.config.TMDb.get_person(values[0]) + if hasattr(item, "biography") and item.biography: + self.summaries[method_name] = item.biography + if hasattr(item, "profile_path") and item.profile_path: + self.posters[method_name] = f"{self.config.TMDb.image_url}{item.profile_path}" + elif method_name.beginswith("tmdb_list"): + item = self.config.TMDb.get_list(values[0]) + if hasattr(item, "description") and item.description: + self.summaries[method_name] = item.description + for value in values: + self.methods.append((method_name[:-8] if method_name.endswith("_details") else method_name, value)) + + def _trakt(self, method_name, method_data): + if method_name.startswith("trakt_list"): + trakt_lists = self.config.Trakt.validate_trakt(method_data, self.library.is_movie) + for trakt_list in trakt_lists: + self.methods.append(("trakt_list", trakt_list)) + if method_name.endswith("_details"): + self.summaries[method_name] = self.config.Trakt.list_description(trakt_lists[0]) + elif method_name in ["trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected"]: + self.methods.append((method_name, util.parse_int(method_name, method_data))) + elif method_name in ["trakt_watchlist", "trakt_collection"]: + for trakt_list in self.config.Trakt.validate_trakt(method_data, self.library.is_movie, trakt_type=method_name[6:]): + self.methods.append((method_name, trakt_list)) + + def _tvdb(self, method_name, method_data): + values = util.get_list(method_data) + if method_name.endswith("_details"): + if method_name.startswith(("tvdb_movie", "tvdb_show")): + item = self.config.TVDb.get_item(self.language, values[0], method_name.startswith("tvdb_movie")) + if hasattr(item, "description") and item.description: + self.summaries[method_name] = item.description + if hasattr(item, "background_path") and item.background_path: + self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.background_path}" + if hasattr(item, "poster_path") and item.poster_path: + self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" + elif method_name.startswith("tvdb_list"): + self.summaries[method_name] = self.config.TVDb.get_list_description(values[0], self.language) + for value in values: + self.methods.append((method_name[:-8] if method_name.endswith("_details") else method_name, value)) + + def _filters(self, method_name, method_data): + for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + validate = True + if "validate" in dict_data: + if dict_data["validate"] is None: + raise Failed("Collection Error: validate filter attribute is blank") + if not isinstance(dict_data["validate"], bool): + raise Failed("Collection Error: validate filter attribute must be either true or false") + validate = dict_data["validate"] + for filter_method, filter_data in dict_data.items(): + filter_attr, modifier, filter_final = self._split(filter_method) + if filter_final not in all_filters: + raise Failed(f"Collection Error: {filter_final} is not a valid filter attribute") + elif filter_final in movie_only_filters and self.library.is_show: + raise Failed(f"Collection Error: {filter_final} filter attribute only works for movie libraries") + elif filter_final in show_only_filters and self.library.is_movie: + raise Failed(f"Collection Error: {filter_final} filter attribute only works for show libraries") + elif filter_final is None: + raise Failed(f"Collection Error: {filter_final} filter attribute is blank") + else: + self.filters.append((filter_final, self.validate_attribute(filter_attr, modifier, f"{filter_final} filter", filter_data, validate))) + def collect_rating_keys(self): filtered_keys = {} name = self.obj.title if self.obj else self.name diff --git a/modules/mal.py b/modules/mal.py index a27a6aea..b77ce012 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -31,12 +31,13 @@ mal_ranked_name = { "mal_popular": "bypopularity", "mal_favorite": "favorite" } -season_sort = { +season_sort_translation = { "anime_score": "anime_score", "anime_num_list_users": "anime_num_list_users", "score": "anime_score", "members": "anime_num_list_users" } +season_sort_options = ["score", "members"] pretty_names = { "anime_score": "Score", "anime_num_list_users": "Members", @@ -51,7 +52,7 @@ pretty_names = { "dropped": "Dropped", "plan_to_watch": "Plan to Watch" } -userlist_sort = { +userlist_sort_translation = { "score": "list_score", "list_score": "list_score", "last_updated": "list_updated_at", @@ -62,6 +63,7 @@ userlist_sort = { "start_date": "anime_start_date", "anime_start_date": "anime_start_date" } +userlist_sort_options = ["score", "last_updated", "title", "start_date"] userlist_status = [ "all", "watching", diff --git a/modules/tvdb.py b/modules/tvdb.py index d338642e..6a27dcef 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -87,7 +87,7 @@ class TVDb: def __init__(self, config): self.config = config - def get_movie_or_series(self, language, tvdb_url, is_movie): + def get_item(self, language, tvdb_url, is_movie): return self.get_movie(language, tvdb_url) if is_movie else self.get_series(language, tvdb_url) def get_series(self, language, tvdb_url): diff --git a/modules/util.py b/modules/util.py index 4f32ae42..390271b7 100644 --- a/modules/util.py +++ b/modules/util.py @@ -458,21 +458,24 @@ def parse_int(method, data, default=10, minimum=1, maximum=None): return list_count return default -def parse_from_dict(parent, method, data, methods, default=None, options=None): +def parse_from_dict(parent, method, data, methods, default=None, options=None, translation=None): message = "" + if options is None and translation is not None: + options = [o for o in translation] if method not in methods: - message = f"Collection Warning: {parent} {method} attribute not found" + message = f"{parent} {method} attribute not found" elif data[methods[method]] is None: - message = f"Collection Warning: {parent} {method} attribute is blank" - elif options is not None and str(data[methods[method]]).lower() not in options: - message = f"Collection Warning: {parent} {method} attribute {data[methods[method]]} must be in {options}" + message = f"{parent} {method} attribute is blank" + elif (translation is not None and str(data[methods[method]]).lower() not in translation) or \ + (options is not None and translation is None and str(data[methods[method]]).lower() not in options): + message = f"{parent} {method} attribute {data[methods[method]]} must be in {options}" else: - return data[methods[method]] + return translation[data[methods[method]]] if translation is not None else data[methods[method]] if default is None: - raise Failed(message) + raise Failed(f"Collection Error: {message}") else: - logger.warning(f"{message} using {default} as default") - return default + logger.warning(f"Collection Warning: {message} using {default} as default") + return translation[default] if translation is not None else default def parse_int_from_dict(parent, method, data, methods, default, minimum=1, maximum=None): if method not in methods: @@ -486,3 +489,8 @@ def parse_int_from_dict(parent, method, data, methods, default, minimum=1, maxim else: return data[methods[method]] return default + +def parse_list(method, data, methods): + if method in methods and data[methods[method]]: + return [i for i in data[methods[method]] if i] if isinstance(data[methods[method]], list) else [str(data[methods[method]])] + return [] \ No newline at end of file From ddc456da2c15ce0055921fae8daadcc0f719e391 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 23 Jul 2021 15:44:21 -0400 Subject: [PATCH 31/95] cleanup --- modules/anilist.py | 15 +---- modules/builder.py | 119 +++++------------------------------- modules/config.py | 36 +++-------- modules/mal.py | 83 ++++++------------------- modules/plex.py | 89 +++++---------------------- modules/radarr.py | 14 +---- modules/sonarr.py | 29 +++++---- modules/tmdb.py | 113 ++++++++-------------------------- modules/trakt.py | 11 +--- modules/tvdb.py | 21 ++----- modules/util.py | 147 +++++++++++---------------------------------- 11 files changed, 149 insertions(+), 528 deletions(-) diff --git a/modules/anilist.py b/modules/anilist.py index 5a31cf02..eac01e5a 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -6,19 +6,10 @@ from retrying import retry logger = logging.getLogger("Plex Meta Manager") builders = [ - "anilist_genre", - "anilist_id", - "anilist_popular", - "anilist_relations", - "anilist_season", - "anilist_studio", - "anilist_tag", - "anilist_top_rated" + "anilist_genre", "anilist_id", "anilist_popular", "anilist_relations", + "anilist_season", "anilist_studio", "anilist_tag", "anilist_top_rated" ] -pretty_names = { - "score": "Average Score", - "popular": "Popularity" -} +pretty_names = {"score": "Average Score", "popular": "Popularity"} base_url = "https://graphql.anilist.co" tag_query = "query{MediaTagCollection {name}}" genre_query = "query{GenreCollection}" diff --git a/modules/builder.py b/modules/builder.py index bf85d610..9ec8d495 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -10,7 +10,6 @@ from urllib.parse import quote logger = logging.getLogger("Plex Meta Manager") string_filters = ["title", "episode_title", "studio"] -image_file_details = ["file_poster", "file_background", "asset_directory"] advance_new_agent = ["item_metadata_language", "item_use_original_title"] advance_show = ["item_episode_sorting", "item_keep_episodes", "item_delete_episodes", "item_season_display", "item_episode_sorting"] method_alias = { @@ -58,116 +57,26 @@ filter_translation = { "writer": "writers" } modifier_alias = {".greater": ".gt", ".less": ".lt"} -all_builders = anidb.builders + anilist.builders + icheckmovies.builders + imdb.builders + letterboxd.builders + mal.builders + plex.builders + tautulli.builders + tmdb.builders + trakt.builders + tvdb.builders -dictionary_builders = [ - "filters", - "anidb_tag", - "anilist_genre", - "anilist_season", - "anilist_tag", - "mal_season", - "mal_userlist", - "plex_collectionless", - "plex_search", - "tautulli_popular", - "tautulli_watched", - "tmdb_discover" -] -show_only_builders = [ - "tmdb_network", - "tmdb_show", - "tmdb_show_details", - "tvdb_show", - "tvdb_show_details" -] +all_builders = anidb.builders + anilist.builders + icheckmovies.builders + imdb.builders + letterboxd.builders + \ + mal.builders + plex.builders + tautulli.builders + tmdb.builders + trakt.builders + tvdb.builders +show_only_builders = ["tmdb_network", "tmdb_show", "tmdb_show_details", "tvdb_show", "tvdb_show_details"] movie_only_builders = [ - "letterboxd_list", - "letterboxd_list_details", - "icheckmovies_list", - "icheckmovies_list_details", - "tmdb_collection", - "tmdb_collection_details", - "tmdb_movie", - "tmdb_movie_details", - "tmdb_now_playing", - "tvdb_movie", - "tvdb_movie_details" -] -numbered_builders = [ - "anidb_popular", - "anilist_popular", - "anilist_top_rated", - "mal_all", - "mal_airing", - "mal_upcoming", - "mal_tv", - "mal_ova", - "mal_movie", - "mal_special", - "mal_popular", - "mal_favorite", - "mal_suggested", - "tmdb_popular", - "tmdb_top_rated", - "tmdb_now_playing", - "tmdb_trending_daily", - "tmdb_trending_weekly", - "trakt_trending", - "trakt_popular", - "trakt_recommended", - "trakt_watched", - "trakt_collected" -] -smart_invalid = ["collection_order"] -smart_url_invalid = [ - "run_again", "sync_mode", "show_filtered", "show_missing", "save_missing", "smart_label", - "radarr_add", "radarr_folder", "radarr_monitor", "radarr_availability", - "radarr_quality", "radarr_tag", "radarr_search", - "sonarr_add", "sonarr_folder", "sonarr_monitor", "sonarr_quality", "sonarr_language", - "sonarr_series", "sonarr_season", "sonarr_tag", "sonarr_search", "sonarr_cutoff_search", - "filters" + "letterboxd_list", "letterboxd_list_details", "icheckmovies_list", "icheckmovies_list_details", + "tmdb_collection", "tmdb_collection_details", "tmdb_movie", "tmdb_movie_details", "tmdb_now_playing", + "tvdb_movie", "tvdb_movie_details" ] summary_details = [ "summary", "tmdb_summary", "tmdb_description", "tmdb_biography", "tvdb_summary", "tvdb_description", "trakt_description", "letterboxd_description", "icheckmovies_description" ] -poster_details = [ - "url_poster", "tmdb_poster", "tmdb_profile", "tvdb_poster", "file_poster" -] -background_details = [ - "url_background", "tmdb_background", "tvdb_background", "file_background" -] -boolean_details = [ - "visible_library", - "visible_home", - "visible_shared", - "show_filtered", - "show_missing", - "save_missing", - "item_assets" -] -string_details = [ - "sort_title", - "content_rating", - "name_mapping" -] -ignored_details = [ - "smart_filter", - "smart_label", - "smart_url", - "run_again", - "schedule", - "sync_mode", - "template", - "test", - "tmdb_person", - "build_collection" -] +poster_details = ["url_poster", "tmdb_poster", "tmdb_profile", "tvdb_poster", "file_poster"] +background_details = ["url_background", "tmdb_background", "tvdb_background", "file_background"] +boolean_details = ["visible_library", "visible_home", "visible_shared", "show_filtered", "show_missing", "save_missing", "item_assets"] +string_details = ["sort_title", "content_rating", "name_mapping"] +ignored_details = ["smart_filter", "smart_label", "smart_url", "run_again", "schedule", "sync_mode", "template", "test", "tmdb_person", "build_collection"] details = ["collection_mode", "collection_order", "label"] + boolean_details + string_details -collectionless_details = [ - "collection_order", "plex_collectionless", - "label", "label_sync_mode", "test" -] + poster_details + background_details + summary_details + string_details +collectionless_details = ["collection_order", "plex_collectionless", "label", "label_sync_mode", "test"] + \ + poster_details + background_details + summary_details + string_details item_details = ["item_label", "item_radarr_tag", "item_sonarr_tag", "item_overlay"] + list(plex.item_advance_keys.keys()) radarr_details = ["radarr_add", "radarr_folder", "radarr_monitor", "radarr_search", "radarr_availability", "radarr_quality", "radarr_tag"] sonarr_details = ["sonarr_add", "sonarr_folder", "sonarr_monitor", "sonarr_language", "sonarr_series", "sonarr_quality", "sonarr_season", "sonarr_search", "sonarr_cutoff_search", "sonarr_tag"] @@ -212,6 +121,8 @@ movie_only_filters = [ "writer", "writer.not" ] show_only_filters = ["network"] +smart_invalid = ["collection_order"] +smart_url_invalid = ["filters", "run_again", "sync_mode", "show_filtered", "show_missing", "save_missing", "smart_label"] + radarr_details + sonarr_details class CollectionBuilder: def __init__(self, config, library, metadata, name, data): diff --git a/modules/config.py b/modules/config.py index ae68493a..0468527c 100644 --- a/modules/config.py +++ b/modules/config.py @@ -1,7 +1,7 @@ import logging, os, requests from datetime import datetime from lxml import html -from modules import util +from modules import util, radarr, sonarr from modules.anidb import AniDB from modules.anilist import AniList from modules.cache import Cache @@ -25,29 +25,7 @@ from ruamel import yaml logger = logging.getLogger("Plex Meta Manager") sync_modes = {"append": "Only Add Items to the Collection", "sync": "Add & Remove Items from the Collection"} -radarr_availabilities = { - "announced": "For Announced", - "cinemas": "For In Cinemas", - "released": "For Released", - "db": "For PreDB" -} -sonarr_monitors = { - "all": "Monitor all episodes except specials", - "future": "Monitor episodes that have not aired yet", - "missing": "Monitor episodes that do not have files or have not aired yet", - "existing": "Monitor episodes that have files or have not aired yet", - "pilot": "Monitor the first episode. All other episodes will be ignored", - "first": "Monitor all episodes of the first season. All other seasons will be ignored", - "latest": "Monitor all episodes of the latest season and future seasons", - "none": "No episodes will be monitored" -} -sonarr_series_types = { - "standard": "Episodes released with SxxEyy pattern", - "daily": "Episodes released daily or less frequently that use year-month-day (2017-05-25)", - "anime": "Episodes released using an absolute episode number" -} mass_update_options = {"tmdb": "Use TMDb Metadata", "omdb": "Use IMDb Metadata through OMDb"} -library_types = {"movie": "For Movie Libraries", "show": "For Show Libraries"} class Config: def __init__(self, default_dir, config_path=None, is_test=False, time_scheduled=None, requested_collections=None, requested_libraries=None, resume_from=None): @@ -317,7 +295,7 @@ class Config: self.general["radarr"]["add"] = check_for_attribute(self.data, "add", parent="radarr", var_type="bool", default=False) self.general["radarr"]["root_folder_path"] = check_for_attribute(self.data, "root_folder_path", parent="radarr", default_is_none=True) self.general["radarr"]["monitor"] = check_for_attribute(self.data, "monitor", parent="radarr", var_type="bool", default=True) - self.general["radarr"]["availability"] = check_for_attribute(self.data, "availability", parent="radarr", test_list=radarr_availabilities, default="announced") + self.general["radarr"]["availability"] = check_for_attribute(self.data, "availability", parent="radarr", test_list=radarr.availability_descriptions, default="announced") self.general["radarr"]["quality_profile"] = check_for_attribute(self.data, "quality_profile", parent="radarr", default_is_none=True) self.general["radarr"]["tag"] = check_for_attribute(self.data, "tag", parent="radarr", var_type="lower_list", default_is_none=True) self.general["radarr"]["search"] = check_for_attribute(self.data, "search", parent="radarr", var_type="bool", default=False) @@ -327,10 +305,10 @@ class Config: self.general["sonarr"]["token"] = check_for_attribute(self.data, "token", parent="sonarr", default_is_none=True) self.general["sonarr"]["add"] = check_for_attribute(self.data, "add", parent="sonarr", var_type="bool", default=False) self.general["sonarr"]["root_folder_path"] = check_for_attribute(self.data, "root_folder_path", parent="sonarr", default_is_none=True) - self.general["sonarr"]["monitor"] = check_for_attribute(self.data, "monitor", parent="sonarr", test_list=sonarr_monitors, default="all") + self.general["sonarr"]["monitor"] = check_for_attribute(self.data, "monitor", parent="sonarr", test_list=sonarr.monitor_descriptions, default="all") self.general["sonarr"]["quality_profile"] = check_for_attribute(self.data, "quality_profile", parent="sonarr", default_is_none=True) self.general["sonarr"]["language_profile"] = check_for_attribute(self.data, "language_profile", parent="sonarr", default_is_none=True) - self.general["sonarr"]["series_type"] = check_for_attribute(self.data, "series_type", parent="sonarr", test_list=sonarr_series_types, default="standard") + self.general["sonarr"]["series_type"] = check_for_attribute(self.data, "series_type", parent="sonarr", test_list=sonarr.series_type_descriptions, default="standard") self.general["sonarr"]["season_folder"] = check_for_attribute(self.data, "season_folder", parent="sonarr", var_type="bool", default=True) self.general["sonarr"]["tag"] = check_for_attribute(self.data, "tag", parent="sonarr", var_type="lower_list", default_is_none=True) self.general["sonarr"]["search"] = check_for_attribute(self.data, "search", parent="sonarr", var_type="bool", default=False) @@ -495,7 +473,7 @@ class Config: radarr_params["add"] = check_for_attribute(lib, "add", parent="radarr", var_type="bool", default=self.general["radarr"]["add"], save=False) radarr_params["root_folder_path"] = check_for_attribute(lib, "root_folder_path", parent="radarr", default=self.general["radarr"]["root_folder_path"], req_default=True, save=False) radarr_params["monitor"] = check_for_attribute(lib, "monitor", parent="radarr", var_type="bool", default=self.general["radarr"]["monitor"], save=False) - radarr_params["availability"] = check_for_attribute(lib, "availability", parent="radarr", test_list=radarr_availabilities, default=self.general["radarr"]["availability"], save=False) + radarr_params["availability"] = check_for_attribute(lib, "availability", parent="radarr", test_list=radarr.availability_descriptions, default=self.general["radarr"]["availability"], save=False) radarr_params["quality_profile"] = check_for_attribute(lib, "quality_profile", parent="radarr", default=self.general["radarr"]["quality_profile"], req_default=True, save=False) radarr_params["tag"] = check_for_attribute(lib, "tag", parent="radarr", var_type="lower_list", default=self.general["radarr"]["tag"], default_is_none=True, save=False) radarr_params["search"] = check_for_attribute(lib, "search", parent="radarr", var_type="bool", default=self.general["radarr"]["search"], save=False) @@ -518,13 +496,13 @@ class Config: sonarr_params["token"] = check_for_attribute(lib, "token", parent="sonarr", default=self.general["sonarr"]["token"], req_default=True, save=False) sonarr_params["add"] = check_for_attribute(lib, "add", parent="sonarr", var_type="bool", default=self.general["sonarr"]["add"], save=False) sonarr_params["root_folder_path"] = check_for_attribute(lib, "root_folder_path", parent="sonarr", default=self.general["sonarr"]["root_folder_path"], req_default=True, save=False) - sonarr_params["monitor"] = check_for_attribute(lib, "monitor", parent="sonarr", test_list=sonarr_monitors, default=self.general["sonarr"]["monitor"], save=False) + sonarr_params["monitor"] = check_for_attribute(lib, "monitor", parent="sonarr", test_list=sonarr.monitor_descriptions, default=self.general["sonarr"]["monitor"], save=False) sonarr_params["quality_profile"] = check_for_attribute(lib, "quality_profile", parent="sonarr", default=self.general["sonarr"]["quality_profile"], req_default=True, save=False) if self.general["sonarr"]["language_profile"]: sonarr_params["language_profile"] = check_for_attribute(lib, "language_profile", parent="sonarr", default=self.general["sonarr"]["language_profile"], save=False) else: sonarr_params["language_profile"] = check_for_attribute(lib, "language_profile", parent="sonarr", default_is_none=True, save=False) - sonarr_params["series_type"] = check_for_attribute(lib, "series_type", parent="sonarr", test_list=sonarr_series_types, default=self.general["sonarr"]["series_type"], save=False) + sonarr_params["series_type"] = check_for_attribute(lib, "series_type", parent="sonarr", test_list=sonarr.series_type_descriptions, default=self.general["sonarr"]["series_type"], save=False) sonarr_params["season_folder"] = check_for_attribute(lib, "season_folder", parent="sonarr", var_type="bool", default=self.general["sonarr"]["season_folder"], save=False) sonarr_params["tag"] = check_for_attribute(lib, "tag", parent="sonarr", var_type="lower_list", default=self.general["sonarr"]["tag"], default_is_none=True, save=False) sonarr_params["search"] = check_for_attribute(lib, "search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["search"], save=False) diff --git a/modules/mal.py b/modules/mal.py index b77ce012..d7e8a3bb 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -6,79 +6,36 @@ from ruamel import yaml logger = logging.getLogger("Plex Meta Manager") builders = [ - "mal_id", - "mal_all", - "mal_airing", - "mal_upcoming", - "mal_tv", - "mal_ova", - "mal_movie", - "mal_special", - "mal_popular", - "mal_favorite", - "mal_season", - "mal_suggested", - "mal_userlist" + "mal_id", "mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_ova", "mal_movie", + "mal_special", "mal_popular", "mal_favorite", "mal_season", "mal_suggested", "mal_userlist" ] mal_ranked_name = { - "mal_all": "all", - "mal_airing": "airing", - "mal_upcoming": "upcoming", - "mal_tv": "tv", - "mal_ova": "ova", - "mal_movie": "movie", - "mal_special": "special", - "mal_popular": "bypopularity", - "mal_favorite": "favorite" -} -season_sort_translation = { - "anime_score": "anime_score", - "anime_num_list_users": "anime_num_list_users", - "score": "anime_score", - "members": "anime_num_list_users" + "mal_all": "all", "mal_airing": "airing", "mal_upcoming": "upcoming", "mal_tv": "tv", "mal_ova": "ova", + "mal_movie": "movie", "mal_special": "special", "mal_popular": "bypopularity", "mal_favorite": "favorite" } +season_sort_translation = {"score": "anime_score", "anime_score": "anime_score", "members": "anime_num_list_users", "anime_num_list_users": "anime_num_list_users"} season_sort_options = ["score", "members"] pretty_names = { - "anime_score": "Score", - "anime_num_list_users": "Members", - "list_score": "Score", - "list_updated_at": "Last Updated", - "anime_title": "Title", - "anime_start_date": "Start Date", - "all": "All Anime", - "watching": "Currently Watching", - "completed": "Completed", - "on_hold": "On Hold", - "dropped": "Dropped", - "plan_to_watch": "Plan to Watch" + "anime_score": "Score", "list_score": "Score", "anime_num_list_users": "Members", "list_updated_at": "Last Updated", + "anime_title": "Title", "anime_start_date": "Start Date", "all": "All Anime", "watching": "Currently Watching", + "completed": "Completed", "on_hold": "On Hold", "dropped": "Dropped", "plan_to_watch": "Plan to Watch" } userlist_sort_translation = { - "score": "list_score", - "list_score": "list_score", - "last_updated": "list_updated_at", - "list_updated": "list_updated_at", - "list_updated_at": "list_updated_at", - "title": "anime_title", - "anime_title": "anime_title", - "start_date": "anime_start_date", - "anime_start_date": "anime_start_date" + "score": "list_score", "list_score": "list_score", + "last_updated": "list_updated_at", "list_updated": "list_updated_at", "list_updated_at": "list_updated_at", + "title": "anime_title", "anime_title": "anime_title", + "start_date": "anime_start_date", "anime_start_date": "anime_start_date" } userlist_sort_options = ["score", "last_updated", "title", "start_date"] -userlist_status = [ - "all", - "watching", - "completed", - "on_hold", - "dropped", - "plan_to_watch" -] +userlist_status = ["all", "watching", "completed", "on_hold", "dropped", "plan_to_watch"] +base_url = "https://myanimelist.net" urls = { - "oauth_token": "https://myanimelist.net/v1/oauth2/token", - "oauth_authorize": "https://myanimelist.net/v1/oauth2/authorize", - "ranking": "https://api.myanimelist.net/v2/anime/ranking", - "season": "https://api.myanimelist.net/v2/anime/season", - "suggestions": "https://api.myanimelist.net/v2/anime/suggestions", - "user": "https://api.myanimelist.net/v2/users" + "oauth_token": f"{base_url}/v1/oauth2/token", + "oauth_authorize": f"{base_url}/v1/oauth2/authorize", + "ranking": f"{base_url}/v2/anime/ranking", + "season": f"{base_url}/v2/anime/season", + "suggestions": f"{base_url}/v2/anime/suggestions", + "user": f"{base_url}/v2/users" } class MyAnimeList: diff --git a/modules/plex.py b/modules/plex.py index 3cc0ccb4..6ee186a2 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -34,16 +34,8 @@ search_translation = { "episode_plays": "episode.viewCount" } modifier_translation = { - "": "", - ".not": "!", - ".gt": "%3E%3E", - ".gte": "%3E", - ".lt": "%3C%3C", - ".lte": "%3C", - ".before": "%3C%3C", - ".after": "%3E%3E", - ".begins": "%3C", - ".ends": "%3E" + "": "", ".not": "!", ".gt": "%3E%3E", ".gte": "%3E", ".lt": "%3C%3C", ".lte": "%3C", + ".before": "%3C%3C", ".after": "%3E%3E", ".begins": "%3C", ".ends": "%3E" } episode_sorting_options = {"default": "-1", "oldest": "0", "newest": "1"} keep_episodes_options = {"all": 0, "5_latest": 5, "3_latest": 3, "latest": 1, "past_3": -3, "past_7": -7, "past_30": -30} @@ -58,17 +50,11 @@ metadata_language_options = {lang.lower(): lang for lang in plex_languages} metadata_language_options["default"] = None use_original_title_options = {"default": -1, "no": 0, "yes": 1} collection_mode_options = { - "default": "default", - "hide": "hide", - "hide_items": "hideItems", - "hideitems": "hideItems", - "show_items": "showItems", - "showitems": "showItems" -} -collection_order_options = { - "release": "release", - "alpha": "alpha" + "default": "default", "hide": "hide", + "hide_items": "hideItems", "hideitems": "hideItems", + "show_items": "showItems", "showitems": "showItems" } +collection_order_options = {"release": "release", "alpha": "alpha"} collection_mode_keys = {-1: "default", 0: "hide", 1: "hideItems", 2: "showItems"} collection_order_keys = {0: "release", 1: "alpha", 2: "custom"} item_advance_keys = { @@ -126,13 +112,9 @@ or_searches = [ "writer", "decade", "resolution", "year", "episode_title", "episode_year" ] movie_only_searches = [ - "country", "country.not", - "director", "director.not", - "producer", "producer.not", - "writer", "writer.not", + "country", "country.not", "director", "director.not", "producer", "producer.not", "writer", "writer.not", "decade", "duplicate", "unplayed", "progress", "trash", - "plays.gt", "plays.gte", "plays.lt", "plays.lte", - "duration.gt", "duration.gte", "duration.lt", "duration.lte" + "plays.gt", "plays.gte", "plays.lt", "plays.lte", "duration.gt", "duration.gte", "duration.lt", "duration.lte" ] show_only_searches = [ "network", "network.not", @@ -152,13 +134,7 @@ boolean_attributes = [ ] tmdb_attributes = ["actor", "director", "producer", "writer"] date_attributes = ["added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played"] -search_display = { - "added": "Date Added", - "release": "Release Date", - "hdr": "HDR", - "progress": "In Progress", - "episode_progress": "Episode In Progress" -} +search_display = {"added": "Date Added", "release": "Release Date", "hdr": "HDR", "progress": "In Progress", "episode_progress": "Episode In Progress"} sorts = { None: None, "title.asc": "titleSort:asc", "title.desc": "titleSort:desc", @@ -169,44 +145,14 @@ sorts = { "duration.asc": "duration:asc", "duration.desc": "duration:desc", "added.asc": "addedAt:asc", "added.desc": "addedAt:desc" } -modifiers = { - ".not": "!", - ".begins": "<", - ".ends": ">", - ".before": "<<", - ".after": ">>", - ".gt": ">>", - ".gte": "__gte", - ".lt": "<<", - ".lte": "__lte" -} +modifiers = {".not": "!", ".begins": "<", ".ends": ">", ".before": "<<", ".after": ">>", ".gt": ">>", ".gte": "__gte", ".lt": "<<", ".lte": "__lte"} mod_displays = { - "": "is", - ".not": "is not", - ".begins": "begins with", - ".ends": "ends with", - ".before": "is before", - ".after": "is after", - ".gt": "is greater than", - ".gte": "is greater than or equal", - ".lt": "is less than", - ".lte": "is less than or equal" + "": "is", ".not": "is not", ".begins": "begins with", ".ends": "ends with", ".before": "is before", ".after": "is after", + ".gt": "is greater than", ".gte": "is greater than or equal", ".lt": "is less than", ".lte": "is less than or equal" } tags = [ - "actor", - "audio_language", - "collection", - "content_rating", - "country", - "director", - "genre", - "label", - "network", - "producer", - "resolution", - "studio", - "subtitle_language", - "writer" + "actor", "audio_language", "collection", "content_rating", "country", "director", "genre", "label", + "network", "producer", "resolution", "studio", "subtitle_language", "writer" ] movie_sorts = { "title.asc": "titleSort", "title.desc": "titleSort%3Adesc", @@ -257,12 +203,7 @@ episode_sorts = { "added.asc": "addedAt", "added.desc": "addedAt%3Adesc", "random": "random" } -sort_types = { - "movies": (1, movie_sorts), - "shows": (2, show_sorts), - "seasons": (3, season_sorts), - "episodes": (4, episode_sorts), -} +sort_types = {"movies": (1, movie_sorts), "shows": (2, show_sorts), "seasons": (3, season_sorts), "episodes": (4, episode_sorts)} class Plex: def __init__(self, config, params): diff --git a/modules/radarr.py b/modules/radarr.py index 1a135aaf..8bb0909a 100644 --- a/modules/radarr.py +++ b/modules/radarr.py @@ -6,17 +6,9 @@ from arrapi.exceptions import ArrException, Invalid logger = logging.getLogger("Plex Meta Manager") -availability_translation = { - "announced": "announced", - "cinemas": "inCinemas", - "released": "released", - "db": "preDB" -} -apply_tags_translation = { - "": "add", - "sync": "replace", - "remove": "remove" -} +availability_translation = {"announced": "announced", "cinemas": "inCinemas", "released": "released", "db": "preDB"} +apply_tags_translation = {"": "add", "sync": "replace", "remove": "remove"} +availability_descriptions = {"announced": "For Announced", "cinemas": "For In Cinemas", "released": "For Released", "db": "For PreDB"} class Radarr: def __init__(self, config, params): diff --git a/modules/sonarr.py b/modules/sonarr.py index 55782b84..bba35783 100644 --- a/modules/sonarr.py +++ b/modules/sonarr.py @@ -8,20 +8,25 @@ logger = logging.getLogger("Plex Meta Manager") series_type = ["standard", "daily", "anime"] monitor_translation = { - "all": "all", - "future": "future", - "missing": "missing", - "existing": "existing", - "pilot": "pilot", - "first": "firstSeason", - "latest": "latestSeason", - "none": "none" + "all": "all", "future": "future", "missing": "missing", "existing": "existing", + "pilot": "pilot", "first": "firstSeason", "latest": "latestSeason", "none": "none" } -apply_tags_translation = { - "": "add", - "sync": "replace", - "remove": "remove" +series_type_descriptions = { + "standard": "Episodes released with SxxEyy pattern", + "daily": "Episodes released daily or less frequently that use year-month-day (2017-05-25)", + "anime": "Episodes released using an absolute episode number" } +monitor_descriptions = { + "all": "Monitor all episodes except specials", + "future": "Monitor episodes that have not aired yet", + "missing": "Monitor episodes that do not have files or have not aired yet", + "existing": "Monitor episodes that have files or have not aired yet", + "pilot": "Monitor the first episode. All other episodes will be ignored", + "first": "Monitor all episodes of the first season. All other seasons will be ignored", + "latest": "Monitor all episodes of the latest season and future seasons", + "none": "No episodes will be monitored" +} +apply_tags_translation = {"": "add", "sync": "replace", "remove": "remove"} class Sonarr: def __init__(self, config, params): diff --git a/modules/tmdb.py b/modules/tmdb.py index a3065403..e2b82dd6 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -7,105 +7,44 @@ from tmdbv3api.exceptions import TMDbException logger = logging.getLogger("Plex Meta Manager") builders = [ - "tmdb_actor", - "tmdb_actor_details", - "tmdb_collection", - "tmdb_collection_details", - "tmdb_company", - "tmdb_crew", - "tmdb_crew_details", - "tmdb_director", - "tmdb_director_details", - "tmdb_discover", - "tmdb_keyword", - "tmdb_list", - "tmdb_list_details", - "tmdb_movie", - "tmdb_movie_details", - "tmdb_network", - "tmdb_now_playing", - "tmdb_popular", - "tmdb_producer", - "tmdb_producer_details", - "tmdb_show", - "tmdb_show_details", - "tmdb_top_rated", - "tmdb_trending_daily", - "tmdb_trending_weekly", - "tmdb_writer", - "tmdb_writer_details" + "tmdb_actor", "tmdb_actor_details", "tmdb_collection", "tmdb_collection_details", "tmdb_company", + "tmdb_crew", "tmdb_crew_details", "tmdb_director", "tmdb_director_details", "tmdb_discover", + "tmdb_keyword", "tmdb_list", "tmdb_list_details", "tmdb_movie", "tmdb_movie_details", "tmdb_network", + "tmdb_now_playing", "tmdb_popular", "tmdb_producer", "tmdb_producer_details", "tmdb_show", "tmdb_show_details", + "tmdb_top_rated", "tmdb_trending_daily", "tmdb_trending_weekly", "tmdb_writer", "tmdb_writer_details" ] type_map = { - "tmdb_actor": "Person", - "tmdb_actor_details": "Person", - "tmdb_collection": "Collection", - "tmdb_collection_details": "Collection", - "tmdb_company": "Company", - "tmdb_crew": "Person", - "tmdb_crew_details": "Person", - "tmdb_director": "Person", - "tmdb_director_details": "Person", - "tmdb_keyword": "Keyword", - "tmdb_list": "List", - "tmdb_list_details": "List", - "tmdb_movie": "Movie", - "tmdb_movie_details": "Movie", - "tmdb_network": "Network", - "tmdb_person": "Person", - "tmdb_producer": "Person", - "tmdb_producer_details": "Person", - "tmdb_show": "Show", - "tmdb_show_details": "Show", - "tmdb_writer": "Person", - "tmdb_writer_details": "Person" + "tmdb_actor": "Person", "tmdb_actor_details": "Person", "tmdb_crew": "Person", "tmdb_crew_details": "Person", + "tmdb_collection": "Collection", "tmdb_collection_details": "Collection", "tmdb_company": "Company", + "tmdb_director": "Person", "tmdb_director_details": "Person", "tmdb_keyword": "Keyword", + "tmdb_list": "List", "tmdb_list_details": "List", "tmdb_movie": "Movie", "tmdb_movie_details": "Movie", + "tmdb_network": "Network", "tmdb_person": "Person", "tmdb_producer": "Person", "tmdb_producer_details": "Person", + "tmdb_show": "Show", "tmdb_show_details": "Show", "tmdb_writer": "Person", "tmdb_writer_details": "Person" } discover_movie = [ - "language", "with_original_language", "region", "sort_by", + "language", "with_original_language", "region", "sort_by", "with_cast", "with_crew", "with_people", "certification_country", "certification", "certification.lte", "certification.gte", - "include_adult", - "primary_release_year", "primary_release_date.gte", "primary_release_date.lte", - "release_date.gte", "release_date.lte", "year", - "vote_count.gte", "vote_count.lte", - "vote_average.gte", "vote_average.lte", - "with_cast", "with_crew", "with_people", - "with_companies", - "with_genres", "without_genres", - "with_keywords", "without_keywords", - "with_runtime.gte", "with_runtime.lte" + "year", "primary_release_year", "primary_release_date.gte", "primary_release_date.lte", + "release_date.gte", "release_date.lte", "vote_count.gte", "vote_count.lte", + "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte", + "with_companies", "with_genres", "without_genres", "with_keywords", "without_keywords", "include_adult" ] discover_tv = [ - "language", "with_original_language", "timezone", "sort_by", - "air_date.gte", "air_date.lte", - "first_air_date.gte", "first_air_date.lte", "first_air_date_year", - "vote_count.gte", "vote_count.lte", - "vote_average.gte", "vote_average.lte", - "with_genres", "without_genres", - "with_keywords", "without_keywords", - "with_networks", "with_companies", - "with_runtime.gte", "with_runtime.lte", - "include_null_first_air_dates", - "screened_theatrically" + "language", "with_original_language", "timezone", "sort_by", "screened_theatrically", "include_null_first_air_dates", + "air_date.gte", "air_date.lte", "first_air_date.gte", "first_air_date.lte", "first_air_date_year", + "vote_count.gte", "vote_count.lte", "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte", + "with_genres", "without_genres", "with_keywords", "without_keywords", "with_networks", "with_companies" ] discover_dates = [ - "primary_release_date.gte", "primary_release_date.lte", - "release_date.gte", "release_date.lte", - "air_date.gte", "air_date.lte", - "first_air_date.gte", "first_air_date.lte" + "primary_release_date.gte", "primary_release_date.lte", "release_date.gte", "release_date.lte", + "air_date.gte", "air_date.lte", "first_air_date.gte", "first_air_date.lte" ] discover_movie_sort = [ - "popularity.asc", "popularity.desc", - "release_date.asc", "release_date.desc", - "revenue.asc", "revenue.desc", - "primary_release_date.asc", "primary_release_date.desc", - "original_title.asc", "original_title.desc", - "vote_average.asc", "vote_average.desc", - "vote_count.asc", "vote_count.desc" -] -discover_tv_sort = [ - "vote_average.desc", "vote_average.asc", - "first_air_date.desc", "first_air_date.asc", - "popularity.desc", "popularity.asc" + "popularity.asc", "popularity.desc", "release_date.asc", "release_date.desc", "revenue.asc", "revenue.desc", + "primary_release_date.asc", "primary_release_date.desc", "original_title.asc", "original_title.desc", + "vote_average.asc", "vote_average.desc", "vote_count.asc", "vote_count.desc" ] +discover_tv_sort = ["vote_average.desc", "vote_average.asc", "first_air_date.desc", "first_air_date.asc", "popularity.desc", "popularity.asc"] class TMDb: def __init__(self, config, params): diff --git a/modules/trakt.py b/modules/trakt.py index 60221049..a5ae584e 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -9,15 +9,8 @@ redirect_uri = "urn:ietf:wg:oauth:2.0:oob" redirect_uri_encoded = redirect_uri.replace(":", "%3A") base_url = "https://api.trakt.tv" builders = [ - "trakt_collected", - "trakt_collection", - "trakt_list", - "trakt_list_details", - "trakt_popular", - "trakt_recommended", - "trakt_trending", - "trakt_watched", - "trakt_watchlist" + "trakt_collected", "trakt_collection", "trakt_list", "trakt_list_details", "trakt_popular", + "trakt_recommended", "trakt_trending", "trakt_watched", "trakt_watchlist" ] class Trakt: diff --git a/modules/tvdb.py b/modules/tvdb.py index 6a27dcef..3d6c2e32 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -4,25 +4,14 @@ from modules.util import Failed logger = logging.getLogger("Plex Meta Manager") -builders = [ - "tvdb_list", - "tvdb_list_details", - "tvdb_movie", - "tvdb_movie_details", - "tvdb_show", - "tvdb_show_details" -] +builders = ["tvdb_list", "tvdb_list_details", "tvdb_movie", "tvdb_movie_details", "tvdb_show", "tvdb_show_details"] base_url = "https://www.thetvdb.com" alt_url = "https://thetvdb.com" urls = { - "list": f"{base_url}/lists/", - "alt_list": f"{alt_url}/lists/", - "series": f"{base_url}/series/", - "alt_series": f"{alt_url}/series/", - "movies": f"{base_url}/movies/", - "alt_movies": f"{alt_url}/movies/", - "series_id": f"{base_url}/dereferrer/series/", - "movie_id": f"{base_url}/dereferrer/movie/" + "list": f"{base_url}/lists/", "alt_list": f"{alt_url}/lists/", + "series": f"{base_url}/series/", "alt_series": f"{alt_url}/series/", + "movies": f"{base_url}/movies/", "alt_movies": f"{alt_url}/movies/", + "series_id": f"{base_url}/dereferrer/series/", "movie_id": f"{base_url}/dereferrer/movie/" } class TVDbObj: diff --git a/modules/util.py b/modules/util.py index 390271b7..6510a74c 100644 --- a/modules/util.py +++ b/modules/util.py @@ -47,122 +47,47 @@ days_alias = { "saturday": 5, "sat": 5, "s": 5, "sunday": 6, "sun": 6, "su": 6, "u": 6 } -pretty_days = { - 0: "Monday", - 1: "Tuesday", - 2: "Wednesday", - 3: "Thursday", - 4: "Friday", - 5: "Saturday", - 6: "Sunday" -} +pretty_days = {0: "Monday", 1: "Tuesday", 2: "Wednesday", 3: "Thursday", 4: "Friday", 5: "Saturday", 6: "Sunday"} pretty_months = { - 1: "January", - 2: "February", - 3: "March", - 4: "April", - 5: "May", - 6: "June", - 7: "July", - 8: "August", - 9: "September", - 10: "October", - 11: "November", - 12: "December" -} -pretty_seasons = { - "winter": "Winter", - "spring": "Spring", - "summer": "Summer", - "fall": "Fall" + 1: "January", 2: "February", 3: "March", 4: "April", 5: "May", 6: "June", + 7: "July", 8: "August", 9: "September", 10: "October", 11: "November", 12: "December" } +pretty_seasons = {"winter": "Winter", "spring": "Spring", "summer": "Summer", "fall": "Fall"} pretty_names = { - "anidb_id": "AniDB ID", - "anidb_relation": "AniDB Relation", - "anidb_popular": "AniDB Popular", - "anilist_genre": "AniList Genre", - "anilist_id": "AniList ID", - "anilist_popular": "AniList Popular", - "anilist_relations": "AniList Relations", - "anilist_season": "AniList Season", - "anilist_studio": "AniList Studio", - "anilist_tag": "AniList Tag", - "anilist_top_rated": "AniList Top Rated", + "anidb_id": "AniDB ID", "anidb_relation": "AniDB Relation", "anidb_popular": "AniDB Popular", + "anilist_genre": "AniList Genre", "anilist_id": "AniList ID", "anilist_popular": "AniList Popular", + "anilist_relations": "AniList Relations", "anilist_season": "AniList Season", "anilist_studio": "AniList Studio", + "anilist_tag": "AniList Tag", "anilist_top_rated": "AniList Top Rated", "icheckmovies_list": "I Check Movies List", - "imdb_list": "IMDb List", - "imdb_id": "IMDb ID", - "letterboxd_list": "Letterboxd List", - "letterboxd_list_details": "Letterboxd List", - "mal_id": "MyAnimeList ID", - "mal_all": "MyAnimeList All", - "mal_airing": "MyAnimeList Airing", - "mal_upcoming": "MyAnimeList Upcoming", - "mal_tv": "MyAnimeList TV", - "mal_ova": "MyAnimeList OVA", - "mal_movie": "MyAnimeList Movie", - "mal_special": "MyAnimeList Special", - "mal_popular": "MyAnimeList Popular", - "mal_favorite": "MyAnimeList Favorite", - "mal_season": "MyAnimeList Season", - "mal_suggested": "MyAnimeList Suggested", - "mal_userlist": "MyAnimeList Userlist", - "plex_all": "Plex All", - "plex_collection": "Plex Collection", - "plex_search": "Plex Search", - "tautulli_popular": "Tautulli Popular", - "tautulli_watched": "Tautulli Watched", - "tmdb_actor": "TMDb Actor", - "tmdb_actor_details": "TMDb Actor", - "tmdb_collection": "TMDb Collection", - "tmdb_collection_details": "TMDb Collection", - "tmdb_company": "TMDb Company", - "tmdb_crew": "TMDb Crew", - "tmdb_crew_details": "TMDb Crew", - "tmdb_director": "TMDb Director", - "tmdb_director_details": "TMDb Director", - "tmdb_discover": "TMDb Discover", - "tmdb_keyword": "TMDb Keyword", - "tmdb_list": "TMDb List", - "tmdb_list_details": "TMDb List", - "tmdb_movie": "TMDb Movie", - "tmdb_movie_details": "TMDb Movie", - "tmdb_network": "TMDb Network", - "tmdb_now_playing": "TMDb Now Playing", - "tmdb_person": "TMDb Person", - "tmdb_popular": "TMDb Popular", - "tmdb_producer": "TMDb Producer", - "tmdb_producer_details": "TMDb Producer", - "tmdb_show": "TMDb Show", - "tmdb_show_details": "TMDb Show", - "tmdb_top_rated": "TMDb Top Rated", - "tmdb_trending_daily": "TMDb Trending Daily", - "tmdb_trending_weekly": "TMDb Trending Weekly", - "tmdb_writer": "TMDb Writer", - "tmdb_writer_details": "TMDb Writer", - "trakt_collected": "Trakt Collected", - "trakt_collection": "Trakt Collection", - "trakt_list": "Trakt List", - "trakt_list_details": "Trakt List", - "trakt_popular": "Trakt Popular", - "trakt_recommended": "Trakt Recommended", - "trakt_trending": "Trakt Trending", - "trakt_watched": "Trakt Watched", - "trakt_watchlist": "Trakt Watchlist", - "tvdb_list": "TVDb List", - "tvdb_list_details": "TVDb List", - "tvdb_movie": "TVDb Movie", - "tvdb_movie_details": "TVDb Movie", - "tvdb_show": "TVDb Show", - "tvdb_show_details": "TVDb Show" -} -pretty_ids = { - "anidbid": "AniDB", - "imdbid": "IMDb", - "mal_id": "MyAnimeList", - "themoviedb_id": "TMDb", - "thetvdb_id": "TVDb", - "tvdbid": "TVDb" + "imdb_list": "IMDb List", "imdb_id": "IMDb ID", + "letterboxd_list": "Letterboxd List", "letterboxd_list_details": "Letterboxd List", + "mal_id": "MyAnimeList ID", "mal_all": "MyAnimeList All", "mal_airing": "MyAnimeList Airing", + "mal_upcoming": "MyAnimeList Upcoming", "mal_tv": "MyAnimeList TV", "mal_ova": "MyAnimeList OVA", + "mal_movie": "MyAnimeList Movie", "mal_special": "MyAnimeList Special", "mal_popular": "MyAnimeList Popular", + "mal_favorite": "MyAnimeList Favorite", "mal_season": "MyAnimeList Season", + "mal_suggested": "MyAnimeList Suggested", "mal_userlist": "MyAnimeList Userlist", + "plex_all": "Plex All", "plex_collection": "Plex Collection", "plex_search": "Plex Search", + "tautulli_popular": "Tautulli Popular", "tautulli_watched": "Tautulli Watched", + "tmdb_actor": "TMDb Actor", "tmdb_actor_details": "TMDb Actor", + "tmdb_collection": "TMDb Collection", "tmdb_collection_details": "TMDb Collection", + "tmdb_company": "TMDb Company", "tmdb_crew": "TMDb Crew", "tmdb_crew_details": "TMDb Crew", + "tmdb_director": "TMDb Director", "tmdb_director_details": "TMDb Director", "tmdb_discover": "TMDb Discover", + "tmdb_keyword": "TMDb Keyword", "tmdb_list": "TMDb List", "tmdb_list_details": "TMDb List", + "tmdb_movie": "TMDb Movie", "tmdb_movie_details": "TMDb Movie", "tmdb_network": "TMDb Network", + "tmdb_now_playing": "TMDb Now Playing", "tmdb_person": "TMDb Person", "tmdb_popular": "TMDb Popular", + "tmdb_producer": "TMDb Producer", "tmdb_producer_details": "TMDb Producer", + "tmdb_show": "TMDb Show", "tmdb_show_details": "TMDb Show", "tmdb_top_rated": "TMDb Top Rated", + "tmdb_trending_daily": "TMDb Trending Daily", "tmdb_trending_weekly": "TMDb Trending Weekly", + "tmdb_writer": "TMDb Writer", "tmdb_writer_details": "TMDb Writer", + "trakt_collected": "Trakt Collected", "trakt_collection": "Trakt Collection", + "trakt_list": "Trakt List", "trakt_list_details": "Trakt List", + "trakt_popular": "Trakt Popular", "trakt_recommended": "Trakt Recommended", "trakt_trending": "Trakt Trending", + "trakt_watched": "Trakt Watched", "trakt_watchlist": "Trakt Watchlist", + "tvdb_list": "TVDb List", "tvdb_list_details": "TVDb List", + "tvdb_movie": "TVDb Movie", "tvdb_movie_details": "TVDb Movie", + "tvdb_show": "TVDb Show", "tvdb_show_details": "TVDb Show" } +pretty_ids = {"anidbid": "AniDB", "imdbid": "IMDb", "mal_id": "MyAnimeList", "themoviedb_id": "TMDb", "thetvdb_id": "TVDb", "tvdbid": "TVDb"} def tab_new_lines(data): return str(data).replace("\n", "\n|\t ") if "\n" in str(data) else str(data) From 7f1593ecd8470d93f6417dbe1cd750029594e6ba Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 26 Jul 2021 10:55:28 -0400 Subject: [PATCH 32/95] fix overlay and icheckmovies issue --- modules/builder.py | 2 +- modules/icheckmovies.py | 5 +++-- modules/plex.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 9ec8d495..af0fe147 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1561,7 +1561,7 @@ class CollectionBuilder: if self.config.Cache: rating_keys = self.config.Cache.query_image_map_overlay(self.library.image_table_name, overlay_name) overlay_folder = os.path.join(self.config.default_dir, "overlays", overlay_name) - overlay_image = Image.open(os.path.join(overlay_folder, "overlay.png")) + overlay_image = Image.open(os.path.join(overlay_folder, "overlay.png")).convert("RGBA") temp_image = os.path.join(overlay_folder, f"temp.png") overlay = (overlay_name, overlay_folder, overlay_image, temp_image) diff --git a/modules/icheckmovies.py b/modules/icheckmovies.py index dc13e201..816fcc43 100644 --- a/modules/icheckmovies.py +++ b/modules/icheckmovies.py @@ -28,9 +28,10 @@ class ICheckMovies: list_url = icheckmovies_list.strip() if not list_url.startswith(base_url): raise Failed(f"ICheckMovies Error: {list_url} must begin with: {base_url}") - if len(self._parse_list(list_url, language)) > 0: + elif len(self._parse_list(list_url, language)) > 0: valid_lists.append(list_url) - raise Failed(f"ICheckMovies Error: {list_url} failed to parse") + else: + raise Failed(f"ICheckMovies Error: {list_url} failed to parse") return valid_lists def get_items(self, method, data, language): diff --git a/modules/plex.py b/modules/plex.py index 6ee186a2..ac1d6d99 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -420,7 +420,7 @@ class Plex: shutil.copyfile(temp_image, os.path.join(overlay_folder, f"{item.ratingKey}.png")) while util.is_locked(temp_image): time.sleep(1) - new_poster = Image.open(temp_image) + new_poster = Image.open(temp_image).convert("RGBA") new_poster = new_poster.resize(overlay_image.size, Image.ANTIALIAS) new_poster.paste(overlay_image, (0, 0), overlay_image) new_poster.save(temp_image) From 1f5b7b5818dad6fa85989888dc2bc7e436f64697 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 26 Jul 2021 11:46:36 -0400 Subject: [PATCH 33/95] fix beginswith --- modules/builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index af0fe147..e07274a1 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -867,7 +867,7 @@ class CollectionBuilder: else: values = self.config.TMDb.validate_tmdb_ids(method_data, method_name) if method_name.endswith("_details"): - if method_name.beginswith(("tmdb_collection", "tmdb_movie", "tmdb_show")): + if method_name.startswith(("tmdb_collection", "tmdb_movie", "tmdb_show")): item = self.config.TMDb.get_movie_show_or_collection(values[0], self.library.is_movie) if hasattr(item, "overview") and item.overview: self.summaries[method_name] = item.overview @@ -875,13 +875,13 @@ class CollectionBuilder: self.backgrounds[method_name] = f"{self.config.TMDb.image_url}{item.backdrop_path}" if hasattr(item, "poster_path") and item.poster_path: self.posters[method_name] = f"{self.config.TMDb.image_url}{item.poster_path}" - elif method_name.beginswith(("tmdb_actor", "tmdb_crew", "tmdb_director", "tmdb_producer", "tmdb_writer")): + elif method_name.startswith(("tmdb_actor", "tmdb_crew", "tmdb_director", "tmdb_producer", "tmdb_writer")): item = self.config.TMDb.get_person(values[0]) if hasattr(item, "biography") and item.biography: self.summaries[method_name] = item.biography if hasattr(item, "profile_path") and item.profile_path: self.posters[method_name] = f"{self.config.TMDb.image_url}{item.profile_path}" - elif method_name.beginswith("tmdb_list"): + elif method_name.startswith("tmdb_list"): item = self.config.TMDb.get_list(values[0]) if hasattr(item, "description") and item.description: self.summaries[method_name] = item.description From 39dc94cca2beddb6846cd4f552afaf9dea454691 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 26 Jul 2021 14:52:32 -0400 Subject: [PATCH 34/95] fix plex search --- modules/builder.py | 1 + modules/plex.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index e07274a1..f54381ef 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1070,6 +1070,7 @@ class CollectionBuilder: def build_url_arg(arg, mod=None, arg_s=None, mod_s=None): arg_key = plex.search_translation[attr] if attr in plex.search_translation else attr + arg_key = plex.show_translation[arg_key] if self.library.is_show and arg_key in plex.show_translation else arg_key if mod is None: mod = plex.modifier_translation[modifier] if modifier in plex.modifier_translation else modifier if arg_s is None: diff --git a/modules/plex.py b/modules/plex.py index ac1d6d99..1bca19b2 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -33,6 +33,12 @@ search_translation = { "episode_user_rating": "episode.userRating", "episode_plays": "episode.viewCount" } +show_translation = { + "hdr": "episode.hdr", + "audioLanguage": "episode.audioLanguage", + "subtitleLanguage": "episode.subtitleLanguage", + "resolution": "episode.resolution" +} modifier_translation = { "": "", ".not": "!", ".gt": "%3E%3E", ".gte": "%3E", ".lt": "%3C%3C", ".lte": "%3C", ".before": "%3C%3C", ".after": "%3E%3E", ".begins": "%3C", ".ends": "%3E" @@ -455,8 +461,7 @@ class Plex: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def get_search_choices(self, search_name, title=True): final_search = search_translation[search_name] if search_name in search_translation else search_name - if final_search == "resolution" and self.is_show: - final_search = "episode.resolution" + final_search = show_translation[final_search] if self.is_show and final_search in show_translation else final_search try: choices = {} for choice in self.Plex.listFilterChoices(final_search): @@ -464,7 +469,8 @@ class Plex: choices[choice.key.lower()] = choice.title if title else choice.key return choices except NotFound: - raise Failed(f"Collection Error: plex search attribute: {search_name} only supported with Plex's New TV Agent") + logger.debug(f"Search Attribute: {final_search}") + raise Failed(f"Collection Error: plex search attribute: {search_name} not supported") @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def get_labels(self): From cff3025aac1742332209c53598fb516c54d86def Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 26 Jul 2021 15:03:17 -0400 Subject: [PATCH 35/95] cleanup and version --- modules/anidb.py | 8 +- modules/anilist.py | 2 +- modules/config.py | 219 +++++++++++++++++++++---------------------- modules/mal.py | 4 +- modules/trakt.py | 4 +- plex_meta_manager.py | 2 +- 6 files changed, 118 insertions(+), 121 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 7cd299f4..6dbcc212 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -23,15 +23,15 @@ class AniDB: if not self._login(self.username, self.password).xpath("//li[@class='sub-menu my']/@title"): raise Failed("AniDB Error: Login failed") - def _request(self, url, language=None, postData=None): - if postData: - return self.config.post_html(url, postData, headers=util.header(language)) + def _request(self, url, language=None, post=None): + if post: + return self.config.post_html(url, post, headers=util.header(language)) else: return self.config.get_html(url, headers=util.header(language)) def _login(self, username, password): data = {"show": "main", "xuser": username, "xpass": password, "xdoautologin": "on"} - return self._request(urls["login"], postData=data) + return self._request(urls["login"], post=data) def _popular(self, language): response = self._request(urls["popular"], language=language) diff --git a/modules/anilist.py b/modules/anilist.py index eac01e5a..ae0b4fa2 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -198,7 +198,7 @@ class AniList: def validate_anilist_ids(self, anilist_ids, studio=False): anilist_id_list = util.get_int_list(anilist_ids, "AniList ID") anilist_values = [] - for anilist_id in anilist_ids: + for anilist_id in anilist_id_list: if studio: query = "query ($id: Int) {Studio(id: $id) {name}}" else: query = "query ($id: Int) {Media(id: $id) {id}}" try: diff --git a/modules/config.py b/modules/config.py index 0468527c..e8e58219 100644 --- a/modules/config.py +++ b/modules/config.py @@ -170,34 +170,34 @@ class Config: self.session = requests.Session() - self.general = {} - self.general["cache"] = check_for_attribute(self.data, "cache", parent="settings", var_type="bool", default=True) - self.general["cache_expiration"] = check_for_attribute(self.data, "cache_expiration", parent="settings", var_type="int", default=60) + self.general = { + "cache": check_for_attribute(self.data, "cache", parent="settings", var_type="bool", default=True), + "cache_expiration": check_for_attribute(self.data, "cache_expiration", parent="settings", var_type="int", default=60), + "asset_directory": check_for_attribute(self.data, "asset_directory", parent="settings", var_type="list_path", default=[os.path.join(default_dir, "assets")]), + "asset_folders": check_for_attribute(self.data, "asset_folders", parent="settings", var_type="bool", default=True), + "assets_for_all": check_for_attribute(self.data, "assets_for_all", parent="settings", var_type="bool", default=False), + "sync_mode": check_for_attribute(self.data, "sync_mode", parent="settings", default="append", test_list=sync_modes), + "run_again_delay": check_for_attribute(self.data, "run_again_delay", parent="settings", var_type="int", default=0), + "show_unmanaged": check_for_attribute(self.data, "show_unmanaged", parent="settings", var_type="bool", default=True), + "show_filtered": check_for_attribute(self.data, "show_filtered", parent="settings", var_type="bool", default=False), + "show_missing": check_for_attribute(self.data, "show_missing", parent="settings", var_type="bool", default=True), + "save_missing": check_for_attribute(self.data, "save_missing", parent="settings", var_type="bool", default=True) + } if self.general["cache"]: util.separator() self.Cache = Cache(self.config_path, self.general["cache_expiration"]) else: self.Cache = None - self.general["asset_directory"] = check_for_attribute(self.data, "asset_directory", parent="settings", var_type="list_path", default=[os.path.join(default_dir, "assets")]) - self.general["asset_folders"] = check_for_attribute(self.data, "asset_folders", parent="settings", var_type="bool", default=True) - self.general["assets_for_all"] = check_for_attribute(self.data, "assets_for_all", parent="settings", var_type="bool", default=False) - self.general["sync_mode"] = check_for_attribute(self.data, "sync_mode", parent="settings", default="append", test_list=sync_modes) - self.general["run_again_delay"] = check_for_attribute(self.data, "run_again_delay", parent="settings", var_type="int", default=0) - self.general["show_unmanaged"] = check_for_attribute(self.data, "show_unmanaged", parent="settings", var_type="bool", default=True) - self.general["show_filtered"] = check_for_attribute(self.data, "show_filtered", parent="settings", var_type="bool", default=False) - self.general["show_missing"] = check_for_attribute(self.data, "show_missing", parent="settings", var_type="bool", default=True) - self.general["save_missing"] = check_for_attribute(self.data, "save_missing", parent="settings", var_type="bool", default=True) util.separator() self.TMDb = None if "tmdb" in self.data: logger.info("Connecting to TMDb...") - self.tmdb = {} - try: self.tmdb["apikey"] = check_for_attribute(self.data, "apikey", parent="tmdb", throw=True) - except Failed as e: raise Failed(e) - self.tmdb["language"] = check_for_attribute(self.data, "language", parent="tmdb", default="en") - self.TMDb = TMDb(self, self.tmdb) + self.TMDb = TMDb(self, { + "apikey": check_for_attribute(self.data, "apikey", parent="tmdb", throw=True), + "language": check_for_attribute(self.data, "language", parent="tmdb", default="en") + }) logger.info(f"TMDb Connection {'Failed' if self.TMDb is None else 'Successful'}") else: raise Failed("Config Error: tmdb attribute not found") @@ -207,10 +207,8 @@ class Config: self.OMDb = None if "omdb" in self.data: logger.info("Connecting to OMDb...") - self.omdb = {} try: - self.omdb["apikey"] = check_for_attribute(self.data, "apikey", parent="omdb", throw=True) - self.OMDb = OMDb(self, self.omdb) + self.OMDb = OMDb(self, {"apikey": check_for_attribute(self.data, "apikey", parent="omdb", throw=True)}) except Failed as e: logger.error(e) logger.info(f"OMDb Connection {'Failed' if self.OMDb is None else 'Successful'}") @@ -222,13 +220,13 @@ class Config: self.Trakt = None if "trakt" in self.data: logger.info("Connecting to Trakt...") - self.trakt = {} try: - self.trakt["client_id"] = check_for_attribute(self.data, "client_id", parent="trakt", throw=True) - self.trakt["client_secret"] = check_for_attribute(self.data, "client_secret", parent="trakt", throw=True) - self.trakt["config_path"] = self.config_path - authorization = self.data["trakt"]["authorization"] if "authorization" in self.data["trakt"] and self.data["trakt"]["authorization"] else None - self.Trakt = Trakt(self, self.trakt, authorization) + self.Trakt = Trakt(self, { + "client_id": check_for_attribute(self.data, "client_id", parent="trakt", throw=True), + "client_secret": check_for_attribute(self.data, "client_secret", parent="trakt", throw=True), + "config_path": self.config_path, + "authorization": self.data["trakt"]["authorization"] if "authorization" in self.data["trakt"] else None + }) except Failed as e: logger.error(e) logger.info(f"Trakt Connection {'Failed' if self.Trakt is None else 'Successful'}") @@ -240,13 +238,13 @@ class Config: self.MyAnimeList = None if "mal" in self.data: logger.info("Connecting to My Anime List...") - self.mal = {} try: - self.mal["client_id"] = check_for_attribute(self.data, "client_id", parent="mal", throw=True) - self.mal["client_secret"] = check_for_attribute(self.data, "client_secret", parent="mal", throw=True) - self.mal["config_path"] = self.config_path - authorization = self.data["mal"]["authorization"] if "authorization" in self.data["mal"] and self.data["mal"]["authorization"] else None - self.MyAnimeList = MyAnimeList(self, self.mal, authorization) + self.MyAnimeList = MyAnimeList(self, { + "client_id": check_for_attribute(self.data, "client_id", parent="mal", throw=True), + "client_secret": check_for_attribute(self.data, "client_secret", parent="mal", throw=True), + "config_path": self.config_path, + "authorization": self.data["mal"]["authorization"] if "authorization" in self.data["mal"] else None + }) except Failed as e: logger.error(e) logger.info(f"My Anime List Connection {'Failed' if self.MyAnimeList is None else 'Successful'}") @@ -259,11 +257,11 @@ class Config: if "anidb" in self.data: util.separator() logger.info("Connecting to AniDB...") - self.anidb = {} try: - self.anidb["username"] = check_for_attribute(self.data, "username", parent="anidb", throw=True) - self.anidb["password"] = check_for_attribute(self.data, "password", parent="anidb", throw=True) - self.AniDB = AniDB(self, self.anidb) + self.AniDB = AniDB(self, { + "username": check_for_attribute(self.data, "username", parent="anidb", throw=True), + "password": check_for_attribute(self.data, "password", parent="anidb", throw=True) + }) except Failed as e: logger.error(e) logger.info(f"My Anime List Connection {'Failed Continuing as Guest ' if self.MyAnimeList is None else 'Successful'}") @@ -281,42 +279,43 @@ class Config: logger.info("Connecting to Plex Libraries...") - self.general["plex"] = {} - self.general["plex"]["url"] = check_for_attribute(self.data, "url", parent="plex", var_type="url", default_is_none=True) - self.general["plex"]["token"] = check_for_attribute(self.data, "token", parent="plex", default_is_none=True) - self.general["plex"]["timeout"] = check_for_attribute(self.data, "timeout", parent="plex", var_type="int", default=60) - self.general["plex"]["clean_bundles"] = check_for_attribute(self.data, "clean_bundles", parent="plex", var_type="bool", default=False) - self.general["plex"]["empty_trash"] = check_for_attribute(self.data, "empty_trash", parent="plex", var_type="bool", default=False) - self.general["plex"]["optimize"] = check_for_attribute(self.data, "optimize", parent="plex", var_type="bool", default=False) - - self.general["radarr"] = {} - self.general["radarr"]["url"] = check_for_attribute(self.data, "url", parent="radarr", var_type="url", default_is_none=True) - self.general["radarr"]["token"] = check_for_attribute(self.data, "token", parent="radarr", default_is_none=True) - self.general["radarr"]["add"] = check_for_attribute(self.data, "add", parent="radarr", var_type="bool", default=False) - self.general["radarr"]["root_folder_path"] = check_for_attribute(self.data, "root_folder_path", parent="radarr", default_is_none=True) - self.general["radarr"]["monitor"] = check_for_attribute(self.data, "monitor", parent="radarr", var_type="bool", default=True) - self.general["radarr"]["availability"] = check_for_attribute(self.data, "availability", parent="radarr", test_list=radarr.availability_descriptions, default="announced") - self.general["radarr"]["quality_profile"] = check_for_attribute(self.data, "quality_profile", parent="radarr", default_is_none=True) - self.general["radarr"]["tag"] = check_for_attribute(self.data, "tag", parent="radarr", var_type="lower_list", default_is_none=True) - self.general["radarr"]["search"] = check_for_attribute(self.data, "search", parent="radarr", var_type="bool", default=False) - - self.general["sonarr"] = {} - self.general["sonarr"]["url"] = check_for_attribute(self.data, "url", parent="sonarr", var_type="url", default_is_none=True) - self.general["sonarr"]["token"] = check_for_attribute(self.data, "token", parent="sonarr", default_is_none=True) - self.general["sonarr"]["add"] = check_for_attribute(self.data, "add", parent="sonarr", var_type="bool", default=False) - self.general["sonarr"]["root_folder_path"] = check_for_attribute(self.data, "root_folder_path", parent="sonarr", default_is_none=True) - self.general["sonarr"]["monitor"] = check_for_attribute(self.data, "monitor", parent="sonarr", test_list=sonarr.monitor_descriptions, default="all") - self.general["sonarr"]["quality_profile"] = check_for_attribute(self.data, "quality_profile", parent="sonarr", default_is_none=True) - self.general["sonarr"]["language_profile"] = check_for_attribute(self.data, "language_profile", parent="sonarr", default_is_none=True) - self.general["sonarr"]["series_type"] = check_for_attribute(self.data, "series_type", parent="sonarr", test_list=sonarr.series_type_descriptions, default="standard") - self.general["sonarr"]["season_folder"] = check_for_attribute(self.data, "season_folder", parent="sonarr", var_type="bool", default=True) - self.general["sonarr"]["tag"] = check_for_attribute(self.data, "tag", parent="sonarr", var_type="lower_list", default_is_none=True) - self.general["sonarr"]["search"] = check_for_attribute(self.data, "search", parent="sonarr", var_type="bool", default=False) - self.general["sonarr"]["cutoff_search"] = check_for_attribute(self.data, "cutoff_search", parent="sonarr", var_type="bool", default=False) - - self.general["tautulli"] = {} - self.general["tautulli"]["url"] = check_for_attribute(self.data, "url", parent="tautulli", var_type="url", default_is_none=True) - self.general["tautulli"]["apikey"] = check_for_attribute(self.data, "apikey", parent="tautulli", default_is_none=True) + self.general["plex"] = { + "url": check_for_attribute(self.data, "url", parent="plex", var_type="url", default_is_none=True), + "token": check_for_attribute(self.data, "token", parent="plex", default_is_none=True), + "timeout": check_for_attribute(self.data, "timeout", parent="plex", var_type="int", default=60), + "clean_bundles": check_for_attribute(self.data, "clean_bundles", parent="plex", var_type="bool", default=False), + "empty_trash": check_for_attribute(self.data, "empty_trash", parent="plex", var_type="bool", default=False), + "optimize": check_for_attribute(self.data, "optimize", parent="plex", var_type="bool", default=False) + } + self.general["radarr"] = { + "url": check_for_attribute(self.data, "url", parent="radarr", var_type="url", default_is_none=True), + "token": check_for_attribute(self.data, "token", parent="radarr", default_is_none=True), + "add": check_for_attribute(self.data, "add", parent="radarr", var_type="bool", default=False), + "root_folder_path": check_for_attribute(self.data, "root_folder_path", parent="radarr", default_is_none=True), + "monitor": check_for_attribute(self.data, "monitor", parent="radarr", var_type="bool", default=True), + "availability": check_for_attribute(self.data, "availability", parent="radarr", test_list=radarr.availability_descriptions, default="announced"), + "quality_profile": check_for_attribute(self.data, "quality_profile", parent="radarr", default_is_none=True), + "tag": check_for_attribute(self.data, "tag", parent="radarr", var_type="lower_list", default_is_none=True), + "search": check_for_attribute(self.data, "search", parent="radarr", var_type="bool", default=False) + } + self.general["sonarr"] = { + "url": check_for_attribute(self.data, "url", parent="sonarr", var_type="url", default_is_none=True), + "token": check_for_attribute(self.data, "token", parent="sonarr", default_is_none=True), + "add": check_for_attribute(self.data, "add", parent="sonarr", var_type="bool", default=False), + "root_folder_path": check_for_attribute(self.data, "root_folder_path", parent="sonarr", default_is_none=True), + "monitor": check_for_attribute(self.data, "monitor", parent="sonarr", test_list=sonarr.monitor_descriptions, default="all"), + "quality_profile": check_for_attribute(self.data, "quality_profile", parent="sonarr", default_is_none=True), + "language_profile": check_for_attribute(self.data, "language_profile", parent="sonarr", default_is_none=True), + "series_type": check_for_attribute(self.data, "series_type", parent="sonarr", test_list=sonarr.series_type_descriptions, default="standard"), + "season_folder": check_for_attribute(self.data, "season_folder", parent="sonarr", var_type="bool", default=True), + "tag": check_for_attribute(self.data, "tag", parent="sonarr", var_type="lower_list", default_is_none=True), + "search": check_for_attribute(self.data, "search", parent="sonarr", var_type="bool", default=False), + "cutoff_search": check_for_attribute(self.data, "cutoff_search", parent="sonarr", var_type="bool", default=False) + } + self.general["tautulli"] = { + "url": check_for_attribute(self.data, "url", parent="tautulli", var_type="url", default_is_none=True), + "apikey": check_for_attribute(self.data, "apikey", parent="tautulli", default_is_none=True) + } self.libraries = [] libs = check_for_attribute(self.data, "libraries", throw=True) @@ -444,13 +443,14 @@ class Config: else: params["metadata_path"] = [("File", os.path.join(default_dir, f"{library_name}.yml"))] params["default_dir"] = default_dir - params["plex"] = {} - params["plex"]["url"] = check_for_attribute(lib, "url", parent="plex", var_type="url", default=self.general["plex"]["url"], req_default=True, save=False) - params["plex"]["token"] = check_for_attribute(lib, "token", parent="plex", default=self.general["plex"]["token"], req_default=True, save=False) - params["plex"]["timeout"] = check_for_attribute(lib, "timeout", parent="plex", var_type="int", default=self.general["plex"]["timeout"], save=False) - params["plex"]["clean_bundles"] = check_for_attribute(lib, "clean_bundles", parent="plex", var_type="bool", default=self.general["plex"]["clean_bundles"], save=False) - params["plex"]["empty_trash"] = check_for_attribute(lib, "empty_trash", parent="plex", var_type="bool", default=self.general["plex"]["empty_trash"], save=False) - params["plex"]["optimize"] = check_for_attribute(lib, "optimize", parent="plex", var_type="bool", default=self.general["plex"]["optimize"], save=False) + params["plex"] = { + "url": check_for_attribute(lib, "url", parent="plex", var_type="url", default=self.general["plex"]["url"], req_default=True, save=False), + "token": check_for_attribute(lib, "token", parent="plex", default=self.general["plex"]["token"], req_default=True, save=False), + "timeout": check_for_attribute(lib, "timeout", parent="plex", var_type="int", default=self.general["plex"]["timeout"], save=False), + "clean_bundles": check_for_attribute(lib, "clean_bundles", parent="plex", var_type="bool", default=self.general["plex"]["clean_bundles"], save=False), + "empty_trash": check_for_attribute(lib, "empty_trash", parent="plex", var_type="bool", default=self.general["plex"]["empty_trash"], save=False), + "optimize": check_for_attribute(lib, "optimize", parent="plex", var_type="bool", default=self.general["plex"]["optimize"], save=False) + } library = Plex(self, params) logger.info("") logger.info(f"{display_name} Library Connection Successful") @@ -466,18 +466,18 @@ class Config: logger.info("") logger.info(f"Connecting to {display_name} library's Radarr...") logger.info("") - radarr_params = {} try: - radarr_params["url"] = check_for_attribute(lib, "url", parent="radarr", var_type="url", default=self.general["radarr"]["url"], req_default=True, save=False) - radarr_params["token"] = check_for_attribute(lib, "token", parent="radarr", default=self.general["radarr"]["token"], req_default=True, save=False) - radarr_params["add"] = check_for_attribute(lib, "add", parent="radarr", var_type="bool", default=self.general["radarr"]["add"], save=False) - radarr_params["root_folder_path"] = check_for_attribute(lib, "root_folder_path", parent="radarr", default=self.general["radarr"]["root_folder_path"], req_default=True, save=False) - radarr_params["monitor"] = check_for_attribute(lib, "monitor", parent="radarr", var_type="bool", default=self.general["radarr"]["monitor"], save=False) - radarr_params["availability"] = check_for_attribute(lib, "availability", parent="radarr", test_list=radarr.availability_descriptions, default=self.general["radarr"]["availability"], save=False) - radarr_params["quality_profile"] = check_for_attribute(lib, "quality_profile", parent="radarr", default=self.general["radarr"]["quality_profile"], req_default=True, save=False) - radarr_params["tag"] = check_for_attribute(lib, "tag", parent="radarr", var_type="lower_list", default=self.general["radarr"]["tag"], default_is_none=True, save=False) - radarr_params["search"] = check_for_attribute(lib, "search", parent="radarr", var_type="bool", default=self.general["radarr"]["search"], save=False) - library.Radarr = Radarr(self, radarr_params) + library.Radarr = Radarr(self, { + "url": check_for_attribute(lib, "url", parent="radarr", var_type="url", default=self.general["radarr"]["url"], req_default=True, save=False), + "token": check_for_attribute(lib, "token", parent="radarr", default=self.general["radarr"]["token"], req_default=True, save=False), + "add": check_for_attribute(lib, "add", parent="radarr", var_type="bool", default=self.general["radarr"]["add"], save=False), + "root_folder_path": check_for_attribute(lib, "root_folder_path", parent="radarr", default=self.general["radarr"]["root_folder_path"], req_default=True, save=False), + "monitor": check_for_attribute(lib, "monitor", parent="radarr", var_type="bool", default=self.general["radarr"]["monitor"], save=False), + "availability": check_for_attribute(lib, "availability", parent="radarr", test_list=radarr.availability_descriptions, default=self.general["radarr"]["availability"], save=False), + "quality_profile": check_for_attribute(lib, "quality_profile", parent="radarr",default=self.general["radarr"]["quality_profile"], req_default=True, save=False), + "tag": check_for_attribute(lib, "tag", parent="radarr", var_type="lower_list", default=self.general["radarr"]["tag"], default_is_none=True, save=False), + "search": check_for_attribute(lib, "search", parent="radarr", var_type="bool", default=self.general["radarr"]["search"], save=False) + }) except Failed as e: util.print_stacktrace() util.print_multiline(e, error=True) @@ -490,24 +490,21 @@ class Config: logger.info("") logger.info(f"Connecting to {display_name} library's Sonarr...") logger.info("") - sonarr_params = {} try: - sonarr_params["url"] = check_for_attribute(lib, "url", parent="sonarr", var_type="url", default=self.general["sonarr"]["url"], req_default=True, save=False) - sonarr_params["token"] = check_for_attribute(lib, "token", parent="sonarr", default=self.general["sonarr"]["token"], req_default=True, save=False) - sonarr_params["add"] = check_for_attribute(lib, "add", parent="sonarr", var_type="bool", default=self.general["sonarr"]["add"], save=False) - sonarr_params["root_folder_path"] = check_for_attribute(lib, "root_folder_path", parent="sonarr", default=self.general["sonarr"]["root_folder_path"], req_default=True, save=False) - sonarr_params["monitor"] = check_for_attribute(lib, "monitor", parent="sonarr", test_list=sonarr.monitor_descriptions, default=self.general["sonarr"]["monitor"], save=False) - sonarr_params["quality_profile"] = check_for_attribute(lib, "quality_profile", parent="sonarr", default=self.general["sonarr"]["quality_profile"], req_default=True, save=False) - if self.general["sonarr"]["language_profile"]: - sonarr_params["language_profile"] = check_for_attribute(lib, "language_profile", parent="sonarr", default=self.general["sonarr"]["language_profile"], save=False) - else: - sonarr_params["language_profile"] = check_for_attribute(lib, "language_profile", parent="sonarr", default_is_none=True, save=False) - sonarr_params["series_type"] = check_for_attribute(lib, "series_type", parent="sonarr", test_list=sonarr.series_type_descriptions, default=self.general["sonarr"]["series_type"], save=False) - sonarr_params["season_folder"] = check_for_attribute(lib, "season_folder", parent="sonarr", var_type="bool", default=self.general["sonarr"]["season_folder"], save=False) - sonarr_params["tag"] = check_for_attribute(lib, "tag", parent="sonarr", var_type="lower_list", default=self.general["sonarr"]["tag"], default_is_none=True, save=False) - sonarr_params["search"] = check_for_attribute(lib, "search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["search"], save=False) - sonarr_params["cutoff_search"] = check_for_attribute(lib, "cutoff_search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["cutoff_search"], save=False) - library.Sonarr = Sonarr(self, sonarr_params) + library.Sonarr = Sonarr(self, { + "url": check_for_attribute(lib, "url", parent="sonarr", var_type="url", default=self.general["sonarr"]["url"], req_default=True, save=False), + "token": check_for_attribute(lib, "token", parent="sonarr", default=self.general["sonarr"]["token"], req_default=True, save=False), + "add": check_for_attribute(lib, "add", parent="sonarr", var_type="bool", default=self.general["sonarr"]["add"], save=False), + "root_folder_path": check_for_attribute(lib, "root_folder_path", parent="sonarr", default=self.general["sonarr"]["root_folder_path"], req_default=True, save=False), + "monitor": check_for_attribute(lib, "monitor", parent="sonarr", test_list=sonarr.monitor_descriptions, default=self.general["sonarr"]["monitor"], save=False), + "quality_profile": check_for_attribute(lib, "quality_profile", parent="sonarr", default=self.general["sonarr"]["quality_profile"], req_default=True, save=False), + "language_profile": check_for_attribute(lib, "language_profile", parent="sonarr", default=self.general["sonarr"]["language_profile"], save=False) if self.general["sonarr"]["language_profile"] else check_for_attribute(lib, "language_profile", parent="sonarr", default_is_none=True, save=False), + "series_type": check_for_attribute(lib, "series_type", parent="sonarr", test_list=sonarr.series_type_descriptions, default=self.general["sonarr"]["series_type"], save=False), + "season_folder": check_for_attribute(lib, "season_folder", parent="sonarr", var_type="bool", default=self.general["sonarr"]["season_folder"], save=False), + "tag": check_for_attribute(lib, "tag", parent="sonarr", var_type="lower_list", default=self.general["sonarr"]["tag"], default_is_none=True, save=False), + "search": check_for_attribute(lib, "search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["search"], save=False), + "cutoff_search": check_for_attribute(lib, "cutoff_search", parent="sonarr", var_type="bool", default=self.general["sonarr"]["cutoff_search"], save=False) + }) except Failed as e: util.print_stacktrace() util.print_multiline(e, error=True) @@ -520,11 +517,11 @@ class Config: logger.info("") logger.info(f"Connecting to {display_name} library's Tautulli...") logger.info("") - tautulli_params = {} try: - tautulli_params["url"] = check_for_attribute(lib, "url", parent="tautulli", var_type="url", default=self.general["tautulli"]["url"], req_default=True, save=False) - tautulli_params["apikey"] = check_for_attribute(lib, "apikey", parent="tautulli", default=self.general["tautulli"]["apikey"], req_default=True, save=False) - library.Tautulli = Tautulli(self, tautulli_params) + library.Tautulli = Tautulli(self, { + "url": check_for_attribute(lib, "url", parent="tautulli", var_type="url", default=self.general["tautulli"]["url"], req_default=True, save=False), + "apikey": check_for_attribute(lib, "apikey", parent="tautulli", default=self.general["tautulli"]["apikey"], req_default=True, save=False) + }) except Failed as e: util.print_stacktrace() util.print_multiline(e, error=True) diff --git a/modules/mal.py b/modules/mal.py index d7e8a3bb..12a163c6 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -39,12 +39,12 @@ urls = { } class MyAnimeList: - def __init__(self, config, params, authorization=None): + def __init__(self, config, params): self.config = config self.client_id = params["client_id"] self.client_secret = params["client_secret"] self.config_path = params["config_path"] - self.authorization = authorization + self.authorization = params["authorization"] if not self._save(self.authorization): if not self._refresh(): self._authorization() diff --git a/modules/trakt.py b/modules/trakt.py index a5ae584e..8d9375e0 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -14,12 +14,12 @@ builders = [ ] class Trakt: - def __init__(self, config, params, authorization=None): + def __init__(self, config, params): self.config = config self.client_id = params["client_id"] self.client_secret = params["client_secret"] self.config_path = params["config_path"] - self.authorization = authorization + self.authorization = params["authorization"] if not self._save(self.authorization): if not self._refresh(): self._authorization() diff --git a/plex_meta_manager.py b/plex_meta_manager.py index cd477c86..2d42ead7 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -105,7 +105,7 @@ def start(config_path, is_test=False, time_scheduled=None, requested_collections logger.info(util.centered("| __/| | __/> < | | | | __/ || (_| | | | | | (_| | | | | (_| | (_| | __/ | ")) logger.info(util.centered("|_| |_|\\___/_/\\_\\ |_| |_|\\___|\\__\\__,_| |_| |_|\\__,_|_| |_|\\__,_|\\__, |\\___|_| ")) logger.info(util.centered(" |___/ ")) - logger.info(util.centered(" Version: 1.11.3-beta1 ")) + logger.info(util.centered(" Version: 1.11.3-beta2 ")) if time_scheduled: start_type = f"{time_scheduled} " elif is_test: start_type = "Test " elif requested_collections: start_type = "Collections " From 306102d0eecfff3e4ad69ca3f2d6938b1dfb3048 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 26 Jul 2021 15:17:33 -0400 Subject: [PATCH 36/95] MyAnimeList fix --- modules/mal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mal.py b/modules/mal.py index 12a163c6..6eecad33 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -28,10 +28,10 @@ userlist_sort_translation = { } userlist_sort_options = ["score", "last_updated", "title", "start_date"] userlist_status = ["all", "watching", "completed", "on_hold", "dropped", "plan_to_watch"] -base_url = "https://myanimelist.net" +base_url = "https://api.myanimelist.net" urls = { - "oauth_token": f"{base_url}/v1/oauth2/token", - "oauth_authorize": f"{base_url}/v1/oauth2/authorize", + "oauth_token": f"https://myanimelist.net/v1/oauth2/token", + "oauth_authorize": f"https://myanimelist.net/v1/oauth2/authorize", "ranking": f"{base_url}/v2/anime/ranking", "season": f"{base_url}/v2/anime/season", "suggestions": f"{base_url}/v2/anime/suggestions", From 4b3056f22ddcbca2897c4e7f91b044d028d4cbad Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 26 Jul 2021 16:29:28 -0400 Subject: [PATCH 37/95] added debugs --- modules/builder.py | 20 ++++++++++---------- modules/plex.py | 21 +++++++++++++++------ modules/util.py | 43 ++++--------------------------------------- 3 files changed, 29 insertions(+), 55 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index f54381ef..a24279fc 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -296,7 +296,7 @@ class CollectionBuilder: if run_time.startswith("hour"): try: if 0 <= int(param) <= 23: - self.schedule += f"\nScheduled to run only on the {util.make_ordinal(param)} hour" + self.schedule += f"\nScheduled to run only on the {util.make_ordinal(int(param))} hour" if self.config.run_hour == int(param): skip_collection = False else: @@ -314,7 +314,7 @@ class CollectionBuilder: elif run_time.startswith("month"): try: if 1 <= int(param) <= 31: - self.schedule += f"\nScheduled monthly on the {util.make_ordinal(param)}" + self.schedule += f"\nScheduled monthly on the {util.make_ordinal(int(param))}" if self.current_time.day == int(param) or (self.current_time.day == last_day.day and int(param) > last_day.day): skip_collection = False else: @@ -349,7 +349,7 @@ class CollectionBuilder: logger.warning(f"Collection Warning: run_again attribute is blank defaulting to false") else: logger.debug(f"Value: {self.data[methods['run_again']]}") - self.run_again = util.get_bool("run_again", self.data[methods["run_again"]]) + self.run_again = util.parse_bool("run_again", self.data[methods["run_again"]]) self.sync = self.library.sync_mode == "sync" if "sync_mode" in methods: @@ -372,7 +372,7 @@ class CollectionBuilder: logger.warning(f"Collection Warning: build_collection attribute is blank defaulting to true") else: logger.debug(f"Value: {self.data[methods['build_collection']]}") - self.build_collection = util.get_bool("build_collection", self.data[methods["build_collection"]]) + self.build_collection = util.parse_bool("build_collection", self.data[methods["build_collection"]]) if "tmdb_person" in methods: logger.info("") @@ -596,7 +596,7 @@ class CollectionBuilder: else: self.details[method_final] = util.get_list(method_data) elif method_name in boolean_details: - self.details[method_name] = util.get_bool(method_name, method_data) + self.details[method_name] = util.parse_bool(method_name, method_data) elif method_name in string_details: self.details[method_name] = str(method_data) @@ -638,11 +638,11 @@ class CollectionBuilder: def _radarr(self, method_name, method_data): if method_name == "radarr_add": - self.add_to_radarr = util.get_bool(method_name, method_data) + self.add_to_radarr = util.parse_bool(method_name, method_data) elif method_name == "radarr_folder": self.radarr_options["folder"] = method_data elif method_name in ["radarr_monitor", "radarr_search"]: - self.radarr_options[method_name[7:]] = util.get_bool(method_name, method_data) + self.radarr_options[method_name[7:]] = util.parse_bool(method_name, method_data) elif method_name == "radarr_availability": if str(method_data).lower() in radarr.availability_translation: self.radarr_options["availability"] = str(method_data).lower() @@ -656,7 +656,7 @@ class CollectionBuilder: def _sonarr(self, method_name, method_data): if method_name == "sonarr_add": - self.add_to_sonarr = util.get_bool(method_name, method_data) + self.add_to_sonarr = util.parse_bool(method_name, method_data) elif method_name == "sonarr_folder": self.sonarr_options["folder"] = method_data elif method_name == "sonarr_monitor": @@ -676,7 +676,7 @@ class CollectionBuilder: else: raise Failed(f"Collection Error: {method_name} attribute must be either standard, daily, or anime") elif method_name in ["sonarr_season", "sonarr_search", "sonarr_cutoff_search"]: - self.sonarr_options[method_name[7:]] = util.get_bool(method_name, method_data) + self.sonarr_options[method_name[7:]] = util.parse_bool(method_name, method_data) elif method_name == "sonarr_tag": self.sonarr_options["tag"] = util.get_list(method_data) @@ -1231,7 +1231,7 @@ class CollectionBuilder: elif attribute in ["decade", "year", "episode_year"] and modifier in ["", ".not"]: return smart_pair(util.get_year_list(data, self.current_year, final)) elif attribute in plex.boolean_attributes: - return util.get_bool(attribute, data) + return util.parse_bool(attribute, data) else: raise Failed(f"Collection Error: {final} attribute not supported") diff --git a/modules/plex.py b/modules/plex.py index 1bca19b2..c885bd95 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -376,6 +376,10 @@ class Plex: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def _upload_image(self, item, image): + logger.debug(item) + logger.debug(image.is_poster) + logger.debug(image.is_url) + logger.debug(image.location) if image.is_poster and image.is_url: item.uploadPoster(url=image.location) elif image.is_poster: @@ -388,6 +392,8 @@ class Plex: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def upload_file_poster(self, item, image): + logger.debug(item) + logger.debug(image) item.uploadPoster(filepath=image) self.reload(item) @@ -546,13 +552,13 @@ class Plex: def get_collection(self, data): if isinstance(data, int): - collection = self.fetchItem(data) + return self.fetchItem(data) elif isinstance(data, Collection): - collection = data + return data else: - collection = util.choose_from_list(self.search(title=str(data), libtype="collection"), "collection", str(data), exact=True) - if collection: - return collection + for d in self.search(title=str(data), libtype="collection"): + if d.title == data: + return d raise Failed(f"Plex Error: Collection {data} not found") def validate_collections(self, collections): @@ -689,7 +695,10 @@ class Plex: kwargs = {} if year is not None: kwargs["year"] = year - return util.choose_from_list(self.search(title=str(data), **kwargs), "movie" if self.is_movie else "show", str(data), exact=True) + for d in self.search(title=str(data), **kwargs): + if d.title == data: + return d + return None def edit_item(self, item, name, item_type, edits, advanced=False): if len(edits) > 0: diff --git a/modules/util.py b/modules/util.py index 6510a74c..57b71ec0 100644 --- a/modules/util.py +++ b/modules/util.py @@ -93,48 +93,13 @@ def tab_new_lines(data): return str(data).replace("\n", "\n|\t ") if "\n" in str(data) else str(data) def make_ordinal(n): - n = int(n) - suffix = ["th", "st", "nd", "rd", "th"][min(n % 10, 4)] - if 11 <= (n % 100) <= 13: - suffix = "th" - return str(n) + suffix - -def choose_from_list(datalist, description, data=None, list_type="title", exact=False): - if len(datalist) > 0: - if len(datalist) == 1 and (description != "collection" or datalist[0].title == data): - return datalist[0] - zero_option = f"Create New Collection: {data}" if description == "collection" else "Do Nothing" - message = f"Multiple {description}s Found\n0) {zero_option}" - for i, d in enumerate(datalist, 1): - if list_type == "title": - if d.title == data: - return d - message += f"\n{i}) {d.title}" - else: - message += f"\n{i}) [{d[0]}] {d[1]}" - if exact: - return None - print_multiline(message, info=True) - while True: - try: - selection = int(logger_input(f"Choose {description} number")) - 1 - if selection >= 0: return datalist[selection] - elif selection == -1: return None - else: logger.info(f"Invalid {description} number") - except IndexError: logger.info(f"Invalid {description} number") - except TimeoutExpired: - if list_type == "title": - logger.warning(f"Input Timeout: using {data}") - return None - else: - logger.warning(f"Input Timeout: using {datalist[0][1]}") - return datalist[0] - else: - return None + return f"{n}{'th' if 11 <= (n % 100) <= 13 else ['th', 'st', 'nd', 'rd', 'th'][min(n % 10, 4)]}" -def get_bool(method_name, method_data): +def parse_bool(method_name, method_data): if isinstance(method_data, bool): return method_data + elif isinstance(method_data, int): + return method_data > 0 elif str(method_data).lower() in ["t", "true"]: return True elif str(method_data).lower() in ["f", "false"]: From ba977e6a3711bb624ab6276da177a0ce957556e2 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 28 Jul 2021 22:26:39 -0400 Subject: [PATCH 38/95] fix imdb to prep for custom sort --- modules/builder.py | 89 +++++++++++++++++++----------- modules/config.py | 4 +- modules/imdb.py | 125 ++++++++++++++++++++----------------------- modules/plex.py | 2 +- modules/util.py | 1 - plex_meta_manager.py | 2 +- 6 files changed, 118 insertions(+), 105 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index a24279fc..3d533d18 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -73,7 +73,7 @@ poster_details = ["url_poster", "tmdb_poster", "tmdb_profile", "tvdb_poster", "f background_details = ["url_background", "tmdb_background", "tvdb_background", "file_background"] boolean_details = ["visible_library", "visible_home", "visible_shared", "show_filtered", "show_missing", "save_missing", "item_assets"] string_details = ["sort_title", "content_rating", "name_mapping"] -ignored_details = ["smart_filter", "smart_label", "smart_url", "run_again", "schedule", "sync_mode", "template", "test", "tmdb_person", "build_collection"] +ignored_details = ["smart_filter", "smart_label", "smart_url", "run_again", "schedule", "sync_mode", "template", "test", "tmdb_person", "build_collection", "collection_order"] details = ["collection_mode", "collection_order", "label"] + boolean_details + string_details collectionless_details = ["collection_order", "plex_collectionless", "label", "label_sync_mode", "test"] + \ poster_details + background_details + summary_details + string_details @@ -123,6 +123,17 @@ movie_only_filters = [ show_only_filters = ["network"] smart_invalid = ["collection_order"] smart_url_invalid = ["filters", "run_again", "sync_mode", "show_filtered", "show_missing", "save_missing", "smart_label"] + radarr_details + sonarr_details +custom_sort_builders = [ + "tmdb_collection", "tmdb_list", "tmdb_popular", "tmdb_now_playing", "tmdb_top_rated", + "tmdb_trending_daily", "tmdb_trending_weekly", "tmdb_discover", + "tvdb_list", "imdb_list", + "trakt_list", "trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected", + "tautulli_popular", "tautulli_watched", "letterboxd_list", "icheckmovies_list", + "anidb_popular", + "anilist_top_rated", "anilist_popular", "anilist_season", "anilist_studio", "anilist_genre", "anilist_tag", + "mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_movie", "mal_ova", "mal_special", + "mal_popular", "mal_favorite", "mal_suggested", "mal_userlist", "mal_season" +] class CollectionBuilder: def __init__(self, config, library, metadata, name, data): @@ -143,7 +154,7 @@ class CollectionBuilder: self.sonarr_options = {} self.missing_movies = [] self.missing_shows = [] - self.methods = [] + self.builders = [] self.filters = [] self.rating_keys = [] self.run_again_movies = [] @@ -374,6 +385,19 @@ class CollectionBuilder: logger.debug(f"Value: {self.data[methods['build_collection']]}") self.build_collection = util.parse_bool("build_collection", self.data[methods["build_collection"]]) + self.sort_collection = False + if "collection_order" in methods: + logger.info("") + logger.info("Validating Method: collection_order") + if self.data[methods["collection_order"]] is None: + raise Failed(f"Collection Warning: collection_order attribute is blank") + elif self.data[methods["collection_order"]].lower() in plex.collection_order_options: + self.details["collection_order"] = self.data[methods["collection_order"]].lower() + if self.data[methods["collection_order"]].lower() == "custom": + self.sort_collection = True + else: + raise Failed(f"Collection Error: {self.data[methods['collection_order']]} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)\n\tcustom (Custom Order Collection)") + if "tmdb_person" in methods: logger.info("") logger.info("Validating Method: tmdb_person") @@ -487,6 +511,12 @@ class CollectionBuilder: elif method_name == "filters": self._filters(method_name, method_data) else: raise Failed(f"Collection Error: {method_final} attribute not supported") + if self.sort_collection and len(self.builders) > 1: + raise Failed("Collection Error: collection_order: custom can only be used with a single builder per collection") + + if self.sort_collection and self.builders[0][0] not in custom_sort_builders: + raise Failed(f"Collection Error: collection_order: custom cannot be used with {self.builders[0][0]}") + if self.add_to_radarr is None: self.add_to_radarr = self.library.Radarr.add if self.library.Radarr else False if self.add_to_sonarr is None: @@ -581,11 +611,6 @@ class CollectionBuilder: self.details[method_name] = plex.collection_mode_options[str(method_data).lower()] else: raise Failed(f"Collection Error: {method_data} collection_mode invalid\n\tdefault (Library default)\n\thide (Hide Collection)\n\thide_items (Hide Items in this Collection)\n\tshow_items (Show this Collection and its Items)") - elif method_name == "collection_order": - if str(method_data).lower() in plex.collection_order_options: - self.details[method_name] = plex.collection_order_options[str(method_data).lower()] - else: - raise Failed(f"Collection Error: {method_data} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)") elif method_name == "label": if "label" in methods and "label.sync" in methods: raise Failed("Collection Error: Cannot use label and label.sync together") @@ -682,10 +707,10 @@ class CollectionBuilder: def _anidb(self, method_name, method_data): if method_name == "anidb_popular": - self.methods.append((method_name, util.parse_int(method_name, method_data, default=30, maximum=30))) + self.builders.append((method_name, util.parse_int(method_name, method_data, default=30, maximum=30))) elif method_name in ["anidb_id", "anidb_relation"]: for anidb_id in self.config.AniDB.validate_anidb_ids(method_data, self.language): - self.methods.append((method_name, anidb_id)) + self.builders.append((method_name, anidb_id)) elif method_name == "anidb_tag": for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): new_dictionary = {} @@ -696,14 +721,14 @@ class CollectionBuilder: else: new_dictionary["tag"] = util.regex_first_int(dict_data[dict_methods["username"]], "AniDB Tag ID") new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 0, minimum=0) - self.methods.append((method_name, new_dictionary)) + self.builders.append((method_name, new_dictionary)) def _anilist(self, method_name, method_data): if method_name in ["anilist_id", "anilist_relations", "anilist_studio"]: for anilist_id in self.config.AniList.validate_anilist_ids(method_data, studio=method_name == "anilist_studio"): - self.methods.append((method_name, anilist_id)) + self.builders.append((method_name, anilist_id)) elif method_name in ["anilist_popular", "anilist_top_rated"]: - self.methods.append((method_name, util.parse_int(method_name, method_data))) + self.builders.append((method_name, util.parse_int(method_name, method_data))) elif method_name in ["anilist_season", "anilist_genre", "anilist_tag"]: for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): new_dictionary = {} @@ -720,13 +745,13 @@ class CollectionBuilder: new_dictionary["tag"] = self.config.AniList.validate_tag(util.parse_from_dict(method_name, "tag", dict_data, dict_methods)) new_dictionary["sort_by"] = util.parse_from_dict(method_name, "sort_by", dict_data, dict_methods, default="score", options=["score", "popular"]) new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 0, maximum=500) - self.methods.append((method_name, new_dictionary)) + self.builders.append((method_name, new_dictionary)) def _icheckmovies(self, method_name, method_data): if method_name.startswith("icheckmovies_list"): icheckmovies_lists = self.config.ICheckMovies.validate_icheckmovies_lists(method_data, self.language) for icheckmovies_list in icheckmovies_lists: - self.methods.append(("icheckmovies_list", icheckmovies_list)) + self.builders.append(("icheckmovies_list", icheckmovies_list)) if method_name.endswith("_details"): self.summaries[method_name] = self.config.ICheckMovies.get_list_description(icheckmovies_lists[0], self.language) @@ -734,27 +759,27 @@ class CollectionBuilder: if method_name == "imdb_id": for value in util.get_list(method_data): if str(value).startswith("tt"): - self.methods.append((method_name, value)) + self.builders.append((method_name, value)) else: raise Failed(f"Collection Error: imdb_id {value} must begin with tt") elif method_name == "imdb_list": for imdb_dict in self.config.IMDb.validate_imdb_lists(method_data, self.language): - self.methods.append((method_name, imdb_dict)) + self.builders.append((method_name, imdb_dict)) def _letterboxd(self, method_name, method_data): if method_name.startswith("letterboxd_list"): letterboxd_lists = self.config.Letterboxd.validate_letterboxd_lists(method_data, self.language) for letterboxd_list in letterboxd_lists: - self.methods.append(("letterboxd_list", letterboxd_list)) + self.builders.append(("letterboxd_list", letterboxd_list)) if method_name.endswith("_details"): self.summaries[method_name] = self.config.Letterboxd.get_list_description(letterboxd_lists[0], self.language) def _mal(self, method_name, method_data): if method_name == "mal_id": for mal_id in util.get_int_list(method_data, "MyAnimeList ID"): - self.methods.append((method_name, mal_id)) + self.builders.append((method_name, mal_id)) elif method_name in ["mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_ova", "mal_movie", "mal_special", "mal_popular", "mal_favorite", "mal_suggested"]: - self.methods.append((method_name, util.parse_int(method_name, method_data))) + self.builders.append((method_name, util.parse_int(method_name, method_data))) elif method_name in ["mal_season", "mal_userlist"]: for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): new_dictionary = {} @@ -772,11 +797,11 @@ class CollectionBuilder: new_dictionary["status"] = util.parse_from_dict(method_name, "status", dict_data, dict_methods, default="all", options=mal.userlist_status) new_dictionary["sort_by"] = util.parse_from_dict(method_name, "sort_by", dict_data, dict_methods, default="score", options=mal.userlist_sort_options, translation=mal.userlist_sort_translation) new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 100, maximum=1000) - self.methods.append((method_name, new_dictionary)) + self.builders.append((method_name, new_dictionary)) def _plex(self, method_name, method_data): if method_name == "plex_all": - self.methods.append((method_name, True)) + self.builders.append((method_name, True)) elif method_name in ["plex_search", "plex_collectionless"]: for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): new_dictionary = {} @@ -790,13 +815,13 @@ class CollectionBuilder: exact_list.append(self.name) new_dictionary["exclude_prefix"] = prefix_list new_dictionary["exclude"] = exact_list - self.methods.append((method_name, new_dictionary)) + self.builders.append((method_name, new_dictionary)) else: - self.methods.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) + self.builders.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) def _tautulli(self, method_name, method_data): for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): - self.methods.append((method_name, { + self.builders.append((method_name, { "list_type": "popular" if method_name == "tautulli_popular" else "watched", "list_days": util.parse_int_from_dict(method_name, "list_days", dict_data, dict_methods, 30), "list_size": util.parse_int_from_dict(method_name, "list_size", dict_data, dict_methods, 10), @@ -859,11 +884,11 @@ class CollectionBuilder: else: raise Failed(f"Collection Error: {method_name} parameter {discover_final} is blank") if len(new_dictionary) > 1: - self.methods.append((method_name, new_dictionary)) + self.builders.append((method_name, new_dictionary)) else: raise Failed(f"Collection Error: {method_name} had no valid fields") elif method_name in ["tmdb_popular", "tmdb_top_rated", "tmdb_now_playing", "tmdb_trending_daily", "tmdb_trending_weekly"]: - self.methods.append((method_name, util.parse_int(method_name, method_data))) + self.builders.append((method_name, util.parse_int(method_name, method_data))) else: values = self.config.TMDb.validate_tmdb_ids(method_data, method_name) if method_name.endswith("_details"): @@ -886,20 +911,20 @@ class CollectionBuilder: if hasattr(item, "description") and item.description: self.summaries[method_name] = item.description for value in values: - self.methods.append((method_name[:-8] if method_name.endswith("_details") else method_name, value)) + self.builders.append((method_name[:-8] if method_name.endswith("_details") else method_name, value)) def _trakt(self, method_name, method_data): if method_name.startswith("trakt_list"): trakt_lists = self.config.Trakt.validate_trakt(method_data, self.library.is_movie) for trakt_list in trakt_lists: - self.methods.append(("trakt_list", trakt_list)) + self.builders.append(("trakt_list", trakt_list)) if method_name.endswith("_details"): self.summaries[method_name] = self.config.Trakt.list_description(trakt_lists[0]) elif method_name in ["trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected"]: - self.methods.append((method_name, util.parse_int(method_name, method_data))) + self.builders.append((method_name, util.parse_int(method_name, method_data))) elif method_name in ["trakt_watchlist", "trakt_collection"]: for trakt_list in self.config.Trakt.validate_trakt(method_data, self.library.is_movie, trakt_type=method_name[6:]): - self.methods.append((method_name, trakt_list)) + self.builders.append((method_name, trakt_list)) def _tvdb(self, method_name, method_data): values = util.get_list(method_data) @@ -915,7 +940,7 @@ class CollectionBuilder: elif method_name.startswith("tvdb_list"): self.summaries[method_name] = self.config.TVDb.get_list_description(values[0], self.language) for value in values: - self.methods.append((method_name[:-8] if method_name.endswith("_details") else method_name, value)) + self.builders.append((method_name[:-8] if method_name.endswith("_details") else method_name, value)) def _filters(self, method_name, method_data): for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): @@ -987,7 +1012,7 @@ class CollectionBuilder: elif show_id not in self.missing_shows: self.missing_shows.append(show_id) return items_found_inside - for method, value in self.methods: + for method, value in self.builders: logger.debug("") logger.debug(f"Builder: {method}: {value}") logger.info("") diff --git a/modules/config.py b/modules/config.py index e8e58219..e9232222 100644 --- a/modules/config.py +++ b/modules/config.py @@ -540,8 +540,8 @@ class Config: util.separator() - def get_html(self, url, headers=None): - return html.fromstring(self.get(url, headers=headers).content) + def get_html(self, url, headers=None, params=None): + return html.fromstring(self.get(url, headers=headers, params=params).content) def get_json(self, url, headers=None): return self.get(url, headers=headers).json() diff --git a/modules/imdb.py b/modules/imdb.py index bfab31e6..507fcf03 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -1,6 +1,7 @@ import logging, math, re, time from modules import util from modules.util import Failed +from urllib.parse import urlparse, parse_qs logger = logging.getLogger("Plex Meta Manager") @@ -8,102 +9,90 @@ builders = ["imdb_list", "imdb_id"] base_url = "https://www.imdb.com" urls = { "list": f"{base_url}/list/ls", - "search": f"{base_url}/search/title/?", - "keyword": f"{base_url}/search/keyword/?" + "search": f"{base_url}/search/title/", + "keyword": f"{base_url}/search/keyword/" } +xpath = { + "imdb_id": "//div[contains(@class, 'lister-item-image')]//@data-tconst", + "list": "//div[@class='desc lister-total-num-results']/text()", + "search": "//div[@class='desc']/span/text()", + "keyword": "//div[@class='desc']/text()" +} +item_counts = {"list": 100, "search": 250, "keyword": 50} class IMDb: def __init__(self, config): self.config = config - def _validate_url(self, imdb_url, language): - imdb_url = imdb_url.strip() - if not imdb_url.startswith(urls["list"]) and not imdb_url.startswith(urls["search"]) and not imdb_url.startswith(urls["keyword"]): - raise Failed(f"IMDb Error: {imdb_url} must begin with either:\n{urls['list']} (For Lists)\n{urls['search']} (For Searches)\n{urls['keyword']} (For Keyword Searches)") - total, _ = self._total(self._fix_url(imdb_url), language) - if total > 0: - return imdb_url - raise Failed(f"IMDb Error: {imdb_url} failed to parse") - def validate_imdb_lists(self, imdb_lists, language): valid_lists = [] - for imdb_list in util.get_list(imdb_lists, split=False): - if isinstance(imdb_list, dict): - dict_methods = {dm.lower(): dm for dm in imdb_list} - if "url" in dict_methods and imdb_list[dict_methods["url"]]: - imdb_url = self._validate_url(imdb_list[dict_methods["url"]], language) - else: - raise Failed("Collection Error: imdb_list attribute url is required") - if "limit" in dict_methods and imdb_list[dict_methods["limit"]]: - list_count = util.regex_first_int(imdb_list[dict_methods["limit"]], "List Limit", default=0) - else: - list_count = 0 - else: - imdb_url = self._validate_url(str(imdb_list), language) - list_count = 0 + for imdb_dict in util.get_list(imdb_lists, split=False): + if not isinstance(imdb_dict, dict): + imdb_dict = {"url": imdb_dict} + dict_methods = {dm.lower(): dm for dm in imdb_dict} + imdb_url = util.parse_from_dict("imdb_list", "url", imdb_dict, dict_methods).strip() + if not imdb_url.startswith((urls["list"], urls["search"], urls["keyword"])): + raise Failed(f"IMDb Error: {imdb_url} must begin with either:\n{urls['list']} (For Lists)\n{urls['search']} (For Searches)\n{urls['keyword']} (For Keyword Searches)") + self._total(imdb_url, language) + list_count = util.parse_int_from_dict("imdb_list", "limit", imdb_dict, dict_methods, 0, minimum=0) if "limit" in dict_methods else 0 valid_lists.append({"url": imdb_url, "limit": list_count}) return valid_lists - def _fix_url(self, imdb_url): - if imdb_url.startswith(urls["list"]): - try: list_id = re.search("(\\d+)", str(imdb_url)).group(1) - except AttributeError: raise Failed(f"IMDb Error: Failed to parse List ID from {imdb_url}") - return f"{urls['search']}lists=ls{list_id}" - elif imdb_url.endswith("/"): - return imdb_url[:-1] - else: - return imdb_url - def _total(self, imdb_url, language): headers = util.header(language) if imdb_url.startswith(urls["keyword"]): - results = self.config.get_html(imdb_url, headers=headers).xpath("//div[@class='desc']/text()") - total = None - for result in results: - if "title" in result: - try: - total = int(re.findall("(\\d+) title", result)[0]) - break - except IndexError: - pass - if total is None: - raise Failed(f"IMDb Error: No Results at URL: {imdb_url}") - return total, 50 + page_type = "keyword" + elif imdb_url.startswith(urls["list"]): + page_type = "list" else: - try: results = self.config.get_html(imdb_url, headers=headers).xpath("//div[@class='desc']/span/text()")[0].replace(",", "") - except IndexError: raise Failed(f"IMDb Error: Failed to parse URL: {imdb_url}") - try: total = int(re.findall("(\\d+) title", results)[0]) - except IndexError: raise Failed(f"IMDb Error: No Results at URL: {imdb_url}") - return total, 250 + page_type = "search" + results = self.config.get_html(imdb_url, headers=headers).xpath(xpath[page_type]) + total = 0 + for result in results: + if "title" in result: + try: + total = int(re.findall("(\\d+) title", result.replace(",", ""))[0]) + break + except IndexError: + pass + if total > 0: + return total, item_counts[page_type] + raise ValueError(f"IMDb Error: Failed to parse URL: {imdb_url}") def _ids_from_url(self, imdb_url, language, limit): - current_url = self._fix_url(imdb_url) - total, item_count = self._total(current_url, language) + total, item_count = self._total(imdb_url, language) headers = util.header(language) imdb_ids = [] - if "&start=" in current_url: current_url = re.sub("&start=\\d+", "", current_url) - if "&count=" in current_url: current_url = re.sub("&count=\\d+", "", current_url) - if "&page=" in current_url: current_url = re.sub("&page=\\d+", "", current_url) - if limit < 1 or total < limit: limit = total + parsed_url = urlparse(imdb_url) + params = parse_qs(parsed_url.query) + imdb_base = parsed_url._replace(query=None).geturl() + params.pop("start", None) + params.pop("count", None) + params.pop("page", None) + if limit < 1 or total < limit: + limit = total remainder = limit % item_count - if remainder == 0: remainder = item_count + if remainder == 0: + remainder = item_count num_of_pages = math.ceil(int(limit) / item_count) for i in range(1, num_of_pages + 1): start_num = (i - 1) * item_count + 1 util.print_return(f"Parsing Page {i}/{num_of_pages} {start_num}-{limit if i == num_of_pages else i * item_count}") - if imdb_url.startswith(urls["keyword"]): - response = self.config.get_html(f"{current_url}&page={i}", headers=headers) - else: - response = self.config.get_html(f"{current_url}&count={remainder if i == num_of_pages else item_count}&start={start_num}", headers=headers) - if imdb_url.startswith(urls["keyword"]) and i == num_of_pages: - imdb_ids.extend(response.xpath("//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst")[:remainder]) + if imdb_base.startswith((urls["list"], urls["keyword"])): + params["page"] = i else: - imdb_ids.extend(response.xpath("//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst")) + params["count"] = remainder if i == num_of_pages else item_count + params["start"] = start_num + ids_found = self.config.get_html(imdb_base, headers=headers, params=params).xpath(xpath["imdb_id"]) + if imdb_base.startswith((urls["list"], urls["keyword"])) and i == num_of_pages: + ids_found = ids_found[:remainder] + imdb_ids.extend(ids_found) time.sleep(2) util.print_end() - if imdb_ids: return imdb_ids - else: raise Failed(f"IMDb Error: No IMDb IDs Found at {imdb_url}") + if len(imdb_ids) > 0: + return imdb_ids + raise ValueError(f"IMDb Error: No IMDb IDs Found at {imdb_url}") def get_items(self, method, data, language, is_movie): pretty = util.pretty_names[method] if method in util.pretty_names else method diff --git a/modules/plex.py b/modules/plex.py index c885bd95..5ac0ce1d 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -60,7 +60,7 @@ collection_mode_options = { "hide_items": "hideItems", "hideitems": "hideItems", "show_items": "showItems", "showitems": "showItems" } -collection_order_options = {"release": "release", "alpha": "alpha"} +collection_order_options = ["release", "alpha", "custom"] collection_mode_keys = {-1: "default", 0: "hide", 1: "hideItems", 2: "showItems"} collection_order_keys = {0: "release", 1: "alpha", 2: "custom"} item_advance_keys = { diff --git a/modules/util.py b/modules/util.py index 57b71ec0..4e057cac 100644 --- a/modules/util.py +++ b/modules/util.py @@ -349,7 +349,6 @@ def parse_int(method, data, default=10, minimum=1, maximum=None): return default def parse_from_dict(parent, method, data, methods, default=None, options=None, translation=None): - message = "" if options is None and translation is not None: options = [o for o in translation] if method not in methods: diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 2d42ead7..3b848c60 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -504,7 +504,7 @@ try: minutes = int((seconds % 3600) // 60) time_str = f"{hours} Hour{'s' if hours > 1 else ''} and " if hours > 0 else "" time_str += f"{minutes} Minute{'s' if minutes > 1 else ''}" - util.print_return(f"Current Time: {current} | {time_str} until the next run at {og_time_str} {times_to_run}") + util.print_return(f"Current Time: {current} | {time_str} until the next run at {og_time_str} {util.compile_list(times_to_run)}") time.sleep(60) except KeyboardInterrupt: util.separator("Exiting Plex Meta Manager") From c55d93bbfbd307e01863ac0d3f0aaef43d72dbc2 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 29 Jul 2021 09:36:30 -0400 Subject: [PATCH 39/95] fix duplication error with imdb_list --- modules/builder.py | 43 +++++++++++++++++++++++++++++-------------- modules/imdb.py | 3 ++- modules/plex.py | 6 ++++++ plex_meta_manager.py | 6 ++++++ requirements.txt | 2 +- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 3d533d18..ad5c5d9a 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -124,15 +124,16 @@ show_only_filters = ["network"] smart_invalid = ["collection_order"] smart_url_invalid = ["filters", "run_again", "sync_mode", "show_filtered", "show_missing", "save_missing", "smart_label"] + radarr_details + sonarr_details custom_sort_builders = [ - "tmdb_collection", "tmdb_list", "tmdb_popular", "tmdb_now_playing", "tmdb_top_rated", - "tmdb_trending_daily", "tmdb_trending_weekly", "tmdb_discover", - "tvdb_list", "imdb_list", - "trakt_list", "trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected", - "tautulli_popular", "tautulli_watched", "letterboxd_list", "icheckmovies_list", - "anidb_popular", - "anilist_top_rated", "anilist_popular", "anilist_season", "anilist_studio", "anilist_genre", "anilist_tag", - "mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_movie", "mal_ova", "mal_special", - "mal_popular", "mal_favorite", "mal_suggested", "mal_userlist", "mal_season" + #"tmdb_collection", "tmdb_list", "tmdb_popular", "tmdb_now_playing", "tmdb_top_rated", + #"tmdb_trending_daily", "tmdb_trending_weekly", "tmdb_discover", + #"tvdb_list", + "imdb_list" + #"trakt_list", "trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected", + #"tautulli_popular", "tautulli_watched", "letterboxd_list", "icheckmovies_list", + #"anidb_popular", + #"anilist_top_rated", "anilist_popular", "anilist_season", "anilist_studio", "anilist_genre", "anilist_tag", + #"mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_movie", "mal_ova", "mal_special", + #"mal_popular", "mal_favorite", "mal_suggested", "mal_userlist", "mal_season" ] class CollectionBuilder: @@ -385,7 +386,7 @@ class CollectionBuilder: logger.debug(f"Value: {self.data[methods['build_collection']]}") self.build_collection = util.parse_bool("build_collection", self.data[methods["build_collection"]]) - self.sort_collection = False + self.custom_sort = False if "collection_order" in methods: logger.info("") logger.info("Validating Method: collection_order") @@ -393,8 +394,8 @@ class CollectionBuilder: raise Failed(f"Collection Warning: collection_order attribute is blank") elif self.data[methods["collection_order"]].lower() in plex.collection_order_options: self.details["collection_order"] = self.data[methods["collection_order"]].lower() - if self.data[methods["collection_order"]].lower() == "custom": - self.sort_collection = True + if self.data[methods["collection_order"]].lower() == "custom" and self.build_collection: + self.custom_sort = True else: raise Failed(f"Collection Error: {self.data[methods['collection_order']]} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)\n\tcustom (Custom Order Collection)") @@ -511,10 +512,10 @@ class CollectionBuilder: elif method_name == "filters": self._filters(method_name, method_data) else: raise Failed(f"Collection Error: {method_final} attribute not supported") - if self.sort_collection and len(self.builders) > 1: + if self.custom_sort and len(self.builders) > 1: raise Failed("Collection Error: collection_order: custom can only be used with a single builder per collection") - if self.sort_collection and self.builders[0][0] not in custom_sort_builders: + if self.custom_sort and self.builders[0][0] not in custom_sort_builders: raise Failed(f"Collection Error: collection_order: custom cannot be used with {self.builders[0][0]}") if self.add_to_radarr is None: @@ -1791,6 +1792,20 @@ class CollectionBuilder: if poster or background: self.library.upload_images(self.obj, poster=poster, background=background) + def sort_collection(self): + items = self.library.get_collection_items(self.obj, self.smart_label_collection) + keys = {item.ratingKey: item for item in items} + previous = None + for ki, key in enumerate(self.rating_keys): + if key != items[ki].ratingKey: + logger.info(f"Moving {keys[key].title} {'after {}'.format(keys[previous].title) if previous else 'to the beginning'}") + self.library.moveItem(self.obj, key, after=previous) + for ii, item in enumerate(items): + if key == item.ratingKey: + items.insert(ki, items.pop(ii)) + break + previous = key + def run_collections_again(self): self.obj = self.library.get_collection(self.name) name, collection_items = self.library.get_collection_name_and_items(self.obj, self.smart_label_collection) diff --git a/modules/imdb.py b/modules/imdb.py index 507fcf03..8df6924d 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -13,7 +13,7 @@ urls = { "keyword": f"{base_url}/search/keyword/" } xpath = { - "imdb_id": "//div[contains(@class, 'lister-item-image')]//@data-tconst", + "imdb_id": "//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst", "list": "//div[@class='desc lister-total-num-results']/text()", "search": "//div[@class='desc']/span/text()", "keyword": "//div[@class='desc']/text()" @@ -91,6 +91,7 @@ class IMDb: time.sleep(2) util.print_end() if len(imdb_ids) > 0: + logger.debug(f"{len(imdb_ids)} IMDb IDs Found: {imdb_ids}") return imdb_ids raise ValueError(f"IMDb Error: No IMDb IDs Found at {imdb_url}") diff --git a/modules/plex.py b/modules/plex.py index 5ac0ce1d..d21c6353 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -489,6 +489,12 @@ class Plex: else: method = None return self.Plex._server.query(key, method=method) + def moveItem(self, collection, item, after=None): + key = f"{collection.key}/items/{item}/move" + if after: + key += f"?after={after}" + self._query(key, put=True) + def smart_label_url(self, title, sort): labels = self.get_labels() if title not in labels: diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 3b848c60..704a4114 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -463,6 +463,12 @@ def run_collection(config, library, metadata, requested_collections): logger.info("") builder.update_details() + if builder.custom_sort: + logger.info("") + util.separator(f"Sorting {mapping_name} Collection", space=False, border=False) + logger.info("") + builder.sort_collection() + logger.info("") util.separator(f"Updating Details of the Items in {mapping_name} Collection", space=False, border=False) logger.info("") diff --git a/requirements.txt b/requirements.txt index 0bda40ca..d913beed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -PlexAPI==4.6.1 +PlexAPI==4.7.0 tmdbv3api==1.7.6 arrapi==1.1.1 lxml From a4409f7f17534701ecdd80001e6acf08eb7bfb8f Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 29 Jul 2021 09:56:32 -0400 Subject: [PATCH 40/95] turn on tmdb_list --- modules/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/builder.py b/modules/builder.py index ad5c5d9a..2bd5bc32 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -127,7 +127,7 @@ custom_sort_builders = [ #"tmdb_collection", "tmdb_list", "tmdb_popular", "tmdb_now_playing", "tmdb_top_rated", #"tmdb_trending_daily", "tmdb_trending_weekly", "tmdb_discover", #"tvdb_list", - "imdb_list" + "imdb_list", "tmdb_list" #"trakt_list", "trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected", #"tautulli_popular", "tautulli_watched", "letterboxd_list", "icheckmovies_list", #"anidb_popular", From b6005b0f0316f50735b9a5d37ee1b8636c6a70a9 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 29 Jul 2021 17:36:26 -0400 Subject: [PATCH 41/95] #308 Custom sorting --- modules/builder.py | 20 ++++++++++---------- modules/plex.py | 2 ++ modules/trakt.py | 6 +++++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 2bd5bc32..077bca61 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -124,16 +124,16 @@ show_only_filters = ["network"] smart_invalid = ["collection_order"] smart_url_invalid = ["filters", "run_again", "sync_mode", "show_filtered", "show_missing", "save_missing", "smart_label"] + radarr_details + sonarr_details custom_sort_builders = [ - #"tmdb_collection", "tmdb_list", "tmdb_popular", "tmdb_now_playing", "tmdb_top_rated", - #"tmdb_trending_daily", "tmdb_trending_weekly", "tmdb_discover", - #"tvdb_list", - "imdb_list", "tmdb_list" - #"trakt_list", "trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected", - #"tautulli_popular", "tautulli_watched", "letterboxd_list", "icheckmovies_list", - #"anidb_popular", - #"anilist_top_rated", "anilist_popular", "anilist_season", "anilist_studio", "anilist_genre", "anilist_tag", - #"mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_movie", "mal_ova", "mal_special", - #"mal_popular", "mal_favorite", "mal_suggested", "mal_userlist", "mal_season" + "tmdb_list", "tmdb_popular", "tmdb_now_playing", "tmdb_top_rated", + "tmdb_trending_daily", "tmdb_trending_weekly", "tmdb_discover", + "tvdb_list", + "imdb_list", + "trakt_list", "trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected", + "tautulli_popular", "tautulli_watched", "letterboxd_list", "icheckmovies_list", + "anidb_popular", + "anilist_top_rated", "anilist_popular", "anilist_season", "anilist_studio", "anilist_genre", "anilist_tag", + "mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_movie", "mal_ova", "mal_special", + "mal_popular", "mal_favorite", "mal_suggested", "mal_userlist", "mal_season" ] class CollectionBuilder: diff --git a/modules/plex.py b/modules/plex.py index d21c6353..d0a9d106 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -423,6 +423,8 @@ class Plex: if self.config.Cache: _, _, image_overlay = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) if poster_uploaded or not image_overlay or image_overlay != overlay_name: + if not item.posterUrl: + raise Failed(f"Overlay Error: No existing poster to Overlay for {item.title}") response = requests.get(item.posterUrl) if response.status_code >= 400: raise Failed(f"Overlay Error: Overlay Failed for {item.title}") diff --git a/modules/trakt.py b/modules/trakt.py index 8d9375e0..d8caa89e 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -12,6 +12,10 @@ builders = [ "trakt_collected", "trakt_collection", "trakt_list", "trakt_list_details", "trakt_popular", "trakt_recommended", "trakt_trending", "trakt_watched", "trakt_watchlist" ] +sorts = [ + "rank", "added", "title", "released", "runtime", "popularity", + "percentage", "votes", "random", "my_rating", "watched", "collected" +] class Trakt: def __init__(self, config, params): @@ -140,7 +144,7 @@ class Trakt: else: return [], [item["show"]["ids"]["tvdb"] for item in items] def validate_trakt(self, trakt_lists, is_movie, trakt_type="list"): - values = util.get_list(trakt_lists) + values = util.get_list(trakt_lists, split=False) trakt_values = [] for value in values: try: From 711ae083723d3b8c0d7c86db4b4bbb657a14255d Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 30 Jul 2021 14:50:08 -0400 Subject: [PATCH 42/95] #348 only run missing when needed --- plex_meta_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 704a4114..3e953f43 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -448,7 +448,9 @@ def run_collection(config, library, metadata, requested_collections): util.separator(f"Adding to {mapping_name} Collection", space=False, border=False) logger.info("") builder.add_to_collection() - if len(builder.missing_movies) > 0 or len(builder.missing_shows) > 0: + if (builder.details["show_missing"] is True or builder.details["save_missing"] is True + or (library.Radarr and builder.add_to_radarr) or (library.Sonarr and builder.add_to_sonarr)) \ + and (len(builder.missing_movies) > 0 or len(builder.missing_shows) > 0): if builder.details["show_missing"] is True: logger.info("") util.separator(f"Missing from Library", space=False, border=False) From 91e525e43d17af11505c903d4c525dfa0f0112a9 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 30 Jul 2021 15:19:43 -0400 Subject: [PATCH 43/95] #341 added validate_builders --- modules/builder.py | 205 ++++++++++++++++++++++++--------------------- modules/imdb.py | 4 +- modules/meta.py | 4 +- modules/util.py | 154 +++++++++++----------------------- 4 files changed, 163 insertions(+), 204 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 077bca61..c2bdaa6b 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -73,7 +73,7 @@ poster_details = ["url_poster", "tmdb_poster", "tmdb_profile", "tvdb_poster", "f background_details = ["url_background", "tmdb_background", "tvdb_background", "file_background"] boolean_details = ["visible_library", "visible_home", "visible_shared", "show_filtered", "show_missing", "save_missing", "item_assets"] string_details = ["sort_title", "content_rating", "name_mapping"] -ignored_details = ["smart_filter", "smart_label", "smart_url", "run_again", "schedule", "sync_mode", "template", "test", "tmdb_person", "build_collection", "collection_order"] +ignored_details = ["smart_filter", "smart_label", "smart_url", "run_again", "schedule", "sync_mode", "template", "test", "tmdb_person", "build_collection", "collection_order", "validate_builders"] details = ["collection_mode", "collection_order", "label"] + boolean_details + string_details collectionless_details = ["collection_order", "plex_collectionless", "label", "label_sync_mode", "test"] + \ poster_details + background_details + summary_details + string_details @@ -350,18 +350,28 @@ class CollectionBuilder: if skip_collection: raise Failed(f"{self.schedule}\n\nCollection {self.name} not scheduled to run") - self.run_again = "run_again" in methods self.collectionless = "plex_collectionless" in methods + self.validate_builders = True + if "validate_builders" in methods: + logger.info("") + logger.info("Validating Method: validate_builders") + logger.info(f"Value: {data[methods['validate_builders']]}") + self.validate_builders = util.parse("validate_builders", self.data, datatype="bool", methods=methods, default=True) + self.run_again = False if "run_again" in methods: logger.info("") logger.info("Validating Method: run_again") - if not self.data[methods["run_again"]]: - logger.warning(f"Collection Warning: run_again attribute is blank defaulting to false") - else: - logger.debug(f"Value: {self.data[methods['run_again']]}") - self.run_again = util.parse_bool("run_again", self.data[methods["run_again"]]) + logger.info(f"Value: {data[methods['run_again']]}") + self.run_again = util.parse("run_again", self.data, datatype="bool", methods=methods, default=False) + + self.build_collection = True + if "build_collection" in methods: + logger.info("") + logger.info("Validating Method: build_collection") + logger.info(f"Value: {data[methods['build_collection']]}") + self.build_collection = util.parse("build_collection", self.data, datatype="bool", methods=methods, default=True) self.sync = self.library.sync_mode == "sync" if "sync_mode" in methods: @@ -376,16 +386,6 @@ class CollectionBuilder: else: self.sync = self.data[methods["sync_mode"]].lower() == "sync" - self.build_collection = True - if "build_collection" in methods: - logger.info("") - logger.info("Validating Method: build_collection") - if self.data[methods["build_collection"]] is None: - logger.warning(f"Collection Warning: build_collection attribute is blank defaulting to true") - else: - logger.debug(f"Value: {self.data[methods['build_collection']]}") - self.build_collection = util.parse_bool("build_collection", self.data[methods["build_collection"]]) - self.custom_sort = False if "collection_order" in methods: logger.info("") @@ -477,40 +477,46 @@ class CollectionBuilder: logger.debug("") logger.debug(f"Validating Method: {method_key}") logger.debug(f"Value: {method_data}") - if method_data is None and method_name in all_builders + plex.searches: raise Failed(f"Collection Error: {method_final} attribute is blank") - elif method_data is None: logger.warning(f"Collection Warning: {method_final} attribute is blank") - elif not self.config.Trakt and "trakt" in method_name: raise Failed(f"Collection Error: {method_final} requires Trakt to be configured") - elif not self.library.Radarr and "radarr" in method_name: raise Failed(f"Collection Error: {method_final} requires Radarr to be configured") - elif not self.library.Sonarr and "sonarr" in method_name: raise Failed(f"Collection Error: {method_final} requires Sonarr to be configured") - elif not self.library.Tautulli and "tautulli" in method_name: raise Failed(f"Collection Error: {method_final} requires Tautulli to be configured") - elif not self.config.MyAnimeList and "mal" in method_name: raise Failed(f"Collection Error: {method_final} requires MyAnimeList to be configured") - elif self.library.is_movie and method_name in show_only_builders: raise Failed(f"Collection Error: {method_final} attribute only works for show libraries") - elif self.library.is_show and method_name in movie_only_builders: raise Failed(f"Collection Error: {method_final} attribute only works for movie libraries") - elif self.library.is_show and method_name in plex.movie_only_searches: raise Failed(f"Collection Error: {method_final} plex search only works for movie libraries") - elif self.library.is_movie and method_name in plex.show_only_searches: raise Failed(f"Collection Error: {method_final} plex search only works for show libraries") - elif self.smart and method_name in smart_invalid: raise Failed(f"Collection Error: {method_final} attribute only works with normal collections") - elif self.collectionless and method_name not in collectionless_details: raise Failed(f"Collection Error: {method_final} attribute does not work for Collectionless collection") - elif self.smart_url and method_name in all_builders + smart_url_invalid: raise Failed(f"Collection Error: {method_final} builder not allowed when using smart_filter") - elif method_name in summary_details: self._summary(method_name, method_data) - elif method_name in poster_details: self._poster(method_name, method_data) - elif method_name in background_details: self._background(method_name, method_data) - elif method_name in details: self._details(method_name, method_data, method_final, methods) - elif method_name in item_details: self._item_details(method_name, method_data, method_mod, method_final, methods) - elif method_name in radarr_details: self._radarr(method_name, method_data) - elif method_name in sonarr_details: self._sonarr(method_name, method_data) - elif method_name in anidb.builders: self._anidb(method_name, method_data) - elif method_name in anilist.builders: self._anilist(method_name, method_data) - elif method_name in icheckmovies.builders: self._icheckmovies(method_name, method_data) - elif method_name in letterboxd.builders: self._letterboxd(method_name, method_data) - elif method_name in imdb.builders: self._imdb(method_name, method_data) - elif method_name in mal.builders: self._mal(method_name, method_data) - elif method_name in plex.builders or method_final in plex.searches: self._plex(method_name, method_data) - elif method_name in tautulli.builders: self._tautulli(method_name, method_data) - elif method_name in tmdb.builders: self._tmdb(method_name, method_data) - elif method_name in trakt.builders: self._trakt(method_name, method_data) - elif method_name in tvdb.builders: self._tvdb(method_name, method_data) - elif method_name == "filters": self._filters(method_name, method_data) - else: raise Failed(f"Collection Error: {method_final} attribute not supported") + try: + if method_data is None and method_name in all_builders + plex.searches: raise Failed(f"Collection Error: {method_final} attribute is blank") + elif method_data is None: logger.warning(f"Collection Warning: {method_final} attribute is blank") + elif not self.config.Trakt and "trakt" in method_name: raise Failed(f"Collection Error: {method_final} requires Trakt to be configured") + elif not self.library.Radarr and "radarr" in method_name: raise Failed(f"Collection Error: {method_final} requires Radarr to be configured") + elif not self.library.Sonarr and "sonarr" in method_name: raise Failed(f"Collection Error: {method_final} requires Sonarr to be configured") + elif not self.library.Tautulli and "tautulli" in method_name: raise Failed(f"Collection Error: {method_final} requires Tautulli to be configured") + elif not self.config.MyAnimeList and "mal" in method_name: raise Failed(f"Collection Error: {method_final} requires MyAnimeList to be configured") + elif self.library.is_movie and method_name in show_only_builders: raise Failed(f"Collection Error: {method_final} attribute only works for show libraries") + elif self.library.is_show and method_name in movie_only_builders: raise Failed(f"Collection Error: {method_final} attribute only works for movie libraries") + elif self.library.is_show and method_name in plex.movie_only_searches: raise Failed(f"Collection Error: {method_final} plex search only works for movie libraries") + elif self.library.is_movie and method_name in plex.show_only_searches: raise Failed(f"Collection Error: {method_final} plex search only works for show libraries") + elif self.smart and method_name in smart_invalid: raise Failed(f"Collection Error: {method_final} attribute only works with normal collections") + elif self.collectionless and method_name not in collectionless_details: raise Failed(f"Collection Error: {method_final} attribute does not work for Collectionless collection") + elif self.smart_url and method_name in all_builders + smart_url_invalid: raise Failed(f"Collection Error: {method_final} builder not allowed when using smart_filter") + elif method_name in summary_details: self._summary(method_name, method_data) + elif method_name in poster_details: self._poster(method_name, method_data) + elif method_name in background_details: self._background(method_name, method_data) + elif method_name in details: self._details(method_name, method_data, method_final, methods) + elif method_name in item_details: self._item_details(method_name, method_data, method_mod, method_final, methods) + elif method_name in radarr_details: self._radarr(method_name, method_data) + elif method_name in sonarr_details: self._sonarr(method_name, method_data) + elif method_name in anidb.builders: self._anidb(method_name, method_data) + elif method_name in anilist.builders: self._anilist(method_name, method_data) + elif method_name in icheckmovies.builders: self._icheckmovies(method_name, method_data) + elif method_name in letterboxd.builders: self._letterboxd(method_name, method_data) + elif method_name in imdb.builders: self._imdb(method_name, method_data) + elif method_name in mal.builders: self._mal(method_name, method_data) + elif method_name in plex.builders or method_final in plex.searches: self._plex(method_name, method_data) + elif method_name in tautulli.builders: self._tautulli(method_name, method_data) + elif method_name in tmdb.builders: self._tmdb(method_name, method_data) + elif method_name in trakt.builders: self._trakt(method_name, method_data) + elif method_name in tvdb.builders: self._tvdb(method_name, method_data) + elif method_name == "filters": self._filters(method_name, method_data) + else: raise Failed(f"Collection Error: {method_final} attribute not supported") + except Failed as e: + if self.validate_builders: + raise + else: + logger.error(e) if self.custom_sort and len(self.builders) > 1: raise Failed("Collection Error: collection_order: custom can only be used with a single builder per collection") @@ -622,7 +628,7 @@ class CollectionBuilder: else: self.details[method_final] = util.get_list(method_data) elif method_name in boolean_details: - self.details[method_name] = util.parse_bool(method_name, method_data) + self.details[method_name] = util.parse(method_name, method_data, datatype="bool") elif method_name in string_details: self.details[method_name] = str(method_data) @@ -664,11 +670,11 @@ class CollectionBuilder: def _radarr(self, method_name, method_data): if method_name == "radarr_add": - self.add_to_radarr = util.parse_bool(method_name, method_data) + self.add_to_radarr = util.parse(method_name, method_data, datatype="bool") elif method_name == "radarr_folder": self.radarr_options["folder"] = method_data elif method_name in ["radarr_monitor", "radarr_search"]: - self.radarr_options[method_name[7:]] = util.parse_bool(method_name, method_data) + self.radarr_options[method_name[7:]] = util.parse(method_name, method_data, datatype="bool") elif method_name == "radarr_availability": if str(method_data).lower() in radarr.availability_translation: self.radarr_options["availability"] = str(method_data).lower() @@ -682,7 +688,7 @@ class CollectionBuilder: def _sonarr(self, method_name, method_data): if method_name == "sonarr_add": - self.add_to_sonarr = util.parse_bool(method_name, method_data) + self.add_to_sonarr = util.parse(method_name, method_data, datatype="bool") elif method_name == "sonarr_folder": self.sonarr_options["folder"] = method_data elif method_name == "sonarr_monitor": @@ -702,18 +708,18 @@ class CollectionBuilder: else: raise Failed(f"Collection Error: {method_name} attribute must be either standard, daily, or anime") elif method_name in ["sonarr_season", "sonarr_search", "sonarr_cutoff_search"]: - self.sonarr_options[method_name[7:]] = util.parse_bool(method_name, method_data) + self.sonarr_options[method_name[7:]] = util.parse(method_name, method_data, datatype="bool") elif method_name == "sonarr_tag": self.sonarr_options["tag"] = util.get_list(method_data) def _anidb(self, method_name, method_data): if method_name == "anidb_popular": - self.builders.append((method_name, util.parse_int(method_name, method_data, default=30, maximum=30))) + self.builders.append((method_name, util.parse(method_name, method_data, datatype="int", default=30, maximum=30))) elif method_name in ["anidb_id", "anidb_relation"]: for anidb_id in self.config.AniDB.validate_anidb_ids(method_data, self.language): self.builders.append((method_name, anidb_id)) elif method_name == "anidb_tag": - for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): new_dictionary = {} if "tag" not in dict_methods: raise Failed("Collection Error: anidb_tag tag attribute is required") @@ -721,7 +727,7 @@ class CollectionBuilder: raise Failed("Collection Error: anidb_tag tag attribute is blank") else: new_dictionary["tag"] = util.regex_first_int(dict_data[dict_methods["username"]], "AniDB Tag ID") - new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 0, minimum=0) + new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=0, parent=method_name, minimum=0) self.builders.append((method_name, new_dictionary)) def _anilist(self, method_name, method_data): @@ -729,23 +735,23 @@ class CollectionBuilder: for anilist_id in self.config.AniList.validate_anilist_ids(method_data, studio=method_name == "anilist_studio"): self.builders.append((method_name, anilist_id)) elif method_name in ["anilist_popular", "anilist_top_rated"]: - self.builders.append((method_name, util.parse_int(method_name, method_data))) + self.builders.append((method_name, util.parse(method_name, method_data, datatype="int", default=10))) elif method_name in ["anilist_season", "anilist_genre", "anilist_tag"]: - for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): new_dictionary = {} if method_name == "anilist_season": if self.current_time.month in [12, 1, 2]: new_dictionary["season"] = "winter" elif self.current_time.month in [3, 4, 5]: new_dictionary["season"] = "spring" elif self.current_time.month in [6, 7, 8]: new_dictionary["season"] = "summer" elif self.current_time.month in [9, 10, 11]: new_dictionary["season"] = "fall" - new_dictionary["season"] = util.parse_from_dict(method_name, "season", dict_data, dict_methods, default=new_dictionary["season"], options=["winter", "spring", "summer", "fall"]) - new_dictionary["year"] = util.parse_int_from_dict(method_name, "year", dict_data, dict_methods, self.current_time.year, minimum=1917, maximum=self.current_time.year + 1) + new_dictionary["season"] = util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=new_dictionary["season"], options=["winter", "spring", "summer", "fall"]) + new_dictionary["year"] = util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_time.year, parent=method_name, minimum=1917, maximum=self.current_time.year + 1) elif method_name == "anilist_genre": - new_dictionary["genre"] = self.config.AniList.validate_genre(util.parse_from_dict(method_name, "genre", dict_data, dict_methods)) + new_dictionary["genre"] = self.config.AniList.validate_genre(util.parse("genre", dict_data, methods=dict_methods, parent=method_name)) elif method_name == "anilist_tag": - new_dictionary["tag"] = self.config.AniList.validate_tag(util.parse_from_dict(method_name, "tag", dict_data, dict_methods)) - new_dictionary["sort_by"] = util.parse_from_dict(method_name, "sort_by", dict_data, dict_methods, default="score", options=["score", "popular"]) - new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 0, maximum=500) + new_dictionary["tag"] = self.config.AniList.validate_tag(util.parse("tag", dict_data, methods=dict_methods, parent=method_name)) + new_dictionary["sort_by"] = util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="score", options=["score", "popular"]) + new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=0, parent=method_name, maximum=500) self.builders.append((method_name, new_dictionary)) def _icheckmovies(self, method_name, method_data): @@ -780,37 +786,37 @@ class CollectionBuilder: for mal_id in util.get_int_list(method_data, "MyAnimeList ID"): self.builders.append((method_name, mal_id)) elif method_name in ["mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_ova", "mal_movie", "mal_special", "mal_popular", "mal_favorite", "mal_suggested"]: - self.builders.append((method_name, util.parse_int(method_name, method_data))) + self.builders.append((method_name, util.parse(method_name, method_data, datatype="int", default=10))) elif method_name in ["mal_season", "mal_userlist"]: - for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): new_dictionary = {} if method_name == "mal_season": if self.current_time.month in [1, 2, 3]: new_dictionary["season"] = "winter" elif self.current_time.month in [4, 5, 6]: new_dictionary["season"] = "spring" elif self.current_time.month in [7, 8, 9]: new_dictionary["season"] = "summer" elif self.current_time.month in [10, 11, 12]: new_dictionary["season"] = "fall" - new_dictionary["season"] = util.parse_from_dict(method_name, "season", dict_data, dict_methods, default=new_dictionary["season"], options=["winter", "spring", "summer", "fall"]) - new_dictionary["sort_by"] = util.parse_from_dict(method_name, "sort_by", dict_data, dict_methods, default="members", options=mal.season_sort_options, translation=mal.season_sort_translation) - new_dictionary["year"] = util.parse_int_from_dict(method_name, "year", dict_data, dict_methods, self.current_time.year, minimum=1917, maximum=self.current_time.year + 1) - new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 100, maximum=500) + new_dictionary["season"] = util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=new_dictionary["season"], options=["winter", "spring", "summer", "fall"]) + new_dictionary["sort_by"] = util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="members", options=mal.season_sort_options, translation=mal.season_sort_translation) + new_dictionary["year"] = util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_time.year, parent=method_name, minimum=1917, maximum=self.current_time.year + 1) + new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=100, parent=method_name, maximum=500) elif method_name == "mal_userlist": - new_dictionary["username"] = util.parse_from_dict(method_name, "username", dict_data, dict_methods) - new_dictionary["status"] = util.parse_from_dict(method_name, "status", dict_data, dict_methods, default="all", options=mal.userlist_status) - new_dictionary["sort_by"] = util.parse_from_dict(method_name, "sort_by", dict_data, dict_methods, default="score", options=mal.userlist_sort_options, translation=mal.userlist_sort_translation) - new_dictionary["limit"] = util.parse_int_from_dict(method_name, "limit", dict_data, dict_methods, 100, maximum=1000) + new_dictionary["username"] = util.parse("username", dict_data, methods=dict_methods, parent=method_name) + new_dictionary["status"] = util.parse("status", dict_data, methods=dict_methods, parent=method_name, default="all", options=mal.userlist_status) + new_dictionary["sort_by"] = util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="score", options=mal.userlist_sort_options, translation=mal.userlist_sort_translation) + new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=100, parent=method_name, maximum=1000) self.builders.append((method_name, new_dictionary)) def _plex(self, method_name, method_data): if method_name == "plex_all": self.builders.append((method_name, True)) elif method_name in ["plex_search", "plex_collectionless"]: - for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): new_dictionary = {} if method_name == "plex_search": new_dictionary = self.build_filter("plex_search", dict_data) elif method_name == "plex_collectionless": - prefix_list = util.parse_list("exclude_prefix", dict_data, dict_methods) - exact_list = util.parse_list("exclude", dict_data, dict_methods) + prefix_list = util.parse("exclude_prefix", dict_data, datatype="list", methods=dict_methods) + exact_list = util.parse("exclude", dict_data, datatype="list", methods=dict_methods) if len(prefix_list) == 0 and len(exact_list) == 0: raise Failed("Collection Error: you must have at least one exclusion") exact_list.append(self.name) @@ -821,17 +827,17 @@ class CollectionBuilder: self.builders.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) def _tautulli(self, method_name, method_data): - for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): self.builders.append((method_name, { "list_type": "popular" if method_name == "tautulli_popular" else "watched", - "list_days": util.parse_int_from_dict(method_name, "list_days", dict_data, dict_methods, 30), - "list_size": util.parse_int_from_dict(method_name, "list_size", dict_data, dict_methods, 10), - "list_buffer": util.parse_int_from_dict(method_name, "list_buffer", dict_data, dict_methods, 20) + "list_days": util.parse("list_days", dict_data, datatype="int", methods=dict_methods, default=30, parent=method_name), + "list_size": util.parse("list_size", dict_data, datatype="int", methods=dict_methods, default=10, parent=method_name), + "list_buffer": util.parse("list_buffer", dict_data, datatype="int", methods=dict_methods, default=20, parent=method_name) })) def _tmdb(self, method_name, method_data): if method_name == "tmdb_discover": - for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): new_dictionary = {"limit": 100} for discover_name, discover_data in dict_data.items(): discover_final = discover_name.lower() @@ -868,9 +874,9 @@ class CollectionBuilder: elif discover_final in tmdb.discover_dates: new_dictionary[discover_final] = util.validate_date(discover_data, f"{method_name} attribute {discover_final}", return_as="%m/%d/%Y") elif discover_final in ["primary_release_year", "year", "first_air_date_year"]: - new_dictionary[discover_final] = util.check_number(discover_data, f"{method_name} attribute {discover_final}", minimum=1800, maximum=self.current_year + 1) + new_dictionary[discover_final] = util.parse(discover_final, discover_data, datatype="int", parent=method_name, minimum=1800, maximum=self.current_year + 1) elif discover_final in ["vote_count.gte", "vote_count.lte", "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte"]: - new_dictionary[discover_final] = util.check_number(discover_data, f"{method_name} attribute {discover_final}", minimum=1) + new_dictionary[discover_final] = util.parse(discover_final, discover_data, datatype="int", parent=method_name) elif discover_final in ["with_cast", "with_crew", "with_people", "with_companies", "with_networks", "with_genres", "without_genres", "with_keywords", "without_keywords", "with_original_language", "timezone"]: new_dictionary[discover_final] = discover_data else: @@ -889,7 +895,7 @@ class CollectionBuilder: else: raise Failed(f"Collection Error: {method_name} had no valid fields") elif method_name in ["tmdb_popular", "tmdb_top_rated", "tmdb_now_playing", "tmdb_trending_daily", "tmdb_trending_weekly"]: - self.builders.append((method_name, util.parse_int(method_name, method_data))) + self.builders.append((method_name, util.parse(method_name, method_data, datatype="int", default=10))) else: values = self.config.TMDb.validate_tmdb_ids(method_data, method_name) if method_name.endswith("_details"): @@ -922,7 +928,7 @@ class CollectionBuilder: if method_name.endswith("_details"): self.summaries[method_name] = self.config.Trakt.list_description(trakt_lists[0]) elif method_name in ["trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected"]: - self.builders.append((method_name, util.parse_int(method_name, method_data))) + self.builders.append((method_name, util.parse(method_name, method_data, datatype="int", default=10))) elif method_name in ["trakt_watchlist", "trakt_collection"]: for trakt_list in self.config.Trakt.validate_trakt(method_data, self.library.is_movie, trakt_type=method_name[6:]): self.builders.append((method_name, trakt_list)) @@ -944,7 +950,7 @@ class CollectionBuilder: self.builders.append((method_name[:-8] if method_name.endswith("_details") else method_name, value)) def _filters(self, method_name, method_data): - for dict_data, dict_methods in util.validate_dict_list(method_name, method_data): + for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): validate = True if "validate" in dict_data: if dict_data["validate"] is None: @@ -1215,7 +1221,7 @@ class CollectionBuilder: return util.get_list(data) elif attribute == "history": try: - return util.check_number(data, final, minimum=1, maximum=30) + return util.parse(final, data, datatype="int", maximum=30) except Failed: if str(data).lower() in ["day", "month"]: return data.lower() @@ -1247,17 +1253,21 @@ class CollectionBuilder: logger.error(error) return valid_list elif attribute in ["year", "episode_year"] and modifier in [".gt", ".gte", ".lt", ".lte"]: - return util.check_year(data, self.current_year, final) + return util.parse(final, data, datatype="int", minimum=1800, maximum=self.current_year) elif attribute in plex.date_attributes and modifier in [".before", ".after"]: return util.validate_date(data, final, return_as="%Y-%m-%d") elif attribute in plex.number_attributes and modifier in ["", ".not", ".gt", ".gte", ".lt", ".lte"]: - return util.check_number(data, final, minimum=1) + return util.parse(final, data, datatype="int") elif attribute in plex.float_attributes and modifier in [".gt", ".gte", ".lt", ".lte"]: - return util.check_number(data, final, number_type="float", minimum=0, maximum=10) + return util.parse(final, data, datatype="float", minimum=0, maximum=10) elif attribute in ["decade", "year", "episode_year"] and modifier in ["", ".not"]: - return smart_pair(util.get_year_list(data, self.current_year, final)) + final_years = [] + values = util.get_list(data) + for value in values: + final_years.append(util.parse(final, value, datatype="int", minimum=1800, maximum=self.current_year)) + return smart_pair(final_years) elif attribute in plex.boolean_attributes: - return util.parse_bool(attribute, data) + return util.parse(attribute, data, datatype="bool") else: raise Failed(f"Collection Error: {final} attribute not supported") @@ -1796,7 +1806,10 @@ class CollectionBuilder: items = self.library.get_collection_items(self.obj, self.smart_label_collection) keys = {item.ratingKey: item for item in items} previous = None + logger.debug(keys) + logger.debug(self.rating_keys) for ki, key in enumerate(self.rating_keys): + logger.debug(items) if key != items[ki].ratingKey: logger.info(f"Moving {keys[key].title} {'after {}'.format(keys[previous].title) if previous else 'to the beginning'}") self.library.moveItem(self.obj, key, after=previous) diff --git a/modules/imdb.py b/modules/imdb.py index 8df6924d..803e6bd5 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -30,11 +30,11 @@ class IMDb: if not isinstance(imdb_dict, dict): imdb_dict = {"url": imdb_dict} dict_methods = {dm.lower(): dm for dm in imdb_dict} - imdb_url = util.parse_from_dict("imdb_list", "url", imdb_dict, dict_methods).strip() + imdb_url = util.parse("url", imdb_dict, methods=dict_methods, parent="imdb_list").strip() if not imdb_url.startswith((urls["list"], urls["search"], urls["keyword"])): raise Failed(f"IMDb Error: {imdb_url} must begin with either:\n{urls['list']} (For Lists)\n{urls['search']} (For Searches)\n{urls['keyword']} (For Keyword Searches)") self._total(imdb_url, language) - list_count = util.parse_int_from_dict("imdb_list", "limit", imdb_dict, dict_methods, 0, minimum=0) if "limit" in dict_methods else 0 + list_count = util.parse("limit", imdb_dict, datatype="int", methods=dict_methods, default=0, parent="imdb_list", minimum=0) if "limit" in dict_methods else 0 valid_lists.append({"url": imdb_url, "limit": list_count}) return valid_lists diff --git a/modules/meta.py b/modules/meta.py index f4805e56..b3afccc2 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -93,7 +93,7 @@ class Metadata: final_value = util.validate_date(value, name, return_as="%Y-%m-%d") current = current[:-9] elif var_type == "float": - final_value = util.check_number(value, name, number_type="float", minimum=0, maximum=10) + final_value = util.parse(name, value, datatype="float", minimum=0, maximum=10) else: final_value = value if current != str(final_value): @@ -170,7 +170,7 @@ class Metadata: logger.info("") year = None if "year" in methods: - year = util.check_number(meta[methods["year"]], "year", minimum=1800, maximum=datetime.now().year + 1) + year = util.parse("year", meta, datatype="int", methods=methods, minimum=1800, maximum=datetime.now().year + 1) title = mapping_name if "title" in methods: diff --git a/modules/util.py b/modules/util.py index 4e057cac..bb06df7d 100644 --- a/modules/util.py +++ b/modules/util.py @@ -95,18 +95,6 @@ def tab_new_lines(data): def make_ordinal(n): return f"{n}{'th' if 11 <= (n % 100) <= 13 else ['th', 'st', 'nd', 'rd', 'th'][min(n % 10, 4)]}" -def parse_bool(method_name, method_data): - if isinstance(method_data, bool): - return method_data - elif isinstance(method_data, int): - return method_data > 0 - elif str(method_data).lower() in ["t", "true"]: - return True - elif str(method_data).lower() in ["f", "false"]: - return False - else: - raise Failed(f"Collection Error: {method_name} attribute: {method_data} invalid must be either true or false") - def compile_list(data): if isinstance(data, list): text = "" @@ -126,40 +114,12 @@ def get_list(data, lower=False, split=True, int_list=False): else: return [d.strip() for d in str(data).split(",")] def get_int_list(data, id_type): - values = get_list(data) int_values = [] - for value in values: + for value in get_list(data): try: int_values.append(regex_first_int(value, id_type)) except Failed as e: logger.error(e) return int_values -def get_year_list(data, current_year, method): - final_years = [] - values = get_list(data) - for value in values: - final_years.append(check_year(value, current_year, method)) - return final_years - -def check_year(year, current_year, method): - return check_number(year, method, minimum=1800, maximum=current_year) - -def check_number(value, method, number_type="int", minimum=None, maximum=None): - if number_type == "int": - try: num_value = int(str(value)) - except ValueError: raise Failed(f"Collection Error: {method}: {value} must be an integer") - elif number_type == "float": - try: num_value = float(str(value)) - except ValueError: raise Failed(f"Collection Error: {method}: {value} must be a number") - else: raise Failed(f"Number Type: {number_type} invalid") - if minimum is not None and maximum is not None and (num_value < minimum or num_value > maximum): - raise Failed(f"Collection Error: {method}: {num_value} must be between {minimum} and {maximum}") - elif minimum is not None and num_value < minimum: - raise Failed(f"Collection Error: {method}: {num_value} is less then {minimum}") - elif maximum is not None and num_value > maximum: - raise Failed(f"Collection Error: {method}: {num_value} is greater then {maximum}") - else: - return num_value - def validate_date(date_text, method, return_as=None): try: date_obg = datetime.strptime(str(date_text), "%Y-%m-%d" if "-" in str(date_text) else "%m/%d/%Y") except ValueError: raise Failed(f"Collection Error: {method}: {date_text} must match pattern YYYY-MM-DD (e.g. 2020-12-25) or MM/DD/YYYY (e.g. 12/25/2020)") @@ -184,22 +144,6 @@ def unix_input(prompt, timeout=60): except EOFError: raise Failed("Input Failed") finally: signal.alarm(0) -def old_windows_input(prompt, timeout=60, timer=time.monotonic): - prompt = f"| {prompt}: " - sys.stdout.write(prompt) - sys.stdout.flush() - endtime = timer() + timeout - result = [] - while timer() < endtime: - if msvcrt.kbhit(): - result.append(msvcrt.getwche()) - if result[-1] == "\n": - out = "".join(result[:-1]) - logger.debug(f"{prompt[2:]}{out}") - return out - time.sleep(0.04) - raise TimeoutExpired - def windows_input(prompt, timeout=5): sys.stdout.write(f"| {prompt}: ") sys.stdout.flush() @@ -322,64 +266,66 @@ def is_locked(filepath): file_object = open(filepath, 'a', 8) if file_object: locked = False - except IOError as message: + except IOError: locked = True finally: if file_object: file_object.close() return locked -def validate_dict_list(method_name, data): - final_list = [] - for dict_data in get_list(data): - if isinstance(dict_data, dict): - final_list.append((dict_data, {dm.lower(): dm for dm in dict_data})) - else: - raise Failed(f"Collection Error: {method_name} attribute is not a dictionary: {dict_data}") - return final_list - -def parse_int(method, data, default=10, minimum=1, maximum=None): - list_count = regex_first_int(data, "List Size", default=default) - if maximum is None and list_count < minimum: - logger.warning(f"Collection Warning: {method} must an integer >= {minimum} using {default} as default") - elif maximum is not None and (list_count < minimum or list_count > maximum): - logger.warning(f"Collection Warning: {method} must an integer between {minimum} and {maximum} using {default} as default") - else: - return list_count - return default - -def parse_from_dict(parent, method, data, methods, default=None, options=None, translation=None): +def parse(attribute, data, datatype=None, methods=None, parent=None, default=None, options=None, translation=None, minimum=1, maximum=None): + display = f"{parent + ' ' if parent else ''}{attribute} attribute" if options is None and translation is not None: options = [o for o in translation] - if method not in methods: - message = f"{parent} {method} attribute not found" - elif data[methods[method]] is None: - message = f"{parent} {method} attribute is blank" - elif (translation is not None and str(data[methods[method]]).lower() not in translation) or \ - (options is not None and translation is None and str(data[methods[method]]).lower() not in options): - message = f"{parent} {method} attribute {data[methods[method]]} must be in {options}" + value = data[methods[attribute]] if methods and attribute in methods else data + + if datatype == "list": + if methods and attribute in methods and data[methods[attribute]]: + return [v for v in value if v] if isinstance(value, list) else [str(value)] + return [] + elif datatype == "dictlist": + final_list = [] + for dict_data in get_list(value): + if isinstance(dict_data, dict): + final_list.append((dict_data, {dm.lower(): dm for dm in dict_data})) + else: + raise Failed(f"Collection Error: {display} {dict_data} is not a dictionary") + return final_list + elif methods and attribute not in methods: + message = f"{display} not found" + elif value is None: + message = f"{display} is blank" + elif datatype == "bool": + if isinstance(value, bool): + return value + elif isinstance(value, int): + return value > 0 + elif str(value).lower() in ["t", "true"]: + return True + elif str(value).lower() in ["f", "false"]: + return False + else: + message = f"{display} must be either true or false" + elif datatype in ["int", "float"]: + try: + value = int(str(value)) if datatype == "int" else float(str(value)) + if (maximum is None and minimum <= value) or (maximum is not None and minimum <= value <= maximum): + return value + except ValueError: + pass + pre = f"{display} {value} must {'an integer' if datatype == 'int' else 'a number'}" + if maximum is None: + message = f"{pre} {minimum} or greater" + else: + message = f"{pre} between {minimum} and {maximum}" + elif (translation is not None and str(value).lower() not in translation) or \ + (options is not None and translation is None and str(value).lower() not in options): + message = f"{display} {value} must be in {options}" else: - return translation[data[methods[method]]] if translation is not None else data[methods[method]] + return translation[value] if translation is not None else value + if default is None: raise Failed(f"Collection Error: {message}") else: logger.warning(f"Collection Warning: {message} using {default} as default") return translation[default] if translation is not None else default - -def parse_int_from_dict(parent, method, data, methods, default, minimum=1, maximum=None): - if method not in methods: - logger.warning(f"Collection Warning: {parent} {method} attribute not found using {default} as default") - elif not data[methods[method]]: - logger.warning(f"Collection Warning: {parent} {methods[method]} attribute is blank using {default} as default") - elif maximum is None and (not isinstance(data[methods[method]], int) or data[methods[method]] < minimum): - logger.warning(f"Collection Warning: {parent} {methods[method]} attribute {data[methods[method]]} must an integer >= {minimum} using {default} as default") - elif maximum is not None and (not isinstance(data[methods[method]], int) or data[methods[method]] < minimum or data[methods[method]] > maximum): - logger.warning(f"Collection Warning: {parent} {methods[method]} attribute {data[methods[method]]} must an integer between {minimum} and {maximum} using {default} as default") - else: - return data[methods[method]] - return default - -def parse_list(method, data, methods): - if method in methods and data[methods[method]]: - return [i for i in data[methods[method]] if i] if isinstance(data[methods[method]], list) else [str(data[methods[method]])] - return [] \ No newline at end of file From ede0325224a5d802837c041aaeefbbd4c1e6b7f4 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 2 Aug 2021 10:24:13 -0400 Subject: [PATCH 44/95] fixed custom sort --- modules/builder.py | 12 +++--------- modules/plex.py | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index c2bdaa6b..31bb0a3e 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1808,15 +1808,9 @@ class CollectionBuilder: previous = None logger.debug(keys) logger.debug(self.rating_keys) - for ki, key in enumerate(self.rating_keys): - logger.debug(items) - if key != items[ki].ratingKey: - logger.info(f"Moving {keys[key].title} {'after {}'.format(keys[previous].title) if previous else 'to the beginning'}") - self.library.moveItem(self.obj, key, after=previous) - for ii, item in enumerate(items): - if key == item.ratingKey: - items.insert(ki, items.pop(ii)) - break + for key in self.rating_keys: + logger.info(f"Moving {keys[key].title} {'after {}'.format(keys[previous].title) if previous else 'to the beginning'}") + self.library.move_item(self.obj, key, after=previous) previous = key def run_collections_again(self): diff --git a/modules/plex.py b/modules/plex.py index d0a9d106..952fffe4 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -491,7 +491,7 @@ class Plex: else: method = None return self.Plex._server.query(key, method=method) - def moveItem(self, collection, item, after=None): + def move_item(self, collection, item, after=None): key = f"{collection.key}/items/{item}/move" if after: key += f"?after={after}" From 9d89d36c2c4438c1da4482e833029ee2ef47192c Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sat, 31 Jul 2021 21:23:17 -0400 Subject: [PATCH 45/95] #317 added stevenlu_popular --- modules/builder.py | 11 ++++++++--- modules/config.py | 2 ++ modules/stevenlu.py | 32 ++++++++++++++++++++++++++++++++ modules/util.py | 2 +- 4 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 modules/stevenlu.py diff --git a/modules/builder.py b/modules/builder.py index 31bb0a3e..43cc4931 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1,6 +1,6 @@ import logging, os, re from datetime import datetime, timedelta -from modules import anidb, anilist, icheckmovies, imdb, letterboxd, mal, plex, radarr, sonarr, tautulli, tmdb, trakt, tvdb, util +from modules import anidb, anilist, icheckmovies, imdb, letterboxd, mal, plex, radarr, sonarr, stevenlu, tautulli, tmdb, trakt, tvdb, util from modules.util import Failed, ImageData from PIL import Image from plexapi.exceptions import BadRequest, NotFound @@ -58,10 +58,10 @@ filter_translation = { } modifier_alias = {".greater": ".gt", ".less": ".lt"} all_builders = anidb.builders + anilist.builders + icheckmovies.builders + imdb.builders + letterboxd.builders + \ - mal.builders + plex.builders + tautulli.builders + tmdb.builders + trakt.builders + tvdb.builders + mal.builders + plex.builders + stevenlu.builders + tautulli.builders + tmdb.builders + trakt.builders + tvdb.builders show_only_builders = ["tmdb_network", "tmdb_show", "tmdb_show_details", "tvdb_show", "tvdb_show_details"] movie_only_builders = [ - "letterboxd_list", "letterboxd_list_details", "icheckmovies_list", "icheckmovies_list_details", + "letterboxd_list", "letterboxd_list_details", "icheckmovies_list", "icheckmovies_list_details", "stevenlu_popular", "tmdb_collection", "tmdb_collection_details", "tmdb_movie", "tmdb_movie_details", "tmdb_now_playing", "tvdb_movie", "tvdb_movie_details" ] @@ -506,6 +506,7 @@ class CollectionBuilder: elif method_name in imdb.builders: self._imdb(method_name, method_data) elif method_name in mal.builders: self._mal(method_name, method_data) elif method_name in plex.builders or method_final in plex.searches: self._plex(method_name, method_data) + elif method_name in stevenlu.builders: self._stevenlu(method_name, method_data) elif method_name in tautulli.builders: self._tautulli(method_name, method_data) elif method_name in tmdb.builders: self._tmdb(method_name, method_data) elif method_name in trakt.builders: self._trakt(method_name, method_data) @@ -826,6 +827,9 @@ class CollectionBuilder: else: self.builders.append(("plex_search", self.build_filter("plex_search", {"any": {method_name: method_data}}))) + def _stevenlu(self, method_name, method_data): + self.builders.append((method_name, util.parse(method_name, method_data, "bool"))) + def _tautulli(self, method_name, method_data): for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): self.builders.append((method_name, { @@ -1032,6 +1036,7 @@ class CollectionBuilder: elif "imdb" in method: check_map(self.config.IMDb.get_items(method, value, self.language, self.library.is_movie)) elif "icheckmovies" in method: check_map(self.config.ICheckMovies.get_items(method, value, self.language)) elif "letterboxd" in method: check_map(self.config.Letterboxd.get_items(method, value, self.language)) + elif "stevenlu" in method: check_map(self.config.StevenLu.get_items(method)) elif "tmdb" in method: check_map(self.config.TMDb.get_items(method, value, self.library.is_movie)) elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) else: logger.error(f"Collection Error: {method} method not supported") diff --git a/modules/config.py b/modules/config.py index e9232222..c055c88d 100644 --- a/modules/config.py +++ b/modules/config.py @@ -14,6 +14,7 @@ from modules.omdb import OMDb from modules.plex import Plex from modules.radarr import Radarr from modules.sonarr import Sonarr +from modules.stevenlu import StevenLu from modules.tautulli import Tautulli from modules.tmdb import TMDb from modules.trakt import Trakt @@ -274,6 +275,7 @@ class Config: self.AniList = AniList(self) self.Letterboxd = Letterboxd(self) self.ICheckMovies = ICheckMovies(self) + self.StevenLu = StevenLu(self) util.separator() diff --git a/modules/stevenlu.py b/modules/stevenlu.py new file mode 100644 index 00000000..85404b5f --- /dev/null +++ b/modules/stevenlu.py @@ -0,0 +1,32 @@ +import logging +from modules import util +from modules.util import Failed + +logger = logging.getLogger("Plex Meta Manager") + +builders = ["stevenlu_popular"] +base_url = "https://s3.amazonaws.com/popular-movies/movies.json" + +class StevenLu: + def __init__(self, config): + self.config = config + + def get_items(self, method): + pretty = util.pretty_names[method] if method in util.pretty_names else method + movie_ids = [] + fail_ids = [] + if method == "stevenlu_popular": + logger.info(f"Processing {pretty} Movies") + for i in self.config.get_json(base_url): + tmdb_id = self.config.Convert.imdb_to_tmdb(i["imdb_id"]) + if tmdb_id: + movie_ids.append(tmdb_id) + else: + logger.error(f"Convert Error: No TMDb ID found for IMDb: {i['imdb_id']}") + fail_ids.append(i["imdb_id"]) + else: + raise Failed(f"StevenLu Error: Method {method} not supported") + logger.debug("") + logger.debug(f"{len(fail_ids)} IMDb IDs Failed to Convert: {fail_ids}") + logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") + return movie_ids, [] diff --git a/modules/util.py b/modules/util.py index bb06df7d..19fd322b 100644 --- a/modules/util.py +++ b/modules/util.py @@ -67,7 +67,7 @@ pretty_names = { "mal_favorite": "MyAnimeList Favorite", "mal_season": "MyAnimeList Season", "mal_suggested": "MyAnimeList Suggested", "mal_userlist": "MyAnimeList Userlist", "plex_all": "Plex All", "plex_collection": "Plex Collection", "plex_search": "Plex Search", - "tautulli_popular": "Tautulli Popular", "tautulli_watched": "Tautulli Watched", + "stevenlu_popular": "Steven Lu Popular", "tautulli_popular": "Tautulli Popular", "tautulli_watched": "Tautulli Watched", "tmdb_actor": "TMDb Actor", "tmdb_actor_details": "TMDb Actor", "tmdb_collection": "TMDb Collection", "tmdb_collection_details": "TMDb Collection", "tmdb_company": "TMDb Company", "tmdb_crew": "TMDb Crew", "tmdb_crew_details": "TMDb Crew", From 74db60bf17d62377dd41f0541d11a7270b322553 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sun, 1 Aug 2021 00:35:42 -0400 Subject: [PATCH 46/95] cleanup --- modules/anidb.py | 16 +++++++------- modules/anilist.py | 28 ++++++++++++++----------- modules/icheckmovies.py | 3 +-- modules/imdb.py | 5 ++--- modules/letterboxd.py | 46 +++++++++++++++++++++-------------------- modules/mal.py | 17 +++++++++------ modules/plex.py | 4 ++-- modules/stevenlu.py | 4 +--- modules/tmdb.py | 2 +- modules/trakt.py | 2 +- modules/tvdb.py | 9 ++++---- modules/util.py | 34 ------------------------------ 12 files changed, 72 insertions(+), 98 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 6dbcc212..82ee48cb 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -74,19 +74,21 @@ class AniDB: return anidb_ids[:limit] def get_items(self, method, data, language): - pretty = util.pretty_names[method] if method in util.pretty_names else method anidb_ids = [] if method == "anidb_popular": - logger.info(f"Processing {pretty}: {data} Anime") + logger.info(f"Processing AniDB Popular: {data} Anime") anidb_ids.extend(self._popular(language)[:data]) elif method == "anidb_tag": + logger.info(f"Processing AniDB Tag: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag ID: {data['tag']}") anidb_ids = self._tag(data["tag"], data["limit"], language) - logger.info(f"Processing {pretty}: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag ID: {data['tag']}") + elif method == "anidb_id": + logger.info(f"Processing AniDB ID: {data}") + anidb_ids.append(data) + elif method == "anidb_relation": + logger.info(f"Processing AniDB Relation: {data}") + anidb_ids.extend(self._relations(data, language)) else: - logger.info(f"Processing {pretty}: {data}") - if method == "anidb_id": anidb_ids.append(data) - elif method == "anidb_relation": anidb_ids.extend(self._relations(data, language)) - else: raise Failed(f"AniDB Error: Method {method} not supported") + raise Failed(f"AniDB Error: Method {method} not supported") movie_ids, show_ids = self.config.Convert.anidb_to_ids(anidb_ids) logger.debug("") logger.debug(f"{len(anidb_ids)} AniDB IDs Found: {anidb_ids}") diff --git a/modules/anilist.py b/modules/anilist.py index ae0b4fa2..d1bc31c9 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -210,27 +210,31 @@ class AniList: raise Failed(f"AniList Error: No valid AniList IDs in {anilist_ids}") def get_items(self, method, data): - pretty = util.pretty_names[method] if method in util.pretty_names else method if method == "anilist_id": + logger.info(f"Processing AniList ID: {data}") anilist_id, name = self._validate(data) anilist_ids = [anilist_id] - logger.info(f"Processing {pretty}: ({data}) {name}") - elif method in ["anilist_popular", "anilist_top_rated"]: - anilist_ids = self._popular(data) if method == "anilist_popular" else self._top_rated(data) - logger.info(f"Processing {pretty}: {data} Anime") + elif method == "anilist_popular": + logger.info(f"Processing AniList Popular: {data} Anime") + anilist_ids = self._popular(data) + elif method == "anilist_top_rated": + logger.info(f"Processing AniList Top Rated: {data} Anime") + anilist_ids = self._top_rated(data) elif method == "anilist_season": + logger.info(f"Processing AniList Season: {data['limit'] if data['limit'] > 0 else 'All'} Anime from {util.pretty_seasons[data['season']]} {data['year']} sorted by {pretty_names[data['sort_by']]}") anilist_ids = self._season(data["season"], data["year"], data["sort_by"], data["limit"]) - logger.info(f"Processing {pretty}: {data['limit'] if data['limit'] > 0 else 'All'} Anime from {util.pretty_seasons[data['season']]} {data['year']} sorted by {pretty_names[data['sort_by']]}") elif method == "anilist_genre": + logger.info(f"Processing AniList Genre: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Genre: {data['genre']} sorted by {pretty_names[data['sort_by']]}") anilist_ids = self._genre(data["genre"], data["sort_by"], data["limit"]) - logger.info(f"Processing {pretty}: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Genre: {data['genre']} sorted by {pretty_names[data['sort_by']]}") elif method == "anilist_tag": + logger.info(f"Processing AniList Tag: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag: {data['tag']} sorted by {pretty_names[data['sort_by']]}") anilist_ids = self._tag(data["tag"], data["sort_by"], data["limit"]) - logger.info(f"Processing {pretty}: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag: {data['tag']} sorted by {pretty_names[data['sort_by']]}") - elif method in ["anilist_studio", "anilist_relations"]: - if method == "anilist_studio": anilist_ids, name = self._studio(data) - else: anilist_ids, _, name = self._relations(data) - logger.info(f"Processing {pretty}: ({data}) {name} ({len(anilist_ids)} Anime)") + elif method == "anilist_studio": + anilist_ids, name = self._studio(data) + logger.info(f"Processing AniList Studio: ({data}) {name} ({len(anilist_ids)} Anime)") + elif method == "anilist_relations": + anilist_ids, _, name = self._relations(data) + logger.info(f"Processing AniList Relations: ({data}) {name} ({len(anilist_ids)} Anime)") else: raise Failed(f"AniList Error: Method {method} not supported") movie_ids, show_ids = self.config.Convert.anilist_to_ids(anilist_ids) diff --git a/modules/icheckmovies.py b/modules/icheckmovies.py index 816fcc43..3d183021 100644 --- a/modules/icheckmovies.py +++ b/modules/icheckmovies.py @@ -35,10 +35,9 @@ class ICheckMovies: return valid_lists def get_items(self, method, data, language): - pretty = util.pretty_names[method] if method in util.pretty_names else method movie_ids = [] if method == "icheckmovies_list": - logger.info(f"Processing {pretty}: {data}") + logger.info(f"Processing ICheckMovies List: {data}") imdb_ids = self._parse_list(data, language) total_ids = len(imdb_ids) for i, imdb_id in enumerate(imdb_ids, 1): diff --git a/modules/imdb.py b/modules/imdb.py index 803e6bd5..11205d65 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -96,7 +96,6 @@ class IMDb: raise ValueError(f"IMDb Error: No IMDb IDs Found at {imdb_url}") def get_items(self, method, data, language, is_movie): - pretty = util.pretty_names[method] if method in util.pretty_names else method show_ids = [] movie_ids = [] fail_ids = [] @@ -110,11 +109,11 @@ class IMDb: fail_ids.append(imdb_id) if method == "imdb_id": - logger.info(f"Processing {pretty}: {data}") + logger.info(f"Processing IMDb ID: {data}") run_convert(data) elif method == "imdb_list": status = f"{data['limit']} Items at " if data['limit'] > 0 else '' - logger.info(f"Processing {pretty}: {status}{data['url']}") + logger.info(f"Processing IMDb List: {status}{data['url']}") imdb_ids = self._ids_from_url(data["url"], language, data["limit"]) total_ids = len(imdb_ids) for i, imdb in enumerate(imdb_ids, 1): diff --git a/modules/letterboxd.py b/modules/letterboxd.py index d936546f..0e9f33e7 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -51,31 +51,33 @@ class Letterboxd: return valid_lists def get_items(self, method, data, language): - pretty = util.pretty_names[method] if method in util.pretty_names else method movie_ids = [] - logger.info(f"Processing {pretty}: {data}") - items = self._parse_list(data, language) - total_items = len(items) - if total_items > 0: - for i, item in enumerate(items, 1): - letterboxd_id, slug = item - util.print_return(f"Finding TMDb ID {i}/{total_items}") - tmdb_id = None - expired = None - if self.config.Cache: - tmdb_id, expired = self.config.Cache.query_letterboxd_map(letterboxd_id) - if not tmdb_id or expired is not False: - try: - tmdb_id = self._tmdb(f"{base_url}{slug}", language) - except Failed as e: - logger.error(e) - continue + if method == "letterboxd_list": + logger.info(f"Processing Letterboxd List: {data}") + items = self._parse_list(data, language) + total_items = len(items) + if total_items > 0: + for i, item in enumerate(items, 1): + letterboxd_id, slug = item + util.print_return(f"Finding TMDb ID {i}/{total_items}") + tmdb_id = None + expired = None if self.config.Cache: - self.config.Cache.update_letterboxd_map(expired, letterboxd_id, tmdb_id) - movie_ids.append(tmdb_id) - logger.info(util.adjust_space(f"Processed {total_items} TMDb IDs")) + tmdb_id, expired = self.config.Cache.query_letterboxd_map(letterboxd_id) + if not tmdb_id or expired is not False: + try: + tmdb_id = self._tmdb(f"{base_url}{slug}", language) + except Failed as e: + logger.error(e) + continue + if self.config.Cache: + self.config.Cache.update_letterboxd_map(expired, letterboxd_id, tmdb_id) + movie_ids.append(tmdb_id) + logger.info(util.adjust_space(f"Processed {total_items} TMDb IDs")) + else: + logger.error(f"Letterboxd Error: No List Items found in {data}") else: - logger.error(f"Letterboxd Error: No List Items found in {data}") + raise Failed(f"Letterboxd Error: Method {method} not supported") logger.debug("") logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") return movie_ids, [] diff --git a/modules/mal.py b/modules/mal.py index 6eecad33..e8ff7d0e 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -13,6 +13,12 @@ mal_ranked_name = { "mal_all": "all", "mal_airing": "airing", "mal_upcoming": "upcoming", "mal_tv": "tv", "mal_ova": "ova", "mal_movie": "movie", "mal_special": "special", "mal_popular": "bypopularity", "mal_favorite": "favorite" } +mal_ranked_pretty = { + "mal_all": "MyAnimeList All", "mal_airing": "MyAnimeList Airing", + "mal_upcoming": "MyAnimeList Upcoming", "mal_tv": "MyAnimeList TV", "mal_ova": "MyAnimeList OVA", + "mal_movie": "MyAnimeList Movie", "mal_special": "MyAnimeList Special", "mal_popular": "MyAnimeList Popular", + "mal_favorite": "MyAnimeList Favorite" +} season_sort_translation = {"score": "anime_score", "anime_score": "anime_score", "members": "anime_num_list_users", "anime_num_list_users": "anime_num_list_users"} season_sort_options = ["score", "members"] pretty_names = { @@ -150,22 +156,21 @@ class MyAnimeList: return self._parse_request(url) def get_items(self, method, data): - pretty = util.pretty_names[method] if method in util.pretty_names else method if method == "mal_id": + logger.info(f"Processing MyAnimeList ID: {data}") mal_ids = [data] - logger.info(f"Processing {pretty}: {data}") elif method in mal_ranked_name: + logger.info(f"Processing {mal_ranked_pretty[method]}: {data} Anime") mal_ids = self._ranked(mal_ranked_name[method], data) - logger.info(f"Processing {pretty}: {data} Anime") elif method == "mal_season": + logger.info(f"Processing MyAnimeList Season: {data['limit']} Anime from {util.pretty_seasons[data['season']]} {data['year']} sorted by {pretty_names[data['sort_by']]}") mal_ids = self._season(data["season"], data["year"], data["sort_by"], data["limit"]) - logger.info(f"Processing {pretty}: {data['limit']} Anime from {util.pretty_seasons[data['season']]} {data['year']} sorted by {pretty_names[data['sort_by']]}") elif method == "mal_suggested": + logger.info(f"Processing MyAnimeList Suggested: {data} Anime") mal_ids = self._suggestions(data) - logger.info(f"Processing {pretty}: {data} Anime") elif method == "mal_userlist": + logger.info(f"Processing MyAnimeList Userlist: {data['limit']} Anime from {self._username() if data['username'] == '@me' else data['username']}'s {pretty_names[data['status']]} list sorted by {pretty_names[data['sort_by']]}") mal_ids = self._userlist(data["username"], data["status"], data["sort_by"], data["limit"]) - logger.info(f"Processing {pretty}: {data['limit']} Anime from {self._username() if data['username'] == '@me' else data['username']}'s {pretty_names[data['status']]} list sorted by {pretty_names[data['sort_by']]}") else: raise Failed(f"MyAnimeList Error: Method {method} not supported") movie_ids, show_ids = self.config.Convert.myanimelist_to_ids(mal_ids) diff --git a/modules/plex.py b/modules/plex.py index 952fffe4..8c019cd8 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -579,17 +579,17 @@ class Plex: return valid_collections def get_items(self, method, data): - pretty = util.pretty_names[method] if method in util.pretty_names else method media_type = "Movie" if self.is_movie else "Show" items = [] if method == "plex_all": - logger.info(f"Processing {pretty} {media_type}s") + logger.info(f"Processing Plex All {media_type}s") items = self.get_all() elif method == "plex_search": util.print_multiline(data[1], info=True) items = self.get_filter_items(data[2]) elif method == "plex_collectionless": good_collections = [] + logger.info(f"Processing Plex Collectionless") logger.info("Collections Excluded") for col in self.get_all_collections(): keep_collection = True diff --git a/modules/stevenlu.py b/modules/stevenlu.py index 85404b5f..7712dcce 100644 --- a/modules/stevenlu.py +++ b/modules/stevenlu.py @@ -1,5 +1,4 @@ import logging -from modules import util from modules.util import Failed logger = logging.getLogger("Plex Meta Manager") @@ -12,11 +11,10 @@ class StevenLu: self.config = config def get_items(self, method): - pretty = util.pretty_names[method] if method in util.pretty_names else method movie_ids = [] fail_ids = [] if method == "stevenlu_popular": - logger.info(f"Processing {pretty} Movies") + logger.info(f"Processing StevenLu Popular Movies") for i in self.config.get_json(base_url): tmdb_id = self.config.Convert.imdb_to_tmdb(i["imdb_id"]) if tmdb_id: diff --git a/modules/tmdb.py b/modules/tmdb.py index e2b82dd6..0e7c44a5 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -232,7 +232,7 @@ class TMDb: return tmdb_id def get_items(self, method, data, is_movie): - pretty = util.pretty_names[method] if method in util.pretty_names else method + pretty = method.replace("_", " ").title().replace("Tmdb", "TMDb") media_type = "Movie" if is_movie else "Show" movie_ids = [] show_ids = [] diff --git a/modules/trakt.py b/modules/trakt.py index d8caa89e..4cd5d121 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -162,7 +162,7 @@ class Trakt: return trakt_values def get_items(self, method, data, is_movie): - pretty = util.pretty_names[method] if method in util.pretty_names else method + pretty = method.replace("_", " ").title() media_type = "Movie" if is_movie else "Show" if method in ["trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected"]: movie_ids, show_ids = self._pagenation(method[6:], data, is_movie) diff --git a/modules/tvdb.py b/modules/tvdb.py index 3d6c2e32..7ee1a918 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -136,18 +136,17 @@ class TVDb: raise Failed(f"TVDb Error: {tvdb_url} must begin with {urls['list']}") def get_items(self, method, data, language): - pretty = util.pretty_names[method] if method in util.pretty_names else method show_ids = [] movie_ids = [] - logger.info(f"Processing {pretty}: {data}") if method == "tvdb_show": + logger.info(f"Processing TVDb Show: {data}") show_ids.append(self.get_series(language, data).id) elif method == "tvdb_movie": + logger.info(f"Processing TVDb Movie: {data}") movie_ids.append(self.get_movie(language, data).tmdb_id) elif method == "tvdb_list": - tmdb_ids, tvdb_ids = self._ids_from_url(data, language) - movie_ids.extend(tmdb_ids) - show_ids.extend(tvdb_ids) + logger.info(f"Processing TVDb List: {data}") + movie_ids, show_ids = self._ids_from_url(data, language) else: raise Failed(f"TVDb Error: Method {method} not supported") logger.debug("") diff --git a/modules/util.py b/modules/util.py index 19fd322b..8fab6fb3 100644 --- a/modules/util.py +++ b/modules/util.py @@ -53,40 +53,6 @@ pretty_months = { 7: "July", 8: "August", 9: "September", 10: "October", 11: "November", 12: "December" } pretty_seasons = {"winter": "Winter", "spring": "Spring", "summer": "Summer", "fall": "Fall"} -pretty_names = { - "anidb_id": "AniDB ID", "anidb_relation": "AniDB Relation", "anidb_popular": "AniDB Popular", - "anilist_genre": "AniList Genre", "anilist_id": "AniList ID", "anilist_popular": "AniList Popular", - "anilist_relations": "AniList Relations", "anilist_season": "AniList Season", "anilist_studio": "AniList Studio", - "anilist_tag": "AniList Tag", "anilist_top_rated": "AniList Top Rated", - "icheckmovies_list": "I Check Movies List", - "imdb_list": "IMDb List", "imdb_id": "IMDb ID", - "letterboxd_list": "Letterboxd List", "letterboxd_list_details": "Letterboxd List", - "mal_id": "MyAnimeList ID", "mal_all": "MyAnimeList All", "mal_airing": "MyAnimeList Airing", - "mal_upcoming": "MyAnimeList Upcoming", "mal_tv": "MyAnimeList TV", "mal_ova": "MyAnimeList OVA", - "mal_movie": "MyAnimeList Movie", "mal_special": "MyAnimeList Special", "mal_popular": "MyAnimeList Popular", - "mal_favorite": "MyAnimeList Favorite", "mal_season": "MyAnimeList Season", - "mal_suggested": "MyAnimeList Suggested", "mal_userlist": "MyAnimeList Userlist", - "plex_all": "Plex All", "plex_collection": "Plex Collection", "plex_search": "Plex Search", - "stevenlu_popular": "Steven Lu Popular", "tautulli_popular": "Tautulli Popular", "tautulli_watched": "Tautulli Watched", - "tmdb_actor": "TMDb Actor", "tmdb_actor_details": "TMDb Actor", - "tmdb_collection": "TMDb Collection", "tmdb_collection_details": "TMDb Collection", - "tmdb_company": "TMDb Company", "tmdb_crew": "TMDb Crew", "tmdb_crew_details": "TMDb Crew", - "tmdb_director": "TMDb Director", "tmdb_director_details": "TMDb Director", "tmdb_discover": "TMDb Discover", - "tmdb_keyword": "TMDb Keyword", "tmdb_list": "TMDb List", "tmdb_list_details": "TMDb List", - "tmdb_movie": "TMDb Movie", "tmdb_movie_details": "TMDb Movie", "tmdb_network": "TMDb Network", - "tmdb_now_playing": "TMDb Now Playing", "tmdb_person": "TMDb Person", "tmdb_popular": "TMDb Popular", - "tmdb_producer": "TMDb Producer", "tmdb_producer_details": "TMDb Producer", - "tmdb_show": "TMDb Show", "tmdb_show_details": "TMDb Show", "tmdb_top_rated": "TMDb Top Rated", - "tmdb_trending_daily": "TMDb Trending Daily", "tmdb_trending_weekly": "TMDb Trending Weekly", - "tmdb_writer": "TMDb Writer", "tmdb_writer_details": "TMDb Writer", - "trakt_collected": "Trakt Collected", "trakt_collection": "Trakt Collection", - "trakt_list": "Trakt List", "trakt_list_details": "Trakt List", - "trakt_popular": "Trakt Popular", "trakt_recommended": "Trakt Recommended", "trakt_trending": "Trakt Trending", - "trakt_watched": "Trakt Watched", "trakt_watchlist": "Trakt Watchlist", - "tvdb_list": "TVDb List", "tvdb_list_details": "TVDb List", - "tvdb_movie": "TVDb Movie", "tvdb_movie_details": "TVDb Movie", - "tvdb_show": "TVDb Show", "tvdb_show_details": "TVDb Show" -} pretty_ids = {"anidbid": "AniDB", "imdbid": "IMDb", "mal_id": "MyAnimeList", "themoviedb_id": "TMDb", "thetvdb_id": "TVDb", "tvdbid": "TVDb"} def tab_new_lines(data): From ed8d7696d71d97bd9df13d5febd58ae6c372a065 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 2 Aug 2021 11:52:43 -0400 Subject: [PATCH 47/95] #336 added released_missing_only to settings --- modules/builder.py | 42 +++++++++++++++++++++++++------- modules/config.py | 58 ++++++++++++-------------------------------- modules/plex.py | 1 + plex_meta_manager.py | 2 +- 4 files changed, 50 insertions(+), 53 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 43cc4931..1c971de2 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -148,6 +148,7 @@ class CollectionBuilder: "show_filtered": self.library.show_filtered, "show_missing": self.library.show_missing, "save_missing": self.library.save_missing, + "released_missing_only": self.library.released_missing_only, "item_assets": False } self.item_details = {} @@ -155,6 +156,8 @@ class CollectionBuilder: self.sonarr_options = {} self.missing_movies = [] self.missing_shows = [] + self.filtered_missing_movies = [] + self.filtered_missing_shows = [] self.builders = [] self.filters = [] self.rating_keys = [] @@ -1013,15 +1016,37 @@ class CollectionBuilder: for movie_id in movie_ids: if movie_id in self.library.movie_map: add_rating_keys(self.library.movie_map[movie_id]) - elif movie_id not in self.missing_movies: - self.missing_movies.append(movie_id) + elif movie_id not in self.missing_movies and movie_id not in self.filtered_missing_movies: + filter_missing = False + if self.details["released_missing_only"]: + try: + movie = self.config.TMDb.get_movie(movie_id) + if util.validate_date(movie.release_date, "") > self.current_time: + filter_missing = True + except Failed: + pass + if filter_missing: + self.filtered_missing_movies.append(movie_id) + else: + self.missing_movies.append(movie_id) if len(show_ids) > 0: items_found_inside += len(show_ids) for show_id in show_ids: if show_id in self.library.show_map: add_rating_keys(self.library.show_map[show_id]) - elif show_id not in self.missing_shows: - self.missing_shows.append(show_id) + elif show_id not in self.missing_shows and show_id not in self.filtered_missing_shows: + filter_missing = False + if self.details["released_missing_only"]: + try: + show = self.config.TMDb.get_show(show_id) + if util.validate_date(show.first_air_date, "") > self.current_time: + filter_missing = True + except Failed: + pass + if filter_missing: + self.filtered_missing_shows.append(show_id) + else: + self.missing_shows.append(show_id) return items_found_inside for method, value in self.builders: logger.debug("") @@ -1329,14 +1354,13 @@ class CollectionBuilder: def check_filters(self, current, display): if self.filters: util.print_return(f"Filtering {display} {current.title}") - current_date = datetime.now() for filter_method, filter_data in self.filters: filter_attr, modifier, filter_final = self._split(filter_method) filter_actual = filter_translation[filter_attr] if filter_attr in filter_translation else filter_attr if filter_attr in ["release", "added", "last_played"] and modifier != ".regex": current_data = getattr(current, filter_actual) if modifier in ["", ".not"]: - threshold_date = current_date - timedelta(days=filter_data) + threshold_date = self.current_time - timedelta(days=filter_data) if (modifier == "" and (current_data is None or current_data < threshold_date)) \ or (modifier == ".not" and current_data and current_data >= threshold_date): return False @@ -1405,15 +1429,15 @@ class CollectionBuilder: if item_date is None: return False elif filter_data == "day": - if item_date.month != current_date.month or item_date.day != current_date.day: + if item_date.month != self.current_time.month or item_date.day != self.current_time.day: return False elif filter_data == "month": - if item_date.month != current_date.month: + if item_date.month != self.current_time.month: return False else: date_match = False for i in range(filter_data): - check_date = current_date - timedelta(days=i) + check_date = self.current_time - timedelta(days=i) if item_date.month == check_date.month and item_date.day == check_date.day: date_match = True if date_match is False: diff --git a/modules/config.py b/modules/config.py index c055c88d..a098b348 100644 --- a/modules/config.py +++ b/modules/config.py @@ -182,7 +182,8 @@ class Config: "show_unmanaged": check_for_attribute(self.data, "show_unmanaged", parent="settings", var_type="bool", default=True), "show_filtered": check_for_attribute(self.data, "show_filtered", parent="settings", var_type="bool", default=False), "show_missing": check_for_attribute(self.data, "show_missing", parent="settings", var_type="bool", default=True), - "save_missing": check_for_attribute(self.data, "save_missing", parent="settings", var_type="bool", default=True) + "save_missing": check_for_attribute(self.data, "save_missing", parent="settings", var_type="bool", default=True), + "released_missing_only": check_for_attribute(self.data, "released_missing_only", parent="settings", var_type="bool", default=False) } if self.general["cache"]: util.separator() @@ -326,14 +327,11 @@ class Config: if self.requested_libraries and library_name not in self.requested_libraries: continue util.separator() - params = {} - params["mapping_name"] = str(library_name) - if lib and "library_name" in lib and lib["library_name"]: - params["name"] = str(lib["library_name"]) - display_name = f"{params['name']} ({params['mapping_name']})" - else: - params["name"] = params["mapping_name"] - display_name = params["mapping_name"] + params = { + "mapping_name": str(library_name), + "name": str(lib["library_name"]) if lib and "library_name" in lib and lib["library_name"] else str(library_name) + } + display_name = f"{params['name']} ({params['mapping_name']})" if lib and "library_name" in lib and lib["library_name"] else params["mapping_name"] util.separator(f"{display_name} Configuration") logger.info("") @@ -343,40 +341,14 @@ class Config: if params["asset_directory"] is None: logger.warning("Config Warning: Assets will not be used asset_directory attribute must be set under config or under this specific Library") - if lib and "settings" in lib and lib["settings"] and "asset_folders" in lib["settings"]: - params["asset_folders"] = check_for_attribute(lib, "asset_folders", parent="settings", var_type="bool", default=self.general["asset_folders"], do_print=False, save=False) - else: - params["asset_folders"] = check_for_attribute(lib, "asset_folders", var_type="bool", default=self.general["asset_folders"], do_print=False, save=False) - - if lib and "settings" in lib and lib["settings"] and "assets_for_all" in lib["settings"]: - params["assets_for_all"] = check_for_attribute(lib, "assets_for_all", parent="settings", var_type="bool", default=self.general["assets_for_all"], do_print=False, save=False) - else: - params["assets_for_all"] = check_for_attribute(lib, "assets_for_all", var_type="bool", default=self.general["assets_for_all"], do_print=False, save=False) - - if lib and "settings" in lib and lib["settings"] and "sync_mode" in lib["settings"]: - params["sync_mode"] = check_for_attribute(lib, "sync_mode", parent="settings", test_list=sync_modes, default=self.general["sync_mode"], do_print=False, save=False) - else: - params["sync_mode"] = check_for_attribute(lib, "sync_mode", test_list=sync_modes, default=self.general["sync_mode"], do_print=False, save=False) - - if lib and "settings" in lib and lib["settings"] and "show_unmanaged" in lib["settings"]: - params["show_unmanaged"] = check_for_attribute(lib, "show_unmanaged", parent="settings", var_type="bool", default=self.general["show_unmanaged"], do_print=False, save=False) - else: - params["show_unmanaged"] = check_for_attribute(lib, "show_unmanaged", var_type="bool", default=self.general["show_unmanaged"], do_print=False, save=False) - - if lib and "settings" in lib and lib["settings"] and "show_filtered" in lib["settings"]: - params["show_filtered"] = check_for_attribute(lib, "show_filtered", parent="settings", var_type="bool", default=self.general["show_filtered"], do_print=False, save=False) - else: - params["show_filtered"] = check_for_attribute(lib, "show_filtered", var_type="bool", default=self.general["show_filtered"], do_print=False, save=False) - - if lib and "settings" in lib and lib["settings"] and "show_missing" in lib["settings"]: - params["show_missing"] = check_for_attribute(lib, "show_missing", parent="settings", var_type="bool", default=self.general["show_missing"], do_print=False, save=False) - else: - params["show_missing"] = check_for_attribute(lib, "show_missing", var_type="bool", default=self.general["show_missing"], do_print=False, save=False) - - if lib and "settings" in lib and lib["settings"] and "save_missing" in lib["settings"]: - params["save_missing"] = check_for_attribute(lib, "save_missing", parent="settings", var_type="bool", default=self.general["save_missing"], do_print=False, save=False) - else: - params["save_missing"] = check_for_attribute(lib, "save_missing", var_type="bool", default=self.general["save_missing"], do_print=False, save=False) + params["asset_folders"] = check_for_attribute(lib, "asset_folders", parent="settings", var_type="bool", default=self.general["asset_folders"], do_print=False, save=False) + params["assets_for_all"] = check_for_attribute(lib, "assets_for_all", parent="settings", var_type="bool", default=self.general["assets_for_all"], do_print=False, save=False) + params["sync_mode"] = check_for_attribute(lib, "sync_mode", parent="settings", test_list=sync_modes, default=self.general["sync_mode"], do_print=False, save=False) + params["show_unmanaged"] = check_for_attribute(lib, "show_unmanaged", parent="settings", var_type="bool", default=self.general["show_unmanaged"], do_print=False, save=False) + params["show_filtered"] = check_for_attribute(lib, "show_filtered", parent="settings", var_type="bool", default=self.general["show_filtered"], do_print=False, save=False) + params["show_missing"] = check_for_attribute(lib, "show_missing", parent="settings", var_type="bool", default=self.general["show_missing"], do_print=False, save=False) + params["save_missing"] = check_for_attribute(lib, "save_missing", parent="settings", var_type="bool", default=self.general["save_missing"], do_print=False, save=False) + params["released_missing_only"] = check_for_attribute(lib, "released_missing_only", parent="settings", var_type="bool", default=self.general["released_missing_only"], do_print=False, save=False) if lib and "mass_genre_update" in lib and lib["mass_genre_update"]: params["mass_genre_update"] = check_for_attribute(lib, "mass_genre_update", test_list=mass_update_options, default_is_none=True, save=False) diff --git a/modules/plex.py b/modules/plex.py index 8c019cd8..df739bab 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -279,6 +279,7 @@ class Plex: self.show_filtered = params["show_filtered"] self.show_missing = params["show_missing"] self.save_missing = params["save_missing"] + self.released_missing_only = params["released_missing_only"] self.mass_genre_update = params["mass_genre_update"] self.mass_audience_rating_update = params["mass_audience_rating_update"] self.mass_critic_rating_update = params["mass_critic_rating_update"] diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 3e953f43..6115e2fd 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -105,7 +105,7 @@ def start(config_path, is_test=False, time_scheduled=None, requested_collections logger.info(util.centered("| __/| | __/> < | | | | __/ || (_| | | | | | (_| | | | | (_| | (_| | __/ | ")) logger.info(util.centered("|_| |_|\\___/_/\\_\\ |_| |_|\\___|\\__\\__,_| |_| |_|\\__,_|_| |_|\\__,_|\\__, |\\___|_| ")) logger.info(util.centered(" |___/ ")) - logger.info(util.centered(" Version: 1.11.3-beta2 ")) + logger.info(util.centered(" Version: 1.11.3-beta3 ")) if time_scheduled: start_type = f"{time_scheduled} " elif is_test: start_type = "Test " elif requested_collections: start_type = "Collections " From 2663cbe4875135026c218503f38d3c834f007645 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 2 Aug 2021 15:00:15 -0400 Subject: [PATCH 48/95] #328 added `create_asset_folders` to settings --- modules/builder.py | 1 + modules/config.py | 4 +++- modules/plex.py | 13 ++++++++++--- plex_meta_manager.py | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 1c971de2..f98ef4b2 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -149,6 +149,7 @@ class CollectionBuilder: "show_missing": self.library.show_missing, "save_missing": self.library.save_missing, "released_missing_only": self.library.released_missing_only, + "create_asset_folders": self.library.create_asset_folders, "item_assets": False } self.item_details = {} diff --git a/modules/config.py b/modules/config.py index a098b348..a18978c1 100644 --- a/modules/config.py +++ b/modules/config.py @@ -183,7 +183,8 @@ class Config: "show_filtered": check_for_attribute(self.data, "show_filtered", parent="settings", var_type="bool", default=False), "show_missing": check_for_attribute(self.data, "show_missing", parent="settings", var_type="bool", default=True), "save_missing": check_for_attribute(self.data, "save_missing", parent="settings", var_type="bool", default=True), - "released_missing_only": check_for_attribute(self.data, "released_missing_only", parent="settings", var_type="bool", default=False) + "released_missing_only": check_for_attribute(self.data, "released_missing_only", parent="settings", var_type="bool", default=False), + "create_asset_folders": check_for_attribute(self.data, "create_asset_folders", parent="settings", var_type="bool", default=False) } if self.general["cache"]: util.separator() @@ -349,6 +350,7 @@ class Config: params["show_missing"] = check_for_attribute(lib, "show_missing", parent="settings", var_type="bool", default=self.general["show_missing"], do_print=False, save=False) params["save_missing"] = check_for_attribute(lib, "save_missing", parent="settings", var_type="bool", default=self.general["save_missing"], do_print=False, save=False) params["released_missing_only"] = check_for_attribute(lib, "released_missing_only", parent="settings", var_type="bool", default=self.general["released_missing_only"], do_print=False, save=False) + params["create_asset_folders"] = check_for_attribute(lib, "create_asset_folders", parent="settings", var_type="bool", default=self.general["create_asset_folders"], do_print=False, save=False) if lib and "mass_genre_update" in lib and lib["mass_genre_update"]: params["mass_genre_update"] = check_for_attribute(lib, "mass_genre_update", test_list=mass_update_options, default_is_none=True, save=False) diff --git a/modules/plex.py b/modules/plex.py index df739bab..0f2168eb 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -280,6 +280,7 @@ class Plex: self.show_missing = params["show_missing"] self.save_missing = params["save_missing"] self.released_missing_only = params["released_missing_only"] + self.create_asset_folders = params["create_asset_folders"] self.mass_genre_update = params["mass_genre_update"] self.mass_audience_rating_update = params["mass_audience_rating_update"] self.mass_critic_rating_update = params["mass_critic_rating_update"] @@ -746,7 +747,7 @@ class Plex: logger.info(f"Detail: {attr.capitalize()} {_remove} removed") return updated - def update_item_from_assets(self, item, overlay=None): + def update_item_from_assets(self, item, overlay=None, create=False): name = os.path.basename(os.path.dirname(str(item.locations[0])) if self.is_movie else str(item.locations[0])) logger.debug(name) found_folder = False @@ -798,12 +799,15 @@ class Plex: self.upload_images(episode, poster=episode_poster) if not poster and overlay: self.upload_images(item, overlay=overlay) - if not overlay and self.asset_folders and not found_folder: + if create and self.asset_folders and not found_folder: + os.makedirs(os.path.join(self.asset_directory[0], name), exist_ok=True) + logger.info(f"Asset Directory Created: {os.path.join(self.asset_directory[0], name)}") + elif not overlay and self.asset_folders and not found_folder: logger.error(f"Asset Warning: No asset folder found called '{name}'") elif not poster and not background: logger.error(f"Asset Warning: No poster or background found in an assets folder for '{name}'") - def find_collection_assets(self, item, name=None): + def find_collection_assets(self, item, name=None, create=False): if name is None: name = item.title for ad in self.asset_directory: @@ -825,4 +829,7 @@ class Plex: background = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_poster=False, is_url=False) if poster or background: return poster, background + if create and self.asset_folders and not os.path.isdir(os.path.join(self.asset_directory[0], name)): + os.makedirs(os.path.join(self.asset_directory[0], name), exist_ok=True) + logger.info(f"Asset Directory Created: {os.path.join(self.asset_directory[0], name)}") return None, None diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 6115e2fd..a1df12d0 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -189,10 +189,10 @@ def update_libraries(config): util.separator(f"All {'Movies' if library.is_movie else 'Shows'} Assets Check for {library.name} Library", space=False, border=False) logger.info("") for col in unmanaged_collections: - poster, background = library.find_collection_assets(col) + poster, background = library.find_collection_assets(col, create=library.create_asset_folders) library.upload_images(col, poster=poster, background=background) for item in library.get_all(): - library.update_item_from_assets(item) + library.update_item_from_assets(item, create=library.create_asset_folders) logger.removeHandler(library_handler) From 07546d75c73545d8b6b214e2358b57a394428890 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 2 Aug 2021 17:01:19 -0400 Subject: [PATCH 49/95] #316 added `mass_trakt_rating_update` to settings --- modules/config.py | 47 +++++++++++++++----------------------------- modules/plex.py | 1 + modules/trakt.py | 34 +++++++++++++++++++++++--------- plex_meta_manager.py | 36 +++++++++++++++++++++++++++++---- 4 files changed, 74 insertions(+), 44 deletions(-) diff --git a/modules/config.py b/modules/config.py index a18978c1..572748fe 100644 --- a/modules/config.py +++ b/modules/config.py @@ -352,44 +352,29 @@ class Config: params["released_missing_only"] = check_for_attribute(lib, "released_missing_only", parent="settings", var_type="bool", default=self.general["released_missing_only"], do_print=False, save=False) params["create_asset_folders"] = check_for_attribute(lib, "create_asset_folders", parent="settings", var_type="bool", default=self.general["create_asset_folders"], do_print=False, save=False) - if lib and "mass_genre_update" in lib and lib["mass_genre_update"]: - params["mass_genre_update"] = check_for_attribute(lib, "mass_genre_update", test_list=mass_update_options, default_is_none=True, save=False) - if self.OMDb is None and params["mass_genre_update"] == "omdb": - params["mass_genre_update"] = None - logger.error("Config Error: mass_genre_update cannot be omdb without a successful OMDb Connection") - else: + params["mass_genre_update"] = check_for_attribute(lib, "mass_genre_update", test_list=mass_update_options, default_is_none=True, save=False, do_print=lib and "mass_genre_update" in lib) + if self.OMDb is None and params["mass_genre_update"] == "omdb": params["mass_genre_update"] = None + logger.error("Config Error: mass_genre_update cannot be omdb without a successful OMDb Connection") - if lib and "mass_audience_rating_update" in lib and lib["mass_audience_rating_update"]: - params["mass_audience_rating_update"] = check_for_attribute(lib, "mass_audience_rating_update", test_list=mass_update_options, default_is_none=True, save=False) - if self.OMDb is None and params["mass_audience_rating_update"] == "omdb": - params["mass_audience_rating_update"] = None - logger.error("Config Error: mass_audience_rating_update cannot be omdb without a successful OMDb Connection") - else: + params["mass_audience_rating_update"] = check_for_attribute(lib, "mass_audience_rating_update", test_list=mass_update_options, default_is_none=True, save=False, do_print=lib and "mass_audience_rating_update" in lib) + if self.OMDb is None and params["mass_audience_rating_update"] == "omdb": params["mass_audience_rating_update"] = None + logger.error("Config Error: mass_audience_rating_update cannot be omdb without a successful OMDb Connection") - if lib and "mass_critic_rating_update" in lib and lib["mass_critic_rating_update"]: - params["mass_critic_rating_update"] = check_for_attribute(lib, "mass_critic_rating_update", test_list=mass_update_options, default_is_none=True, save=False) - if self.OMDb is None and params["mass_critic_rating_update"] == "omdb": - params["mass_critic_rating_update"] = None - logger.error("Config Error: mass_critic_rating_update cannot be omdb without a successful OMDb Connection") - else: + params["mass_critic_rating_update"] = check_for_attribute(lib, "mass_critic_rating_update", test_list=mass_update_options, default_is_none=True, save=False, do_print=lib and "mass_audience_rating_update" in lib) + if self.OMDb is None and params["mass_critic_rating_update"] == "omdb": params["mass_critic_rating_update"] = None + logger.error("Config Error: mass_critic_rating_update cannot be omdb without a successful OMDb Connection") - if lib and "split_duplicates" in lib and lib["split_duplicates"]: - params["split_duplicates"] = check_for_attribute(lib, "split_duplicates", var_type="bool", default=False, save=False) - else: - params["split_duplicates"] = None + params["mass_trakt_rating_update"] = check_for_attribute(lib, "mass_trakt_rating_update", var_type="bool", default=False, save=False, do_print=lib and "mass_trakt_rating_update" in lib) + if self.Trakt is None and params["mass_trakt_rating_update"]: + params["mass_trakt_rating_update"] = None + logger.error("Config Error: mass_trakt_rating_update cannot run without a successful Trakt Connection") - if lib and "radarr_add_all" in lib and lib["radarr_add_all"]: - params["radarr_add_all"] = check_for_attribute(lib, "radarr_add_all", var_type="bool", default=False, save=False) - else: - params["radarr_add_all"] = None - - if lib and "sonarr_add_all" in lib and lib["sonarr_add_all"]: - params["sonarr_add_all"] = check_for_attribute(lib, "sonarr_add_all", var_type="bool", default=False, save=False) - else: - params["sonarr_add_all"] = None + params["split_duplicates"] = check_for_attribute(lib, "split_duplicates", var_type="bool", default=False, save=False, do_print=lib and "split_duplicates" in lib) + params["radarr_add_all"] = check_for_attribute(lib, "radarr_add_all", var_type="bool", default=False, save=False, do_print=lib and "radarr_add_all" in lib) + params["sonarr_add_all"] = check_for_attribute(lib, "sonarr_add_all", var_type="bool", default=False, save=False, do_print=lib and "sonarr_add_all" in lib) try: if lib and "metadata_path" in lib: diff --git a/modules/plex.py b/modules/plex.py index 0f2168eb..335e3fd8 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -284,6 +284,7 @@ class Plex: self.mass_genre_update = params["mass_genre_update"] self.mass_audience_rating_update = params["mass_audience_rating_update"] self.mass_critic_rating_update = params["mass_critic_rating_update"] + self.mass_trakt_rating_update = params["mass_trakt_rating_update"] self.split_duplicates = params["split_duplicates"] self.radarr_add_all = params["radarr_add_all"] self.sonarr_add_all = params["sonarr_add_all"] diff --git a/modules/trakt.py b/modules/trakt.py index 4cd5d121..e747dfd0 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -101,31 +101,47 @@ class Trakt: "trakt-api-version": "2", "trakt-api-key": self.client_id } - response = self.config.get(url, headers=headers) - if response.status_code == 200: - return response.json() - else: - raise Failed(f"({response.status_code}) {response.reason}") + output_json = [] + pages = 1 + current = 1 + while current <= pages: + if pages == 1: + response = self.config.get(f"{base_url}{url}", headers=headers) + if "X-Pagination-Page-Count" in response.headers: + pages = int(response.headers["X-Pagination-Page-Count"]) + else: + response = self.config.get(f"{base_url}{url}?page={current}", headers=headers) + if response.status_code == 200: + output_json.extend(response.json()) + else: + raise Failed(f"({response.status_code}) {response.reason}") + current += 1 + return output_json + + def user_ratings(self, is_movie): + media = "movie" if is_movie else "show" + id_type = "tmdb" if is_movie else "tvdb" + return {int(i[media]["ids"][id_type]): i["rating"] for i in self._request(f"/users/me/ratings/{media}s")} def convert(self, external_id, from_source, to_source, media_type): path = f"/search/{from_source}/{external_id}" if from_source in ["tmdb", "tvdb"]: path = f"{path}?type={media_type}" - lookup = self._request(f"{base_url}{path}") + lookup = self._request(path) if lookup and media_type in lookup[0] and to_source in lookup[0][media_type]["ids"]: return lookup[0][media_type]["ids"][to_source] raise Failed(f"Trakt Error: No {to_source.upper().replace('B', 'b')} ID found for {from_source.upper().replace('B', 'b')} ID: {external_id}") def list_description(self, data): try: - return self._request(f"{base_url}{requests.utils.urlparse(data).path}")["description"] + return self._request(requests.utils.urlparse(data).path)["description"] except Failed: raise Failed(f"Trakt Error: List {data} not found") def _user_list(self, list_type, data, is_movie): path = f"{requests.utils.urlparse(data).path}/items" if list_type == "list" else f"/users/{data}/{list_type}" try: - items = self._request(f"{base_url}{path}/{'movies' if is_movie else 'shows'}") + items = self._request(f"{path}/{'movies' if is_movie else 'shows'}") except Failed: raise Failed(f"Trakt Error: {'List' if list_type == 'list' else 'User'} {data} not found") if len(items) == 0: @@ -137,7 +153,7 @@ class Trakt: else: return [], [item["show"]["ids"]["tvdb"] for item in items] def _pagenation(self, pagenation, amount, is_movie): - items = self._request(f"{base_url}/{'movies' if is_movie else 'shows'}/{pagenation}?limit={amount}") + items = self._request(f"/{'movies' if is_movie else 'shows'}/{pagenation}?limit={amount}") if pagenation == "popular" and is_movie: return [item["ids"]["tmdb"] for item in items], [] elif pagenation == "popular": return [], [item["ids"]["tvdb"] for item in items] elif is_movie: return [item["movie"]["ids"]["tmdb"] for item in items], [] diff --git a/plex_meta_manager.py b/plex_meta_manager.py index a1df12d0..d3a1694f 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -105,7 +105,7 @@ def start(config_path, is_test=False, time_scheduled=None, requested_collections logger.info(util.centered("| __/| | __/> < | | | | __/ || (_| | | | | | (_| | | | | (_| | (_| | __/ | ")) logger.info(util.centered("|_| |_|\\___/_/\\_\\ |_| |_|\\___|\\__\\__,_| |_| |_|\\__,_|_| |_|\\__,_|\\__, |\\___|_| ")) logger.info(util.centered(" |___/ ")) - logger.info(util.centered(" Version: 1.11.3-beta3 ")) + logger.info(util.centered(" Version: 1.11.3-beta4 ")) if time_scheduled: start_type = f"{time_scheduled} " elif is_test: start_type = "Test " elif requested_collections: start_type = "Collections " @@ -256,6 +256,8 @@ def mass_metadata(config, library): logger.info(util.adjust_space(f"{item.title[:25]:<25} | Splitting")) radarr_adds = [] sonarr_adds = [] + trakt_ratings = config.Trakt.user_ratings(library.is_movie) if library.mass_trakt_rating_update else [] + items = library.get_all() for i, item in enumerate(items, 1): library.reload(item) @@ -339,11 +341,11 @@ def mass_metadata(config, library): logger.info(util.adjust_space(f"{item.title[:25]:<25} | Genres | {display_str}")) except Failed: pass - if library.mass_audience_rating_update or library.mass_critic_rating_update: + if library.mass_audience_rating_update: try: - if tmdb_item and library.mass_genre_update == "tmdb": + if tmdb_item and library.mass_audience_rating_update == "tmdb": new_rating = tmdb_item.vote_average - elif omdb_item and library.mass_genre_update in ["omdb", "imdb"]: + elif omdb_item and library.mass_audience_rating_update in ["omdb", "imdb"]: new_rating = omdb_item.imdb_rating else: raise Failed @@ -353,11 +355,37 @@ def mass_metadata(config, library): if library.mass_audience_rating_update and str(item.audienceRating) != str(new_rating): library.edit_query(item, {"audienceRating.value": new_rating, "audienceRating.locked": 1}) logger.info(util.adjust_space(f"{item.title[:25]:<25} | Audience Rating | {new_rating}")) + except Failed: + pass + if library.mass_critic_rating_update: + try: + if tmdb_item and library.mass_critic_rating_update == "tmdb": + new_rating = tmdb_item.vote_average + elif omdb_item and library.mass_critic_rating_update in ["omdb", "imdb"]: + new_rating = omdb_item.imdb_rating + else: + raise Failed + if new_rating is None: + logger.info(util.adjust_space(f"{item.title[:25]:<25} | No Rating Found")) + else: if library.mass_critic_rating_update and str(item.rating) != str(new_rating): library.edit_query(item, {"rating.value": new_rating, "rating.locked": 1}) logger.info(util.adjust_space(f"{item.title[:25]:<25} | Critic Rating | {new_rating}")) except Failed: pass + if library.mass_trakt_rating_update: + try: + if library.is_movie and tmdb_id in trakt_ratings: + new_rating = trakt_ratings[tmdb_id] + elif library.is_show and tvdb_id in trakt_ratings: + new_rating = trakt_ratings[tvdb_id] + else: + raise Failed + if str(item.userRating) != str(new_rating): + library.edit_query(item, {"userRating.value": new_rating, "userRating.locked": 1}) + logger.info(util.adjust_space(f"{item.title[:25]:<25} | User Rating | {new_rating}")) + except Failed: + pass if library.Radarr and library.radarr_add_all: try: From 7e9a8ce598e8d46f2bc036fe1b4b9f85f3fb0f18 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 3 Aug 2021 09:09:00 -0400 Subject: [PATCH 50/95] trakt list fix --- modules/trakt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/trakt.py b/modules/trakt.py index e747dfd0..72c7e371 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -112,7 +112,11 @@ class Trakt: else: response = self.config.get(f"{base_url}{url}?page={current}", headers=headers) if response.status_code == 200: - output_json.extend(response.json()) + json_data = response.json() + if isinstance(json_data, dict): + return json_data + else: + output_json.extend(response.json()) else: raise Failed(f"({response.status_code}) {response.reason}") current += 1 From 09f3ba40f8e147a945f300f6f8f9a4a894b5a7db Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 4 Aug 2021 09:42:21 -0400 Subject: [PATCH 51/95] fix no ID trakt Errors --- modules/trakt.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/trakt.py b/modules/trakt.py index 72c7e371..be0b5525 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -142,6 +142,16 @@ class Trakt: except Failed: raise Failed(f"Trakt Error: List {data} not found") + def _parse(self, items, top=True, is_movie=True): + ids = [] + for item in items: + data = item["movie" if is_movie else "show"] if top else item + if data["ids"]["tmdb" if is_movie else "tvdb"]: + ids.append(data["ids"]["tmdb" if is_movie else "tvdb"]) + else: + logger.error(f"Trakt Error: No {'TMDb' if is_movie else 'TVDb'} ID found for {data['title']} ({data['year']})") + return (ids, []) if is_movie else ([], ids) + def _user_list(self, list_type, data, is_movie): path = f"{requests.utils.urlparse(data).path}/items" if list_type == "list" else f"/users/{data}/{list_type}" try: @@ -153,15 +163,11 @@ class Trakt: raise Failed(f"Trakt Error: List {data} is empty") else: raise Failed(f"Trakt Error: {data}'s {list_type.capitalize()} is empty") - if is_movie: return [item["movie"]["ids"]["tmdb"] for item in items], [] - else: return [], [item["show"]["ids"]["tvdb"] for item in items] + return self._parse(items, is_movie=is_movie) def _pagenation(self, pagenation, amount, is_movie): items = self._request(f"/{'movies' if is_movie else 'shows'}/{pagenation}?limit={amount}") - if pagenation == "popular" and is_movie: return [item["ids"]["tmdb"] for item in items], [] - elif pagenation == "popular": return [], [item["ids"]["tvdb"] for item in items] - elif is_movie: return [item["movie"]["ids"]["tmdb"] for item in items], [] - else: return [], [item["show"]["ids"]["tvdb"] for item in items] + return self._parse(items, top=pagenation != "popular", is_movie=is_movie) def validate_trakt(self, trakt_lists, is_movie, trakt_type="list"): values = util.get_list(trakt_lists, split=False) @@ -185,14 +191,14 @@ class Trakt: pretty = method.replace("_", " ").title() media_type = "Movie" if is_movie else "Show" if method in ["trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected"]: - movie_ids, show_ids = self._pagenation(method[6:], data, is_movie) logger.info(f"Processing {pretty}: {data} {media_type}{'' if data == 1 else 's'}") + movie_ids, show_ids = self._pagenation(method[6:], data, is_movie) elif method in ["trakt_collection", "trakt_watchlist"]: - movie_ids, show_ids = self._user_list(method[6:], data, is_movie) logger.info(f"Processing {pretty} {media_type}s for {data}") - elif method == "trakt_list": movie_ids, show_ids = self._user_list(method[6:], data, is_movie) + elif method == "trakt_list": logger.info(f"Processing {pretty}: {data}") + movie_ids, show_ids = self._user_list(method[6:], data, is_movie) else: raise Failed(f"Trakt Error: Method {method} not supported") logger.debug("") From 9c9d3aa051ba6b8c4073b30c884a95f3ed89faa2 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 4 Aug 2021 09:46:42 -0400 Subject: [PATCH 52/95] Dockerfile update --- .dockerignore | 9 +++++++++ Dockerfile | 31 +++++++++++++------------------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.dockerignore b/.dockerignore index 689d5bfd..f20e7a38 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,13 @@ README.md LICENSE .gitignore +.dockerignore .git +.github +*.psd +config/**/* +config +Dockerfile +venv +.idea +test.py \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6541b903..d0256ffc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,15 @@ -FROM python:3-slim -VOLUME /config +FROM python:3.9-slim +RUN echo "**** install system packages ****" \ + && apt-get update \ + && apt-get upgrade -y --no-install-recommends \ + && apt-get install -y tzdata --no-install-recommends \ + && apt-get install -y gcc g++ libxml2-dev libxslt-dev libz-dev +COPY requirements.txt / +RUN echo "**** install python packages ****" \ + && pip3 install --no-cache-dir --upgrade --requirement /requirements.txt \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /requirements.txt /tmp/* /var/tmp/* /var/lib/apt/lists/* COPY . / -RUN \ - echo "**** install system packages ****" && \ - apt-get update && \ - apt-get upgrade -y --no-install-recommends && \ - apt-get install -y tzdata --no-install-recommends && \ - apt-get install -y gcc g++ libxml2-dev libxslt-dev libz-dev && \ - echo "**** install python packages ****" && \ - pip3 install --no-cache-dir --upgrade --requirement /requirements.txt && \ - echo "**** cleanup ****" && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf \ - /requirements.txt \ - /tmp/* \ - /var/tmp/* \ - /var/lib/apt/lists/* +VOLUME /config ENTRYPOINT ["python3", "plex_meta_manager.py"] From 49f2a06361b89fd71dd2ce8aa626eb9b7ddbf552 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 4 Aug 2021 10:20:52 -0400 Subject: [PATCH 53/95] cleanup --- modules/anidb.py | 5 +- modules/anilist.py | 2 - modules/convert.py | 146 ++++++++++++++++++------------------------- modules/plex.py | 18 ++---- modules/util.py | 7 +++ plex_meta_manager.py | 11 ++-- requirements.txt | 2 +- 7 files changed, 80 insertions(+), 111 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 82ee48cb..06aa4748 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -19,9 +19,8 @@ class AniDB: self.config = config self.username = params["username"] if params else None self.password = params["password"] if params else None - if params: - if not self._login(self.username, self.password).xpath("//li[@class='sub-menu my']/@title"): - raise Failed("AniDB Error: Login failed") + if params and not self._login(self.username, self.password).xpath("//li[@class='sub-menu my']/@title"): + raise Failed("AniDB Error: Login failed") def _request(self, url, language=None, post=None): if post: diff --git a/modules/anilist.py b/modules/anilist.py index d1bc31c9..bdefc1fc 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -1,7 +1,6 @@ import logging, time from modules import util from modules.util import Failed -from retrying import retry logger = logging.getLogger("Plex Meta Manager") @@ -22,7 +21,6 @@ class AniList: self.tags = {t["name"].lower(): t["name"] for t in self._request(tag_query, {})["data"]["MediaTagCollection"]} self.genres = {g.lower(): g for g in self._request(genre_query, {})["data"]["GenreCollection"]} - @retry(stop_max_attempt_number=2, retry_on_exception=util.retry_if_not_failed) def _request(self, query, variables): response = self.config.post(base_url, json={"query": query, "variables": variables}) json_obj = response.json() diff --git a/modules/convert.py b/modules/convert.py index ac0ad97d..50d6a875 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -13,20 +13,31 @@ class Convert: self.config = config self.AniDBIDs = self.config.get_html(anidb_url) - def _anidb(self, input_id, to_id, fail=False): - ids = self.AniDBIDs.xpath(f"//anime[contains(@anidbid, '{input_id}')]/@{to_id}") - if len(ids) > 0: - try: - if len(ids[0]) > 0: - return util.get_list(ids[0]) if to_id == "imdbid" else int(ids[0]) - raise ValueError - except ValueError: - fail_text = f"Convert Error: No {util.pretty_ids[to_id]} ID found for AniDB ID: {input_id}" + def _anidb(self, anidb_id, fail=False): + tvdbid = self.AniDBIDs.xpath(f"//anime[contains(@anidbid, '{anidb_id}')]/@tvdbid") + imdbid = self.AniDBIDs.xpath(f"//anime[contains(@anidbid, '{anidb_id}')]/@imdbid") + if len(tvdbid) > 0: + if len(imdbid[0]) > 0: + imdb_ids = util.get_list(imdbid[0]) + tmdb_ids = [] + for imdb in imdb_ids: + tmdb_id = self.imdb_to_tmdb(imdb) + if tmdb_id: + tmdb_ids.append(tmdb_id) + if tmdb_ids: + return None, imdb_ids, tmdb_ids + else: + fail_text = f"Convert Error: No TMDb ID found for AniDB ID: {anidb_id}" + else: + try: + return int(tvdbid[0]), [], [] + except ValueError: + fail_text = f"Convert Error: No TVDb ID or IMDb ID found for AniDB ID: {anidb_id}" else: - fail_text = f"Convert Error: AniDB ID: {input_id} not found" + fail_text = f"Convert Error: AniDB ID: {anidb_id} not found" if fail: raise Failed(fail_text) - return [] if to_id == "imdbid" else None + return None, [], [] def _arms_ids(self, anilist_ids=None, anidb_ids=None, mal_ids=None): all_ids = [] @@ -70,20 +81,14 @@ class Convert: show_ids = [] movie_ids = [] for anidb_id in anidb_list: - imdb_ids = self.anidb_to_imdb(anidb_id) - tmdb_ids = [] - if imdb_ids: - for imdb_id in imdb_ids: - tmdb_id = self.imdb_to_tmdb(imdb_id) - if tmdb_id: - tmdb_ids.append(tmdb_id) - tvdb_id = self.anidb_to_tvdb(anidb_id) - if tvdb_id: - show_ids.append(tvdb_id) - if tmdb_ids: - movie_ids.extend(tmdb_ids) - if not tvdb_id and not tmdb_ids: - logger.error(f"Convert Error: No TVDb ID or IMDb ID found for AniDB ID: {anidb_id}") + try: + tvdb_id, _, tmdb_ids = self._anidb(anidb_id, fail=True) + if tvdb_id: + show_ids.append(tvdb_id) + if tmdb_ids: + movie_ids.extend(tmdb_ids) + except Failed as e: + logger.error(e) return movie_ids, show_ids def anilist_to_ids(self, anilist_ids): @@ -104,12 +109,6 @@ class Convert: logger.error(f"Convert Error: AniDB ID not found for MyAnimeList ID: {id_set['myanimelist']}") return self.anidb_to_ids(anidb_ids) - def anidb_to_tvdb(self, anidb_id, fail=False): - return self._anidb(anidb_id, "tvdbid", fail=fail) - - def anidb_to_imdb(self, anidb_id, fail=False): - return self._anidb(anidb_id, "imdbid", fail=fail) - def tmdb_to_imdb(self, tmdb_id, is_movie=True, fail=False): media_type = "movie" if is_movie else "show" expired = False @@ -162,11 +161,11 @@ class Convert: return cache_id tvdb_id = None try: - tvdb_id = self.config.TMDb.convert_from(tmdb_id, "tvdb_id", False) + tvdb_id = int(self.config.TMDb.convert_from(tmdb_id, "tvdb_id", False)) except Failed: if self.config.Trakt: try: - tvdb_id = self.config.Trakt.convert(tmdb_id, "tmdb", "tvdb", "show") + tvdb_id = int(self.config.Trakt.convert(tmdb_id, "tmdb", "tvdb", "show")) except Failed: pass if fail and tvdb_id is None: @@ -240,24 +239,21 @@ class Convert: def get_id(self, item, library): expired = None + tmdb_id = [] + tvdb_id = [] + imdb_id = [] + anidb_id = None if self.config.Cache: cache_id, media_type, expired = self.config.Cache.query_guid_map(item.guid) if cache_id and not expired: media_id_type = "movie" if "movie" in media_type else "show" return media_id_type, util.get_list(cache_id, int_list=True) try: - tmdb_id = None - imdb_id = None - tvdb_id = None - anidb_id = None guid = requests.utils.urlparse(item.guid) item_type = guid.scheme.split(".")[-1] check_id = guid.netloc if item_type == "plex": - tmdb_id = [] - imdb_id = [] - tvdb_id = [] try: for guid_tag in library.get_guids(item): url_parsed = requests.utils.urlparse(guid_tag.id) @@ -269,12 +265,13 @@ class Convert: util.print_stacktrace() raise Failed("No External GUIDs found") if not tvdb_id and not imdb_id and not tmdb_id: + library.query(item.refresh) raise Failed("Refresh Metadata") - elif item_type == "imdb": imdb_id = check_id - elif item_type == "thetvdb": tvdb_id = int(check_id) - elif item_type == "themoviedb": tmdb_id = int(check_id) + elif item_type == "imdb": imdb_id.append(check_id) + elif item_type == "thetvdb": tvdb_id.append(int(check_id)) + elif item_type == "themoviedb": tmdb_id.append(int(check_id)) elif item_type == "hama": - if check_id.startswith("tvdb"): tvdb_id = int(re.search("-(.*)", check_id).group(1)) + if check_id.startswith("tvdb"): tvdb_id.append(int(re.search("-(.*)", check_id).group(1))) elif check_id.startswith("anidb"): anidb_id = re.search("-(.*)", check_id).group(1) else: raise Failed(f"Hama Agent ID: {check_id} not supported") elif item_type == "myanimelist": @@ -285,51 +282,28 @@ class Convert: else: raise Failed(f"Agent {item_type} not supported") if anidb_id: - tvdb_id = self.anidb_to_tvdb(anidb_id) - if not tvdb_id: - imdb_id = self.anidb_to_imdb(anidb_id) - if not imdb_id and not tvdb_id: - raise Failed(f"Unable to convert AniDB ID: {anidb_id} to TVDb ID or IMDb ID") - - if not tmdb_id and imdb_id: - if isinstance(imdb_id, list): - tmdb_id = [] + ani_tvdb, ani_imdb, ani_tmdb = self._anidb(anidb_id, fail=True) + if ani_imdb: + imdb_id.extend(ani_imdb) + if ani_tmdb: + tmdb_id.extend(ani_tmdb) + if ani_tvdb: + tvdb_id.append(ani_tvdb) + else: + if not tmdb_id and imdb_id: for imdb in imdb_id: - try: - tmdb_id.append(self.imdb_to_tmdb(imdb, fail=True)) - except Failed: - continue - else: - tmdb_id = self.imdb_to_tmdb(imdb_id) - if not tmdb_id: - raise Failed(f"Unable to convert IMDb ID: {util.compile_list(imdb_id)} to TMDb ID") - if not anidb_id and not tvdb_id and tmdb_id and library.is_show: - if isinstance(tmdb_id, list): - tvdb_id = [] + tmdb = self.imdb_to_tmdb(imdb, is_movie=library.is_movie) + if tmdb: + tmdb_id.append(tmdb) + + if not tvdb_id and tmdb_id and library.is_show: for tmdb in tmdb_id: - try: - tvdb_id.append(self.tmdb_to_tvdb(tmdb, fail=True)) - except Failed: - continue - else: - tvdb_id = self.tmdb_to_tvdb(tmdb_id) - if not tvdb_id: - raise Failed(f"Unable to convert TMDb ID: {util.compile_list(tmdb_id)} to TVDb ID") + tvdb = self.tmdb_to_tvdb(tmdb) + if tvdb: + tvdb_id.append(tvdb) + if not tvdb_id: + raise Failed(f"Unable to convert TMDb ID: {util.compile_list(tmdb_id)} to TVDb ID") - if tvdb_id: - if isinstance(tvdb_id, list): - new_tvdb_id = [] - for tvdb in tvdb_id: - try: - new_tvdb_id.append(int(tvdb)) - except ValueError: - continue - tvdb_id = new_tvdb_id - else: - try: - tvdb_id = int(tvdb_id) - except ValueError: - tvdb_id = None def update_cache(cache_ids, id_type, guid_type): if self.config.Cache: diff --git a/modules/plex.py b/modules/plex.py index 335e3fd8..09f04c4f 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -288,7 +288,8 @@ class Plex: self.split_duplicates = params["split_duplicates"] self.radarr_add_all = params["radarr_add_all"] self.sonarr_add_all = params["sonarr_add_all"] - self.mass_update = self.mass_genre_update or self.mass_audience_rating_update or self.mass_critic_rating_update or self.split_duplicates or self.radarr_add_all or self.sonarr_add_all + self.mass_update = self.mass_genre_update or self.mass_audience_rating_update or self.mass_critic_rating_update \ + or self.mass_trakt_rating_update or self.split_duplicates or self.radarr_add_all or self.sonarr_add_all self.clean_bundles = params["plex"]["clean_bundles"] self.empty_trash = params["plex"]["empty_trash"] self.optimize = params["plex"]["optimize"] @@ -677,24 +678,15 @@ class Plex: if item.ratingKey not in self.movie_rating_key_map and item.ratingKey not in self.show_rating_key_map: id_type, main_id = self.config.Convert.get_id(item, self) if main_id: - if not isinstance(main_id, list): - main_id = [main_id] if id_type == "movie": self.movie_rating_key_map[item.ratingKey] = main_id[0] - for m in main_id: - if m in self.movie_map: - self.movie_map[m].append(item.ratingKey) - else: - self.movie_map[m] = [item.ratingKey] + util.add_dict_list(main_id, item.ratingKey, self.movie_map) elif id_type == "show": self.show_rating_key_map[item.ratingKey] = main_id[0] - for m in main_id: - if m in self.show_map: - self.show_map[m].append(item.ratingKey) - else: - self.show_map[m] = [item.ratingKey] + util.add_dict_list(main_id, item.ratingKey, self.show_map) logger.info("") logger.info(util.adjust_space(f"Processed {len(items)} {'Movies' if self.is_movie else 'Shows'}")) + return items def get_tmdb_from_map(self, item): return self.movie_rating_key_map[item.ratingKey] if item.ratingKey in self.movie_rating_key_map else None diff --git a/modules/util.py b/modules/util.py index 8fab6fb3..d2347cd1 100644 --- a/modules/util.py +++ b/modules/util.py @@ -61,6 +61,13 @@ def tab_new_lines(data): def make_ordinal(n): return f"{n}{'th' if 11 <= (n % 100) <= 13 else ['th', 'st', 'nd', 'rd', 'th'][min(n % 10, 4)]}" +def add_dict_list(keys, value, dict_map): + for key in keys: + if key in dict_map: + dict_map[key].append(value) + else: + dict_map[key] = [value] + def compile_list(data): if isinstance(data, list): text = "" diff --git a/plex_meta_manager.py b/plex_meta_manager.py index d3a1694f..62fb0c3c 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -144,9 +144,9 @@ def update_libraries(config): logger.info("") util.separator(f"Mapping {library.name} Library", space=False, border=False) logger.info("") - library.map_guids() + items = library.map_guids() if not config.test_mode and not config.resume_from and not collection_only and library.mass_update: - mass_metadata(config, library) + mass_metadata(config, library, items) for metadata in library.metadata_files: logger.info("") util.separator(f"Running Metadata File\n{metadata.path}") @@ -245,7 +245,7 @@ def update_libraries(config): if library.optimize: library.query(library.PlexServer.library.optimize) -def mass_metadata(config, library): +def mass_metadata(config, library, items): logger.info("") util.separator(f"Mass Editing {'Movie' if library.is_movie else 'Show'} Library: {library.name}") logger.info("") @@ -258,7 +258,6 @@ def mass_metadata(config, library): sonarr_adds = [] trakt_ratings = config.Trakt.user_ratings(library.is_movie) if library.mass_trakt_rating_update else [] - items = library.get_all() for i, item in enumerate(items, 1): library.reload(item) util.print_return(f"Processing: {i}/{len(items)} {item.title}") @@ -290,7 +289,7 @@ def mass_metadata(config, library): try: tmdb_item = config.TMDb.get_movie(tmdb_id) if library.is_movie else config.TMDb.get_show(tmdb_id) except Failed as e: - logger.info(util.adjust_space(str(e))) + logger.error(util.adjust_space(str(e))) else: logger.info(util.adjust_space(f"{item.title[:25]:<25} | No TMDb ID for Guid: {item.guid}")) @@ -305,7 +304,7 @@ def mass_metadata(config, library): try: omdb_item = config.OMDb.get_omdb(imdb_id) except Failed as e: - logger.info(util.adjust_space(str(e))) + logger.error(util.adjust_space(str(e))) except Exception: logger.error(f"IMDb ID: {imdb_id}") raise diff --git a/requirements.txt b/requirements.txt index d913beed..1d9c3b89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PlexAPI==4.7.0 tmdbv3api==1.7.6 -arrapi==1.1.1 +arrapi==1.1.2 lxml requests>=2.4.2 ruamel.yaml From ad89db3ca4d03e5018a8cb473044668226964a04 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 4 Aug 2021 15:17:34 -0400 Subject: [PATCH 54/95] added `last_episode_aired` show date filter --- modules/builder.py | 74 +++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index f98ef4b2..4c29c871 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -95,6 +95,7 @@ all_filters = [ "release", "release.not", "release.before", "release.after", "release.regex", "history", "added", "added.not", "added.before", "added.after", "added.regex", "last_played", "last_played.not", "last_played.before", "last_played.after", "last_played.regex", + "last_episode_aired", "last_episode_aired.not", "last_episode_aired.before", "last_episode_aired.after", "last_episode_aired.regex", "title", "title.not", "title.begins", "title.ends", "title.regex", "plays.gt", "plays.gte", "plays.lt", "plays.lte", "tmdb_vote_count.gt", "tmdb_vote_count.gte", "tmdb_vote_count.lt", "tmdb_vote_count.lte", @@ -120,7 +121,7 @@ movie_only_filters = [ "resolution", "resolution.not", "writer", "writer.not" ] -show_only_filters = ["network"] +show_only_filters = ["last_episode_aired", "network"] smart_invalid = ["collection_order"] smart_url_invalid = ["filters", "run_again", "sync_mode", "show_filtered", "show_missing", "save_missing", "smart_label"] + radarr_details + sonarr_details custom_sort_builders = [ @@ -1358,30 +1359,45 @@ class CollectionBuilder: for filter_method, filter_data in self.filters: filter_attr, modifier, filter_final = self._split(filter_method) filter_actual = filter_translation[filter_attr] if filter_attr in filter_translation else filter_attr - if filter_attr in ["release", "added", "last_played"] and modifier != ".regex": - current_data = getattr(current, filter_actual) + if filter_attr in ["tmdb_vote_count", "original_language", "last_episode_aired"]: + if current.ratingKey not in self.library.movie_rating_key_map: + logger.warning(f"Filter Error: No TMDb ID found for {current.title}") + continue + if self.library.is_movie: + tmdb_item = self.config.TMDb.get_movie(self.library.movie_rating_key_map[current.ratingKey]) + else: + tmdb_item = self.config.TMDb.get_show(self.library.show_rating_key_map[current.ratingKey]) + else: + tmdb_item = None + if filter_attr in ["release", "added", "last_played", "last_episode_aired"] and modifier != ".regex": + if filter_attr == "last_episode_aired": + current_data = tmdb_item.last_air_date + if current_data is None: + return False + current_data = util.validate_date(current_data, "TMDB Last Air Date") + else: + current_data = getattr(current, filter_actual) + if current_data is None: + return False + if filter_attr == "last_episode_aired": + current_data = util.validate_date(current_data, "TMDB First Air Date") if modifier in ["", ".not"]: threshold_date = self.current_time - timedelta(days=filter_data) if (modifier == "" and (current_data is None or current_data < threshold_date)) \ or (modifier == ".not" and current_data and current_data >= threshold_date): return False elif modifier in [".before", ".after"]: - if current_data is None: - return False filter_date = util.validate_date(filter_data, filter_final) if (modifier == ".before" and current_data >= filter_date) or (modifier == ".after" and current_data <= filter_date): return False - elif filter_attr in ["release", "added", "last_played"] and modifier == ".regex": - jailbreak = False - current_data = getattr(current, filter_actual) - if current_data is None: - return False - for check_data in filter_data: - if re.compile(check_data).match(current_data.strftime("%m/%d/%Y")): - jailbreak = True - break - if not jailbreak: - return False + elif modifier == ".regex": + jailbreak = False + for check_data in filter_data: + if re.compile(check_data).match(current_data.strftime("%m/%d/%Y")): + jailbreak = True + break + if not jailbreak: + return False elif filter_attr == "audio_track_title": jailbreak = False for media in current.media: @@ -1444,33 +1460,11 @@ class CollectionBuilder: if date_match is False: return False elif filter_attr == "original_language": - movie = None - for key, value in self.library.movie_map.items(): - if current.ratingKey in value: - try: - movie = self.config.TMDb.get_movie(key) - break - except Failed: - pass - if movie is None: - logger.warning(f"Filter Error: No TMDb ID found for {current.title}") - continue - if (modifier == ".not" and movie.original_language in filter_data) \ - or (modifier == "" and movie.original_language not in filter_data): + if (modifier == ".not" and tmdb_item.original_language in filter_data) \ + or (modifier == "" and tmdb_item.original_language not in filter_data): return False elif modifier in [".gt", ".gte", ".lt", ".lte"]: if filter_attr == "tmdb_vote_count": - tmdb_item = None - for key, value in self.library.movie_map.items(): - if current.ratingKey in value: - try: - tmdb_item = self.config.TMDb.get_movie(key) if self.library.is_movie else self.config.TMDb.get_show(key) - break - except Failed: - pass - if tmdb_item is None: - logger.warning(f"Filter Error: No TMDb ID found for {current.title}") - continue attr = tmdb_item.vote_count elif filter_attr == "duration": attr = getattr(current, filter_actual) / 60000 From a9f9cca61e46a919219f962a6b7f432a472c47c2 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 4 Aug 2021 15:17:56 -0400 Subject: [PATCH 55/95] more cleanup --- modules/convert.py | 104 +++++++++++++++++++++------------------------ modules/tmdb.py | 2 +- requirements.txt | 16 +++---- 3 files changed, 58 insertions(+), 64 deletions(-) diff --git a/modules/convert.py b/modules/convert.py index 50d6a875..2982455c 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -120,16 +120,15 @@ class Convert: try: imdb_id = self.config.TMDb.convert_from(tmdb_id, "imdb_id", is_movie) except Failed: - if self.config.Trakt: - try: - imdb_id = self.config.Trakt.convert(tmdb_id, "tmdb", "imdb", "movie" if is_movie else "show") - except Failed: - pass - if fail and imdb_id is None: + pass + if imdb_id: + if self.config.Cache: + self.config.Cache.update_imdb_to_tmdb_map(media_type, expired, imdb_id, tmdb_id) + return imdb_id + elif fail: raise Failed(f"Convert Error: No IMDb ID Found for TMDb ID: {tmdb_id}") - if self.config.Cache and imdb_id: - self.config.Cache.update_imdb_to_tmdb_map(media_type, expired, imdb_id, tmdb_id) - return imdb_id + else: + return None def imdb_to_tmdb(self, imdb_id, is_movie=True, fail=False): media_type = "movie" if is_movie else "show" @@ -142,16 +141,15 @@ class Convert: try: tmdb_id = self.config.TMDb.convert_to(imdb_id, "imdb_id", is_movie) except Failed: - if self.config.Trakt: - try: - tmdb_id = self.config.Trakt.convert(imdb_id, "imdb", "tmdb", media_type) - except Failed: - pass - if fail and tmdb_id is None: + pass + if tmdb_id: + if self.config.Cache: + self.config.Cache.update_imdb_to_tmdb_map(media_type, expired, imdb_id, tmdb_id) + return tmdb_id + elif fail: raise Failed(f"Convert Error: No TMDb ID Found for IMDb ID: {imdb_id}") - if self.config.Cache and tmdb_id: - self.config.Cache.update_imdb_to_tmdb_map(media_type, expired, imdb_id, tmdb_id) - return tmdb_id + else: + return None def tmdb_to_tvdb(self, tmdb_id, fail=False): expired = False @@ -161,18 +159,17 @@ class Convert: return cache_id tvdb_id = None try: - tvdb_id = int(self.config.TMDb.convert_from(tmdb_id, "tvdb_id", False)) + tvdb_id = self.config.TMDb.convert_from(tmdb_id, "tvdb_id", False) except Failed: - if self.config.Trakt: - try: - tvdb_id = int(self.config.Trakt.convert(tmdb_id, "tmdb", "tvdb", "show")) - except Failed: - pass - if fail and tvdb_id is None: + pass + if tvdb_id: + if self.config.Cache: + self.config.Cache.update_tmdb_to_tvdb_map(expired, tmdb_id, tvdb_id) + return tvdb_id + elif fail: raise Failed(f"Convert Error: No TVDb ID Found for TMDb ID: {tmdb_id}") - if self.config.Cache and tvdb_id: - self.config.Cache.update_tmdb_to_tvdb_map(expired, tmdb_id, tvdb_id) - return tvdb_id + else: + return None def tvdb_to_tmdb(self, tvdb_id, fail=False): expired = False @@ -184,16 +181,15 @@ class Convert: try: tmdb_id = self.config.TMDb.convert_to(tvdb_id, "tvdb_id", False) except Failed: - if self.config.Trakt: - try: - tmdb_id = self.config.Trakt.convert(tvdb_id, "tvdb", "tmdb", "show") - except Failed: - pass - if fail and tmdb_id is None: + pass + if tmdb_id: + if self.config.Cache: + self.config.Cache.update_tmdb_to_tvdb_map(expired, tmdb_id, tvdb_id) + return tmdb_id + elif fail: raise Failed(f"Convert Error: No TMDb ID Found for TVDb ID: {tvdb_id}") - if self.config.Cache and tmdb_id: - self.config.Cache.update_tmdb_to_tvdb_map(expired, tmdb_id, tvdb_id) - return tmdb_id + else: + return None def tvdb_to_imdb(self, tvdb_id, fail=False): expired = False @@ -205,16 +201,15 @@ class Convert: try: imdb_id = self.tmdb_to_imdb(self.tvdb_to_tmdb(tvdb_id, fail=True), is_movie=False, fail=True) except Failed: - if self.config.Trakt: - try: - imdb_id = self.config.Trakt.convert(tvdb_id, "tvdb", "imdb", "show") - except Failed: - pass - if fail and imdb_id is None: + pass + if imdb_id: + if self.config.Cache: + self.config.Cache.update_imdb_to_tvdb_map(expired, imdb_id, tvdb_id) + return imdb_id + elif fail: raise Failed(f"Convert Error: No IMDb ID Found for TVDb ID: {tvdb_id}") - if self.config.Cache and imdb_id: - self.config.Cache.update_imdb_to_tvdb_map(expired, imdb_id, tvdb_id) - return imdb_id + else: + return None def imdb_to_tvdb(self, imdb_id, fail=False): expired = False @@ -226,16 +221,15 @@ class Convert: try: tvdb_id = self.tmdb_to_tvdb(self.imdb_to_tmdb(imdb_id, is_movie=False, fail=True), fail=True) except Failed: - if self.config.Trakt: - try: - tvdb_id = self.config.Trakt.convert(imdb_id, "imdb", "tvdb", "show") - except Failed: - pass - if fail and tvdb_id is None: + pass + if tvdb_id: + if self.config.Cache: + self.config.Cache.update_imdb_to_tvdb_map(expired, imdb_id, tvdb_id) + return tvdb_id + elif fail: raise Failed(f"Convert Error: No TVDb ID Found for IMDb ID: {imdb_id}") - if self.config.Cache and tvdb_id: - self.config.Cache.update_imdb_to_tvdb_map(expired, imdb_id, tvdb_id) - return tvdb_id + else: + return None def get_id(self, item, library): expired = None diff --git a/modules/tmdb.py b/modules/tmdb.py index 0e7c44a5..fd2fbb84 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -75,7 +75,7 @@ class TMDb: id_to_return = self.Movie.external_ids(tmdb_id)[convert_to] if is_movie else self.TV.external_ids(tmdb_id)[convert_to] if not id_to_return or (convert_to == "tvdb_id" and id_to_return == 0): raise Failed(f"TMDb Error: No {convert_to.upper().replace('B_', 'b ')} found for TMDb ID {tmdb_id}") - return id_to_return + return id_to_return if convert_to == "imdb_id" else int(id_to_return) except TMDbException: raise Failed(f"TMDb Error: TMDb {'Movie' if is_movie else 'Show'} ID: {tmdb_id} not found") diff --git a/requirements.txt b/requirements.txt index 1d9c3b89..b8f6cf3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ PlexAPI==4.7.0 tmdbv3api==1.7.6 arrapi==1.1.2 -lxml -requests>=2.4.2 -ruamel.yaml -schedule -retrying -pathvalidate -pillow -python-slugify \ No newline at end of file +lxml==4.6.3 +requests==2.26.0 +ruamel.yaml==0.17.10 +schedule==1.1.0 +retrying==1.3.3 +pathvalidate==2.4.1 +pillow==8.3.1 +python-slugify==5.0.2 \ No newline at end of file From b76b54ade5a94ea411eaa57e4a43d5e523bfc3e3 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 10:59:45 -0400 Subject: [PATCH 56/95] fixed missing filters --- modules/builder.py | 178 +++++++++++++++++++++---------------------- modules/cache.py | 2 +- modules/convert.py | 1 + modules/meta.py | 8 +- modules/plex.py | 7 +- plex_meta_manager.py | 2 +- 6 files changed, 97 insertions(+), 101 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 4c29c871..c30981d6 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -110,6 +110,7 @@ all_filters = [ "writer", "writer.not", "year", "year.gt", "year.gte", "year.lt", "year.lte", "year.not" ] +tmdb_filters = ("original_language", "tmdb_vote_count", "year") movie_only_filters = [ "audio_language", "audio_language.not", "audio_track_title", "audio_track_title.not", "audio_track_title.begins", "audio_track_title.ends", "audio_track_title.regex", @@ -158,10 +159,9 @@ class CollectionBuilder: self.sonarr_options = {} self.missing_movies = [] self.missing_shows = [] - self.filtered_missing_movies = [] - self.filtered_missing_shows = [] self.builders = [] self.filters = [] + self.tmdb_filters = [] self.rating_keys = [] self.run_again_movies = [] self.run_again_shows = [] @@ -177,8 +177,8 @@ class CollectionBuilder: methods = {m.lower(): m for m in self.data} if "template" in methods: - logger.info("") - logger.info("Validating Method: template") + logger.debug("") + logger.debug("Validating Method: template") if not self.metadata.templates: raise Failed("Collection Error: No templates found") elif not self.data[methods["template"]]: @@ -290,8 +290,8 @@ class CollectionBuilder: continue if "schedule" in methods: - logger.info("") - logger.info("Validating Method: schedule") + logger.debug("") + logger.debug("Validating Method: schedule") if not self.data[methods["schedule"]]: raise Failed("Collection Error: schedule attribute is blank") else: @@ -359,29 +359,29 @@ class CollectionBuilder: self.validate_builders = True if "validate_builders" in methods: - logger.info("") - logger.info("Validating Method: validate_builders") - logger.info(f"Value: {data[methods['validate_builders']]}") + logger.debug("") + logger.debug("Validating Method: validate_builders") + logger.debug(f"Value: {data[methods['validate_builders']]}") self.validate_builders = util.parse("validate_builders", self.data, datatype="bool", methods=methods, default=True) self.run_again = False if "run_again" in methods: - logger.info("") - logger.info("Validating Method: run_again") - logger.info(f"Value: {data[methods['run_again']]}") + logger.debug("") + logger.debug("Validating Method: run_again") + logger.debug(f"Value: {data[methods['run_again']]}") self.run_again = util.parse("run_again", self.data, datatype="bool", methods=methods, default=False) self.build_collection = True if "build_collection" in methods: - logger.info("") - logger.info("Validating Method: build_collection") - logger.info(f"Value: {data[methods['build_collection']]}") + logger.debug("") + logger.debug("Validating Method: build_collection") + logger.debug(f"Value: {data[methods['build_collection']]}") self.build_collection = util.parse("build_collection", self.data, datatype="bool", methods=methods, default=True) self.sync = self.library.sync_mode == "sync" if "sync_mode" in methods: - logger.info("") - logger.info("Validating Method: sync_mode") + logger.debug("") + logger.debug("Validating Method: sync_mode") if not self.data[methods["sync_mode"]]: logger.warning(f"Collection Warning: sync_mode attribute is blank using general: {self.library.sync_mode}") else: @@ -393,20 +393,22 @@ class CollectionBuilder: self.custom_sort = False if "collection_order" in methods: - logger.info("") - logger.info("Validating Method: collection_order") + logger.debug("") + logger.debug("Validating Method: collection_order") if self.data[methods["collection_order"]] is None: raise Failed(f"Collection Warning: collection_order attribute is blank") - elif self.data[methods["collection_order"]].lower() in plex.collection_order_options: - self.details["collection_order"] = self.data[methods["collection_order"]].lower() - if self.data[methods["collection_order"]].lower() == "custom" and self.build_collection: - self.custom_sort = True else: - raise Failed(f"Collection Error: {self.data[methods['collection_order']]} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)\n\tcustom (Custom Order Collection)") + logger.debug(f"Value: {self.data[methods['collection_order']]}") + if self.data[methods["collection_order"]].lower() in plex.collection_order_options: + self.details["collection_order"] = self.data[methods["collection_order"]].lower() + if self.data[methods["collection_order"]].lower() == "custom" and self.build_collection: + self.custom_sort = True + else: + raise Failed(f"Collection Error: {self.data[methods['collection_order']]} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)\n\tcustom (Custom Order Collection)") if "tmdb_person" in methods: - logger.info("") - logger.info("Validating Method: tmdb_person") + logger.debug("") + logger.debug("Validating Method: tmdb_person") if not self.data[methods["tmdb_person"]]: raise Failed("Collection Error: tmdb_person attribute is blank") else: @@ -427,8 +429,8 @@ class CollectionBuilder: self.smart_sort = "random" self.smart_label_collection = False if "smart_label" in methods: - logger.info("") - logger.info("Validating Method: smart_label") + logger.debug("") + logger.debug("Validating Method: smart_label") self.smart_label_collection = True if not self.data[methods["smart_label"]]: logger.warning("Collection Error: smart_label attribute is blank defaulting to random") @@ -444,8 +446,8 @@ class CollectionBuilder: self.smart_type_key = None self.smart_filter_details = "" if "smart_url" in methods: - logger.info("") - logger.info("Validating Method: smart_url") + logger.debug("") + logger.debug("Validating Method: smart_url") if not self.data[methods["smart_url"]]: raise Failed("Collection Error: smart_url attribute is blank") else: @@ -524,6 +526,8 @@ class CollectionBuilder: else: logger.error(e) + self.tmdb_filters = [(fm, fd) for fm, fd in self.filters if fm.startswith(tmdb_filters)] + if self.custom_sort and len(self.builders) > 1: raise Failed("Collection Error: collection_order: custom can only be used with a single builder per collection") @@ -969,16 +973,22 @@ class CollectionBuilder: validate = dict_data["validate"] for filter_method, filter_data in dict_data.items(): filter_attr, modifier, filter_final = self._split(filter_method) + message = None if filter_final not in all_filters: - raise Failed(f"Collection Error: {filter_final} is not a valid filter attribute") + message = f"Collection Error: {filter_final} is not a valid filter attribute" elif filter_final in movie_only_filters and self.library.is_show: - raise Failed(f"Collection Error: {filter_final} filter attribute only works for movie libraries") + message = f"Collection Error: {filter_final} filter attribute only works for movie libraries" elif filter_final in show_only_filters and self.library.is_movie: - raise Failed(f"Collection Error: {filter_final} filter attribute only works for show libraries") + message = f"Collection Error: {filter_final} filter attribute only works for show libraries" elif filter_final is None: - raise Failed(f"Collection Error: {filter_final} filter attribute is blank") + message = f"Collection Error: {filter_final} filter attribute is blank" else: self.filters.append((filter_final, self.validate_attribute(filter_attr, modifier, f"{filter_final} filter", filter_data, validate))) + if message: + if validate: + raise Failed(message) + else: + logger.error(message) def collect_rating_keys(self): filtered_keys = {} @@ -1018,37 +1028,15 @@ class CollectionBuilder: for movie_id in movie_ids: if movie_id in self.library.movie_map: add_rating_keys(self.library.movie_map[movie_id]) - elif movie_id not in self.missing_movies and movie_id not in self.filtered_missing_movies: - filter_missing = False - if self.details["released_missing_only"]: - try: - movie = self.config.TMDb.get_movie(movie_id) - if util.validate_date(movie.release_date, "") > self.current_time: - filter_missing = True - except Failed: - pass - if filter_missing: - self.filtered_missing_movies.append(movie_id) - else: - self.missing_movies.append(movie_id) + elif movie_id not in self.missing_movies: + self.missing_movies.append(movie_id) if len(show_ids) > 0: items_found_inside += len(show_ids) for show_id in show_ids: if show_id in self.library.show_map: add_rating_keys(self.library.show_map[show_id]) - elif show_id not in self.missing_shows and show_id not in self.filtered_missing_shows: - filter_missing = False - if self.details["released_missing_only"]: - try: - show = self.config.TMDb.get_show(show_id) - if util.validate_date(show.first_air_date, "") > self.current_time: - filter_missing = True - except Failed: - pass - if filter_missing: - self.filtered_missing_shows.append(show_id) - else: - self.missing_shows.append(show_id) + elif show_id not in self.missing_shows: + self.missing_shows.append(show_id) return items_found_inside for method, value in self.builders: logger.debug("") @@ -1068,6 +1056,31 @@ class CollectionBuilder: elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) else: logger.error(f"Collection Error: {method} method not supported") + def tmdb_filter(self, item_id, is_movie, item=None): + filter_missing = False + if self.tmdb_filters or self.details["released_missing_only"]: + try: + if item is None: + item = self.config.TMDb.get_movie(item_id) if is_movie else self.config.TMDb.get_movie(self.config.Convert.tvdb_to_tmdb(item_id)) + if self.details["released_missing_only"]: + if util.validate_date(item.release_date if is_movie else item.first_air_date, "") > self.current_time: + return True + for filter_method, filter_data in self.tmdb_filters: + if (filter_method == "original_language" and item.original_language not in filter_data) \ + or (filter_method == "original_language.not" and item.original_language in filter_data) \ + or (filter_method == "tmdb_vote_count.gt" and item.vote_count <= filter_data) \ + or (filter_method == "tmdb_vote_count.gte" and item.vote_count < filter_data) \ + or (filter_method == "tmdb_vote_count.lt" and item.vote_count >= filter_data) \ + or (filter_method == "tmdb_vote_count.lte" and item.vote_count > filter_data) \ + or (filter_method == "year.gt" and item.year <= filter_data) \ + or (filter_method == "year.gte" and item.year < filter_data) \ + or (filter_method == "year.lt" and item.year >= filter_data) \ + or (filter_method == "year.lte" and item.year > filter_data): + return True + except Failed: + return True + return filter_missing + def build_filter(self, method, plex_filter, smart=False): if smart: logger.info("") @@ -1500,10 +1513,6 @@ class CollectionBuilder: return True def run_missing(self): - arr_filters = [] - for filter_method, filter_data in self.filters: - if (filter_method.startswith("original_language") and self.library.is_movie) or filter_method.startswith("tmdb_vote_count"): - arr_filters.append((filter_method, filter_data)) if len(self.missing_movies) > 0: missing_movies_with_names = [] for missing_id in self.missing_movies: @@ -1512,23 +1521,14 @@ class CollectionBuilder: except Failed as e: logger.error(e) continue - match = True - for filter_method, filter_data in arr_filters: - if (filter_method == "original_language" and movie.original_language not in filter_data) \ - or (filter_method == "original_language.not" and movie.original_language in filter_data) \ - or (filter_method == "tmdb_vote_count.gt" and movie.vote_count <= filter_data) \ - or (filter_method == "tmdb_vote_count.gte" and movie.vote_count < filter_data) \ - or (filter_method == "tmdb_vote_count.lt" and movie.vote_count >= filter_data) \ - or (filter_method == "tmdb_vote_count.lte" and movie.vote_count > filter_data): - match = False - break current_title = f"{movie.title} ({util.validate_date(movie.release_date, 'test').year})" if movie.release_date else movie.title - if match: + if self.tmdb_filter(missing_id, True, item=movie): + if self.details["show_filtered"] is True and self.details["show_missing"] is True: + logger.info(f"{self.name} Collection | X | {current_title} (TMDb: {missing_id})") + else: missing_movies_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: logger.info(f"{self.name} Collection | ? | {current_title} (TMDb: {missing_id})") - elif self.details["show_filtered"] is True: - logger.info(f"{self.name} Collection | X | {current_title} (TMDb: {missing_id})") logger.info("") logger.info(f"{len(missing_movies_with_names)} Movie{'s' if len(missing_movies_with_names) > 1 else ''} Missing") if self.details["save_missing"] is True: @@ -1546,26 +1546,18 @@ class CollectionBuilder: missing_shows_with_names = [] for missing_id in self.missing_shows: try: - title = str(self.config.TVDb.get_series(self.language, missing_id).title.encode("ascii", "replace").decode()) + show = self.config.TVDb.get_series(self.language, missing_id) except Failed as e: logger.error(e) continue - match = True - if arr_filters: - show = self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(missing_id)) - for filter_method, filter_data in arr_filters: - if (filter_method == "tmdb_vote_count.gt" and show.vote_count <= filter_data) \ - or (filter_method == "tmdb_vote_count.gte" and show.vote_count < filter_data) \ - or (filter_method == "tmdb_vote_count.lt" and show.vote_count >= filter_data) \ - or (filter_method == "tmdb_vote_count.lte" and show.vote_count > filter_data): - match = False - break - if match: - missing_shows_with_names.append((title, missing_id)) + current_title = str(show.title.encode("ascii", "replace").decode()) + if self.tmdb_filter(missing_id, False): + if self.details["show_filtered"] is True and self.details["show_missing"] is True: + logger.info(f"{self.name} Collection | X | {current_title} (TVDb: {missing_id})") + else: + missing_shows_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: - logger.info(f"{self.name} Collection | ? | {title} (TVDB: {missing_id})") - elif self.details["show_filtered"] is True: - logger.info(f"{self.name} Collection | X | {title} (TVDb: {missing_id})") + logger.info(f"{self.name} Collection | ? | {current_title} (TVDB: {missing_id})") logger.info("") logger.info(f"{len(missing_shows_with_names)} Show{'s' if len(missing_shows_with_names) > 1 else ''} Missing") if self.details["save_missing"] is True: diff --git a/modules/cache.py b/modules/cache.py index ecf1865c..562d1a39 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -108,7 +108,7 @@ class Cache: row = cursor.fetchone() if row: time_between_insertion = datetime.now() - datetime.strptime(row["expiration_date"], "%Y-%m-%d") - id_to_return = row["t_id"] + id_to_return = int(row["t_id"]) media_type = row["media_type"] expired = time_between_insertion.days > self.expiration return id_to_return, media_type, expired diff --git a/modules/convert.py b/modules/convert.py index 2982455c..578f0c88 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -315,6 +315,7 @@ class Convert: update_cache(tmdb_id, "TMDb", "show_movie") return "movie", tmdb_id else: + logger.debug(f"TMDb: {tmdb_id}, IMDb: {imdb_id}, TVDb: {tvdb_id}") raise Failed(f"No ID to convert") except Failed as e: logger.info(util.adjust_space(f"Mapping Error | {item.guid:<46} | {e} for {item.title}")) diff --git a/modules/meta.py b/modules/meta.py index b3afccc2..d15ecf8e 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -67,7 +67,7 @@ class Metadata: else: return self.collections - def update_metadata(self, TMDb, test): + def update_metadata(self): if not self.metadata: return None logger.info("") @@ -75,7 +75,7 @@ class Metadata: logger.info("") for mapping_name, meta in self.metadata.items(): methods = {mm.lower(): mm for mm in meta} - if test and ("test" not in methods or meta[methods["test"]] is not True): + if self.config.test and ("test" not in methods or meta[methods["test"]] is not True): continue updated = False @@ -212,13 +212,13 @@ class Metadata: logger.error("Metadata Error: tmdb_show attribute is blank") else: tmdb_is_movie = False - tmdb_item = TMDb.get_show(util.regex_first_int(data, "Show")) + tmdb_item = self.config.TMDb.get_show(util.regex_first_int(data, "Show")) elif "tmdb_movie" in methods: if meta[methods["tmdb_movie"]] is None: logger.error("Metadata Error: tmdb_movie attribute is blank") else: tmdb_is_movie = True - tmdb_item = TMDb.get_movie(util.regex_first_int(meta[methods["tmdb_movie"]], "Movie")) + tmdb_item = self.config.TMDb.get_movie(util.regex_first_int(meta[methods["tmdb_movie"]], "Movie")) except Failed as e: logger.error(e) diff --git a/modules/plex.py b/modules/plex.py index 09f04c4f..ecea22e1 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -132,14 +132,17 @@ show_only_searches = [ "episode_user_rating.gt", "episode_user_rating.gte", "episode_user_rating.lt", "episode_user_rating.lte", "episode_year", "episode_year.not", "episode_year.gt", "episode_year.gte", "episode_year.lt", "episode_year.lte" ] -number_attributes = ["plays", "episode_plays", "added", "episode_added", "release", "episode_air_date", "duration", "tmdb_vote_count"] +number_attributes = [ + "plays", "episode_plays", "duration", "tmdb_vote_count", "last_episode_aired" + "added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played" +] float_attributes = ["user_rating", "episode_user_rating", "critic_rating", "audience_rating"] boolean_attributes = [ "hdr", "unmatched", "duplicate", "unplayed", "progress", "trash", "unplayed_episodes", "episode_unplayed", "episode_duplicate", "episode_progress", "episode_unmatched", ] tmdb_attributes = ["actor", "director", "producer", "writer"] -date_attributes = ["added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played"] +date_attributes = ["added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played", "last_episode_aired"] search_display = {"added": "Date Added", "release": "Release Date", "hdr": "HDR", "progress": "In Progress", "episode_progress": "Episode In Progress"} sorts = { None: None, diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 62fb0c3c..c5a346e0 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -152,7 +152,7 @@ def update_libraries(config): util.separator(f"Running Metadata File\n{metadata.path}") if not config.test_mode and not config.resume_from and not collection_only: try: - metadata.update_metadata(config.TMDb, config.test_mode) + metadata.update_metadata() except Failed as e: logger.error(e) collections_to_run = metadata.get_collections(config.requested_collections) From c2badff8e286fafb6c2298d86f3e4e9d626948f3 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 11:18:00 -0400 Subject: [PATCH 57/95] fixed int error --- modules/cache.py | 3 ++- modules/convert.py | 2 +- plex_meta_manager.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/cache.py b/modules/cache.py index 562d1a39..eaeab55d 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -1,6 +1,7 @@ import logging, os, random, sqlite3 from contextlib import closing from datetime import datetime, timedelta +from modules.util import Failed logger = logging.getLogger("Plex Meta Manager") @@ -108,7 +109,7 @@ class Cache: row = cursor.fetchone() if row: time_between_insertion = datetime.now() - datetime.strptime(row["expiration_date"], "%Y-%m-%d") - id_to_return = int(row["t_id"]) + id_to_return = util.get_list(row["t_id"], int_list=True) media_type = row["media_type"] expired = time_between_insertion.days > self.expiration return id_to_return, media_type, expired diff --git a/modules/convert.py b/modules/convert.py index 578f0c88..7cdd3680 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -241,7 +241,7 @@ class Convert: cache_id, media_type, expired = self.config.Cache.query_guid_map(item.guid) if cache_id and not expired: media_id_type = "movie" if "movie" in media_type else "show" - return media_id_type, util.get_list(cache_id, int_list=True) + return media_id_type, cache_id try: guid = requests.utils.urlparse(item.guid) item_type = guid.scheme.split(".")[-1] diff --git a/plex_meta_manager.py b/plex_meta_manager.py index c5a346e0..71dc9f32 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -268,9 +268,9 @@ def mass_metadata(config, library, items): t_id, guid_media_type, _ = config.Cache.query_guid_map(item.guid) if t_id: if "movie" in guid_media_type: - tmdb_id = t_id + tmdb_id = t_id[0] else: - tvdb_id = t_id + tvdb_id = t_id[0] if not tmdb_id and not tvdb_id: tmdb_id = library.get_tmdb_from_map(item) if not tmdb_id and not tvdb_id and library.is_show: From 3a66e6f042a250abcb23d8a47bdb31b40cf65d08 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 11:21:11 -0400 Subject: [PATCH 58/95] imported wrong module --- modules/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cache.py b/modules/cache.py index eaeab55d..21a03d51 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -1,7 +1,7 @@ import logging, os, random, sqlite3 from contextlib import closing from datetime import datetime, timedelta -from modules.util import Failed +from modules import util logger = logging.getLogger("Plex Meta Manager") From 65ae1b1905a834546fd49a872dd8c8e9065edb5a Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 11:41:14 -0400 Subject: [PATCH 59/95] add debug --- modules/builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/builder.py b/modules/builder.py index c30981d6..6c53121a 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1243,6 +1243,7 @@ class CollectionBuilder: def validate_attribute(self, attribute, modifier, final, data, validate, pairs=False): def smart_pair(list_to_pair): return [(t, t) for t in list_to_pair] if pairs else list_to_pair + logger.debug(f"{attribute} {modifier}") if modifier == ".regex": regex_list = util.get_list(data, split=False) valid_regex = [] From f3ae3967b6fd683e0f0077f98837078ef61a352b Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 11:56:27 -0400 Subject: [PATCH 60/95] added "show_filtered", "show_missing", "save_missing", "item_assets", "create_asset_folders", "released_missing_only" as collection level details --- modules/builder.py | 6 +++--- modules/plex.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 6c53121a..2694998c 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -71,7 +71,7 @@ summary_details = [ ] poster_details = ["url_poster", "tmdb_poster", "tmdb_profile", "tvdb_poster", "file_poster"] background_details = ["url_background", "tmdb_background", "tvdb_background", "file_background"] -boolean_details = ["visible_library", "visible_home", "visible_shared", "show_filtered", "show_missing", "save_missing", "item_assets"] +boolean_details = ["visible_library", "visible_home", "visible_shared", "show_filtered", "show_missing", "save_missing", "item_assets", "create_asset_folders", "released_missing_only"] string_details = ["sort_title", "content_rating", "name_mapping"] ignored_details = ["smart_filter", "smart_label", "smart_url", "run_again", "schedule", "sync_mode", "template", "test", "tmdb_person", "build_collection", "collection_order", "validate_builders"] details = ["collection_mode", "collection_order", "label"] + boolean_details + string_details @@ -638,7 +638,8 @@ class CollectionBuilder: else: self.details[method_final] = util.get_list(method_data) elif method_name in boolean_details: - self.details[method_name] = util.parse(method_name, method_data, datatype="bool") + default = self.details[method_name] if method_name in self.details else None + self.details[method_name] = util.parse(method_name, method_data, datatype="bool", default=default) elif method_name in string_details: self.details[method_name] = str(method_data) @@ -1243,7 +1244,6 @@ class CollectionBuilder: def validate_attribute(self, attribute, modifier, final, data, validate, pairs=False): def smart_pair(list_to_pair): return [(t, t) for t in list_to_pair] if pairs else list_to_pair - logger.debug(f"{attribute} {modifier}") if modifier == ".regex": regex_list = util.get_list(data, split=False) valid_regex = [] diff --git a/modules/plex.py b/modules/plex.py index ecea22e1..a5c57216 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -133,7 +133,7 @@ show_only_searches = [ "episode_year", "episode_year.not", "episode_year.gt", "episode_year.gte", "episode_year.lt", "episode_year.lte" ] number_attributes = [ - "plays", "episode_plays", "duration", "tmdb_vote_count", "last_episode_aired" + "plays", "episode_plays", "duration", "tmdb_vote_count", "last_episode_aired", "added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played" ] float_attributes = ["user_rating", "episode_user_rating", "critic_rating", "audience_rating"] From f8f2c9d745969af014dde5fa08a491948b41715e Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 12:05:10 -0400 Subject: [PATCH 61/95] checked wrong map --- modules/builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/builder.py b/modules/builder.py index 2694998c..257e2541 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1374,7 +1374,8 @@ class CollectionBuilder: filter_attr, modifier, filter_final = self._split(filter_method) filter_actual = filter_translation[filter_attr] if filter_attr in filter_translation else filter_attr if filter_attr in ["tmdb_vote_count", "original_language", "last_episode_aired"]: - if current.ratingKey not in self.library.movie_rating_key_map: + if (self.library.is_movie and current.ratingKey not in self.library.movie_rating_key_map) \ + or (self.library.is_show and current.ratingKey not in self.library.show_rating_key_map): logger.warning(f"Filter Error: No TMDb ID found for {current.title}") continue if self.library.is_movie: From a3f2a027a340c86abbd3e0af3bd5c7f558c0f967 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 13:15:15 -0400 Subject: [PATCH 62/95] catch failed conversion --- modules/builder.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 257e2541..25fdcbf9 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1376,12 +1376,16 @@ class CollectionBuilder: if filter_attr in ["tmdb_vote_count", "original_language", "last_episode_aired"]: if (self.library.is_movie and current.ratingKey not in self.library.movie_rating_key_map) \ or (self.library.is_show and current.ratingKey not in self.library.show_rating_key_map): - logger.warning(f"Filter Error: No TMDb ID found for {current.title}") - continue - if self.library.is_movie: - tmdb_item = self.config.TMDb.get_movie(self.library.movie_rating_key_map[current.ratingKey]) - else: - tmdb_item = self.config.TMDb.get_show(self.library.show_rating_key_map[current.ratingKey]) + logger.warning(f"Filter Error: No {'TMDb' if self.library.is_movie else 'TVDb'} ID found for {current.title}") + return False + try: + if self.library.is_movie: + tmdb_item = self.config.TMDb.get_movie(self.library.movie_rating_key_map[current.ratingKey]) + else: + tmdb_item = self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(self.library.show_rating_key_map[current.ratingKey])) + except Failed as e: + logger.error(e) + return False else: tmdb_item = None if filter_attr in ["release", "added", "last_played", "last_episode_aired"] and modifier != ".regex": From e8bbfa6b50a982d69c67993b0c25268db7893db5 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 13:25:16 -0400 Subject: [PATCH 63/95] fix date error --- modules/builder.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 25fdcbf9..d32db6e5 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1390,16 +1390,13 @@ class CollectionBuilder: tmdb_item = None if filter_attr in ["release", "added", "last_played", "last_episode_aired"] and modifier != ".regex": if filter_attr == "last_episode_aired": - current_data = tmdb_item.last_air_date - if current_data is None: + if tmdb_item.last_air_date is None: return False - current_data = util.validate_date(current_data, "TMDB Last Air Date") + current_data = util.validate_date(tmdb_item.last_air_date, "TMDB Last Air Date") else: current_data = getattr(current, filter_actual) if current_data is None: return False - if filter_attr == "last_episode_aired": - current_data = util.validate_date(current_data, "TMDB First Air Date") if modifier in ["", ".not"]: threshold_date = self.current_time - timedelta(days=filter_data) if (modifier == "" and (current_data is None or current_data < threshold_date)) \ From 942b7efa88847badb691d1427c8c04bb3cf9e3a3 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 13:48:57 -0400 Subject: [PATCH 64/95] manged keys better --- modules/builder.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index d32db6e5..bc5607e3 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1026,18 +1026,22 @@ class CollectionBuilder: items_found_inside = 0 if len(movie_ids) > 0: items_found_inside += len(movie_ids) + movie_rating_keys = [] for movie_id in movie_ids: if movie_id in self.library.movie_map: - add_rating_keys(self.library.movie_map[movie_id]) + movie_rating_keys.append(self.library.movie_map[movie_id]) elif movie_id not in self.missing_movies: self.missing_movies.append(movie_id) + add_rating_keys(movie_rating_keys) if len(show_ids) > 0: items_found_inside += len(show_ids) + show_rating_keys = [] for show_id in show_ids: if show_id in self.library.show_map: - add_rating_keys(self.library.show_map[show_id]) + show_rating_keys.append(self.library.show_map[show_id]) elif show_id not in self.missing_shows: self.missing_shows.append(show_id) + add_rating_keys(show_rating_keys) return items_found_inside for method, value in self.builders: logger.debug("") From bd4c59f28654a78e7f6571bcaeeadd7cd017b4b5 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 14:17:52 -0400 Subject: [PATCH 65/95] fix unhashable error --- modules/builder.py | 4 ++-- modules/trakt.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index bc5607e3..403627f6 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1029,7 +1029,7 @@ class CollectionBuilder: movie_rating_keys = [] for movie_id in movie_ids: if movie_id in self.library.movie_map: - movie_rating_keys.append(self.library.movie_map[movie_id]) + movie_rating_keys.append(self.library.movie_map[movie_id][0]) elif movie_id not in self.missing_movies: self.missing_movies.append(movie_id) add_rating_keys(movie_rating_keys) @@ -1038,7 +1038,7 @@ class CollectionBuilder: show_rating_keys = [] for show_id in show_ids: if show_id in self.library.show_map: - show_rating_keys.append(self.library.show_map[show_id]) + show_rating_keys.append(self.library.show_map[show_id][0]) elif show_id not in self.missing_shows: self.missing_shows.append(show_id) add_rating_keys(show_rating_keys) diff --git a/modules/trakt.py b/modules/trakt.py index be0b5525..6c8391d3 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -107,7 +107,7 @@ class Trakt: while current <= pages: if pages == 1: response = self.config.get(f"{base_url}{url}", headers=headers) - if "X-Pagination-Page-Count" in response.headers: + if "X-Pagination-Page-Count" in response.headers and "?" not in url: pages = int(response.headers["X-Pagination-Page-Count"]) else: response = self.config.get(f"{base_url}{url}?page={current}", headers=headers) From 01ac6d713e3dcd3a486999a57073d888ce6b40f3 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 14:55:39 -0400 Subject: [PATCH 66/95] add debug --- modules/builder.py | 10 +++++++++- modules/plex.py | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/builder.py b/modules/builder.py index 403627f6..5706e780 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1607,12 +1607,20 @@ class CollectionBuilder: items = self.library.get_collection_items(self.obj, self.smart_label_collection) else: items = [] + logger.info("") + util.separator(f"Items Found for {self.name} Collection", space=False, border=False) + logger.info("") for rk in self.rating_keys: try: - items.append(self.fetch_item(rk)) + item = self.fetch_item(rk) + logger.info(f"{item.title} (Rating Key: {rk})") + items.append(item) except Failed as e: logger.error(e) + logger.info("") + util.separator(f"Updating Details of the Items in {self.name} Collection", space=False, border=False) + logger.info("") overlay = None overlay_folder = None rating_keys = [] diff --git a/modules/plex.py b/modules/plex.py index a5c57216..71363ff7 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -733,6 +733,8 @@ class Plex: _item_tags = [] _add = [f"{t[:1].upper()}{t[1:]}" for t in _add_tags + _sync_tags if t.lower() not in _item_tags] _remove = [t for t in _item_tags if (_sync_tags and t not in _sync_tags) or t in _remove_tags] + logger.debug(_add) + logger.debug(_remove) if _add: updated = True self.query_data(getattr(obj, f"add{attr.capitalize()}"), _add) From 9b18dc3b095c18b221c2041ea36de684c4362dd2 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 14:57:50 -0400 Subject: [PATCH 67/95] mass_trakt_rating_update fix --- plex_meta_manager.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 71dc9f32..8bd11038 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -276,6 +276,20 @@ def mass_metadata(config, library, items): if not tmdb_id and not tvdb_id and library.is_show: tmdb_id = library.get_tvdb_from_map(item) + if library.mass_trakt_rating_update: + try: + if library.is_movie and tmdb_id in trakt_ratings: + new_rating = trakt_ratings[tmdb_id] + elif library.is_show and tvdb_id in trakt_ratings: + new_rating = trakt_ratings[tvdb_id] + else: + raise Failed + if str(item.userRating) != str(new_rating): + library.edit_query(item, {"userRating.value": new_rating, "userRating.locked": 1}) + logger.info(util.adjust_space(f"{item.title[:25]:<25} | User Rating | {new_rating}")) + except Failed: + pass + if library.Radarr and library.radarr_add_all and tmdb_id: radarr_adds.append(tmdb_id) if library.Sonarr and library.sonarr_add_all and tvdb_id: @@ -372,19 +386,6 @@ def mass_metadata(config, library, items): logger.info(util.adjust_space(f"{item.title[:25]:<25} | Critic Rating | {new_rating}")) except Failed: pass - if library.mass_trakt_rating_update: - try: - if library.is_movie and tmdb_id in trakt_ratings: - new_rating = trakt_ratings[tmdb_id] - elif library.is_show and tvdb_id in trakt_ratings: - new_rating = trakt_ratings[tvdb_id] - else: - raise Failed - if str(item.userRating) != str(new_rating): - library.edit_query(item, {"userRating.value": new_rating, "userRating.locked": 1}) - logger.info(util.adjust_space(f"{item.title[:25]:<25} | User Rating | {new_rating}")) - except Failed: - pass if library.Radarr and library.radarr_add_all: try: @@ -498,9 +499,6 @@ def run_collection(config, library, metadata, requested_collections): logger.info("") builder.sort_collection() - logger.info("") - util.separator(f"Updating Details of the Items in {mapping_name} Collection", space=False, border=False) - logger.info("") builder.update_item_details() if builder.run_again and (len(builder.run_again_movies) > 0 or len(builder.run_again_shows) > 0): From ce911b78acc7941222b5567ba61970a23086ff58 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 15:41:59 -0400 Subject: [PATCH 68/95] fixed remove and sync labels --- modules/plex.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/plex.py b/modules/plex.py index 71363ff7..475f7e2d 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -725,8 +725,8 @@ class Plex: key = builder.filter_translation[attr] if attr in builder.filter_translation else attr if add_tags or remove_tags or sync_tags: _add_tags = add_tags if add_tags else [] - _remove_tags = remove_tags if remove_tags else [] - _sync_tags = sync_tags if sync_tags else [] + _remove_tags = [t.lower() for t in remove_tags] if remove_tags else [] + _sync_tags = [t.lower() for t in sync_tags] if sync_tags else [] try: _item_tags = [item_tag.tag.lower() for item_tag in getattr(obj, key)] except BadRequest: @@ -738,11 +738,11 @@ class Plex: if _add: updated = True self.query_data(getattr(obj, f"add{attr.capitalize()}"), _add) - logger.info(f"Detail: {attr.capitalize()} {_add} added") + logger.info(f"Detail: {attr.capitalize()} {util.compile_list(_add)} added") if _remove: updated = True self.query_data(getattr(obj, f"remove{attr.capitalize()}"), _remove) - logger.info(f"Detail: {attr.capitalize()} {_remove} removed") + logger.info(f"Detail: {attr.capitalize()} {util.compile_list(_remove)} removed") return updated def update_item_from_assets(self, item, overlay=None, create=False): From 6920e39d32481f7493636374202972a661195895 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 6 Aug 2021 13:46:13 -0400 Subject: [PATCH 69/95] fixed year filter and test error --- modules/builder.py | 58 ++++++++++++++++++++++++++-------------------- modules/meta.py | 2 +- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 5706e780..f222bfa6 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1061,30 +1061,38 @@ class CollectionBuilder: elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) else: logger.error(f"Collection Error: {method} method not supported") - def tmdb_filter(self, item_id, is_movie, item=None): - filter_missing = False + def check_tmdb_filter(self, item_id, is_movie, item=None): if self.tmdb_filters or self.details["released_missing_only"]: try: if item is None: - item = self.config.TMDb.get_movie(item_id) if is_movie else self.config.TMDb.get_movie(self.config.Convert.tvdb_to_tmdb(item_id)) + item = self.config.TMDb.get_movie(item_id) if is_movie else self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(item_id)) if self.details["released_missing_only"]: if util.validate_date(item.release_date if is_movie else item.first_air_date, "") > self.current_time: - return True + return False for filter_method, filter_data in self.tmdb_filters: - if (filter_method == "original_language" and item.original_language not in filter_data) \ - or (filter_method == "original_language.not" and item.original_language in filter_data) \ - or (filter_method == "tmdb_vote_count.gt" and item.vote_count <= filter_data) \ - or (filter_method == "tmdb_vote_count.gte" and item.vote_count < filter_data) \ - or (filter_method == "tmdb_vote_count.lt" and item.vote_count >= filter_data) \ - or (filter_method == "tmdb_vote_count.lte" and item.vote_count > filter_data) \ - or (filter_method == "year.gt" and item.year <= filter_data) \ - or (filter_method == "year.gte" and item.year < filter_data) \ - or (filter_method == "year.lt" and item.year >= filter_data) \ - or (filter_method == "year.lte" and item.year > filter_data): - return True + filter_attr, modifier, filter_final = self._split(filter_method) + if filter_attr == "original_language": + if (modifier == ".not" and item.original_language in filter_data) \ + or (modifier == "" and item.original_language not in filter_data): + return False + elif modifier in [".gt", ".gte", ".lt", ".lte"]: + attr = None + if filter_attr == "tmdb_vote_count": + attr = item.vote_count + elif filter_attr == "year" and is_movie: + attr = item.year + elif filter_attr == "year" and not is_movie: + air_date = item.first_air_date + if air_date: + attr = util.validate_date(air_date, "Year Filter").year + if attr is None or (modifier == ".gt" and attr <= filter_data) \ + or (modifier == ".gte" and attr < filter_data) \ + or (modifier == ".lt" and attr >= filter_data) \ + or (modifier == ".lte" and attr > filter_data): + return False except Failed: - return True - return filter_missing + return False + return True def build_filter(self, method, plex_filter, smart=False): if smart: @@ -1529,13 +1537,13 @@ class CollectionBuilder: logger.error(e) continue current_title = f"{movie.title} ({util.validate_date(movie.release_date, 'test').year})" if movie.release_date else movie.title - if self.tmdb_filter(missing_id, True, item=movie): - if self.details["show_filtered"] is True and self.details["show_missing"] is True: - logger.info(f"{self.name} Collection | X | {current_title} (TMDb: {missing_id})") - else: + if self.check_tmdb_filter(missing_id, True, item=movie): missing_movies_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: logger.info(f"{self.name} Collection | ? | {current_title} (TMDb: {missing_id})") + else: + if self.details["show_filtered"] is True and self.details["show_missing"] is True: + logger.info(f"{self.name} Collection | X | {current_title} (TMDb: {missing_id})") logger.info("") logger.info(f"{len(missing_movies_with_names)} Movie{'s' if len(missing_movies_with_names) > 1 else ''} Missing") if self.details["save_missing"] is True: @@ -1558,13 +1566,13 @@ class CollectionBuilder: logger.error(e) continue current_title = str(show.title.encode("ascii", "replace").decode()) - if self.tmdb_filter(missing_id, False): - if self.details["show_filtered"] is True and self.details["show_missing"] is True: - logger.info(f"{self.name} Collection | X | {current_title} (TVDb: {missing_id})") - else: + if self.check_tmdb_filter(missing_id, False): missing_shows_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: logger.info(f"{self.name} Collection | ? | {current_title} (TVDB: {missing_id})") + else: + if self.details["show_filtered"] is True and self.details["show_missing"] is True: + logger.info(f"{self.name} Collection | X | {current_title} (TVDb: {missing_id})") logger.info("") logger.info(f"{len(missing_shows_with_names)} Show{'s' if len(missing_shows_with_names) > 1 else ''} Missing") if self.details["save_missing"] is True: diff --git a/modules/meta.py b/modules/meta.py index d15ecf8e..a93632ac 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -75,7 +75,7 @@ class Metadata: logger.info("") for mapping_name, meta in self.metadata.items(): methods = {mm.lower(): mm for mm in meta} - if self.config.test and ("test" not in methods or meta[methods["test"]] is not True): + if self.config.test_mode and ("test" not in methods or meta[methods["test"]] is not True): continue updated = False From d20df5f2585f3ce5b38d07ca77659cbc7a1d5881 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 6 Aug 2021 14:45:15 -0400 Subject: [PATCH 70/95] fix glob error --- modules/plex.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/plex.py b/modules/plex.py index 475f7e2d..b5926b68 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -733,8 +733,6 @@ class Plex: _item_tags = [] _add = [f"{t[:1].upper()}{t[1:]}" for t in _add_tags + _sync_tags if t.lower() not in _item_tags] _remove = [t for t in _item_tags if (_sync_tags and t not in _sync_tags) or t in _remove_tags] - logger.debug(_add) - logger.debug(_remove) if _add: updated = True self.query_data(getattr(obj, f"add{attr.capitalize()}"), _add) @@ -747,6 +745,7 @@ class Plex: def update_item_from_assets(self, item, overlay=None, create=False): name = os.path.basename(os.path.dirname(str(item.locations[0])) if self.is_movie else str(item.locations[0])) + glob_name = name.translate({ord("["): "[[]", ord("]"): "[]]"}) if "[" in name else name logger.debug(name) found_folder = False poster = None @@ -757,7 +756,7 @@ class Plex: if os.path.isdir(os.path.join(ad, name)): item_dir = os.path.join(ad, name) else: - matches = glob.glob(os.path.join(ad, "*", name)) + matches = glob.glob(os.path.join(ad, "*", glob_name)) if len(matches) > 0: item_dir = os.path.abspath(matches[0]) if item_dir is None: @@ -766,8 +765,8 @@ class Plex: poster_filter = os.path.join(item_dir, "poster.*") background_filter = os.path.join(item_dir, "background.*") else: - poster_filter = os.path.join(ad, f"{name}.*") - background_filter = os.path.join(ad, f"{name}_background.*") + poster_filter = os.path.join(ad, f"{glob_name}.*") + background_filter = os.path.join(ad, f"{glob_name}_background.*") matches = glob.glob(poster_filter) if len(matches) > 0: poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_url=False) @@ -781,7 +780,7 @@ class Plex: if item_dir: season_filter = os.path.join(item_dir, f"Season{'0' if season.seasonNumber < 10 else ''}{season.seasonNumber}.*") else: - season_filter = os.path.join(ad, f"{name}_Season{'0' if season.seasonNumber < 10 else ''}{season.seasonNumber}.*") + season_filter = os.path.join(ad, f"{glob_name}_Season{'0' if season.seasonNumber < 10 else ''}{season.seasonNumber}.*") matches = glob.glob(season_filter) if len(matches) > 0: season_poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title} Season {season.seasonNumber}'s ", is_url=False) @@ -790,7 +789,7 @@ class Plex: if item_dir: episode_filter = os.path.join(item_dir, f"{episode.seasonEpisode.upper()}.*") else: - episode_filter = os.path.join(ad, f"{name}_{episode.seasonEpisode.upper()}.*") + episode_filter = os.path.join(ad, f"{glob_name}_{episode.seasonEpisode.upper()}.*") matches = glob.glob(episode_filter) if len(matches) > 0: episode_poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title} {episode.seasonEpisode.upper()}'s ", is_url=False) @@ -808,17 +807,18 @@ class Plex: def find_collection_assets(self, item, name=None, create=False): if name is None: name = item.title + glob_name = name.translate({ord("["): "[[]", ord("]"): "[]]"}) if "[" in name else name for ad in self.asset_directory: poster = None background = None if self.asset_folders: if not os.path.isdir(os.path.join(ad, name)): continue - poster_filter = os.path.join(ad, name, "poster.*") - background_filter = os.path.join(ad, name, "background.*") + poster_filter = os.path.join(ad, glob_name, "poster.*") + background_filter = os.path.join(ad, glob_name, "background.*") else: - poster_filter = os.path.join(ad, f"{name}.*") - background_filter = os.path.join(ad, f"{name}_background.*") + poster_filter = os.path.join(ad, f"{glob_name}.*") + background_filter = os.path.join(ad, f"{glob_name}_background.*") matches = glob.glob(poster_filter) if len(matches) > 0: poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_url=False) From 77fbad70292cc58641940c82b617591215fe692f Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 6 Aug 2021 19:02:33 -0400 Subject: [PATCH 71/95] cleaned up filters --- modules/builder.py | 89 +++++++++++++--------------------------------- modules/util.py | 36 ++++++++++++++++++- 2 files changed, 60 insertions(+), 65 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index f222bfa6..cd438ed8 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1061,12 +1061,12 @@ class CollectionBuilder: elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) else: logger.error(f"Collection Error: {method} method not supported") - def check_tmdb_filter(self, item_id, is_movie, item=None): - if self.tmdb_filters or self.details["released_missing_only"]: + def check_tmdb_filter(self, item_id, is_movie, item=None, check_released=False): + if self.tmdb_filters or check_released: try: if item is None: item = self.config.TMDb.get_movie(item_id) if is_movie else self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(item_id)) - if self.details["released_missing_only"]: + if check_released: if util.validate_date(item.release_date if is_movie else item.first_air_date, "") > self.current_time: return False for filter_method, filter_data in self.tmdb_filters: @@ -1075,6 +1075,12 @@ class CollectionBuilder: if (modifier == ".not" and item.original_language in filter_data) \ or (modifier == "" and item.original_language not in filter_data): return False + elif filter_attr in ["last_episode_aired"]: + tmdb_date = None + if filter_attr == "last_episode_aired": + tmdb_date = item.last_air_date + if not util.date_filter(tmdb_date, modifier, filter_data, filter_final, self.current_time): + return False elif modifier in [".gt", ".gte", ".lt", ".lte"]: attr = None if filter_attr == "tmdb_vote_count": @@ -1085,10 +1091,7 @@ class CollectionBuilder: air_date = item.first_air_date if air_date: attr = util.validate_date(air_date, "Year Filter").year - if attr is None or (modifier == ".gt" and attr <= filter_data) \ - or (modifier == ".gte" and attr < filter_data) \ - or (modifier == ".lt" and attr >= filter_data) \ - or (modifier == ".lte" and attr > filter_data): + if util.number_filter(attr, modifier, filter_data): return False except Failed: return False @@ -1386,46 +1389,22 @@ class CollectionBuilder: filter_attr, modifier, filter_final = self._split(filter_method) filter_actual = filter_translation[filter_attr] if filter_attr in filter_translation else filter_attr if filter_attr in ["tmdb_vote_count", "original_language", "last_episode_aired"]: - if (self.library.is_movie and current.ratingKey not in self.library.movie_rating_key_map) \ - or (self.library.is_show and current.ratingKey not in self.library.show_rating_key_map): + if current.ratingKey not in self.library.movie_rating_key_map and current.ratingKey not in self.library.show_rating_key_map: logger.warning(f"Filter Error: No {'TMDb' if self.library.is_movie else 'TVDb'} ID found for {current.title}") return False try: - if self.library.is_movie: - tmdb_item = self.config.TMDb.get_movie(self.library.movie_rating_key_map[current.ratingKey]) + if current.ratingKey in self.library.movie_rating_key_map: + t_id = self.library.movie_rating_key_map[current.ratingKey] else: - tmdb_item = self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(self.library.show_rating_key_map[current.ratingKey])) + t_id = self.library.show_rating_key_map[current.ratingKey] except Failed as e: logger.error(e) return False - else: - tmdb_item = None - if filter_attr in ["release", "added", "last_played", "last_episode_aired"] and modifier != ".regex": - if filter_attr == "last_episode_aired": - if tmdb_item.last_air_date is None: - return False - current_data = util.validate_date(tmdb_item.last_air_date, "TMDB Last Air Date") - else: - current_data = getattr(current, filter_actual) - if current_data is None: + if not self.check_tmdb_filter(t_id, current.ratingKey in self.library.movie_rating_key_map): + return False + if filter_attr in ["release", "added", "last_played"]: + if not util.date_filter(getattr(current, filter_actual), modifier, filter_data, filter_final, self.current_time): return False - if modifier in ["", ".not"]: - threshold_date = self.current_time - timedelta(days=filter_data) - if (modifier == "" and (current_data is None or current_data < threshold_date)) \ - or (modifier == ".not" and current_data and current_data >= threshold_date): - return False - elif modifier in [".before", ".after"]: - filter_date = util.validate_date(filter_data, filter_final) - if (modifier == ".before" and current_data >= filter_date) or (modifier == ".after" and current_data <= filter_date): - return False - elif modifier == ".regex": - jailbreak = False - for check_data in filter_data: - if re.compile(check_data).match(current_data.strftime("%m/%d/%Y")): - jailbreak = True - break - if not jailbreak: - return False elif filter_attr == "audio_track_title": jailbreak = False for media in current.media: @@ -1433,10 +1412,7 @@ class CollectionBuilder: for audio in part.audioStreams(): for check_title in filter_data: title = audio.title if audio.title else "" - if (modifier in ["", ".not"] and check_title.lower() in title.lower()) \ - or (modifier == ".begins" and title.lower().startswith(check_title.lower())) \ - or (modifier == ".ends" and title.lower().endswith(check_title.lower())) \ - or (modifier == ".regex" and re.compile(check_title).match(title)): + if util.string_filter(title, modifier, check_title): jailbreak = True break if jailbreak: break @@ -1448,10 +1424,7 @@ class CollectionBuilder: jailbreak = False for location in current.locations: for check_text in filter_data: - if (modifier in ["", ".not"] and check_text.lower() in location.lower()) \ - or (modifier == ".begins" and location.lower().startswith(check_text.lower())) \ - or (modifier == ".ends" and location.lower().endswith(check_text.lower())) \ - or (modifier == ".regex" and re.compile(check_text).match(location)): + if util.string_filter(location, modifier, check_text): jailbreak = True break if jailbreak: break @@ -1461,10 +1434,7 @@ class CollectionBuilder: jailbreak = False current_data = getattr(current, filter_actual) for check_data in filter_data: - if (modifier in ["", ".not"] and check_data.lower() in current_data.lower()) \ - or (modifier == ".begins" and current_data.lower().startswith(check_data.lower())) \ - or (modifier == ".ends" and current_data.lower().endswith(check_data.lower())) \ - or (modifier == ".regex" and re.compile(check_data).match(current_data)): + if util.string_filter(current_data, modifier, check_data): jailbreak = True break if (jailbreak and modifier == ".not") or (not jailbreak and modifier in ["", ".begins", ".ends", ".regex"]): @@ -1487,21 +1457,12 @@ class CollectionBuilder: date_match = True if date_match is False: return False - elif filter_attr == "original_language": - if (modifier == ".not" and tmdb_item.original_language in filter_data) \ - or (modifier == "" and tmdb_item.original_language not in filter_data): - return False elif modifier in [".gt", ".gte", ".lt", ".lte"]: - if filter_attr == "tmdb_vote_count": - attr = tmdb_item.vote_count - elif filter_attr == "duration": + if filter_attr == "duration": attr = getattr(current, filter_actual) / 60000 else: attr = getattr(current, filter_actual) - if attr is None or (modifier == ".gt" and attr <= filter_data) \ - or (modifier == ".gte" and attr < filter_data) \ - or (modifier == ".lt" and attr >= filter_data) \ - or (modifier == ".lte" and attr > filter_data): + if util.number_filter(attr, modifier, filter_data): return False else: attrs = [] @@ -1537,7 +1498,7 @@ class CollectionBuilder: logger.error(e) continue current_title = f"{movie.title} ({util.validate_date(movie.release_date, 'test').year})" if movie.release_date else movie.title - if self.check_tmdb_filter(missing_id, True, item=movie): + if self.check_tmdb_filter(missing_id, True, item=movie, check_released=self.details["released_missing_only"]): missing_movies_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: logger.info(f"{self.name} Collection | ? | {current_title} (TMDb: {missing_id})") @@ -1566,7 +1527,7 @@ class CollectionBuilder: logger.error(e) continue current_title = str(show.title.encode("ascii", "replace").decode()) - if self.check_tmdb_filter(missing_id, False): + if self.check_tmdb_filter(missing_id, False, check_released=self.details["released_missing_only"]): missing_shows_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: logger.info(f"{self.name} Collection | ? | {current_title} (TVDB: {missing_id})") diff --git a/modules/util.py b/modules/util.py index d2347cd1..274967ea 100644 --- a/modules/util.py +++ b/modules/util.py @@ -1,5 +1,5 @@ import logging, os, re, signal, sys, time, traceback -from datetime import datetime +from datetime import datetime, timedelta from pathvalidate import is_valid_filename, sanitize_filename from plexapi.exceptions import BadRequest, NotFound, Unauthorized @@ -246,6 +246,40 @@ def is_locked(filepath): file_object.close() return locked +def date_filter(current, modifier, data, final, current_time): + if current is None: + return False + if modifier in ["", ".not"]: + threshold_date = current_time - timedelta(days=data) + if (modifier == "" and (current is None or current < threshold_date)) \ + or (modifier == ".not" and current and current >= threshold_date): + return False + elif modifier in [".before", ".after"]: + filter_date = validate_date(data, final) + if (modifier == ".before" and current >= filter_date) or (modifier == ".after" and current <= filter_date): + return False + elif modifier == ".regex": + jailbreak = False + for check_data in data: + if re.compile(check_data).match(current.strftime("%m/%d/%Y")): + jailbreak = True + break + if not jailbreak: + return False + return True + +def number_filter(current, modifier, data): + return current is None or (modifier == ".gt" and current <= data) \ + or (modifier == ".gte" and current < data) \ + or (modifier == ".lt" and current >= data) \ + or (modifier == ".lte" and current > data) + +def string_filter(current, modifier, data): + return (modifier in ["", ".not"] and data.lower() in current.lower()) \ + or (modifier == ".begins" and current.lower().startswith(data.lower())) \ + or (modifier == ".ends" and current.lower().endswith(data.lower())) \ + or (modifier == ".regex" and re.compile(data).match(current)) + def parse(attribute, data, datatype=None, methods=None, parent=None, default=None, options=None, translation=None, minimum=1, maximum=None): display = f"{parent + ' ' if parent else ''}{attribute} attribute" if options is None and translation is not None: From 9dd7c7910d3347c38fcc5156c867a854fba67b21 Mon Sep 17 00:00:00 2001 From: James Hu Date: Fri, 6 Aug 2021 16:17:19 -0700 Subject: [PATCH 72/95] rebase --- modules/builder.py | 5 +++-- modules/plex.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index cd438ed8..2d31a910 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -95,6 +95,7 @@ all_filters = [ "release", "release.not", "release.before", "release.after", "release.regex", "history", "added", "added.not", "added.before", "added.after", "added.regex", "last_played", "last_played.not", "last_played.before", "last_played.after", "last_played.regex", + "first_episode_aired", "first_episode_aired.not", "first_episode_aired.before", "first_episode_aired.after", "first_episode_aired.regex", "last_episode_aired", "last_episode_aired.not", "last_episode_aired.before", "last_episode_aired.after", "last_episode_aired.regex", "title", "title.not", "title.begins", "title.ends", "title.regex", "plays.gt", "plays.gte", "plays.lt", "plays.lte", @@ -122,7 +123,7 @@ movie_only_filters = [ "resolution", "resolution.not", "writer", "writer.not" ] -show_only_filters = ["last_episode_aired", "network"] +show_only_filters = ["first_episode_aired", "last_episode_aired", "network"] smart_invalid = ["collection_order"] smart_url_invalid = ["filters", "run_again", "sync_mode", "show_filtered", "show_missing", "save_missing", "smart_label"] + radarr_details + sonarr_details custom_sort_builders = [ @@ -1388,7 +1389,7 @@ class CollectionBuilder: for filter_method, filter_data in self.filters: filter_attr, modifier, filter_final = self._split(filter_method) filter_actual = filter_translation[filter_attr] if filter_attr in filter_translation else filter_attr - if filter_attr in ["tmdb_vote_count", "original_language", "last_episode_aired"]: + if filter_attr in ["tmdb_vote_count", "original_language", "first_episode_aired", "last_episode_aired"]: if current.ratingKey not in self.library.movie_rating_key_map and current.ratingKey not in self.library.show_rating_key_map: logger.warning(f"Filter Error: No {'TMDb' if self.library.is_movie else 'TVDb'} ID found for {current.title}") return False diff --git a/modules/plex.py b/modules/plex.py index b5926b68..a876b210 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -142,7 +142,7 @@ boolean_attributes = [ "unplayed_episodes", "episode_unplayed", "episode_duplicate", "episode_progress", "episode_unmatched", ] tmdb_attributes = ["actor", "director", "producer", "writer"] -date_attributes = ["added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played", "last_episode_aired"] +date_attributes = ["added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played", "first_episode_aired", "last_episode_aired"] search_display = {"added": "Date Added", "release": "Release Date", "hdr": "HDR", "progress": "In Progress", "episode_progress": "Episode In Progress"} sorts = { None: None, From d4e6b1ae2db101d5afcc06828511ace968b24d22 Mon Sep 17 00:00:00 2001 From: James Hu Date: Fri, 6 Aug 2021 16:19:18 -0700 Subject: [PATCH 73/95] try this --- modules/builder.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 2d31a910..fa977977 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1076,9 +1076,11 @@ class CollectionBuilder: if (modifier == ".not" and item.original_language in filter_data) \ or (modifier == "" and item.original_language not in filter_data): return False - elif filter_attr in ["last_episode_aired"]: + elif filter_attr in ["first_episode_aired", "last_episode_aired"]: tmdb_date = None - if filter_attr == "last_episode_aired": + if filter_attr == "first_episode_aired": + tmdb_date = item.first_air_date + elif filter_attr == "last_episode_aired": tmdb_date = item.last_air_date if not util.date_filter(tmdb_date, modifier, filter_data, filter_final, self.current_time): return False From 0d539e10c1fe7c149edadf75f66b91169bb28e4b Mon Sep 17 00:00:00 2001 From: James Hu Date: Fri, 6 Aug 2021 16:37:11 -0700 Subject: [PATCH 74/95] Fix conditional --- modules/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/builder.py b/modules/builder.py index fa977977..b1dfa546 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1405,7 +1405,7 @@ class CollectionBuilder: return False if not self.check_tmdb_filter(t_id, current.ratingKey in self.library.movie_rating_key_map): return False - if filter_attr in ["release", "added", "last_played"]: + elif filter_attr in ["release", "added", "last_played"]: if not util.date_filter(getattr(current, filter_actual), modifier, filter_data, filter_final, self.current_time): return False elif filter_attr == "audio_track_title": From eb22e6c832182eafec2b01ae2ce262b517bd4f21 Mon Sep 17 00:00:00 2001 From: James Hu Date: Fri, 6 Aug 2021 17:38:06 -0700 Subject: [PATCH 75/95] Add to number attributes --- modules/plex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/plex.py b/modules/plex.py index a876b210..89957132 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -133,7 +133,7 @@ show_only_searches = [ "episode_year", "episode_year.not", "episode_year.gt", "episode_year.gte", "episode_year.lt", "episode_year.lte" ] number_attributes = [ - "plays", "episode_plays", "duration", "tmdb_vote_count", "last_episode_aired", + "plays", "episode_plays", "duration", "tmdb_vote_count", "first_episode_aired", "last_episode_aired", "added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played" ] float_attributes = ["user_rating", "episode_user_rating", "critic_rating", "audience_rating"] From 382c156b66656a3b4dbbf830c2b64ec3c2b7194f Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sat, 7 Aug 2021 02:01:21 -0400 Subject: [PATCH 76/95] reorg --- modules/anidb.py | 7 +- modules/anilist.py | 7 +- modules/builder.py | 297 +++++++++++++++++++++------------------- modules/cache.py | 37 +++-- modules/convert.py | 141 ++++++++++--------- modules/icheckmovies.py | 18 +-- modules/imdb.py | 28 +--- modules/letterboxd.py | 12 +- modules/mal.py | 7 +- modules/plex.py | 17 ++- modules/stevenlu.py | 16 +-- modules/tautulli.py | 4 +- modules/tmdb.py | 106 +++++++------- modules/trakt.py | 66 +++++---- modules/tvdb.py | 58 ++++---- modules/util.py | 55 ++++---- plex_meta_manager.py | 6 +- 17 files changed, 443 insertions(+), 439 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 06aa4748..315a9cd8 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -72,7 +72,7 @@ class AniDB: current_url = f"{base_url}{next_page_list[0]}" return anidb_ids[:limit] - def get_items(self, method, data, language): + def get_anidb_ids(self, method, data, language): anidb_ids = [] if method == "anidb_popular": logger.info(f"Processing AniDB Popular: {data} Anime") @@ -88,9 +88,6 @@ class AniDB: anidb_ids.extend(self._relations(data, language)) else: raise Failed(f"AniDB Error: Method {method} not supported") - movie_ids, show_ids = self.config.Convert.anidb_to_ids(anidb_ids) logger.debug("") logger.debug(f"{len(anidb_ids)} AniDB IDs Found: {anidb_ids}") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids + return anidb_ids diff --git a/modules/anilist.py b/modules/anilist.py index bdefc1fc..ef001832 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -207,7 +207,7 @@ class AniList: return anilist_values raise Failed(f"AniList Error: No valid AniList IDs in {anilist_ids}") - def get_items(self, method, data): + def get_anilist_ids(self, method, data): if method == "anilist_id": logger.info(f"Processing AniList ID: {data}") anilist_id, name = self._validate(data) @@ -235,9 +235,6 @@ class AniList: logger.info(f"Processing AniList Relations: ({data}) {name} ({len(anilist_ids)} Anime)") else: raise Failed(f"AniList Error: Method {method} not supported") - movie_ids, show_ids = self.config.Convert.anilist_to_ids(anilist_ids) logger.debug("") logger.debug(f"{len(anilist_ids)} AniList IDs Found: {anilist_ids}") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids + return anilist_ids diff --git a/modules/builder.py b/modules/builder.py index b1dfa546..6aeaf209 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -164,6 +164,7 @@ class CollectionBuilder: self.filters = [] self.tmdb_filters = [] self.rating_keys = [] + self.filtered_keys = [] self.run_again_movies = [] self.run_again_shows = [] self.posters = {} @@ -992,113 +993,112 @@ class CollectionBuilder: else: logger.error(message) - def collect_rating_keys(self): - filtered_keys = {} - name = self.obj.title if self.obj else self.name - def add_rating_keys(keys): - if not isinstance(keys, list): - keys = [keys] - total = len(keys) - max_length = len(str(total)) - if self.filters and self.details["show_filtered"] is True: - logger.info("") - logger.info("Filtering Builder:") - for i, key in enumerate(keys, 1): - if key not in self.rating_keys: - if key in filtered_keys: - if self.details["show_filtered"] is True: - logger.info(f"{name} Collection | X | {filtered_keys[key]}") - else: - try: - current = self.fetch_item(key) - except Failed as e: - logger.error(e) - continue - current_title = f"{current.title} ({current.year})" if current.year else current.title - if self.check_filters(current, f"{(' ' * (max_length - len(str(i))))}{i}/{total}"): - self.rating_keys.append(key) - else: - if key not in filtered_keys: - filtered_keys[key] = current_title - if self.details["show_filtered"] is True: - logger.info(f"{name} Collection | X | {current_title}") - def check_map(input_ids): - movie_ids, show_ids = input_ids - items_found_inside = 0 - if len(movie_ids) > 0: - items_found_inside += len(movie_ids) - movie_rating_keys = [] - for movie_id in movie_ids: - if movie_id in self.library.movie_map: - movie_rating_keys.append(self.library.movie_map[movie_id][0]) - elif movie_id not in self.missing_movies: - self.missing_movies.append(movie_id) - add_rating_keys(movie_rating_keys) - if len(show_ids) > 0: - items_found_inside += len(show_ids) - show_rating_keys = [] - for show_id in show_ids: - if show_id in self.library.show_map: - show_rating_keys.append(self.library.show_map[show_id][0]) - elif show_id not in self.missing_shows: - self.missing_shows.append(show_id) - add_rating_keys(show_rating_keys) - return items_found_inside + def find_rating_keys(self): for method, value in self.builders: + ids = [] + rating_keys = [] logger.debug("") logger.debug(f"Builder: {method}: {value}") logger.info("") - if "plex" in method: add_rating_keys(self.library.get_items(method, value)) - elif "tautulli" in method: add_rating_keys(self.library.Tautulli.get_items(self.library, value)) - elif "anidb" in method: check_map(self.config.AniDB.get_items(method, value, self.language)) - elif "anilist" in method: check_map(self.config.AniList.get_items(method, value)) - elif "mal" in method: check_map(self.config.MyAnimeList.get_items(method, value)) - elif "tvdb" in method: check_map(self.config.TVDb.get_items(method, value, self.language)) - elif "imdb" in method: check_map(self.config.IMDb.get_items(method, value, self.language, self.library.is_movie)) - elif "icheckmovies" in method: check_map(self.config.ICheckMovies.get_items(method, value, self.language)) - elif "letterboxd" in method: check_map(self.config.Letterboxd.get_items(method, value, self.language)) - elif "stevenlu" in method: check_map(self.config.StevenLu.get_items(method)) - elif "tmdb" in method: check_map(self.config.TMDb.get_items(method, value, self.library.is_movie)) - elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) - else: logger.error(f"Collection Error: {method} method not supported") - - def check_tmdb_filter(self, item_id, is_movie, item=None, check_released=False): - if self.tmdb_filters or check_released: - try: - if item is None: - item = self.config.TMDb.get_movie(item_id) if is_movie else self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(item_id)) - if check_released: - if util.validate_date(item.release_date if is_movie else item.first_air_date, "") > self.current_time: - return False - for filter_method, filter_data in self.tmdb_filters: - filter_attr, modifier, filter_final = self._split(filter_method) - if filter_attr == "original_language": - if (modifier == ".not" and item.original_language in filter_data) \ - or (modifier == "" and item.original_language not in filter_data): - return False - elif filter_attr in ["first_episode_aired", "last_episode_aired"]: - tmdb_date = None - if filter_attr == "first_episode_aired": - tmdb_date = item.first_air_date - elif filter_attr == "last_episode_aired": - tmdb_date = item.last_air_date - if not util.date_filter(tmdb_date, modifier, filter_data, filter_final, self.current_time): - return False - elif modifier in [".gt", ".gte", ".lt", ".lte"]: - attr = None - if filter_attr == "tmdb_vote_count": - attr = item.vote_count - elif filter_attr == "year" and is_movie: - attr = item.year - elif filter_attr == "year" and not is_movie: - air_date = item.first_air_date - if air_date: - attr = util.validate_date(air_date, "Year Filter").year - if util.number_filter(attr, modifier, filter_data): - return False - except Failed: - return False - return True + if "plex" in method: + rating_keys = self.library.get_rating_keys(method, value) + elif "tautulli" in method: + rating_keys = self.library.Tautulli.get_rating_keys(self.library, value) + elif "anidb" in method: + anidb_ids = self.config.AniDB.get_anidb_ids(method, value, self.language) + ids = self.config.Convert.anidb_to_ids(anidb_ids) + elif "anilist" in method: + anilist_ids = self.config.AniList.get_anilist_ids(method, value) + ids = self.config.Convert.anilist_to_ids(anilist_ids) + elif "mal" in method: + mal_ids = self.config.MyAnimeList.get_mal_ids(method, value) + ids = self.config.Convert.myanimelist_to_ids(mal_ids) + elif "tvdb" in method: + ids = self.config.TVDb.get_tvdb_ids(method, value, self.language) + elif "imdb" in method: + ids = self.config.IMDb.get_imdb_ids(method, value, self.language) + elif "icheckmovies" in method: + ids = self.config.ICheckMovies.get_icheckmovies_ids(method, value, self.language) + elif "letterboxd" in method: + ids = self.config.Letterboxd.get_tmdb_ids(method, value, self.language) + elif "stevenlu" in method: + ids = self.config.StevenLu.get_stevenlu_ids(method) + elif "tmdb" in method: + ids = self.config.TMDb.get_tmdb_ids(method, value, self.library.is_movie) + elif "trakt" in method: + ids = self.config.Trakt.get_trakt_ids(method, value, self.library.is_movie) + else: + logger.error(f"Collection Error: {method} method not supported") + + if len(ids) > 0: + logger.debug("") + logger.debug(f"{len(ids)} IDs Found: {ids}") + total_ids = len(ids) + if total_ids > 0: + for i, input_data in enumerate(ids, 1): + input_id, id_type = input_data + util.print_return(f"Parsing ID {i}/{total_ids}") + if id_type == "tmdb": + if input_id in self.library.movie_map: + rating_keys.append(self.library.movie_map[input_id][0]) + elif input_id not in self.missing_movies: + self.missing_movies.append(input_id) + elif id_type in ["tvdb", "tmdb_show"]: + if id_type == "tmdb_show": + try: + input_id = self.config.Convert.tmdb_to_tvdb(input_id, fail=True) + except Failed as e: + logger.error(e) + continue + if input_id in self.library.show_map: + rating_keys.append(self.library.show_map[input_id][0]) + elif input_id not in self.missing_shows: + self.missing_shows.append(input_id) + elif id_type == "imdb": + if input_id in self.library.imdb_map: + rating_keys.append(self.library.imdb_map[input_id][0]) + else: + try: + tmdb_id, tmdb_type = self.config.Convert.imdb_to_tmdb(input_id) + if tmdb_type == "movie": + if tmdb_id not in self.missing_movies: + self.missing_movies.append(tmdb_id) + else: + tvdb_id = self.config.Convert.tmdb_to_tvdb(tmdb_id) + if tvdb_id not in self.missing_shows: + self.missing_shows.append(tvdb_id) + except Failed as e: + logger.error(e) + continue + util.print_end() + + if len(rating_keys) > 0: + name = self.obj.title if self.obj else self.name + if not isinstance(rating_keys, list): + rating_keys = [rating_keys] + total = len(rating_keys) + max_length = len(str(total)) + if self.filters and self.details["show_filtered"] is True: + logger.info("") + logger.info("Filtering Builder:") + for i, key in enumerate(rating_keys, 1): + if key not in self.rating_keys: + if key in self.filtered_keys: + if self.details["show_filtered"] is True: + logger.info(f"{name} Collection | X | {self.filtered_keys[key]}") + else: + try: + current = self.fetch_item(key) + except Failed as e: + logger.error(e) + continue + current_title = f"{current.title} ({current.year})" if current.year else current.title + if self.check_filters(current, f"{(' ' * (max_length - len(str(i))))}{i}/{total}"): + self.rating_keys.append(key) + else: + self.filtered_keys[key] = current_title + if self.details["show_filtered"] is True: + logger.info(f"{name} Collection | X | {current_title}") def build_filter(self, method, plex_filter, smart=False): if smart: @@ -1385,6 +1385,44 @@ class CollectionBuilder: logger.info("") logger.info(f"{total} {media_type} Processed") + def check_tmdb_filter(self, item_id, is_movie, item=None, check_released=False): + if self.tmdb_filters or check_released: + try: + if item is None: + item = self.config.TMDb.get_movie(item_id) if is_movie else self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(item_id)) + if check_released: + if util.validate_date(item.release_date if is_movie else item.first_air_date, "") > self.current_time: + return False + for filter_method, filter_data in self.tmdb_filters: + filter_attr, modifier, filter_final = self._split(filter_method) + if filter_attr == "original_language": + if (modifier == ".not" and item.original_language in filter_data) \ + or (modifier == "" and item.original_language not in filter_data): + return False + elif filter_attr in ["first_episode_aired", "last_episode_aired"]: + tmdb_date = None + if filter_attr == "first_episode_aired": + tmdb_date = util.validate_date(item.first_air_date, "TMDB First Air Date") + elif filter_attr == "last_episode_aired": + tmdb_date = util.validate_date(item.last_air_date, "TMDB Last Air Date") + if util.is_date_filter(tmdb_date, modifier, filter_data, filter_final, self.current_time): + return False + elif modifier in [".gt", ".gte", ".lt", ".lte"]: + attr = None + if filter_attr == "tmdb_vote_count": + attr = item.vote_count + elif filter_attr == "year" and is_movie: + attr = item.year + elif filter_attr == "year" and not is_movie: + air_date = item.first_air_date + if air_date: + attr = util.validate_date(air_date, "Year Filter").year + if util.is_number_filter(attr, modifier, filter_data): + return False + except Failed: + return False + return True + def check_filters(self, current, display): if self.filters: util.print_return(f"Filtering {display} {current.title}") @@ -1406,41 +1444,19 @@ class CollectionBuilder: if not self.check_tmdb_filter(t_id, current.ratingKey in self.library.movie_rating_key_map): return False elif filter_attr in ["release", "added", "last_played"]: - if not util.date_filter(getattr(current, filter_actual), modifier, filter_data, filter_final, self.current_time): + if util.is_date_filter(getattr(current, filter_actual), modifier, filter_data, filter_final, self.current_time): return False - elif filter_attr == "audio_track_title": - jailbreak = False - for media in current.media: - for part in media.parts: - for audio in part.audioStreams(): - for check_title in filter_data: - title = audio.title if audio.title else "" - if util.string_filter(title, modifier, check_title): - jailbreak = True - break - if jailbreak: break - if jailbreak: break - if jailbreak: break - if (jailbreak and modifier == ".not") or (not jailbreak and modifier in ["", ".begins", ".ends", ".regex"]): - return False - elif filter_attr == "filepath": - jailbreak = False - for location in current.locations: - for check_text in filter_data: - if util.string_filter(location, modifier, check_text): - jailbreak = True - break - if jailbreak: break - if (jailbreak and modifier == ".not") or (not jailbreak and modifier in ["", ".begins", ".ends", ".regex"]): - return False - elif filter_attr in ["title", "studio"]: - jailbreak = False - current_data = getattr(current, filter_actual) - for check_data in filter_data: - if util.string_filter(current_data, modifier, check_data): - jailbreak = True - break - if (jailbreak and modifier == ".not") or (not jailbreak and modifier in ["", ".begins", ".ends", ".regex"]): + elif filter_attr in ["audio_track_title", "filepath", "title", "studio"]: + values = [] + if filter_attr == "audio_track_title": + for media in current.media: + for part in media.parts: + values.extend([a.title for a in part.audioStreams() if a.title]) + elif filter_attr == "filepath": + values = [loc for loc in current.locations] + elif filter_attr in ["title", "studio"]: + values = [getattr(current, filter_actual)] + if util.is_string_filter(values, modifier, filter_data): return False elif filter_attr == "history": item_date = current.originallyAvailableAt @@ -1461,11 +1477,8 @@ class CollectionBuilder: if date_match is False: return False elif modifier in [".gt", ".gte", ".lt", ".lte"]: - if filter_attr == "duration": - attr = getattr(current, filter_actual) / 60000 - else: - attr = getattr(current, filter_actual) - if util.number_filter(attr, modifier, filter_data): + divider = 60000 if filter_attr == "duration" else 1 + if util.is_number_filter(getattr(current, filter_actual) / divider, modifier, filter_data): return False else: attrs = [] diff --git a/modules/cache.py b/modules/cache.py index 21a03d51..398bfbd5 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -18,14 +18,16 @@ class Cache: else: logger.info(f"Using cache database at {self.cache_path}") cursor.execute("DROP TABLE IF EXISTS guids") + cursor.execute("DROP TABLE IF EXISTS guid_map") cursor.execute("DROP TABLE IF EXISTS imdb_to_tvdb_map") cursor.execute("DROP TABLE IF EXISTS tmdb_to_tvdb_map") cursor.execute("DROP TABLE IF EXISTS imdb_map") cursor.execute( - """CREATE TABLE IF NOT EXISTS guid_map ( + """CREATE TABLE IF NOT EXISTS guids_map ( key INTEGER PRIMARY KEY, plex_guid TEXT UNIQUE, t_id TEXT, + imdb_id TEXT, media_type TEXT, expiration_date TEXT)""" ) @@ -100,27 +102,39 @@ class Cache: def query_guid_map(self, plex_guid): id_to_return = None + imdb_id = None media_type = None expired = None with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: - cursor.execute(f"SELECT * FROM guid_map WHERE plex_guid = ?", (plex_guid,)) + cursor.execute(f"SELECT * FROM guids_map WHERE plex_guid = ?", (plex_guid,)) row = cursor.fetchone() if row: time_between_insertion = datetime.now() - datetime.strptime(row["expiration_date"], "%Y-%m-%d") id_to_return = util.get_list(row["t_id"], int_list=True) + imdb_id = util.get_list(row["imdb_id"]) media_type = row["media_type"] expired = time_between_insertion.days > self.expiration - return id_to_return, media_type, expired + return id_to_return, imdb_id, media_type, expired - def update_guid_map(self, media_type, plex_guid, t_id, expired): - self._update_map("guid_map", "plex_guid", plex_guid, "t_id", t_id, expired, media_type=media_type) + def update_guid_map(self, plex_guid, t_id, imdb_id, expired, media_type): + expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, self.expiration))) + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute(f"INSERT OR IGNORE INTO guids_map(plex_guid) VALUES(?)", (plex_guid,)) + if media_type is None: + sql = f"UPDATE guids_map SET t_id = ?, imdb_id = ?, expiration_date = ? WHERE plex_guid = ?" + cursor.execute(sql, (t_id, imdb_id, expiration_date.strftime("%Y-%m-%d"), plex_guid)) + else: + sql = f"UPDATE guids_map SET t_id = ?, imdb_id = ?, expiration_date = ?, media_type = ? WHERE plex_guid = ?" + cursor.execute(sql, (t_id, imdb_id, expiration_date.strftime("%Y-%m-%d"), media_type, plex_guid)) - def query_imdb_to_tmdb_map(self, media_type, _id, imdb=True): + def query_imdb_to_tmdb_map(self, _id, imdb=True, media_type=None, return_type=False): from_id = "imdb_id" if imdb else "tmdb_id" to_id = "tmdb_id" if imdb else "imdb_id" - return self._query_map("imdb_to_tmdb_map", _id, from_id, to_id, media_type=media_type) + return self._query_map("imdb_to_tmdb_map", _id, from_id, to_id, media_type=media_type, return_type=return_type) def update_imdb_to_tmdb_map(self, media_type, expired, imdb_id, tmdb_id): self._update_map("imdb_to_tmdb_map", "imdb_id", imdb_id, "tmdb_id", tmdb_id, expired, media_type=media_type) @@ -147,9 +161,10 @@ class Cache: def update_letterboxd_map(self, expired, letterboxd_id, tmdb_id): self._update_map("letterboxd_map", "letterboxd_id", letterboxd_id, "tmdb_id", tmdb_id, expired) - def _query_map(self, map_name, _id, from_id, to_id, media_type=None): + def _query_map(self, map_name, _id, from_id, to_id, media_type=None, return_type=False): id_to_return = None expired = None + out_type = None with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: @@ -163,7 +178,11 @@ class Cache: time_between_insertion = datetime.now() - datetime_object id_to_return = row[to_id] if to_id == "imdb_id" else int(row[to_id]) expired = time_between_insertion.days > self.expiration - return id_to_return, expired + out_type = row["media_type"] if return_type else None + if out_type: + return id_to_return, out_type, expired + else: + return id_to_return, expired def _update_map(self, map_name, val1_name, val1, val2_name, val2, expired, media_type=None): expiration_date = datetime.now() if expired is True else (datetime.now() - timedelta(days=random.randint(1, self.expiration))) diff --git a/modules/convert.py b/modules/convert.py index 7cdd3680..d0395f40 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -21,8 +21,8 @@ class Convert: imdb_ids = util.get_list(imdbid[0]) tmdb_ids = [] for imdb in imdb_ids: - tmdb_id = self.imdb_to_tmdb(imdb) - if tmdb_id: + tmdb_id, tmdb_type = self.imdb_to_tmdb(imdb) + if tmdb_id and tmdb_type == "movie": tmdb_ids.append(tmdb_id) if tmdb_ids: return None, imdb_ids, tmdb_ids @@ -78,18 +78,17 @@ class Convert: return converted_ids def anidb_to_ids(self, anidb_list): - show_ids = [] - movie_ids = [] + ids = [] for anidb_id in anidb_list: try: tvdb_id, _, tmdb_ids = self._anidb(anidb_id, fail=True) if tvdb_id: - show_ids.append(tvdb_id) + ids.append((tvdb_id, "tvdb")) if tmdb_ids: - movie_ids.extend(tmdb_ids) + ids.extend((tmdb_ids, "tmdb")) except Failed as e: logger.error(e) - return movie_ids, show_ids + return ids def anilist_to_ids(self, anilist_ids): anidb_ids = [] @@ -113,43 +112,40 @@ class Convert: media_type = "movie" if is_movie else "show" expired = False if self.config.Cache and is_movie: - cache_id, expired = self.config.Cache.query_imdb_to_tmdb_map(media_type, tmdb_id, imdb=False) + cache_id, expired = self.config.Cache.query_imdb_to_tmdb_map(tmdb_id, imdb=False, media_type=media_type) if cache_id and not expired: return cache_id - imdb_id = None try: imdb_id = self.config.TMDb.convert_from(tmdb_id, "imdb_id", is_movie) + if imdb_id: + if self.config.Cache: + self.config.Cache.update_imdb_to_tmdb_map(media_type, expired, imdb_id, tmdb_id) + return imdb_id except Failed: pass - if imdb_id: - if self.config.Cache: - self.config.Cache.update_imdb_to_tmdb_map(media_type, expired, imdb_id, tmdb_id) - return imdb_id - elif fail: + if fail: raise Failed(f"Convert Error: No IMDb ID Found for TMDb ID: {tmdb_id}") else: return None - def imdb_to_tmdb(self, imdb_id, is_movie=True, fail=False): - media_type = "movie" if is_movie else "show" + def imdb_to_tmdb(self, imdb_id, fail=False): expired = False - if self.config.Cache and is_movie: - cache_id, expired = self.config.Cache.query_imdb_to_tmdb_map(media_type, imdb_id, imdb=True) + if self.config.Cache: + cache_id, cache_type, expired = self.config.Cache.query_imdb_to_tmdb_map(imdb_id, imdb=True, return_type=True) if cache_id and not expired: - return cache_id - tmdb_id = None + return cache_id, cache_type try: - tmdb_id = self.config.TMDb.convert_to(imdb_id, "imdb_id", is_movie) + tmdb_id, tmdb_type = self.config.TMDb.convert_imdb_to(imdb_id) + if tmdb_id: + if self.config.Cache: + self.config.Cache.update_imdb_to_tmdb_map(tmdb_type, expired, imdb_id, tmdb_id) + return tmdb_id, tmdb_type except Failed: pass - if tmdb_id: - if self.config.Cache: - self.config.Cache.update_imdb_to_tmdb_map(media_type, expired, imdb_id, tmdb_id) - return tmdb_id - elif fail: + if fail: raise Failed(f"Convert Error: No TMDb ID Found for IMDb ID: {imdb_id}") else: - return None + return None, None def tmdb_to_tvdb(self, tmdb_id, fail=False): expired = False @@ -157,16 +153,15 @@ class Convert: cache_id, expired = self.config.Cache.query_tmdb_to_tvdb_map(tmdb_id, tmdb=True) if cache_id and not expired: return cache_id - tvdb_id = None try: tvdb_id = self.config.TMDb.convert_from(tmdb_id, "tvdb_id", False) + if tvdb_id: + if self.config.Cache: + self.config.Cache.update_tmdb_to_tvdb_map(expired, tmdb_id, tvdb_id) + return tvdb_id except Failed: pass - if tvdb_id: - if self.config.Cache: - self.config.Cache.update_tmdb_to_tvdb_map(expired, tmdb_id, tvdb_id) - return tvdb_id - elif fail: + if fail: raise Failed(f"Convert Error: No TVDb ID Found for TMDb ID: {tmdb_id}") else: return None @@ -177,16 +172,15 @@ class Convert: cache_id, expired = self.config.Cache.query_tmdb_to_tvdb_map(tvdb_id, tmdb=False) if cache_id and not expired: return cache_id - tmdb_id = None try: - tmdb_id = self.config.TMDb.convert_to(tvdb_id, "tvdb_id", False) + tmdb_id = self.config.TMDb.convert_tvdb_to(tvdb_id) + if tmdb_id: + if self.config.Cache: + self.config.Cache.update_tmdb_to_tvdb_map(expired, tmdb_id, tvdb_id) + return tmdb_id except Failed: pass - if tmdb_id: - if self.config.Cache: - self.config.Cache.update_tmdb_to_tvdb_map(expired, tmdb_id, tvdb_id) - return tmdb_id - elif fail: + if fail: raise Failed(f"Convert Error: No TMDb ID Found for TVDb ID: {tvdb_id}") else: return None @@ -197,16 +191,15 @@ class Convert: cache_id, expired = self.config.Cache.query_imdb_to_tvdb_map(tvdb_id, imdb=False) if cache_id and not expired: return cache_id - imdb_id = None try: imdb_id = self.tmdb_to_imdb(self.tvdb_to_tmdb(tvdb_id, fail=True), is_movie=False, fail=True) + if imdb_id: + if self.config.Cache: + self.config.Cache.update_imdb_to_tvdb_map(expired, imdb_id, tvdb_id) + return imdb_id except Failed: pass - if imdb_id: - if self.config.Cache: - self.config.Cache.update_imdb_to_tvdb_map(expired, imdb_id, tvdb_id) - return imdb_id - elif fail: + if fail: raise Failed(f"Convert Error: No IMDb ID Found for TVDb ID: {tvdb_id}") else: return None @@ -217,16 +210,17 @@ class Convert: cache_id, expired = self.config.Cache.query_imdb_to_tvdb_map(imdb_id, imdb=True) if cache_id and not expired: return cache_id - tvdb_id = None try: - tvdb_id = self.tmdb_to_tvdb(self.imdb_to_tmdb(imdb_id, is_movie=False, fail=True), fail=True) + tmdb_id, tmdb_type = self.imdb_to_tmdb(imdb_id, fail=True) + if tmdb_type == "show": + tvdb_id = self.tmdb_to_tvdb(tmdb_id, fail=True) + if tvdb_id: + if self.config.Cache: + self.config.Cache.update_imdb_to_tvdb_map(expired, imdb_id, tvdb_id) + return tvdb_id except Failed: pass - if tvdb_id: - if self.config.Cache: - self.config.Cache.update_imdb_to_tvdb_map(expired, imdb_id, tvdb_id) - return tvdb_id - elif fail: + if fail: raise Failed(f"Convert Error: No TVDb ID Found for IMDb ID: {imdb_id}") else: return None @@ -238,10 +232,10 @@ class Convert: imdb_id = [] anidb_id = None if self.config.Cache: - cache_id, media_type, expired = self.config.Cache.query_guid_map(item.guid) + cache_id, imdb_check, media_type, expired = self.config.Cache.query_guid_map(item.guid) if cache_id and not expired: media_id_type = "movie" if "movie" in media_type else "show" - return media_id_type, cache_id + return media_id_type, cache_id, imdb_check try: guid = requests.utils.urlparse(item.guid) item_type = guid.scheme.split(".")[-1] @@ -286,10 +280,16 @@ class Convert: else: if not tmdb_id and imdb_id: for imdb in imdb_id: - tmdb = self.imdb_to_tmdb(imdb, is_movie=library.is_movie) - if tmdb: + tmdb, tmdb_type = self.imdb_to_tmdb(imdb) + if tmdb and ((tmdb_type == "movie" and library.is_movie) or (tmdb_type == "show" and library.is_show)): tmdb_id.append(tmdb) + if not imdb_id and tmdb_id and library.is_movie: + for tmdb in tmdb_id: + imdb = self.tmdb_to_imdb(tmdb) + if imdb: + imdb_id.append(imdb) + if not tvdb_id and tmdb_id and library.is_show: for tmdb in tmdb_id: tvdb = self.tmdb_to_tvdb(tmdb) @@ -298,22 +298,29 @@ class Convert: if not tvdb_id: raise Failed(f"Unable to convert TMDb ID: {util.compile_list(tmdb_id)} to TVDb ID") + if not imdb_id and tvdb_id: + for tvdb in tvdb_id: + imdb = self.tvdb_to_imdb(tvdb) + if imdb: + imdb_id.append(imdb) - def update_cache(cache_ids, id_type, guid_type): + def update_cache(cache_ids, id_type, imdb_in, guid_type): if self.config.Cache: cache_ids = util.compile_list(cache_ids) - logger.info(util.adjust_space(f" Cache | {'^' if expired else '+'} | {item.guid:<46} | {id_type} ID: {cache_ids:<6} | {item.title}")) - self.config.Cache.update_guid_map(guid_type, item.guid, cache_ids, expired) + imdb_in = util.compile_list(imdb_in) if imdb_in else None + ids = f"{item.guid:<46} | {id_type} ID: {cache_ids:<7} | IMDb ID: {str(imdb_in):<10}" + logger.info(util.adjust_space(f" Cache | {'^' if expired else '+'} | {ids} | {item.title}")) + self.config.Cache.update_guid_map(item.guid, cache_ids, imdb_in, expired, guid_type) if tmdb_id and library.is_movie: - update_cache(tmdb_id, "TMDb", "movie") - return "movie", tmdb_id + update_cache(tmdb_id, "TMDb", imdb_id, "movie") + return "movie", tmdb_id, imdb_id elif tvdb_id and library.is_show: - update_cache(tvdb_id, "TVDb", "show") - return "show", tvdb_id + update_cache(tvdb_id, "TVDb", imdb_id, "show") + return "show", tvdb_id, imdb_id elif anidb_id and tmdb_id and library.is_show: - update_cache(tmdb_id, "TMDb", "show_movie") - return "movie", tmdb_id + update_cache(tmdb_id, "TMDb", imdb_id, "show_movie") + return "movie", tmdb_id, imdb_id else: logger.debug(f"TMDb: {tmdb_id}, IMDb: {imdb_id}, TVDb: {tvdb_id}") raise Failed(f"No ID to convert") @@ -322,4 +329,4 @@ class Convert: except BadRequest: util.print_stacktrace() logger.info(util.adjust_space(f"Mapping Error | {item.guid:<46} | Bad Request for {item.title}")) - return None, None + return None, None, None diff --git a/modules/icheckmovies.py b/modules/icheckmovies.py index 3d183021..ebccc9bc 100644 --- a/modules/icheckmovies.py +++ b/modules/icheckmovies.py @@ -16,7 +16,7 @@ class ICheckMovies: def _parse_list(self, list_url, language): imdb_urls = self._request(list_url, language, "//a[@class='optionIcon optionIMDB external']/@href") - return [t[t.find("/tt") + 1:-1] for t in imdb_urls] + return [(t[t.find("/tt") + 1:-1], "imdb") for t in imdb_urls] def get_list_description(self, list_url, language): descriptions = self._request(list_url, language, "//div[@class='span-19 last']/p/em/text()") @@ -34,21 +34,9 @@ class ICheckMovies: raise Failed(f"ICheckMovies Error: {list_url} failed to parse") return valid_lists - def get_items(self, method, data, language): - movie_ids = [] + def get_icheckmovies_ids(self, method, data, language): if method == "icheckmovies_list": logger.info(f"Processing ICheckMovies List: {data}") - imdb_ids = self._parse_list(data, language) - total_ids = len(imdb_ids) - for i, imdb_id in enumerate(imdb_ids, 1): - try: - util.print_return(f"Converting IMDb ID {i}/{total_ids}") - movie_ids.append(self.config.Convert.imdb_to_tmdb(imdb_id)) - except Failed as e: - logger.error(e) - logger.info(util.adjust_space(f"Processed {total_ids} IMDb IDs")) + return self._parse_list(data, language) else: raise Failed(f"ICheckMovies Error: Method {method} not supported") - logger.debug("") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - return movie_ids, [] diff --git a/modules/imdb.py b/modules/imdb.py index 11205d65..e1fb9f3d 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -95,35 +95,13 @@ class IMDb: return imdb_ids raise ValueError(f"IMDb Error: No IMDb IDs Found at {imdb_url}") - def get_items(self, method, data, language, is_movie): - show_ids = [] - movie_ids = [] - fail_ids = [] - def run_convert(imdb_id): - tvdb_id = self.config.Convert.imdb_to_tvdb(imdb_id) if not is_movie else None - tmdb_id = self.config.Convert.imdb_to_tmdb(imdb_id) if tvdb_id is None else None - if tmdb_id: movie_ids.append(tmdb_id) - elif tvdb_id: show_ids.append(tvdb_id) - else: - logger.error(f"Convert Error: No {'' if is_movie else 'TVDb ID or '}TMDb ID found for IMDb: {imdb_id}") - fail_ids.append(imdb_id) - + def get_imdb_ids(self, method, data, language): if method == "imdb_id": logger.info(f"Processing IMDb ID: {data}") - run_convert(data) + return [(data, "imdb")] elif method == "imdb_list": status = f"{data['limit']} Items at " if data['limit'] > 0 else '' logger.info(f"Processing IMDb List: {status}{data['url']}") - imdb_ids = self._ids_from_url(data["url"], language, data["limit"]) - total_ids = len(imdb_ids) - for i, imdb in enumerate(imdb_ids, 1): - util.print_return(f"Converting IMDb ID {i}/{total_ids}") - run_convert(imdb) - logger.info(util.adjust_space(f"Processed {total_ids} IMDb IDs")) + return [(i, "imdb") for i in self._ids_from_url(data["url"], language, data["limit"])] else: raise Failed(f"IMDb Error: Method {method} not supported") - logger.debug("") - logger.debug(f"{len(fail_ids)} IMDb IDs Failed to Convert: {fail_ids}") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids diff --git a/modules/letterboxd.py b/modules/letterboxd.py index 0e9f33e7..7ad7d0c5 100644 --- a/modules/letterboxd.py +++ b/modules/letterboxd.py @@ -50,13 +50,13 @@ class Letterboxd: raise Failed(f"Letterboxd Error: {list_url} failed to parse") return valid_lists - def get_items(self, method, data, language): - movie_ids = [] + def get_tmdb_ids(self, method, data, language): if method == "letterboxd_list": logger.info(f"Processing Letterboxd List: {data}") items = self._parse_list(data, language) total_items = len(items) if total_items > 0: + ids = [] for i, item in enumerate(items, 1): letterboxd_id, slug = item util.print_return(f"Finding TMDb ID {i}/{total_items}") @@ -72,12 +72,10 @@ class Letterboxd: continue if self.config.Cache: self.config.Cache.update_letterboxd_map(expired, letterboxd_id, tmdb_id) - movie_ids.append(tmdb_id) + ids.append((tmdb_id, "tmdb")) logger.info(util.adjust_space(f"Processed {total_items} TMDb IDs")) + return ids else: - logger.error(f"Letterboxd Error: No List Items found in {data}") + raise Failed(f"Letterboxd Error: No List Items found in {data}") else: raise Failed(f"Letterboxd Error: Method {method} not supported") - logger.debug("") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - return movie_ids, [] diff --git a/modules/mal.py b/modules/mal.py index e8ff7d0e..1df5c2fa 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -155,7 +155,7 @@ class MyAnimeList: url = f"{urls['user']}/{username}/animelist?{final_status}sort={sort_by}&limit={limit}" return self._parse_request(url) - def get_items(self, method, data): + def get_mal_ids(self, method, data): if method == "mal_id": logger.info(f"Processing MyAnimeList ID: {data}") mal_ids = [data] @@ -173,9 +173,6 @@ class MyAnimeList: mal_ids = self._userlist(data["username"], data["status"], data["sort_by"], data["limit"]) else: raise Failed(f"MyAnimeList Error: Method {method} not supported") - movie_ids, show_ids = self.config.Convert.myanimelist_to_ids(mal_ids) logger.debug("") logger.debug(f"{len(mal_ids)} MyAnimeList IDs Found: {mal_ids}") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids + return mal_ids diff --git a/modules/plex.py b/modules/plex.py index 89957132..d38e149d 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -132,10 +132,6 @@ show_only_searches = [ "episode_user_rating.gt", "episode_user_rating.gte", "episode_user_rating.lt", "episode_user_rating.lte", "episode_year", "episode_year.not", "episode_year.gt", "episode_year.gte", "episode_year.lt", "episode_year.lte" ] -number_attributes = [ - "plays", "episode_plays", "duration", "tmdb_vote_count", "first_episode_aired", "last_episode_aired", - "added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played" -] float_attributes = ["user_rating", "episode_user_rating", "critic_rating", "audience_rating"] boolean_attributes = [ "hdr", "unmatched", "duplicate", "unplayed", "progress", "trash", @@ -143,6 +139,7 @@ boolean_attributes = [ ] tmdb_attributes = ["actor", "director", "producer", "writer"] date_attributes = ["added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played", "first_episode_aired", "last_episode_aired"] +number_attributes = ["plays", "episode_plays", "duration", "tmdb_vote_count"] + date_attributes search_display = {"added": "Date Added", "release": "Release Date", "hdr": "HDR", "progress": "In Progress", "episode_progress": "Episode In Progress"} sorts = { None: None, @@ -299,6 +296,7 @@ class Plex: self.missing = {} self.movie_map = {} self.show_map = {} + self.imdb_map = {} self.movie_rating_key_map = {} self.show_rating_key_map = {} self.run_again = [] @@ -585,7 +583,7 @@ class Plex: raise Failed(f"Collection Error: No valid Plex Collections in {collections}") return valid_collections - def get_items(self, method, data): + def get_rating_keys(self, method, data): media_type = "Movie" if self.is_movie else "Show" items = [] if method == "plex_all": @@ -634,7 +632,10 @@ class Plex: else: raise Failed(f"Plex Error: Method {method} not supported") if len(items) > 0: - return [item.ratingKey for item in items] + ids = [item.ratingKey for item in items] + logger.debug("") + logger.debug(f"{len(ids)} Keys Found: {ids}") + return ids else: raise Failed("Plex Error: No Items found in Plex") @@ -679,7 +680,7 @@ class Plex: for i, item in enumerate(items, 1): util.print_return(f"Processing: {i}/{len(items)} {item.title}") if item.ratingKey not in self.movie_rating_key_map and item.ratingKey not in self.show_rating_key_map: - id_type, main_id = self.config.Convert.get_id(item, self) + id_type, main_id, imdb_id = self.config.Convert.get_id(item, self) if main_id: if id_type == "movie": self.movie_rating_key_map[item.ratingKey] = main_id[0] @@ -687,6 +688,8 @@ class Plex: elif id_type == "show": self.show_rating_key_map[item.ratingKey] = main_id[0] util.add_dict_list(main_id, item.ratingKey, self.show_map) + if imdb_id: + util.add_dict_list(imdb_id, item.ratingKey, self.imdb_map) logger.info("") logger.info(util.adjust_space(f"Processed {len(items)} {'Movies' if self.is_movie else 'Shows'}")) return items diff --git a/modules/stevenlu.py b/modules/stevenlu.py index 7712dcce..4356c7ae 100644 --- a/modules/stevenlu.py +++ b/modules/stevenlu.py @@ -10,21 +10,9 @@ class StevenLu: def __init__(self, config): self.config = config - def get_items(self, method): - movie_ids = [] - fail_ids = [] + def get_stevenlu_ids(self, method): if method == "stevenlu_popular": logger.info(f"Processing StevenLu Popular Movies") - for i in self.config.get_json(base_url): - tmdb_id = self.config.Convert.imdb_to_tmdb(i["imdb_id"]) - if tmdb_id: - movie_ids.append(tmdb_id) - else: - logger.error(f"Convert Error: No TMDb ID found for IMDb: {i['imdb_id']}") - fail_ids.append(i["imdb_id"]) + return [(i["imdb_id"], "imdb") for i in self.config.get_json(base_url)] else: raise Failed(f"StevenLu Error: Method {method} not supported") - logger.debug("") - logger.debug(f"{len(fail_ids)} IMDb IDs Failed to Convert: {fail_ids}") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - return movie_ids, [] diff --git a/modules/tautulli.py b/modules/tautulli.py index c00e6335..a953c8a0 100644 --- a/modules/tautulli.py +++ b/modules/tautulli.py @@ -20,7 +20,7 @@ class Tautulli: if response["response"]["result"] != "success": raise Failed(f"Tautulli Error: {response['response']['message']}") - def get_items(self, library, params): + def get_rating_keys(self, library, params): query_size = int(params["list_size"]) + int(params["list_buffer"]) logger.info(f"Processing Tautulli Most {params['list_type'].capitalize()}: {params['list_size']} {'Movies' if library.is_movie else 'Shows'}") response = self._request(f"{self.url}/api/v2?apikey={self.apikey}&cmd=get_home_stats&time_range={params['list_days']}&stats_count={query_size}") @@ -50,6 +50,8 @@ class Tautulli: logger.error(f"Plex Error: Item {item} not found") continue count += 1 + logger.debug("") + logger.debug(f"{len(rating_keys)} Keys Found: {rating_keys}") return rating_keys def _section_id(self, library_name): diff --git a/modules/tmdb.py b/modules/tmdb.py index fd2fbb84..9b56e222 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -80,11 +80,24 @@ class TMDb: raise Failed(f"TMDb Error: TMDb {'Movie' if is_movie else 'Show'} ID: {tmdb_id} not found") @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) - def convert_to(self, external_id, external_source, is_movie): - search_results = self.Movie.external(external_id=external_id, external_source=external_source) - search = search_results["movie_results" if is_movie else "tv_results"] - if len(search) == 1: return int(search[0]["id"]) - else: raise Failed(f"TMDb Error: No TMDb ID found for {external_source.upper().replace('B_', 'b ')} {external_id}") + def convert_to(self, external_id, external_source): + return self.Movie.external(external_id=external_id, external_source=external_source) + + def convert_tvdb_to(self, tvdb_id): + search = self.convert_to(tvdb_id, "tvdb_id") + if len(search["tv_results"]) == 1: + return int(search["tv_results"][0]["id"]) + else: + raise Failed(f"TMDb Error: No TMDb ID found for TVDb ID {tvdb_id}") + + def convert_imdb_to(self, imdb_id): + search = self.convert_to(imdb_id, "imdb_id") + if len(search["movie_results"]) > 0: + return int(search["movie_results"][0]["id"]), "movie" + elif len(search["tv_results"]) > 0: + return int(search["tv_results"][0]["id"]), "show" + else: + raise Failed(f"TMDb Error: No TMDb ID found for IMDb ID {imdb_id}") def get_movie_show_or_collection(self, tmdb_id, is_movie): if is_movie: @@ -140,35 +153,27 @@ class TMDb: except TMDbException as e: raise Failed(f"TMDb Error: No List found for TMDb ID {tmdb_id}: {e}") def _credits(self, tmdb_id, actor=False, crew=False, director=False, producer=False, writer=False): - movie_ids = [] - show_ids = [] + ids = [] actor_credits = self._person_credits(tmdb_id) if actor: for credit in actor_credits.cast: if credit.media_type == "movie": - movie_ids.append(credit.id) + ids.append((credit.id, "tmdb")) elif credit.media_type == "tv": - try: - show_ids.append(self.config.Convert.tmdb_to_tvdb(credit.id, fail=True)) - except Failed as e: - logger.warning(e) + ids.append((credit.id, "tmdb_show")) for credit in actor_credits.crew: if crew or \ (director and credit.department == "Directing") or \ (producer and credit.department == "Production") or \ (writer and credit.department == "Writing"): if credit.media_type == "movie": - movie_ids.append(credit.id) + ids.append((credit.id, "tmdb")) elif credit.media_type == "tv": - try: - show_ids.append(self.config.Convert.tmdb_to_tvdb(credit.id, fail=True)) - except Failed as e: - logger.warning(e) - return movie_ids, show_ids + ids.append((credit.id, "tmdb_show")) + return ids def _pagenation(self, method, amount, is_movie): ids = [] - count = 0 for x in range(int(amount / 20) + 1): if method == "tmdb_popular": tmdb_items = self.Movie.popular(x + 1) if is_movie else self.TV.popular(x + 1) elif method == "tmdb_top_rated": tmdb_items = self.Movie.top_rated(x + 1) if is_movie else self.TV.top_rated(x + 1) @@ -178,18 +183,15 @@ class TMDb: else: raise Failed(f"TMDb Error: {method} method not supported") for tmdb_item in tmdb_items: try: - ids.append(tmdb_item.id if is_movie else self.config.Convert.tmdb_to_tvdb(tmdb_item.id, fail=True)) - count += 1 + ids.append((tmdb_item.id, "tmdb" if is_movie else "tmdb_show")) except Failed as e: logger.error(e) - pass - if count == amount: break - if count == amount: break + if len(ids) == amount: break + if len(ids) == amount: break return ids def _discover(self, attrs, amount, is_movie): ids = [] - count = 0 for date_attr in discover_dates: if date_attr in attrs: attrs[date_attr] = util.validate_date(attrs[date_attr], f"tmdb_discover attribute {date_attr}", return_as="%Y-%m-%d") @@ -202,13 +204,11 @@ class TMDb: tmdb_items = self.Discover.discover_movies(attrs) if is_movie else self.Discover.discover_tv_shows(attrs) for tmdb_item in tmdb_items: try: - ids.append(tmdb_item.id if is_movie else self.config.Convert.tmdb_to_tvdb(tmdb_item.id, fail=True)) - count += 1 + ids.append((tmdb_item.id, "tmdb" if is_movie else "tmdb_show")) except Failed as e: logger.error(e) - pass - if count == amount: break - if count == amount: break + if len(ids) == amount: break + if len(ids) == amount: break return ids, amount def validate_tmdb_ids(self, tmdb_ids, tmdb_method): @@ -231,11 +231,10 @@ class TMDb: elif tmdb_type == "List": self.get_list(tmdb_id) return tmdb_id - def get_items(self, method, data, is_movie): + def get_tmdb_ids(self, method, data, is_movie): pretty = method.replace("_", " ").title().replace("Tmdb", "TMDb") media_type = "Movie" if is_movie else "Show" - movie_ids = [] - show_ids = [] + ids = [] if method in ["tmdb_discover", "tmdb_company", "tmdb_keyword"] or (method == "tmdb_network" and not is_movie): attrs = None tmdb_id = "" @@ -255,8 +254,7 @@ class TMDb: else: attrs = data.copy() limit = int(attrs.pop("limit")) - if is_movie: movie_ids, amount = self._discover(attrs, limit, is_movie) - else: show_ids, amount = self._discover(attrs, limit, is_movie) + ids, amount = self._discover(attrs, limit, is_movie) if method in ["tmdb_company", "tmdb_network", "tmdb_keyword"]: logger.info(f"Processing {pretty}: ({tmdb_id}) {tmdb_name} ({amount} {media_type}{'' if amount == 1 else 's'})") elif method == "tmdb_discover": @@ -264,8 +262,7 @@ class TMDb: for attr, value in attrs.items(): logger.info(f" {attr}: {value}") elif method in ["tmdb_popular", "tmdb_top_rated", "tmdb_now_playing", "tmdb_trending_daily", "tmdb_trending_weekly"]: - if is_movie: movie_ids = self._pagenation(method, data, is_movie) - else: show_ids = self._pagenation(method, data, is_movie) + ids = self._pagenation(method, data, is_movie) logger.info(f"Processing {pretty}: {data} {media_type}{'' if data == 1 else 's'}") else: tmdb_id = int(data) @@ -274,34 +271,31 @@ class TMDb: tmdb_name = tmdb_list.name for tmdb_item in tmdb_list.items: if tmdb_item.media_type == "movie": - movie_ids.append(tmdb_item.id) + ids.append((tmdb_item.id, "tmdb")) elif tmdb_item.media_type == "tv": - try: show_ids.append(self.config.Convert.tmdb_to_tvdb(tmdb_item.id, fail=True)) - except Failed: pass + try: + ids.append((tmdb_item.id, "tmdb_show")) + except Failed: + pass elif method == "tmdb_movie": tmdb_name = str(self.get_movie(tmdb_id).title) - movie_ids.append(tmdb_id) + ids.append((tmdb_id, "tmdb")) elif method == "tmdb_collection": tmdb_items = self.get_collection(tmdb_id) tmdb_name = str(tmdb_items.name) for tmdb_item in tmdb_items.parts: - movie_ids.append(tmdb_item["id"]) + ids.append((tmdb_item["id"], "tmdb")) elif method == "tmdb_show": tmdb_name = str(self.get_show(tmdb_id).name) - show_ids.append(self.config.Convert.tmdb_to_tvdb(tmdb_id, fail=True)) + ids.append((tmdb_id, "tmdb_show")) else: tmdb_name = str(self.get_person(tmdb_id).name) - if method == "tmdb_actor": movie_ids, show_ids = self._credits(tmdb_id, actor=True) - elif method == "tmdb_director": movie_ids, show_ids = self._credits(tmdb_id, director=True) - elif method == "tmdb_producer": movie_ids, show_ids = self._credits(tmdb_id, producer=True) - elif method == "tmdb_writer": movie_ids, show_ids = self._credits(tmdb_id, writer=True) - elif method == "tmdb_crew": movie_ids, show_ids = self._credits(tmdb_id, crew=True) + if method == "tmdb_actor": ids = self._credits(tmdb_id, actor=True) + elif method == "tmdb_director": ids = self._credits(tmdb_id, director=True) + elif method == "tmdb_producer": ids = self._credits(tmdb_id, producer=True) + elif method == "tmdb_writer": ids = self._credits(tmdb_id, writer=True) + elif method == "tmdb_crew": ids = self._credits(tmdb_id, crew=True) else: raise Failed(f"TMDb Error: Method {method} not supported") - if len(movie_ids) > 0: - logger.info(f"Processing {pretty}: ({tmdb_id}) {tmdb_name} ({len(movie_ids)} Movie{'' if len(movie_ids) == 1 else 's'})") - if not is_movie and len(show_ids) > 0: - logger.info(f"Processing {pretty}: ({tmdb_id}) {tmdb_name} ({len(show_ids)} Show{'' if len(show_ids) == 1 else 's'})") - logger.debug("") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids + if len(ids) > 0: + logger.info(f"Processing {pretty}: ({tmdb_id}) {tmdb_name} ({len(ids)} Item{'' if len(ids) == 1 else 's'})") + return ids diff --git a/modules/trakt.py b/modules/trakt.py index 6c8391d3..f1886281 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -142,39 +142,59 @@ class Trakt: except Failed: raise Failed(f"Trakt Error: List {data} not found") - def _parse(self, items, top=True, is_movie=True): + def _parse(self, items, top=True, item_type=None): ids = [] for item in items: - data = item["movie" if is_movie else "show"] if top else item - if data["ids"]["tmdb" if is_movie else "tvdb"]: - ids.append(data["ids"]["tmdb" if is_movie else "tvdb"]) + if top: + if item_type: + data = item[item_type] + elif item["type"] in ["movie", "show"]: + data = item[item["type"]] + else: + continue + else: + data = item + if item_type: + id_type = "TMDb" if item_type == "movie" else "TVDb" + else: + id_type = "TMDb" if item["type"] == "movie" else "TVDb" + if data["ids"][id_type.lower()]: + ids.append((data["ids"][id_type.lower()], id_type.lower())) else: - logger.error(f"Trakt Error: No {'TMDb' if is_movie else 'TVDb'} ID found for {data['title']} ({data['year']})") - return (ids, []) if is_movie else ([], ids) + logger.error(f"Trakt Error: No {id_type} ID found for {data['title']} ({data['year']})") + return ids - def _user_list(self, list_type, data, is_movie): - path = f"{requests.utils.urlparse(data).path}/items" if list_type == "list" else f"/users/{data}/{list_type}" + def _user_list(self, data): try: - items = self._request(f"{path}/{'movies' if is_movie else 'shows'}") + items = self._request(f"{requests.utils.urlparse(data).path}/items") except Failed: - raise Failed(f"Trakt Error: {'List' if list_type == 'list' else 'User'} {data} not found") + raise Failed(f"Trakt Error: List {data} not found") if len(items) == 0: - if list_type == "list": - raise Failed(f"Trakt Error: List {data} is empty") - else: - raise Failed(f"Trakt Error: {data}'s {list_type.capitalize()} is empty") - return self._parse(items, is_movie=is_movie) + raise Failed(f"Trakt Error: List {data} is empty") + return self._parse(items) + + def _user_items(self, list_type, data, is_movie): + try: + items = self._request(f"/users/{data}/{list_type}/{'movies' if is_movie else 'shows'}") + except Failed: + raise Failed(f"Trakt Error: User {data} not found") + if len(items) == 0: + raise Failed(f"Trakt Error: {data}'s {list_type.capitalize()} is empty") + return self._parse(items, item_type="movie" if is_movie else "show") def _pagenation(self, pagenation, amount, is_movie): items = self._request(f"/{'movies' if is_movie else 'shows'}/{pagenation}?limit={amount}") - return self._parse(items, top=pagenation != "popular", is_movie=is_movie) + return self._parse(items, top=pagenation != "popular", item_type="movie" if is_movie else "show") def validate_trakt(self, trakt_lists, is_movie, trakt_type="list"): values = util.get_list(trakt_lists, split=False) trakt_values = [] for value in values: try: - self._user_list(trakt_type, value, is_movie) + if trakt_type == "list": + self._user_list(value) + else: + self._user_items(trakt_type, value, is_movie) trakt_values.append(value) except Failed as e: logger.error(e) @@ -187,21 +207,17 @@ class Trakt: raise Failed(f"Trakt Error: No valid Trakt Lists in {values}") return trakt_values - def get_items(self, method, data, is_movie): + def get_trakt_ids(self, method, data, is_movie): pretty = method.replace("_", " ").title() media_type = "Movie" if is_movie else "Show" if method in ["trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected"]: logger.info(f"Processing {pretty}: {data} {media_type}{'' if data == 1 else 's'}") - movie_ids, show_ids = self._pagenation(method[6:], data, is_movie) + return self._pagenation(method[6:], data, is_movie) elif method in ["trakt_collection", "trakt_watchlist"]: logger.info(f"Processing {pretty} {media_type}s for {data}") - movie_ids, show_ids = self._user_list(method[6:], data, is_movie) + return self._user_items(method[6:], data, is_movie) elif method == "trakt_list": logger.info(f"Processing {pretty}: {data}") - movie_ids, show_ids = self._user_list(method[6:], data, is_movie) + return self._user_list(data) else: raise Failed(f"Trakt Error: Method {method} not supported") - logger.debug("") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids diff --git a/modules/tvdb.py b/modules/tvdb.py index 7ee1a918..d2f2843a 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -54,6 +54,7 @@ class TVDbObj: self.summary = results[0] if len(results) > 0 and len(results[0]) > 0 else None tmdb_id = None + imdb_id = None if self.is_movie: results = response.xpath("//*[text()='TheMovieDB.com']/@href") if len(results) > 0: @@ -61,16 +62,16 @@ class TVDbObj: tmdb_id = util.regex_first_int(results[0], "TMDb ID") except Failed: pass - if tmdb_id is None: - results = response.xpath("//*[text()='IMDB']/@href") - if len(results) > 0: - try: - tmdb_id = self.config.Convert.imdb_to_tmdb(util.get_id_from_imdb_url(results[0]), fail=True) - except Failed: - pass - if tmdb_id is None: - raise Failed(f"TVDB Error: No TMDb ID found for {self.title}") + results = response.xpath("//*[text()='IMDB']/@href") + if len(results) > 0: + try: + imdb_id = util.get_id_from_imdb_url(results[0]) + except Failed: + pass + if tmdb_id is None and imdb_id is None: + raise Failed(f"TVDB Error: No TMDb ID or IMDb ID found for {self.title}") self.tmdb_id = tmdb_id + self.imdb_id = imdb_id class TVDb: def __init__(self, config): @@ -99,8 +100,7 @@ class TVDb: return description[0] if len(description) > 0 and len(description[0]) > 0 else "" def _ids_from_url(self, tvdb_url, language): - show_ids = [] - movie_ids = [] + ids = [] tvdb_url = tvdb_url.strip() if tvdb_url.startswith((urls["list"], urls["alt_list"])): try: @@ -111,23 +111,23 @@ class TVDb: item_url = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/@href")[0] if item_url.startswith("/series/"): try: - show_ids.append(self.get_series(language, f"{base_url}{item_url}").id) + ids.append((self.get_series(language, f"{base_url}{item_url}").id, "tvdb")) except Failed as e: logger.error(f"{e} for series {title}") elif item_url.startswith("/movies/"): try: - tmdb_id = self.get_movie(language, f"{base_url}{item_url}").tmdb_id - if tmdb_id: - movie_ids.append(tmdb_id) - else: - raise Failed(f"TVDb Error: TMDb ID not found from TVDb URL: {tvdb_url}") + movie = self.get_movie(language, f"{base_url}{item_url}") + if movie.tmdb_id: + ids.append((movie.tmdb_id, "tmdb")) + elif movie.imdb_id: + ids.append((movie.imdb_id, "imdb")) except Failed as e: - logger.error(f"{e} for series {title}") + logger.error(e) else: logger.error(f"TVDb Error: Skipping Movie: {title}") time.sleep(2) - if len(show_ids) > 0 or len(movie_ids) > 0: - return movie_ids, show_ids + if len(ids) > 0: + return ids raise Failed(f"TVDb Error: No TVDb IDs found at {tvdb_url}") except requests.exceptions.MissingSchema: util.print_stacktrace() @@ -135,21 +135,19 @@ class TVDb: else: raise Failed(f"TVDb Error: {tvdb_url} must begin with {urls['list']}") - def get_items(self, method, data, language): - show_ids = [] - movie_ids = [] + def get_tvdb_ids(self, method, data, language): if method == "tvdb_show": logger.info(f"Processing TVDb Show: {data}") - show_ids.append(self.get_series(language, data).id) + return [(self.get_series(language, data).id, "tvdb")] elif method == "tvdb_movie": logger.info(f"Processing TVDb Movie: {data}") - movie_ids.append(self.get_movie(language, data).tmdb_id) + movie = self.get_movie(language, data) + if movie.tmdb_id: + return [(movie.tmdb_id, "tmdb")] + elif movie.imdb_id: + return [(movie.imdb_id, "imdb")] elif method == "tvdb_list": logger.info(f"Processing TVDb List: {data}") - movie_ids, show_ids = self._ids_from_url(data, language) + return self._ids_from_url(data, language) else: raise Failed(f"TVDb Error: Method {method} not supported") - logger.debug("") - logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}") - logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}") - return movie_ids, show_ids diff --git a/modules/util.py b/modules/util.py index 274967ea..d5dc6d7e 100644 --- a/modules/util.py +++ b/modules/util.py @@ -246,39 +246,46 @@ def is_locked(filepath): file_object.close() return locked -def date_filter(current, modifier, data, final, current_time): - if current is None: - return False +def is_date_filter(value, modifier, data, final, current_time): + if value is None: + return True if modifier in ["", ".not"]: threshold_date = current_time - timedelta(days=data) - if (modifier == "" and (current is None or current < threshold_date)) \ - or (modifier == ".not" and current and current >= threshold_date): - return False + if (modifier == "" and (value is None or value < threshold_date)) \ + or (modifier == ".not" and value and value >= threshold_date): + return True elif modifier in [".before", ".after"]: filter_date = validate_date(data, final) - if (modifier == ".before" and current >= filter_date) or (modifier == ".after" and current <= filter_date): - return False + if (modifier == ".before" and value >= filter_date) or (modifier == ".after" and value <= filter_date): + return True elif modifier == ".regex": - jailbreak = False + jailbreak = True for check_data in data: - if re.compile(check_data).match(current.strftime("%m/%d/%Y")): + if re.compile(check_data).match(value.strftime("%m/%d/%Y")): jailbreak = True break if not jailbreak: - return False - return True - -def number_filter(current, modifier, data): - return current is None or (modifier == ".gt" and current <= data) \ - or (modifier == ".gte" and current < data) \ - or (modifier == ".lt" and current >= data) \ - or (modifier == ".lte" and current > data) - -def string_filter(current, modifier, data): - return (modifier in ["", ".not"] and data.lower() in current.lower()) \ - or (modifier == ".begins" and current.lower().startswith(data.lower())) \ - or (modifier == ".ends" and current.lower().endswith(data.lower())) \ - or (modifier == ".regex" and re.compile(data).match(current)) + return True + return False + +def is_number_filter(value, modifier, data): + return value is None or (modifier == ".gt" and value <= data) \ + or (modifier == ".gte" and value < data) \ + or (modifier == ".lt" and value >= data) \ + or (modifier == ".lte" and value > data) + +def is_string_filter(values, modifier, data): + jailbreak = False + for value in values: + for check_value in data: + if (modifier in ["", ".not"] and check_value.lower() in value.lower()) \ + or (modifier == ".begins" and value.lower().startswith(check_value.lower())) \ + or (modifier == ".ends" and value.lower().endswith(check_value.lower())) \ + or (modifier == ".regex" and re.compile(check_value).match(value)): + jailbreak = True + break + if jailbreak: break + return (jailbreak and modifier == ".not") or (not jailbreak and modifier in ["", ".begins", ".ends", ".regex"]) def parse(attribute, data, datatype=None, methods=None, parent=None, default=None, options=None, translation=None, minimum=1, maximum=None): display = f"{parent + ' ' if parent else ''}{attribute} attribute" diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 8bd11038..50cd055e 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -265,12 +265,14 @@ def mass_metadata(config, library, items): tvdb_id = None imdb_id = None if config.Cache: - t_id, guid_media_type, _ = config.Cache.query_guid_map(item.guid) + t_id, i_id, guid_media_type, _ = config.Cache.query_guid_map(item.guid) if t_id: if "movie" in guid_media_type: tmdb_id = t_id[0] else: tvdb_id = t_id[0] + if i_id: + imdb_id = i_id[0] if not tmdb_id and not tvdb_id: tmdb_id = library.get_tmdb_from_map(item) if not tmdb_id and not tvdb_id and library.is_show: @@ -469,7 +471,7 @@ def run_collection(config, library, metadata, requested_collections): for filter_key, filter_value in builder.filters: logger.info(f"Collection Filter {filter_key}: {filter_value}") - builder.collect_rating_keys() + builder.find_rating_keys() if len(builder.rating_keys) > 0 and builder.build_collection: logger.info("") From 3b88d7749218499c8e03d8c00e76e5beada2ff59 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sun, 8 Aug 2021 14:21:47 -0400 Subject: [PATCH 77/95] add --no-missing --- modules/builder.py | 31 +++++++++++++++++-------------- plex_meta_manager.py | 8 +++++--- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 6aeaf209..d58913ee 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -993,7 +993,7 @@ class CollectionBuilder: else: logger.error(message) - def find_rating_keys(self): + def find_rating_keys(self, no_missing): for method, value in self.builders: ids = [] rating_keys = [] @@ -1058,20 +1058,23 @@ class CollectionBuilder: if input_id in self.library.imdb_map: rating_keys.append(self.library.imdb_map[input_id][0]) else: - try: - tmdb_id, tmdb_type = self.config.Convert.imdb_to_tmdb(input_id) - if tmdb_type == "movie": - if tmdb_id not in self.missing_movies: - self.missing_movies.append(tmdb_id) - else: - tvdb_id = self.config.Convert.tmdb_to_tvdb(tmdb_id) - if tvdb_id not in self.missing_shows: - self.missing_shows.append(tvdb_id) - except Failed as e: - logger.error(e) - continue + if (self.details["show_missing"] or self.details["save_missing"] + or (self.library.Radarr and self.add_to_radarr) + or (self.library.Sonarr and self.add_to_sonarr)) and not no_missing: + try: + tmdb_id, tmdb_type = self.config.Convert.imdb_to_tmdb(input_id) + if tmdb_type == "movie": + if tmdb_id not in self.missing_movies: + self.missing_movies.append(tmdb_id) + else: + tvdb_id = self.config.Convert.tmdb_to_tvdb(tmdb_id) + if tvdb_id not in self.missing_shows: + self.missing_shows.append(tvdb_id) + except Failed as e: + logger.error(e) + continue util.print_end() - + if len(rating_keys) > 0: name = self.obj.title if self.obj else self.name if not isinstance(rating_keys, list): diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 50cd055e..34921081 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -26,6 +26,7 @@ parser.add_argument("-lo", "--library-only", "--libraries-only", dest="library_o parser.add_argument("-rc", "-cl", "--collection", "--collections", "--run-collection", "--run-collections", dest="collections", help="Process only specified collections (comma-separated list)", type=str) parser.add_argument("-rl", "-l", "--library", "--libraries", "--run-library", "--run-libraries", dest="libraries", help="Process only specified libraries (comma-separated list)", type=str) parser.add_argument("-nc", "--no-countdown", dest="no_countdown", help="Run without displaying the countdown", action="store_true", default=False) +parser.add_argument("-nm", "--no-missing", dest="no_missing", help="Run without running the midding section", action="store_true", default=False) parser.add_argument("-d", "--divider", dest="divider", help="Character that divides the sections (Default: '=')", default="=", type=str) parser.add_argument("-w", "--width", dest="width", help="Screen Width (Default: 100)", default=100, type=int) args = parser.parse_args() @@ -46,6 +47,7 @@ test = check_bool("PMM_TEST", args.test) debug = check_bool("PMM_DEBUG", args.debug) run = check_bool("PMM_RUN", args.run) no_countdown = check_bool("PMM_NO_COUNTDOWN", args.no_countdown) +no_missing = check_bool("PMM_NO_MISSING", args.no_missing) library_only = check_bool("PMM_LIBRARIES_ONLY", args.library_only) collection_only = check_bool("PMM_COLLECTIONS_ONLY", args.collection_only) collections = os.environ.get("PMM_COLLECTIONS") if os.environ.get("PMM_COLLECTIONS") else args.collections @@ -471,16 +473,16 @@ def run_collection(config, library, metadata, requested_collections): for filter_key, filter_value in builder.filters: logger.info(f"Collection Filter {filter_key}: {filter_value}") - builder.find_rating_keys() + builder.find_rating_keys(no_missing) if len(builder.rating_keys) > 0 and builder.build_collection: logger.info("") util.separator(f"Adding to {mapping_name} Collection", space=False, border=False) logger.info("") builder.add_to_collection() - if (builder.details["show_missing"] is True or builder.details["save_missing"] is True + if (builder.details["show_missing"] or builder.details["save_missing"] or (library.Radarr and builder.add_to_radarr) or (library.Sonarr and builder.add_to_sonarr)) \ - and (len(builder.missing_movies) > 0 or len(builder.missing_shows) > 0): + and (len(builder.missing_movies) > 0 or len(builder.missing_shows) > 0) and not no_missing: if builder.details["show_missing"] is True: logger.info("") util.separator(f"Missing from Library", space=False, border=False) From 94933b79fc135433b26fd2381ffb33cff7f28247 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sun, 8 Aug 2021 18:36:45 -0400 Subject: [PATCH 78/95] fix unpack error --- modules/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cache.py b/modules/cache.py index 398bfbd5..1f93a3b0 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -179,7 +179,7 @@ class Cache: id_to_return = row[to_id] if to_id == "imdb_id" else int(row[to_id]) expired = time_between_insertion.days > self.expiration out_type = row["media_type"] if return_type else None - if out_type: + if return_type: return id_to_return, out_type, expired else: return id_to_return, expired From 1a79eef18357095f0841ad4b7a634a7f34e26937 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 9 Aug 2021 11:40:44 -0400 Subject: [PATCH 79/95] added tmdb_year --- modules/builder.py | 49 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index d58913ee..781edba6 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -110,8 +110,9 @@ all_filters = [ "resolution", "resolution.not", "writer", "writer.not", "year", "year.gt", "year.gte", "year.lt", "year.lte", "year.not" + "tmdb_year", "tmdb_year.gt", "tmdb_year.gte", "tmdb_year.lt", "tmdb_year.lte", "tmdb_year.not" ] -tmdb_filters = ("original_language", "tmdb_vote_count", "year") +tmdb_filters = ["original_language", "tmdb_vote_count", "tmdb_year", "first_episode_aired", "last_episode_aired"] movie_only_filters = [ "audio_language", "audio_language.not", "audio_track_title", "audio_track_title.not", "audio_track_title.begins", "audio_track_title.ends", "audio_track_title.regex", @@ -528,8 +529,6 @@ class CollectionBuilder: else: logger.error(e) - self.tmdb_filters = [(fm, fd) for fm, fd in self.filters if fm.startswith(tmdb_filters)] - if self.custom_sort and len(self.builders) > 1: raise Failed("Collection Error: collection_order: custom can only be used with a single builder per collection") @@ -985,6 +984,8 @@ class CollectionBuilder: message = f"Collection Error: {filter_final} filter attribute only works for show libraries" elif filter_final is None: message = f"Collection Error: {filter_final} filter attribute is blank" + elif filter_attr in tmdb_filters: + self.tmdb_filters.append((filter_final, self.validate_attribute(filter_attr, modifier, f"{filter_final} filter", filter_data, validate))) else: self.filters.append((filter_final, self.validate_attribute(filter_attr, modifier, f"{filter_final} filter", filter_data, validate))) if message: @@ -1319,7 +1320,7 @@ class CollectionBuilder: else: logger.error(error) return valid_list - elif attribute in ["year", "episode_year"] and modifier in [".gt", ".gte", ".lt", ".lte"]: + elif attribute in ["year", "episode_year", "tmdb_year"] and modifier in [".gt", ".gte", ".lt", ".lte"]: return util.parse(final, data, datatype="int", minimum=1800, maximum=self.current_year) elif attribute in plex.date_attributes and modifier in [".before", ".after"]: return util.validate_date(data, final, return_as="%Y-%m-%d") @@ -1327,7 +1328,7 @@ class CollectionBuilder: return util.parse(final, data, datatype="int") elif attribute in plex.float_attributes and modifier in [".gt", ".gte", ".lt", ".lte"]: return util.parse(final, data, datatype="float", minimum=0, maximum=10) - elif attribute in ["decade", "year", "episode_year"] and modifier in ["", ".not"]: + elif attribute in ["decade", "year", "episode_year", "tmdb_year"] and modifier in ["", ".not"]: final_years = [] values = util.get_list(data) for value in values: @@ -1414,12 +1415,12 @@ class CollectionBuilder: attr = None if filter_attr == "tmdb_vote_count": attr = item.vote_count - elif filter_attr == "year" and is_movie: + elif filter_attr == "tmdb_year" and is_movie: attr = item.year - elif filter_attr == "year" and not is_movie: + elif filter_attr == "tmdb_year" and not is_movie: air_date = item.first_air_date if air_date: - attr = util.validate_date(air_date, "Year Filter").year + attr = util.validate_date(air_date, "TMDb Year Filter").year if util.is_number_filter(attr, modifier, filter_data): return False except Failed: @@ -1427,26 +1428,26 @@ class CollectionBuilder: return True def check_filters(self, current, display): - if self.filters: + if self.filters or self.tmdb_filters: util.print_return(f"Filtering {display} {current.title}") + if self.tmdb_filters: + if current.ratingKey not in self.library.movie_rating_key_map and current.ratingKey not in self.library.show_rating_key_map: + logger.warning(f"Filter Error: No {'TMDb' if self.library.is_movie else 'TVDb'} ID found for {current.title}") + return False + try: + if current.ratingKey in self.library.movie_rating_key_map: + t_id = self.library.movie_rating_key_map[current.ratingKey] + else: + t_id = self.library.show_rating_key_map[current.ratingKey] + except Failed as e: + logger.error(e) + return False + if not self.check_tmdb_filter(t_id, current.ratingKey in self.library.movie_rating_key_map): + return False for filter_method, filter_data in self.filters: filter_attr, modifier, filter_final = self._split(filter_method) filter_actual = filter_translation[filter_attr] if filter_attr in filter_translation else filter_attr - if filter_attr in ["tmdb_vote_count", "original_language", "first_episode_aired", "last_episode_aired"]: - if current.ratingKey not in self.library.movie_rating_key_map and current.ratingKey not in self.library.show_rating_key_map: - logger.warning(f"Filter Error: No {'TMDb' if self.library.is_movie else 'TVDb'} ID found for {current.title}") - return False - try: - if current.ratingKey in self.library.movie_rating_key_map: - t_id = self.library.movie_rating_key_map[current.ratingKey] - else: - t_id = self.library.show_rating_key_map[current.ratingKey] - except Failed as e: - logger.error(e) - return False - if not self.check_tmdb_filter(t_id, current.ratingKey in self.library.movie_rating_key_map): - return False - elif filter_attr in ["release", "added", "last_played"]: + if filter_attr in ["release", "added", "last_played"]: if util.is_date_filter(getattr(current, filter_actual), modifier, filter_data, filter_final, self.current_time): return False elif filter_attr in ["audio_track_title", "filepath", "title", "studio"]: From 4f4999d6b8bf60fc17362c766e024061a29980ad Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 9 Aug 2021 11:50:41 -0400 Subject: [PATCH 80/95] changed list to dict --- modules/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/builder.py b/modules/builder.py index 781edba6..95194131 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -165,7 +165,7 @@ class CollectionBuilder: self.filters = [] self.tmdb_filters = [] self.rating_keys = [] - self.filtered_keys = [] + self.filtered_keys = {} self.run_again_movies = [] self.run_again_shows = [] self.posters = {} From 33a65f74cd5a68b8f97e7b68bfe6e1f95bfb024e Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 9 Aug 2021 15:45:20 -0400 Subject: [PATCH 81/95] cleanup --- plex_meta_manager.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 34921081..7f0fdf98 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -1,5 +1,6 @@ import argparse, logging, os, re, sys, time from datetime import datetime +from logging.handlers import RotatingFileHandler try: import schedule from modules import util @@ -26,7 +27,7 @@ parser.add_argument("-lo", "--library-only", "--libraries-only", dest="library_o parser.add_argument("-rc", "-cl", "--collection", "--collections", "--run-collection", "--run-collections", dest="collections", help="Process only specified collections (comma-separated list)", type=str) parser.add_argument("-rl", "-l", "--library", "--libraries", "--run-library", "--run-libraries", dest="libraries", help="Process only specified libraries (comma-separated list)", type=str) parser.add_argument("-nc", "--no-countdown", dest="no_countdown", help="Run without displaying the countdown", action="store_true", default=False) -parser.add_argument("-nm", "--no-missing", dest="no_missing", help="Run without running the midding section", action="store_true", default=False) +parser.add_argument("-nm", "--no-missing", dest="no_missing", help="Run without running the missing section", action="store_true", default=False) parser.add_argument("-d", "--divider", dest="divider", help="Character that divides the sections (Default: '=')", default="=", type=str) parser.add_argument("-w", "--width", dest="width", help="Screen Width (Default: 100)", default=100, type=int) args = parser.parse_args() @@ -93,7 +94,7 @@ sys.excepthook = util.my_except_hook def start(config_path, is_test=False, time_scheduled=None, requested_collections=None, requested_libraries=None, resume_from=None): file_logger = os.path.join(default_dir, "logs", "meta.log") should_roll_over = os.path.isfile(file_logger) - file_handler = logging.handlers.RotatingFileHandler(file_logger, delay=True, mode="w", backupCount=10, encoding="utf-8") + file_handler = RotatingFileHandler(file_logger, delay=True, mode="w", backupCount=10, encoding="utf-8") util.apply_formatter(file_handler) file_handler.addFilter(fmt_filter) if should_roll_over: @@ -134,7 +135,7 @@ def update_libraries(config): os.makedirs(os.path.join(default_dir, "logs", library.mapping_name, "collections"), exist_ok=True) col_file_logger = os.path.join(default_dir, "logs", library.mapping_name, "library.log") should_roll_over = os.path.isfile(col_file_logger) - library_handler = logging.handlers.RotatingFileHandler(col_file_logger, delay=True, mode="w", backupCount=3, encoding="utf-8") + library_handler = RotatingFileHandler(col_file_logger, delay=True, mode="w", backupCount=3, encoding="utf-8") util.apply_formatter(library_handler) if should_roll_over: library_handler.doRollover() @@ -216,7 +217,7 @@ def update_libraries(config): for library in config.libraries: if library.run_again: col_file_logger = os.path.join(default_dir, "logs", library.mapping_name, f"library.log") - library_handler = logging.handlers.RotatingFileHandler(col_file_logger, mode="w", backupCount=3, encoding="utf-8") + library_handler = RotatingFileHandler(col_file_logger, mode="w", backupCount=3, encoding="utf-8") util.apply_formatter(library_handler) logger.addHandler(library_handler) library_handler.addFilter(fmt_filter) @@ -437,7 +438,7 @@ def run_collection(config, library, metadata, requested_collections): os.makedirs(collection_log_folder, exist_ok=True) col_file_logger = os.path.join(collection_log_folder, f"collection.log") should_roll_over = os.path.isfile(col_file_logger) - collection_handler = logging.handlers.RotatingFileHandler(col_file_logger, delay=True, mode="w", backupCount=3, encoding="utf-8") + collection_handler = RotatingFileHandler(col_file_logger, delay=True, mode="w", backupCount=3, encoding="utf-8") util.apply_formatter(collection_handler) if should_roll_over: collection_handler.doRollover() From 0c537afaf6f621b82c8d12137640315569cb7f5a Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 10 Aug 2021 09:34:13 -0400 Subject: [PATCH 82/95] #347 Added mal_genre and mal_producer builders --- modules/builder.py | 42 +++++++++++++++++++++++------------- modules/mal.py | 53 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 76 insertions(+), 19 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 95194131..3bc8a45e 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -137,7 +137,7 @@ custom_sort_builders = [ "anidb_popular", "anilist_top_rated", "anilist_popular", "anilist_season", "anilist_studio", "anilist_genre", "anilist_tag", "mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_movie", "mal_ova", "mal_special", - "mal_popular", "mal_favorite", "mal_suggested", "mal_userlist", "mal_season" + "mal_popular", "mal_favorite", "mal_suggested", "mal_userlist", "mal_season", "mal_genre", "mal_producer" ] class CollectionBuilder: @@ -801,22 +801,34 @@ class CollectionBuilder: self.builders.append((method_name, util.parse(method_name, method_data, datatype="int", default=10))) elif method_name in ["mal_season", "mal_userlist"]: for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): - new_dictionary = {} if method_name == "mal_season": - if self.current_time.month in [1, 2, 3]: new_dictionary["season"] = "winter" - elif self.current_time.month in [4, 5, 6]: new_dictionary["season"] = "spring" - elif self.current_time.month in [7, 8, 9]: new_dictionary["season"] = "summer" - elif self.current_time.month in [10, 11, 12]: new_dictionary["season"] = "fall" - new_dictionary["season"] = util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=new_dictionary["season"], options=["winter", "spring", "summer", "fall"]) - new_dictionary["sort_by"] = util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="members", options=mal.season_sort_options, translation=mal.season_sort_translation) - new_dictionary["year"] = util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_time.year, parent=method_name, minimum=1917, maximum=self.current_time.year + 1) - new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=100, parent=method_name, maximum=500) + if self.current_time.month in [1, 2, 3]: default_season = "winter" + elif self.current_time.month in [4, 5, 6]: default_season = "spring" + elif self.current_time.month in [7, 8, 9]: default_season = "summer" + else: default_season = "fall" + self.builders.append((method_name, { + "season": util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=default_season, options=["winter", "spring", "summer", "fall"]), + "sort_by": util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="members", options=mal.season_sort_options, translation=mal.season_sort_translation), + "year": util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_time.year, parent=method_name, minimum=1917, maximum=self.current_time.year + 1), + "limit": util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=100, parent=method_name, maximum=500) + })) elif method_name == "mal_userlist": - new_dictionary["username"] = util.parse("username", dict_data, methods=dict_methods, parent=method_name) - new_dictionary["status"] = util.parse("status", dict_data, methods=dict_methods, parent=method_name, default="all", options=mal.userlist_status) - new_dictionary["sort_by"] = util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="score", options=mal.userlist_sort_options, translation=mal.userlist_sort_translation) - new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=100, parent=method_name, maximum=1000) - self.builders.append((method_name, new_dictionary)) + self.builders.append((method_name, { + "username": util.parse("username", dict_data, methods=dict_methods, parent=method_name), + "status": util.parse("status", dict_data, methods=dict_methods, parent=method_name, default="all", options=mal.userlist_status), + "sort_by": util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="score", options=mal.userlist_sort_options, translation=mal.userlist_sort_translation), + "limit": util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=100, parent=method_name, maximum=1000) + })) + elif method_name in ["mal_genre", "mal_producer"]: + id_name = f"{method_name[4:]}_id" + final_data = [] + for data in util.get_list(method_data): + final_data.append(data if isinstance(data, dict) else {id_name: data, "limit": 0}) + for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): + self.builders.append((method_name, { + id_name: util.parse(id_name, dict_data, datatype="int", methods=dict_methods, parent=method_name, maximum=999999), + "limit": util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=0, parent=method_name) + })) def _plex(self, method_name, method_data): if method_name == "plex_all": diff --git a/modules/mal.py b/modules/mal.py index 1df5c2fa..b86f6f13 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -1,4 +1,4 @@ -import logging, re, secrets, webbrowser +import logging, re, secrets, time, webbrowser from modules import util from modules.util import Failed, TimeoutExpired from ruamel import yaml @@ -6,8 +6,8 @@ from ruamel import yaml logger = logging.getLogger("Plex Meta Manager") builders = [ - "mal_id", "mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_ova", "mal_movie", - "mal_special", "mal_popular", "mal_favorite", "mal_season", "mal_suggested", "mal_userlist" + "mal_id", "mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_ova", "mal_movie", "mal_special", + "mal_popular", "mal_favorite", "mal_season", "mal_suggested", "mal_userlist", "mal_genre", "mal_producer" ] mal_ranked_name = { "mal_all": "all", "mal_airing": "airing", "mal_upcoming": "upcoming", "mal_tv": "tv", "mal_ova": "ova", @@ -17,7 +17,7 @@ mal_ranked_pretty = { "mal_all": "MyAnimeList All", "mal_airing": "MyAnimeList Airing", "mal_upcoming": "MyAnimeList Upcoming", "mal_tv": "MyAnimeList TV", "mal_ova": "MyAnimeList OVA", "mal_movie": "MyAnimeList Movie", "mal_special": "MyAnimeList Special", "mal_popular": "MyAnimeList Popular", - "mal_favorite": "MyAnimeList Favorite" + "mal_favorite": "MyAnimeList Favorite", "mal_genre": "MyAnimeList Genre", "mal_producer": "MyAnimeList Producer" } season_sort_translation = {"score": "anime_score", "anime_score": "anime_score", "members": "anime_num_list_users", "anime_num_list_users": "anime_num_list_users"} season_sort_options = ["score", "members"] @@ -35,6 +35,7 @@ userlist_sort_translation = { userlist_sort_options = ["score", "last_updated", "title", "start_date"] userlist_status = ["all", "watching", "completed", "on_hold", "dropped", "plan_to_watch"] base_url = "https://api.myanimelist.net" +jiken_base_url = "https://api.jikan.moe/v3" urls = { "oauth_token": f"https://myanimelist.net/v1/oauth2/token", "oauth_authorize": f"https://myanimelist.net/v1/oauth2/authorize", @@ -131,6 +132,11 @@ class MyAnimeList: if "error" in response: raise Failed(f"MyAnimeList Error: {response['error']}") else: return response + def _jiken_request(self, url): + data = self.config.get_json(f"{jiken_base_url}{url}") + time.sleep(2) + return data + def _parse_request(self, url): data = self._request(url) return [d["node"]["id"] for d in data["data"]] if "data" in data else [] @@ -155,6 +161,39 @@ class MyAnimeList: url = f"{urls['user']}/{username}/animelist?{final_status}sort={sort_by}&limit={limit}" return self._parse_request(url) + def _genre(self, genre_id, limit): + data = self._jiken_request(f"/genre/anime/{genre_id}") + if "item_count" not in data: + raise Failed(f"MyAnimeList Error: No MyAnimeList IDs for Genre ID: {genre_id}") + total_items = data["item_count"] + if total_items < limit or limit <= 0: + limit = total_items + mal_ids = [] + for i in range(1, int(((total_items - 1) / 100) + 2)): + if i > 1: + data = self._jiken_request(f"/genre/anime/{genre_id}/{i}") + mal_ids.extend([anime["mal_id"] for anime in data["anime"]]) + if len(mal_ids) > limit: + return mal_ids[:limit] + return mal_ids + + def _producer(self, producer_id, limit): + data = self._jiken_request(f"/producer/{producer_id}") + if "anime" not in data: + raise Failed(f"MyAnimeList Error: No MyAnimeList IDs for Producer ID: {producer_id}") + mal_ids = [] + count = 1 + while True: + if count > 1: + data = self._jiken_request(f"/producer/{producer_id}/{count}") + if "anime" not in data: + break + mal_ids.extend([anime["mal_id"] for anime in data["anime"]]) + if len(mal_ids) > limit > 0: + return mal_ids[:limit] + count += 1 + return mal_ids + def get_mal_ids(self, method, data): if method == "mal_id": logger.info(f"Processing MyAnimeList ID: {data}") @@ -162,6 +201,12 @@ class MyAnimeList: elif method in mal_ranked_name: logger.info(f"Processing {mal_ranked_pretty[method]}: {data} Anime") mal_ids = self._ranked(mal_ranked_name[method], data) + elif method == "mal_genre": + logger.info(f"Processing {mal_ranked_pretty[method]} ID: {data['genre_id']}") + mal_ids = self._genre(data["genre_id"], data["limit"]) + elif method == "mal_producer": + logger.info(f"Processing {mal_ranked_pretty[method]} ID: {data['producer_id']}") + mal_ids = self._producer(data["producer_id"], data["limit"]) elif method == "mal_season": logger.info(f"Processing MyAnimeList Season: {data['limit']} Anime from {util.pretty_seasons[data['season']]} {data['year']} sorted by {pretty_names[data['sort_by']]}") mal_ids = self._season(data["season"], data["year"], data["sort_by"], data["limit"]) From 325921ea83a7989fd79abdc4dc40e4bb76f8423a Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 10 Aug 2021 11:18:43 -0400 Subject: [PATCH 83/95] mass_genre_update can now also use tvdb --- modules/tvdb.py | 36 +++++++++++++++++++++++------------- plex_meta_manager.py | 16 ++++++++++++++-- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/modules/tvdb.py b/modules/tvdb.py index d2f2843a..5cbc0c2d 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -38,20 +38,30 @@ class TVDbObj: else: raise Failed(f"TVDb Error: Could not find a TVDb {self.media_type} ID at the URL {self.tvdb_url}") - results = response.xpath("//div[@class='change_translation_text' and @data-language='eng']/@data-title") - if len(results) > 0 and len(results[0]) > 0: - self.title = results[0] + def parse_page(xpath, fail=None, multi=False): + parse_results = response.xpath(xpath) + if len(parse_results) > 0: + parse_results = [r.strip() for r in parse_results if len(r) > 0] + if not multi and len(parse_results) > 0: + return parse_results[0] + elif len(parse_results) > 0: + return parse_results + elif fail is not None: + raise Failed(f"TVDb Error: {fail} not found from TVDb URL: {self.tvdb_url}") + else: + return None + + self.title = parse_page("//div[@class='change_translation_text' and not(@style='display:none')]/@data-title", fail="Name") + self.poster_path = parse_page("//div[@class='row hidden-xs hidden-sm']/div/img/@src") + self.background_path = parse_page("(//h2[@class='mt-4' and text()='Backgrounds']/following::div/a/@href)[1]") + self.summary = parse_page("//div[@class='change_translation_text' and not(@style='display:none')]/p/text()[normalize-space()]") + if self.is_movie: + self.directors = parse_page("//strong[text()='Directors']/parent::li/span/a/text()[normalize-space()]") + self.writers = parse_page("//strong[text()='Writers']/parent::li/span/a/text()[normalize-space()]") + self.studios = parse_page("//strong[text()='Studio']/parent::li/span/a/text()[normalize-space()]") else: - raise Failed(f"TVDb Error: Name not found from TVDb URL: {self.tvdb_url}") - - results = response.xpath("//div[@class='row hidden-xs hidden-sm']/div/img/@src") - self.poster_path = results[0] if len(results) > 0 and len(results[0]) > 0 else None - - results = response.xpath("(//h2[@class='mt-4' and text()='Backgrounds']/following::div/a/@href)[1]") - self.background_path = results[0] if len(results) > 0 and len(results[0]) > 0 else None - - results = response.xpath("//div[@class='block']/div[not(@style='display:none')]/p/text()") - self.summary = results[0] if len(results) > 0 and len(results[0]) > 0 else None + self.networks = parse_page("//strong[text()='Networks']/parent::li/span/a/text()[normalize-space()]") + self.genres = parse_page("//strong[text()='Genres']/parent::li/span/a/text()[normalize-space()]") tmdb_id = None imdb_id = None diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 7f0fdf98..1efca2e1 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -279,7 +279,7 @@ def mass_metadata(config, library, items): if not tmdb_id and not tvdb_id: tmdb_id = library.get_tmdb_from_map(item) if not tmdb_id and not tvdb_id and library.is_show: - tmdb_id = library.get_tvdb_from_map(item) + tvdb_id = library.get_tvdb_from_map(item) if library.mass_trakt_rating_update: try: @@ -330,7 +330,17 @@ def mass_metadata(config, library, items): else: logger.info(util.adjust_space(f"{item.title[:25]:<25} | No IMDb ID for Guid: {item.guid}")) - if not tmdb_item and not omdb_item: + tvdb_item = None + if library.mass_genre_update == "tvdb": + if tvdb_id: + try: + tvdb_item = config.TVDb.get_item(tvdb_id, library.is_movie) + except Failed as e: + logger.error(util.adjust_space(str(e))) + else: + logger.info(util.adjust_space(f"{item.title[:25]:<25} | No TVDb ID for Guid: {item.guid}")) + + if not tmdb_item and not omdb_item and not tvdb_item: continue if library.mass_genre_update: @@ -339,6 +349,8 @@ def mass_metadata(config, library, items): new_genres = [genre.name for genre in tmdb_item.genres] elif omdb_item and library.mass_genre_update in ["omdb", "imdb"]: new_genres = omdb_item.genres + elif tvdb_item and library.mass_genre_update == "tvdb": + new_genres = tvdb_item.genres else: raise Failed item_genres = [genre.tag for genre in item.genres] From 67e599c20bba3e1997506fd90c5704ebae839675 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Tue, 10 Aug 2021 11:33:32 -0400 Subject: [PATCH 84/95] fix for [] in filenames --- modules/plex.py | 32 +++++++++++++++----------------- modules/util.py | 9 +++++++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/modules/plex.py b/modules/plex.py index d38e149d..bc9494af 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -748,7 +748,6 @@ class Plex: def update_item_from_assets(self, item, overlay=None, create=False): name = os.path.basename(os.path.dirname(str(item.locations[0])) if self.is_movie else str(item.locations[0])) - glob_name = name.translate({ord("["): "[[]", ord("]"): "[]]"}) if "[" in name else name logger.debug(name) found_folder = False poster = None @@ -759,7 +758,7 @@ class Plex: if os.path.isdir(os.path.join(ad, name)): item_dir = os.path.join(ad, name) else: - matches = glob.glob(os.path.join(ad, "*", glob_name)) + matches = util.glob_filter(os.path.join(ad, "*", name)) if len(matches) > 0: item_dir = os.path.abspath(matches[0]) if item_dir is None: @@ -768,12 +767,12 @@ class Plex: poster_filter = os.path.join(item_dir, "poster.*") background_filter = os.path.join(item_dir, "background.*") else: - poster_filter = os.path.join(ad, f"{glob_name}.*") - background_filter = os.path.join(ad, f"{glob_name}_background.*") - matches = glob.glob(poster_filter) + poster_filter = os.path.join(ad, f"{name}.*") + background_filter = os.path.join(ad, f"{name}_background.*") + matches = util.glob_filter(poster_filter) if len(matches) > 0: poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_url=False) - matches = glob.glob(background_filter) + matches = util.glob_filter(background_filter) if len(matches) > 0: background = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_poster=False, is_url=False) if poster or background: @@ -783,8 +782,8 @@ class Plex: if item_dir: season_filter = os.path.join(item_dir, f"Season{'0' if season.seasonNumber < 10 else ''}{season.seasonNumber}.*") else: - season_filter = os.path.join(ad, f"{glob_name}_Season{'0' if season.seasonNumber < 10 else ''}{season.seasonNumber}.*") - matches = glob.glob(season_filter) + season_filter = os.path.join(ad, f"{name}_Season{'0' if season.seasonNumber < 10 else ''}{season.seasonNumber}.*") + matches = util.glob_filter(season_filter) if len(matches) > 0: season_poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title} Season {season.seasonNumber}'s ", is_url=False) self.upload_images(season, poster=season_poster) @@ -792,8 +791,8 @@ class Plex: if item_dir: episode_filter = os.path.join(item_dir, f"{episode.seasonEpisode.upper()}.*") else: - episode_filter = os.path.join(ad, f"{glob_name}_{episode.seasonEpisode.upper()}.*") - matches = glob.glob(episode_filter) + episode_filter = os.path.join(ad, f"{name}_{episode.seasonEpisode.upper()}.*") + matches = util.glob_filter(episode_filter) if len(matches) > 0: episode_poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title} {episode.seasonEpisode.upper()}'s ", is_url=False) self.upload_images(episode, poster=episode_poster) @@ -810,22 +809,21 @@ class Plex: def find_collection_assets(self, item, name=None, create=False): if name is None: name = item.title - glob_name = name.translate({ord("["): "[[]", ord("]"): "[]]"}) if "[" in name else name for ad in self.asset_directory: poster = None background = None if self.asset_folders: if not os.path.isdir(os.path.join(ad, name)): continue - poster_filter = os.path.join(ad, glob_name, "poster.*") - background_filter = os.path.join(ad, glob_name, "background.*") + poster_filter = os.path.join(ad, name, "poster.*") + background_filter = os.path.join(ad, name, "background.*") else: - poster_filter = os.path.join(ad, f"{glob_name}.*") - background_filter = os.path.join(ad, f"{glob_name}_background.*") - matches = glob.glob(poster_filter) + poster_filter = os.path.join(ad, f"{name}.*") + background_filter = os.path.join(ad, f"{name}_background.*") + matches = util.glob_filter(poster_filter) if len(matches) > 0: poster = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_url=False) - matches = glob.glob(background_filter) + matches = util.glob_filter(background_filter) if len(matches) > 0: background = ImageData("asset_directory", os.path.abspath(matches[0]), prefix=f"{item.title}'s ", is_poster=False, is_url=False) if poster or background: diff --git a/modules/util.py b/modules/util.py index d5dc6d7e..af024ef6 100644 --- a/modules/util.py +++ b/modules/util.py @@ -1,5 +1,6 @@ -import logging, os, re, signal, sys, time, traceback +import glob, logging, os, re, signal, sys, time, traceback from datetime import datetime, timedelta +from logging.handlers import RotatingFileHandler from pathvalidate import is_valid_filename, sanitize_filename from plexapi.exceptions import BadRequest, NotFound, Unauthorized @@ -203,7 +204,7 @@ def separator(text=None, space=True, border=True, debug=False): def apply_formatter(handler, border=True): text = f"| %(message)-{screen_width - 2}s |" if border else f"%(message)-{screen_width - 2}s" - if isinstance(handler, logging.handlers.RotatingFileHandler): + if isinstance(handler, RotatingFileHandler): text = f"[%(asctime)s] %(filename)-27s %(levelname)-10s {text}" handler.setFormatter(logging.Formatter(text)) @@ -246,6 +247,10 @@ def is_locked(filepath): file_object.close() return locked +def glob_filter(filter_in): + filter_in = filter_in.translate({ord("["): "[[]", ord("]"): "[]]"}) if "[" in filter_in else filter_in + return glob.glob(filter_in) + def is_date_filter(value, modifier, data, final, current_time): if value is None: return True From c8394e4bdbf30cffcc20a1ba37416d60feabca5c Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 11 Aug 2021 09:43:07 -0400 Subject: [PATCH 85/95] overlay fix --- modules/plex.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/plex.py b/modules/plex.py index bc9494af..c071d089 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -403,14 +403,16 @@ class Plex: self.reload(item) def upload_images(self, item, poster=None, background=None, overlay=None): + image = None + image_compare = None poster_uploaded = False + if self.config.Cache: + image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) + if poster is not None: try: - image = None - if self.config.Cache: - image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) - if str(poster.compare) != str(image_compare): - image = None + if image_compare and str(poster.compare) != str(image_compare): + image = None if image is None or image != item.thumb: self._upload_image(item, poster) poster_uploaded = True @@ -427,6 +429,8 @@ class Plex: image_overlay = None if self.config.Cache: _, _, image_overlay = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) + if image is None or image != item.thumb: + image_overlay = None if poster_uploaded or not image_overlay or image_overlay != overlay_name: if not item.posterUrl: raise Failed(f"Overlay Error: No existing poster to Overlay for {item.title}") From 20d8f4dae8fc20ed9bbb85a01d0078fa6ec6a8ea Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 12 Aug 2021 09:37:30 -0400 Subject: [PATCH 86/95] item_radarr_tag and item_sonarr_tag now work with missing items --- modules/builder.py | 58 ++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 3bc8a45e..816d1107 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1539,17 +1539,24 @@ class CollectionBuilder: logger.info(f"{self.name} Collection | X | {current_title} (TMDb: {missing_id})") logger.info("") logger.info(f"{len(missing_movies_with_names)} Movie{'s' if len(missing_movies_with_names) > 1 else ''} Missing") - if self.details["save_missing"] is True: - self.library.add_missing(self.name, missing_movies_with_names, True) - if (self.add_to_radarr and self.library.Radarr) or self.run_again: - missing_tmdb_ids = [missing_id for title, missing_id in missing_movies_with_names] - if self.add_to_radarr and self.library.Radarr: - try: - self.library.Radarr.add_tmdb(missing_tmdb_ids, **self.radarr_options) - except Failed as e: - logger.error(e) - if self.run_again: - self.run_again_movies.extend(missing_tmdb_ids) + if len(missing_movies_with_names) > 0: + if self.details["save_missing"] is True: + self.library.add_missing(self.name, missing_movies_with_names, True) + if self.run_again or (self.library.Radarr and (self.add_to_radarr or "item_radarr_tag" in self.item_details)): + missing_tmdb_ids = [missing_id for title, missing_id in missing_movies_with_names] + if self.library.Radarr: + if self.add_to_radarr: + try: + self.library.Radarr.add_tmdb(missing_tmdb_ids, **self.radarr_options) + except Failed as e: + logger.error(e) + if "item_radarr_tag" in self.item_details: + try: + self.library.Radarr.edit_tags(missing_tmdb_ids, self.item_details["item_radarr_tag"], self.item_details["apply_tags"]) + except Failed as e: + logger.error(e) + if self.run_again: + self.run_again_movies.extend(missing_tmdb_ids) if len(self.missing_shows) > 0 and self.library.is_show: missing_shows_with_names = [] for missing_id in self.missing_shows: @@ -1568,17 +1575,24 @@ class CollectionBuilder: logger.info(f"{self.name} Collection | X | {current_title} (TVDb: {missing_id})") logger.info("") logger.info(f"{len(missing_shows_with_names)} Show{'s' if len(missing_shows_with_names) > 1 else ''} Missing") - if self.details["save_missing"] is True: - self.library.add_missing(self.name, missing_shows_with_names, False) - if (self.add_to_sonarr and self.library.Sonarr) or self.run_again: - missing_tvdb_ids = [missing_id for title, missing_id in missing_shows_with_names] - if self.add_to_sonarr and self.library.Sonarr: - try: - self.library.Sonarr.add_tvdb(missing_tvdb_ids, **self.sonarr_options) - except Failed as e: - logger.error(e) - if self.run_again: - self.run_again_shows.extend(missing_tvdb_ids) + if len(missing_shows_with_names) > 0: + if self.details["save_missing"] is True: + self.library.add_missing(self.name, missing_shows_with_names, False) + if self.run_again or (self.library.Sonarr and (self.add_to_sonarr or "item_sonarr_tag" in self.item_details)): + missing_tvdb_ids = [missing_id for title, missing_id in missing_shows_with_names] + if self.library.Sonarr: + if self.add_to_sonarr: + try: + self.library.Sonarr.add_tvdb(missing_tvdb_ids, **self.sonarr_options) + except Failed as e: + logger.error(e) + if "item_sonarr_tag" in self.item_details: + try: + self.library.Sonarr.edit_tags(missing_tvdb_ids, self.item_details["item_sonarr_tag"], self.item_details["apply_tags"]) + except Failed as e: + logger.error(e) + if self.run_again: + self.run_again_shows.extend(missing_tvdb_ids) def sync_collection(self): count_removed = 0 From 98d1146e610236c15fa052aaba0e907f3983cb18 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 12 Aug 2021 16:36:38 -0400 Subject: [PATCH 87/95] changes overlay record keeping to labels --- modules/builder.py | 16 ++++++++++++++-- modules/cache.py | 14 ++++++++++---- modules/plex.py | 41 ++++++++++++++++++++++------------------- plex_meta_manager.py | 6 +++++- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 816d1107..d2f528df 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -1638,11 +1638,22 @@ class CollectionBuilder: logger.info("") overlay = None overlay_folder = None + overlay_name = "" rating_keys = [] if "item_overlay" in self.item_details: overlay_name = self.item_details["item_overlay"] if self.config.Cache: - rating_keys = self.config.Cache.query_image_map_overlay(self.library.image_table_name, overlay_name) + cache_keys = self.config.Cache.query_image_map_overlay(self.library.image_table_name, overlay_name) + if cache_keys: + for rating_key in cache_keys: + try: + item = self.fetch_item(rating_key) + except Failed as e: + logger.error(e) + continue + self.library.edit_tags("label", item, add_tags=[f"{overlay_name} Overlay"]) + self.config.Cache.update_remove_overlay(self.library.image_table_name, overlay_name) + rating_keys = [int(item.ratingKey) for item in self.library.get_labeled_items(f"{overlay_name} Overlay")] overlay_folder = os.path.join(self.config.default_dir, "overlays", overlay_name) overlay_image = Image.open(os.path.join(overlay_folder, "overlay.png")).convert("RGBA") temp_image = os.path.join(overlay_folder, f"temp.png") @@ -1683,11 +1694,12 @@ class CollectionBuilder: except Failed as e: logger.error(e) continue + self.library.edit_tags("label", item, remove_tags=[f"{overlay_name} Overlay"]) og_image = os.path.join(overlay_folder, f"{rating_key}.png") if os.path.exists(og_image): self.library.upload_file_poster(item, og_image) os.remove(og_image) - self.config.Cache.update_image_map(item.ratingKey, self.library.image_table_name, "", "", "") + self.config.Cache.update_image_map(item.ratingKey, self.library.image_table_name, "", "") def update_details(self): if not self.obj and self.smart_url: diff --git a/modules/cache.py b/modules/cache.py index 1f93a3b0..1c0e0fcd 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -302,6 +302,12 @@ class Cache: rks.append(int(row["rating_key"])) return rks + def update_remove_overlay(self, table_name, overlay): + with sqlite3.connect(self.cache_path) as connection: + connection.row_factory = sqlite3.Row + with closing(connection.cursor()) as cursor: + cursor.execute(f"UPDATE {table_name} SET overlay = ? WHERE overlay = ?", ("", overlay)) + def query_image_map(self, rating_key, table_name): with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row @@ -309,12 +315,12 @@ class Cache: cursor.execute(f"SELECT * FROM {table_name} WHERE rating_key = ?", (rating_key,)) row = cursor.fetchone() if row and row["location"]: - return row["location"], row["compare"], row["overlay"] - return None, None, None + return row["location"], row["compare"] + return None, None - def update_image_map(self, rating_key, table_name, location, compare, overlay): + def update_image_map(self, rating_key, table_name, location, compare): with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: cursor.execute(f"INSERT OR IGNORE INTO {table_name}(rating_key) VALUES(?)", (rating_key,)) - cursor.execute(f"UPDATE {table_name} SET location = ?, compare = ?, overlay = ? WHERE rating_key = ?", (location, compare, overlay, rating_key)) + cursor.execute(f"UPDATE {table_name} SET location = ?, compare = ?, overlay = ? WHERE rating_key = ?", (location, compare, "", rating_key)) diff --git a/modules/plex.py b/modules/plex.py index c071d089..10204d0b 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -1,4 +1,4 @@ -import glob, logging, os, plexapi, requests, shutil, time +import logging, os, plexapi, requests, shutil, time from modules import builder, util from modules.meta import Metadata from modules.util import Failed, ImageData @@ -365,11 +365,15 @@ class Plex: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def reload(self, item): - item.reload(checkFiles=False, includeAllConcerts=False, includeBandwidths=False, includeChapters=False, - includeChildren=False, includeConcerts=False, includeExternalMedia=False, includeExtras=False, - includeFields=False, includeGeolocation=False, includeLoudnessRamps=False, includeMarkers=False, - includeOnDeck=False, includePopularLeaves=False, includeRelated=False, - includeRelatedCount=0, includeReviews=False, includeStations=False) + try: + item.reload(checkFiles=False, includeAllConcerts=False, includeBandwidths=False, includeChapters=False, + includeChildren=False, includeConcerts=False, includeExternalMedia=False, includeExtras=False, + includeFields=False, includeGeolocation=False, includeLoudnessRamps=False, includeMarkers=False, + includeOnDeck=False, includePopularLeaves=False, includeRelated=False, + includeRelatedCount=0, includeReviews=False, includeStations=False) + except (BadRequest, NotFound) as e: + util.print_stacktrace() + raise Failed(f"Item Failed to Load: {e}") @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_plex) def edit_query(self, item, edits, advanced=False): @@ -407,7 +411,7 @@ class Plex: image_compare = None poster_uploaded = False if self.config.Cache: - image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) + image, image_compare = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) if poster is not None: try: @@ -423,15 +427,13 @@ class Plex: util.print_stacktrace() logger.error(f"Detail: {poster.attribute} failed to update {poster.message}") - overlay_name = "" if overlay is not None: overlay_name, overlay_folder, overlay_image, temp_image = overlay - image_overlay = None - if self.config.Cache: - _, _, image_overlay = self.config.Cache.query_image_map(item.ratingKey, self.image_table_name) - if image is None or image != item.thumb: - image_overlay = None - if poster_uploaded or not image_overlay or image_overlay != overlay_name: + item_labels = {item_tag.tag.lower(): item_tag.tag for item_tag in item.labels} + for item_label in item_labels: + if item_label.endswith(" overlay") and item_label != f"{overlay_name.lower()} overlay": + raise Failed(f"Overlay Error: Poster already has an existing Overlay: {item_labels[item_label]}") + if poster_uploaded or image is None or image != item.thumb or f"{overlay_name.lower()} overlay" not in item_labels: if not item.posterUrl: raise Failed(f"Overlay Error: No existing poster to Overlay for {item.title}") response = requests.get(item.posterUrl) @@ -448,6 +450,7 @@ class Plex: new_poster.paste(overlay_image, (0, 0), overlay_image) new_poster.save(temp_image) self.upload_file_poster(item, temp_image) + self.edit_tags("label", item, add_tags=[f"{overlay_name} Overlay"]) poster_uploaded = True logger.info(f"Detail: Overlay: {overlay_name} applied to {item.title}") @@ -456,7 +459,7 @@ class Plex: try: image = None if self.config.Cache: - image, image_compare, _ = self.config.Cache.query_image_map(item.ratingKey, f"{self.image_table_name}_backgrounds") + image, image_compare = self.config.Cache.query_image_map(item.ratingKey, f"{self.image_table_name}_backgrounds") if str(background.compare) != str(image_compare): image = None if image is None or image != item.art: @@ -471,9 +474,9 @@ class Plex: if self.config.Cache: if poster_uploaded: - self.config.Cache.update_image_map(item.ratingKey, self.image_table_name, item.thumb, poster.compare if poster else "", overlay_name) + self.config.Cache.update_image_map(item.ratingKey, self.image_table_name, item.thumb, poster.compare if poster else "") if background_uploaded: - self.config.Cache.update_image_map(item.ratingKey, f"{self.image_table_name}_backgrounds", item.art, background.compare, "") + self.config.Cache.update_image_map(item.ratingKey, f"{self.image_table_name}_backgrounds", item.art, background.compare) @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def get_search_choices(self, search_name, title=True): @@ -743,11 +746,11 @@ class Plex: if _add: updated = True self.query_data(getattr(obj, f"add{attr.capitalize()}"), _add) - logger.info(f"Detail: {attr.capitalize()} {util.compile_list(_add)} added") + logger.info(f"Detail: {attr.capitalize()} {util.compile_list(_add)} added to {obj.title}") if _remove: updated = True self.query_data(getattr(obj, f"remove{attr.capitalize()}"), _remove) - logger.info(f"Detail: {attr.capitalize()} {util.compile_list(_remove)} removed") + logger.info(f"Detail: {attr.capitalize()} {util.compile_list(_remove)} removed to {obj.title}") return updated def update_item_from_assets(self, item, overlay=None, create=False): diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 1efca2e1..b1bdf942 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -262,7 +262,11 @@ def mass_metadata(config, library, items): trakt_ratings = config.Trakt.user_ratings(library.is_movie) if library.mass_trakt_rating_update else [] for i, item in enumerate(items, 1): - library.reload(item) + try: + library.reload(item) + except Failed as e: + logger.error(e) + continue util.print_return(f"Processing: {i}/{len(items)} {item.title}") tmdb_id = None tvdb_id = None From 26eb57587f22ea1168a0b608048353e8f313b13f Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 12 Aug 2021 16:37:03 -0400 Subject: [PATCH 88/95] run custom sorts at the end of a library run --- modules/plex.py | 1 + plex_meta_manager.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/modules/plex.py b/modules/plex.py index 10204d0b..854f87b3 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -300,6 +300,7 @@ class Plex: self.movie_rating_key_map = {} self.show_rating_key_map = {} self.run_again = [] + self.run_sort = [] self.overlays = [] def get_all_collections(self): diff --git a/plex_meta_manager.py b/plex_meta_manager.py index b1bdf942..10d47b5f 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -169,6 +169,15 @@ def update_libraries(config): logger.removeHandler(library_handler) run_collection(config, library, metadata, collections_to_run) logger.addHandler(library_handler) + if library.run_sort: + logger.info("") + util.separator(f"Sorting {library.name} Library's Collections", space=False, border=False) + logger.info("") + for builder in library.run_sort: + logger.info("") + util.separator(f"Sorting {builder.name} Collection", space=False, border=False) + logger.info("") + builder.sort_collection() if not config.test_mode and not config.requested_collections and ((library.show_unmanaged and not library_only) or (library.assets_for_all and not collection_only)): logger.info("") @@ -515,10 +524,11 @@ def run_collection(config, library, metadata, requested_collections): builder.update_details() if builder.custom_sort: - logger.info("") - util.separator(f"Sorting {mapping_name} Collection", space=False, border=False) - logger.info("") - builder.sort_collection() + library.run_sort.append(builder) + # logger.info("") + # util.separator(f"Sorting {mapping_name} Collection", space=False, border=False) + # logger.info("") + # builder.sort_collection() builder.update_item_details() From 0cc9d81283de11fe0289fe2a9bcd2e10e869b4f9 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 13 Aug 2021 10:18:05 -0400 Subject: [PATCH 89/95] add Folder option for metadata files and cleanup --- modules/anilist.py | 110 ++++++++++++++++++------------------------- modules/config.py | 25 ++++------ modules/convert.py | 6 +-- modules/plex.py | 17 ++++++- modules/util.py | 9 ---- plex_meta_manager.py | 14 ++---- 6 files changed, 80 insertions(+), 101 deletions(-) diff --git a/modules/anilist.py b/modules/anilist.py index ef001832..f88e2289 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -9,22 +9,33 @@ builders = [ "anilist_season", "anilist_studio", "anilist_tag", "anilist_top_rated" ] pretty_names = {"score": "Average Score", "popular": "Popularity"} +search_translation = { + "season": "MediaSeason", "seasonYear": "Int", "isAdult": "Boolean", + "startDate_greater": "FuzzyDateInt", "startDate_lesser": "FuzzyDateInt", "endDate_greater": "FuzzyDateInt", "endDate_lesser": "FuzzyDateInt", + "format_in": "[MediaFormat]", "format_not_in": "[MediaFormat]", "status_in": "[MediaStatus]", "status_not_in": "[MediaStatus]", + "episodes_greater": "Int", "episodes_lesser": "Int", "duration_greater": "Int", "duration_lesser": "Int", + "genre_in": "[String]", "genre_not_in": "[String]", "tag_in": "[String]", "tag_not_in": "[String]", + "averageScore_greater": "Int", "averageScore_lesser": "Int", "popularity_greater": "Int", "popularity_lesser": "Int" +} base_url = "https://graphql.anilist.co" -tag_query = "query{MediaTagCollection {name}}" +tag_query = "query{MediaTagCollection {name, category}}" genre_query = "query{GenreCollection}" class AniList: def __init__(self, config): self.config = config self.tags = {} - self.genres = {} - self.tags = {t["name"].lower(): t["name"] for t in self._request(tag_query, {})["data"]["MediaTagCollection"]} - self.genres = {g.lower(): g for g in self._request(genre_query, {})["data"]["GenreCollection"]} + self.categories = {} + for media_tag in self._request(tag_query, {})["data"]["MediaTagCollection"]: + self.tags[media_tag["name"].lower().replace(" ", "-")] = media_tag["name"] + self.categories[media_tag["category"].lower().replace(" ", "-")] = media_tag["category"] + self.genres = {g.lower().replace(" ", "-"): g for g in self._request(genre_query, {})["data"]["GenreCollection"]} def _request(self, query, variables): response = self.config.post(base_url, json={"query": query, "variables": variables}) json_obj = response.json() if "errors" in json_obj: + logger.debug(json_obj) if json_obj['errors'][0]['message'] == "Too Many Requests.": if "Retry-After" in response.headers: time.sleep(int(response.headers["Retry-After"])) @@ -35,7 +46,7 @@ class AniList: time.sleep(0.4) return json_obj - def _validate(self, anilist_id): + def _validate_id(self, anilist_id): query = "query ($id: Int) {Media(id: $id) {id title{romaji english}}}" media = self._request(query, {"id": anilist_id})["data"]["Media"] if media["id"]: @@ -65,62 +76,31 @@ class AniList: return anilist_ids def _top_rated(self, limit): - query = """ - query ($page: Int) { - Page(page: $page) { - pageInfo {hasNextPage} - media(averageScore_greater: 3, sort: SCORE_DESC, type: ANIME) {id} - } - } - """ - return self._pagenation(query, limit=limit) + return self._search(limit=limit, averageScore_greater=3) def _popular(self, limit): - query = """ - query ($page: Int) { - Page(page: $page) { - pageInfo {hasNextPage} - media(popularity_greater: 1000, sort: POPULARITY_DESC, type: ANIME) {id} - } - } - """ - return self._pagenation(query, limit=limit) + return self._search(sort="popular", limit=limit, popularity_greater=1000) def _season(self, season, year, sort, limit): - query = """ - query ($page: Int, $season: MediaSeason, $year: Int, $sort: [MediaSort]) { - Page(page: $page){ - pageInfo {hasNextPage} - media(season: $season, seasonYear: $year, type: ANIME, sort: $sort){id} - } - } - """ - variables = {"season": season.upper(), "year": year, "sort": "SCORE_DESC" if sort == "score" else "POPULARITY_DESC"} + return self._search(sort=sort, limit=limit, season=season.upper(), year=year) + + def _search(self, sort="score", limit=0, **kwargs): + query_vars = "$page: Int, $sort: [MediaSort]" + media_vars = "sort: $sort, type: ANIME" + variables = {"sort": "SCORE_DESC" if sort == "score" else "POPULARITY_DESC"} + for key, value in kwargs.items(): + query_vars += f", ${key}: {search_translation[key]}" + media_vars += f", {key}: ${key}" + variables[key] = value + query = f"query ({query_vars}) {{Page(page: $page){{pageInfo {{hasNextPage}}media({media_vars}){{id}}}}}}" + logger.info(query) return self._pagenation(query, limit=limit, variables=variables) def _genre(self, genre, sort, limit): - query = """ - query ($page: Int, $genre: String, $sort: [MediaSort]) { - Page(page: $page){ - pageInfo {hasNextPage} - media(genre: $genre, sort: $sort){id} - } - } - """ - variables = {"genre": genre, "sort": "SCORE_DESC" if sort == "score" else "POPULARITY_DESC"} - return self._pagenation(query, limit=limit, variables=variables) + return self._search(sort=sort, limit=limit, genre=genre) def _tag(self, tag, sort, limit): - query = """ - query ($page: Int, $tag: String, $sort: [MediaSort]) { - Page(page: $page){ - pageInfo {hasNextPage} - media(tag: $tag, sort: $sort){id} - } - } - """ - variables = {"tag": tag, "sort": "SCORE_DESC" if sort == "score" else "POPULARITY_DESC"} - return self._pagenation(query, limit=limit, variables=variables) + return self._search(sort=sort, limit=limit, tag=tag) def _studio(self, studio_id): query = """ @@ -166,7 +146,7 @@ class AniList: name = "" if not ignore_ids: ignore_ids = [anilist_id] - anilist_id, name = self._validate(anilist_id) + anilist_id, name = self._validate_id(anilist_id) anilist_ids.append(anilist_id) json_obj = self._request(query, {"id": anilist_id}) edges = [media["node"]["id"] for media in json_obj["data"]["Media"]["relations"]["edges"] @@ -183,22 +163,26 @@ class AniList: return anilist_ids, ignore_ids, name + def validate_tag(self, tag): + return self._validate(tag, self.tags, "Tag") + + def validate_category(self, category): + return self._validate(category, self.categories, "Category") + def validate_genre(self, genre): - if genre.lower() in self.genres: - return self.genres[genre.lower()] - raise Failed(f"AniList Error: Genre: {genre} does not exist") + return self._validate(genre, self.genres, "Genre") - def validate_tag(self, tag): - if tag.lower() in self.tags: - return self.tags[tag.lower()] - raise Failed(f"AniList Error: Tag: {tag} does not exist") + def _validate(self, data, options, name): + data_check = data.lower().replace(" / ", "-").replace(" ", "-") + if data_check in options: + return options[data_check] + raise Failed(f"AniList Error: {name}: {data} does not exist\nOptions: {', '.join([v for k, v in options.items()])}") def validate_anilist_ids(self, anilist_ids, studio=False): anilist_id_list = util.get_int_list(anilist_ids, "AniList ID") anilist_values = [] + query = f"query ($id: Int) {{{'Studio(id: $id) {name}' if studio else 'Media(id: $id) {id}'}}}" for anilist_id in anilist_id_list: - if studio: query = "query ($id: Int) {Studio(id: $id) {name}}" - else: query = "query ($id: Int) {Media(id: $id) {id}}" try: self._request(query, {"id": anilist_id}) anilist_values.append(anilist_id) @@ -210,7 +194,7 @@ class AniList: def get_anilist_ids(self, method, data): if method == "anilist_id": logger.info(f"Processing AniList ID: {data}") - anilist_id, name = self._validate(data) + anilist_id, name = self._validate_id(data) anilist_ids = [anilist_id] elif method == "anilist_popular": logger.info(f"Processing AniList Popular: {data} Anime") diff --git a/modules/config.py b/modules/config.py index 572748fe..4386d4be 100644 --- a/modules/config.py +++ b/modules/config.py @@ -384,21 +384,16 @@ class Config: paths_to_check = lib["metadata_path"] if isinstance(lib["metadata_path"], list) else [lib["metadata_path"]] for path in paths_to_check: if isinstance(path, dict): - if "url" in path: - if path["url"] is None: - logger.error("Config Error: metadata_path url is blank") - else: - params["metadata_path"].append(("URL", path["url"])) - if "git" in path: - if path["git"] is None: - logger.error("Config Error: metadata_path git is blank") - else: - params["metadata_path"].append(("Git", path['git'])) - if "file" in path: - if path["file"] is None: - logger.error("Config Error: metadata_path file is blank") - else: - params["metadata_path"].append(("File", path['file'])) + def check_dict(attr, name): + if attr in path: + if path[attr] is None: + logger.error(f"Config Error: metadata_path {attr} is blank") + else: + params["metadata_path"].append((name, path[attr])) + check_dict("url", "URL") + check_dict("git", "Git") + check_dict("file", "File") + check_dict("folder", "Folder") else: params["metadata_path"].append(("File", path)) else: diff --git a/modules/convert.py b/modules/convert.py index d0395f40..aa7744d5 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -296,7 +296,7 @@ class Convert: if tvdb: tvdb_id.append(tvdb) if not tvdb_id: - raise Failed(f"Unable to convert TMDb ID: {util.compile_list(tmdb_id)} to TVDb ID") + raise Failed(f"Unable to convert TMDb ID: {', '.join(tmdb_id)} to TVDb ID") if not imdb_id and tvdb_id: for tvdb in tvdb_id: @@ -306,8 +306,8 @@ class Convert: def update_cache(cache_ids, id_type, imdb_in, guid_type): if self.config.Cache: - cache_ids = util.compile_list(cache_ids) - imdb_in = util.compile_list(imdb_in) if imdb_in else None + cache_ids = ",".join(cache_ids) + imdb_in = ",".join(imdb_in) if imdb_in else None ids = f"{item.guid:<46} | {id_type} ID: {cache_ids:<7} | IMDb ID: {str(imdb_in):<10}" logger.info(util.adjust_space(f" Cache | {'^' if expired else '+'} | {ids} | {item.title}")) self.config.Cache.update_guid_map(item.guid, cache_ids, imdb_in, expired, guid_type) diff --git a/modules/plex.py b/modules/plex.py index 854f87b3..b34c2ebf 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -240,7 +240,20 @@ class Plex: self.metadatas = [] self.metadata_files = [] + metadata = [] for file_type, metadata_file in params["metadata_path"]: + if file_type == "folder": + if os.path.isdir(metadata_file): + yml_files = util.glob_filter(os.path.join(metadata_file, "*.yml")) + if yml_files: + metadata.extend([("File", yml) for yml in yml_files]) + else: + logger.error(f"Config Error: No YAML (.yml) files found in {metadata_file}") + else: + logger.error(f"Config Error: Folder not found: {metadata_file}") + else: + metadata.append((file_type, metadata_file)) + for file_type, metadata_file in metadata: try: meta_obj = Metadata(config, self, file_type, metadata_file) if meta_obj.collections: @@ -747,11 +760,11 @@ class Plex: if _add: updated = True self.query_data(getattr(obj, f"add{attr.capitalize()}"), _add) - logger.info(f"Detail: {attr.capitalize()} {util.compile_list(_add)} added to {obj.title}") + logger.info(f"Detail: {attr.capitalize()} {','.join(_add)} added to {obj.title}") if _remove: updated = True self.query_data(getattr(obj, f"remove{attr.capitalize()}"), _remove) - logger.info(f"Detail: {attr.capitalize()} {util.compile_list(_remove)} removed to {obj.title}") + logger.info(f"Detail: {attr.capitalize()} {','.join(_remove)} removed to {obj.title}") return updated def update_item_from_assets(self, item, overlay=None, create=False): diff --git a/modules/util.py b/modules/util.py index af024ef6..597edfcc 100644 --- a/modules/util.py +++ b/modules/util.py @@ -69,15 +69,6 @@ def add_dict_list(keys, value, dict_map): else: dict_map[key] = [value] -def compile_list(data): - if isinstance(data, list): - text = "" - for item in data: - text += f"{',' if len(text) > 0 else ''}{item}" - return text - else: - return data - def get_list(data, lower=False, split=True, int_list=False): if data is None: return None elif isinstance(data, list): return data diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 10d47b5f..62e9e029 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -368,17 +368,13 @@ def mass_metadata(config, library, items): raise Failed item_genres = [genre.tag for genre in item.genres] display_str = "" - add_genre = [] - for genre in (g for g in new_genres if g not in item_genres): - add_genre.append(genre) - display_str += f"{', ' if len(display_str) > 0 else ''}+{genre}" + add_genre = [genre for genre in (g for g in new_genres if g not in item_genres)] if len(add_genre) > 0: + display_str += f"+{', +'.join(add_genre)}" library.query_data(item.addGenre, add_genre) - remove_genre = [] - for genre in (g for g in item_genres if g not in new_genres): - remove_genre.append(genre) - display_str += f"{', ' if len(display_str) > 0 else ''}-{genre}" + remove_genre = [genre for genre in (g for g in item_genres if g not in new_genres)] if len(remove_genre) > 0: + display_str += f"-{', -'.join(remove_genre)}" library.query_data(item.removeGenre, remove_genre) if len(display_str) > 0: logger.info(util.adjust_space(f"{item.title[:25]:<25} | Genres | {display_str}")) @@ -568,7 +564,7 @@ try: minutes = int((seconds % 3600) // 60) time_str = f"{hours} Hour{'s' if hours > 1 else ''} and " if hours > 0 else "" time_str += f"{minutes} Minute{'s' if minutes > 1 else ''}" - util.print_return(f"Current Time: {current} | {time_str} until the next run at {og_time_str} {util.compile_list(times_to_run)}") + util.print_return(f"Current Time: {current} | {time_str} until the next run at {og_time_str} | Runs: {', '.join(times_to_run)}") time.sleep(60) except KeyboardInterrupt: util.separator("Exiting Plex Meta Manager") From a404cc86b4763934ac463037d1dcaaf4328b054a Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 13 Aug 2021 10:20:14 -0400 Subject: [PATCH 90/95] upped the version --- plex_meta_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 62e9e029..19292c21 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -108,7 +108,7 @@ def start(config_path, is_test=False, time_scheduled=None, requested_collections logger.info(util.centered("| __/| | __/> < | | | | __/ || (_| | | | | | (_| | | | | (_| | (_| | __/ | ")) logger.info(util.centered("|_| |_|\\___/_/\\_\\ |_| |_|\\___|\\__\\__,_| |_| |_|\\__,_|_| |_|\\__,_|\\__, |\\___|_| ")) logger.info(util.centered(" |___/ ")) - logger.info(util.centered(" Version: 1.11.3-beta4 ")) + logger.info(util.centered(" Version: 1.11.3-beta5 ")) if time_scheduled: start_type = f"{time_scheduled} " elif is_test: start_type = "Test " elif requested_collections: start_type = "Collections " From a2016d81b2a75d809985832172fa4d61bb3801cb Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 13 Aug 2021 10:33:56 -0400 Subject: [PATCH 91/95] fixed joins --- modules/convert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/convert.py b/modules/convert.py index aa7744d5..015d0411 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -296,7 +296,7 @@ class Convert: if tvdb: tvdb_id.append(tvdb) if not tvdb_id: - raise Failed(f"Unable to convert TMDb ID: {', '.join(tmdb_id)} to TVDb ID") + raise Failed(f"Unable to convert TMDb ID: {', '.join([str(t) for t in tmdb_id])} to TVDb ID") if not imdb_id and tvdb_id: for tvdb in tvdb_id: @@ -306,8 +306,8 @@ class Convert: def update_cache(cache_ids, id_type, imdb_in, guid_type): if self.config.Cache: - cache_ids = ",".join(cache_ids) - imdb_in = ",".join(imdb_in) if imdb_in else None + cache_ids = ",".join([str(c) for c in cache_ids]) + imdb_in = ",".join([str(i) for i in imdb_in]) if imdb_in else None ids = f"{item.guid:<46} | {id_type} ID: {cache_ids:<7} | IMDb ID: {str(imdb_in):<10}" logger.info(util.adjust_space(f" Cache | {'^' if expired else '+'} | {ids} | {item.title}")) self.config.Cache.update_guid_map(item.guid, cache_ids, imdb_in, expired, guid_type) From 9d27f959fdf3b1ede6c7ac51b60cf5f613fd1875 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 13 Aug 2021 23:32:26 -0400 Subject: [PATCH 92/95] #360 fixed folder --- modules/anilist.py | 13 +++++++------ modules/builder.py | 1 + modules/plex.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/modules/anilist.py b/modules/anilist.py index f88e2289..f2acbee4 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -31,19 +31,20 @@ class AniList: self.categories[media_tag["category"].lower().replace(" ", "-")] = media_tag["category"] self.genres = {g.lower().replace(" ", "-"): g for g in self._request(genre_query, {})["data"]["GenreCollection"]} - def _request(self, query, variables): + def _request(self, query, variables, level=1): response = self.config.post(base_url, json={"query": query, "variables": variables}) json_obj = response.json() if "errors" in json_obj: - logger.debug(json_obj) if json_obj['errors'][0]['message'] == "Too Many Requests.": - if "Retry-After" in response.headers: - time.sleep(int(response.headers["Retry-After"])) - raise ValueError + wait_time = int(response.headers["Retry-After"]) if "Retry-After" in response.headers else 0 + time.sleep(wait_time if wait_time > 0 else 10) + if level < 6: + return self._request(query, variables, level=level + 1) + raise Failed(f"AniList Error: Connection Failed") else: raise Failed(f"AniList Error: {json_obj['errors'][0]['message']}") else: - time.sleep(0.4) + time.sleep(60 / 90) return json_obj def _validate_id(self, anilist_id): diff --git a/modules/builder.py b/modules/builder.py index d2f528df..95ac721e 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -26,6 +26,7 @@ method_alias = { "rating": "critic_rating", "show_user_rating": "user_rating", "video_resolution": "resolution", + "tmdb_trending": "tmdb_trending_daily", "play": "plays", "show_plays": "plays", "show_play": "plays", "episode_play": "episode_plays", "originally_available": "release", "episode_originally_available": "episode_air_date", "episode_release": "episode_air_date", "episode_released": "episode_air_date", diff --git a/modules/plex.py b/modules/plex.py index b34c2ebf..192c99d9 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -242,7 +242,7 @@ class Plex: self.metadata_files = [] metadata = [] for file_type, metadata_file in params["metadata_path"]: - if file_type == "folder": + if file_type == "Folder": if os.path.isdir(metadata_file): yml_files = util.glob_filter(os.path.join(metadata_file, "*.yml")) if yml_files: From fcbfcf5ecd2c8f9a781449f108cb1b1260e65b5a Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sat, 14 Aug 2021 18:59:35 -0400 Subject: [PATCH 93/95] added anilist_search --- modules/anilist.py | 138 +++++++++++++++++++++++--------------------- modules/builder.py | 140 ++++++++++++++++++++++++++------------------- modules/mal.py | 2 +- modules/plex.py | 4 -- modules/tmdb.py | 18 +++--- modules/util.py | 16 +++++- 6 files changed, 179 insertions(+), 139 deletions(-) diff --git a/modules/anilist.py b/modules/anilist.py index f2acbee4..d73ecc76 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -6,17 +6,27 @@ logger = logging.getLogger("Plex Meta Manager") builders = [ "anilist_genre", "anilist_id", "anilist_popular", "anilist_relations", - "anilist_season", "anilist_studio", "anilist_tag", "anilist_top_rated" + "anilist_season", "anilist_studio", "anilist_tag", "anilist_top_rated", "anilist_search" ] pretty_names = {"score": "Average Score", "popular": "Popularity"} -search_translation = { - "season": "MediaSeason", "seasonYear": "Int", "isAdult": "Boolean", - "startDate_greater": "FuzzyDateInt", "startDate_lesser": "FuzzyDateInt", "endDate_greater": "FuzzyDateInt", "endDate_lesser": "FuzzyDateInt", - "format_in": "[MediaFormat]", "format_not_in": "[MediaFormat]", "status_in": "[MediaStatus]", "status_not_in": "[MediaStatus]", - "episodes_greater": "Int", "episodes_lesser": "Int", "duration_greater": "Int", "duration_lesser": "Int", - "genre_in": "[String]", "genre_not_in": "[String]", "tag_in": "[String]", "tag_not_in": "[String]", - "averageScore_greater": "Int", "averageScore_lesser": "Int", "popularity_greater": "Int", "popularity_lesser": "Int" +attr_translation = {"year": "seasonYear", "adult": "isAdult", "start": "startDate", "end": "endDate", "tag_category": "tagCategory", "score": "averageScore", "min_tag_percent": "minimumTagRank"} +mod_translation = {"": "in", "not": "not_in", "before": "greater", "after": "lesser", "gt": "greater", "gte": "greater", "lt": "lesser", "lte": "lesser"} +mod_searches = [ + "start.before", "start.after", "end.before", "end.after", + "format", "format.not", "status", "status.not", "genre", "genre.not", "tag", "tag.not", "tag_category", "tag_category.not", + "episodes.gt", "episodes.gte", "episodes.lt", "episodes.lte", "duration.gt", "duration.gte", "duration.lt", "duration.lte", + "score.gt", "score.gte", "score.lt", "score.lte", "popularity.gt", "popularity.gte", "popularity.lt", "popularity.lte" +] +no_mod_searches = ["season", "year", "adult", "min_tag_percent"] +searches = mod_searches + no_mod_searches +search_types = { + "season": "MediaSeason", "seasonYear": "Int", "isAdult": "Boolean", "startDate": "FuzzyDateInt", "endDate": "FuzzyDateInt", + "format": "[MediaFormat]", "status": "[MediaStatus]", "genre": "[String]", "tag": "[String]", "tagCategory": "[String]", + "episodes": "Int", "duration": "Int", "averageScore": "Int", "popularity": "Int", "minimumTagRank": "Int" } +media_season = {"winter": "WINTER", "spring": "SPRING", "summer": "SUMMER", "fall": "FALL"} +media_format = {"tv": "TV", "short": "TV_SHORT", "movie": "MOVIE", "special": "SPECIAL", "ova": "OVA", "ona": "ONA", "music": "MUSIC"} +media_status = {"finished": "FINISHED", "airing": "RELEASING", "not_yet_aired": "NOT_YET_RELEASED", "cancelled": "CANCELLED", "hiatus": "HIATUS"} base_url = "https://graphql.anilist.co" tag_query = "query{MediaTagCollection {name, category}}" genre_query = "query{GenreCollection}" @@ -24,12 +34,14 @@ genre_query = "query{GenreCollection}" class AniList: def __init__(self, config): self.config = config - self.tags = {} - self.categories = {} + self.options = { + "Tag": {}, "Tag Category": {}, + "Genre": {g.lower().replace(" ", "-"): g for g in self._request(genre_query, {})["data"]["GenreCollection"]}, + "Season": media_season, "Format": media_format, "Status": media_status + } for media_tag in self._request(tag_query, {})["data"]["MediaTagCollection"]: - self.tags[media_tag["name"].lower().replace(" ", "-")] = media_tag["name"] - self.categories[media_tag["category"].lower().replace(" ", "-")] = media_tag["category"] - self.genres = {g.lower().replace(" ", "-"): g for g in self._request(genre_query, {})["data"]["GenreCollection"]} + self.options["Tag"][media_tag["name"].lower().replace(" ", "-")] = media_tag["name"] + self.options["Tag Category"][media_tag["category"].lower().replace(" ", "-")] = media_tag["category"] def _request(self, query, variables, level=1): response = self.config.post(base_url, json={"query": query, "variables": variables}) @@ -76,32 +88,31 @@ class AniList: break return anilist_ids - def _top_rated(self, limit): - return self._search(limit=limit, averageScore_greater=3) - - def _popular(self, limit): - return self._search(sort="popular", limit=limit, popularity_greater=1000) - - def _season(self, season, year, sort, limit): - return self._search(sort=sort, limit=limit, season=season.upper(), year=year) - - def _search(self, sort="score", limit=0, **kwargs): + def _search(self, **kwargs): query_vars = "$page: Int, $sort: [MediaSort]" media_vars = "sort: $sort, type: ANIME" - variables = {"sort": "SCORE_DESC" if sort == "score" else "POPULARITY_DESC"} + variables = {"sort": "SCORE_DESC" if kwargs['sort_by'] == "score" else "POPULARITY_DESC"} for key, value in kwargs.items(): - query_vars += f", ${key}: {search_translation[key]}" - media_vars += f", {key}: ${key}" - variables[key] = value + if key not in ["sort_by", "limit"]: + if "." in key: + attr, mod = key.split(".") + else: + attr = key + mod = "" + ani_attr = attr_translation[attr] if attr in attr_translation else attr + final = ani_attr if attr in no_mod_searches else f"{ani_attr}_{mod_translation[mod]}" + if attr in ["start", "end"]: + value = int(util.validate_date(value, f"anilist_search {key}", return_as="%Y%m%d")) + if mod == "gte": + value -= 1 + elif mod == "lte": + value += 1 + query_vars += f", ${final}: {search_types[ani_attr]}" + media_vars += f", {final}: ${final}" + variables[key] = value query = f"query ({query_vars}) {{Page(page: $page){{pageInfo {{hasNextPage}}media({media_vars}){{id}}}}}}" - logger.info(query) - return self._pagenation(query, limit=limit, variables=variables) - - def _genre(self, genre, sort, limit): - return self._search(sort=sort, limit=limit, genre=genre) - - def _tag(self, tag, sort, limit): - return self._search(sort=sort, limit=limit, tag=tag) + logger.debug(query) + return self._pagenation(query, limit=kwargs["limit"], variables=variables) def _studio(self, studio_id): query = """ @@ -164,20 +175,15 @@ class AniList: return anilist_ids, ignore_ids, name - def validate_tag(self, tag): - return self._validate(tag, self.tags, "Tag") - - def validate_category(self, category): - return self._validate(category, self.categories, "Category") - - def validate_genre(self, genre): - return self._validate(genre, self.genres, "Genre") - - def _validate(self, data, options, name): - data_check = data.lower().replace(" / ", "-").replace(" ", "-") - if data_check in options: - return options[data_check] - raise Failed(f"AniList Error: {name}: {data} does not exist\nOptions: {', '.join([v for k, v in options.items()])}") + def validate(self, name, data): + valid = [] + for d in util.get_list(data): + data_check = d.lower().replace(" / ", "-").replace(" ", "-") + if data_check in self.options[name]: + valid.append(self.options[name][data_check]) + if len(valid) > 0: + return valid + raise Failed(f"AniList Error: {name}: {data} does not exist\nOptions: {', '.join([v for k, v in self.options[name].items()])}") def validate_anilist_ids(self, anilist_ids, studio=False): anilist_id_list = util.get_int_list(anilist_ids, "AniList ID") @@ -197,21 +203,6 @@ class AniList: logger.info(f"Processing AniList ID: {data}") anilist_id, name = self._validate_id(data) anilist_ids = [anilist_id] - elif method == "anilist_popular": - logger.info(f"Processing AniList Popular: {data} Anime") - anilist_ids = self._popular(data) - elif method == "anilist_top_rated": - logger.info(f"Processing AniList Top Rated: {data} Anime") - anilist_ids = self._top_rated(data) - elif method == "anilist_season": - logger.info(f"Processing AniList Season: {data['limit'] if data['limit'] > 0 else 'All'} Anime from {util.pretty_seasons[data['season']]} {data['year']} sorted by {pretty_names[data['sort_by']]}") - anilist_ids = self._season(data["season"], data["year"], data["sort_by"], data["limit"]) - elif method == "anilist_genre": - logger.info(f"Processing AniList Genre: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Genre: {data['genre']} sorted by {pretty_names[data['sort_by']]}") - anilist_ids = self._genre(data["genre"], data["sort_by"], data["limit"]) - elif method == "anilist_tag": - logger.info(f"Processing AniList Tag: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag: {data['tag']} sorted by {pretty_names[data['sort_by']]}") - anilist_ids = self._tag(data["tag"], data["sort_by"], data["limit"]) elif method == "anilist_studio": anilist_ids, name = self._studio(data) logger.info(f"Processing AniList Studio: ({data}) {name} ({len(anilist_ids)} Anime)") @@ -219,7 +210,24 @@ class AniList: anilist_ids, _, name = self._relations(data) logger.info(f"Processing AniList Relations: ({data}) {name} ({len(anilist_ids)} Anime)") else: - raise Failed(f"AniList Error: Method {method} not supported") + if method == "anilist_popular": + data = {"limit": data, "popularity.gt": 3, "sort_by": "popular"} + elif method == "anilist_top_rated": + data = {"limit": data, "score.gt": 3, "sort_by": "score"} + elif method not in builders: + raise Failed(f"AniList Error: Method {method} not supported") + message = f"Processing {method.replace('_', ' ').title().replace('Anilist', 'AniList')}:\nSort By: {pretty_names[data['sort_by']]}" + if data['limit'] > 0: + message += f"\nLimit: {data['limit']}" + for key, value in data.items(): + if "." in key: + attr, mod = key.split(".") + else: + attr = key + mod = "" + message += f"\n{attr.replace('_', ' ').title()} {util.mod_displays[mod]} {value}" + util.print_multiline(message) + anilist_ids = self._search(**data) logger.debug("") logger.debug(f"{len(anilist_ids)} AniList IDs Found: {anilist_ids}") return anilist_ids diff --git a/modules/builder.py b/modules/builder.py index 95ac721e..950650c9 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -37,7 +37,9 @@ method_alias = { "producers": "producer", "writers": "writer", "years": "year", "show_year": "year", "show_years": "year", - "show_title": "title" + "show_title": "title", + "seasonyear": "year", "isadult": "adult", "startdate": "start", "enddate": "end", "averagescore": "score", + "minimum_tag_percentage": "min_tag_percent", "minimumtagrank": "min_tag_percent", "minimum_tag_rank": "min_tag_percent" } filter_translation = { "actor": "actors", @@ -757,12 +759,45 @@ class CollectionBuilder: elif self.current_time.month in [3, 4, 5]: new_dictionary["season"] = "spring" elif self.current_time.month in [6, 7, 8]: new_dictionary["season"] = "summer" elif self.current_time.month in [9, 10, 11]: new_dictionary["season"] = "fall" - new_dictionary["season"] = util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=new_dictionary["season"], options=["winter", "spring", "summer", "fall"]) - new_dictionary["year"] = util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_time.year, parent=method_name, minimum=1917, maximum=self.current_time.year + 1) + new_dictionary["season"] = util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=new_dictionary["season"], options=util.seasons) + new_dictionary["year"] = util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_year, parent=method_name, minimum=1917, maximum=self.current_year + 1) elif method_name == "anilist_genre": - new_dictionary["genre"] = self.config.AniList.validate_genre(util.parse("genre", dict_data, methods=dict_methods, parent=method_name)) + new_dictionary["genre"] = self.config.AniList.validate("Genre", util.parse("genre", dict_data, methods=dict_methods, parent=method_name)) elif method_name == "anilist_tag": - new_dictionary["tag"] = self.config.AniList.validate_tag(util.parse("tag", dict_data, methods=dict_methods, parent=method_name)) + new_dictionary["tag"] = self.config.AniList.validate("Tag", util.parse("tag", dict_data, methods=dict_methods, parent=method_name)) + elif method_name == "anilist_search": + for search_method, search_data in dict_data.items(): + search_attr, modifier, search_final = self._split(search_method) + if search_data is None: + raise Failed(f"Collection Error: {method_name} {search_final} attribute is blank") + elif search_final not in anilist.searches: + raise Failed(f"Collection Error: {method_name} {search_final} attribute not supported") + elif search_attr == "season": + if self.current_time.month in [12, 1, 2]: new_dictionary["season"] = "winter" + elif self.current_time.month in [3, 4, 5]: new_dictionary["season"] = "spring" + elif self.current_time.month in [6, 7, 8]: new_dictionary["season"] = "summer" + elif self.current_time.month in [9, 10, 11]: new_dictionary["season"] = "fall" + new_dictionary["season"] = util.parse("season", dict_data, parent=method_name, default=new_dictionary["season"], options=util.seasons) + if "year" not in dict_methods: + logger.warning(f"Collection Warning: {method_name} {search_final} attribute must be used with the year attribute using this year by default") + elif search_attr == "year": + if "season" not in dict_methods: + raise Failed(f"Collection Error: {method_name} {search_final} attribute must be used with the season attribute") + new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="int", parent=method_name, default=self.current_year, minimum=1917, maximum=self.current_year + 1) + elif search_attr == "adult": + new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="bool", parent=method_name) + elif search_attr in ["episodes", "duration", "score", "popularity"]: + new_dictionary[search_final] = util.parse(search_final, search_data, datatype="int", parent=method_name) + elif search_attr in ["format", "status", "genre", "tag", "tag_category"]: + new_dictionary[search_final] = self.config.AniList.validate(search_attr.replace("_", " ").title(), util.parse(search_final, search_data)) + elif search_attr in ["start", "end"]: + new_dictionary[search_final] = util.validate_date(search_data, f"{method_name} {search_final} attribute", return_as="%m/%d/%Y") + elif search_attr == "min_tag_percent": + new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="int", parent=method_name, minimum=0, maximum=100) + elif search_final not in ["sort_by", "limit"]: + raise Failed(f"Collection Error: {method_name} {search_final} attribute not supported") + if len(new_dictionary) > 0: + raise Failed(f"Collection Error: {method_name} must have at least one valid search option") new_dictionary["sort_by"] = util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="score", options=["score", "popular"]) new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=0, parent=method_name, maximum=500) self.builders.append((method_name, new_dictionary)) @@ -808,9 +843,9 @@ class CollectionBuilder: elif self.current_time.month in [7, 8, 9]: default_season = "summer" else: default_season = "fall" self.builders.append((method_name, { - "season": util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=default_season, options=["winter", "spring", "summer", "fall"]), + "season": util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=default_season, options=util.seasons), "sort_by": util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="members", options=mal.season_sort_options, translation=mal.season_sort_translation), - "year": util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_time.year, parent=method_name, minimum=1917, maximum=self.current_time.year + 1), + "year": util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_year, parent=method_name, minimum=1917, maximum=self.current_year + 1), "limit": util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=100, parent=method_name, maximum=500) })) elif method_name == "mal_userlist": @@ -866,58 +901,45 @@ class CollectionBuilder: def _tmdb(self, method_name, method_data): if method_name == "tmdb_discover": for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): - new_dictionary = {"limit": 100} - for discover_name, discover_data in dict_data.items(): - discover_final = discover_name.lower() - if discover_data: - if (self.library.is_movie and discover_final in tmdb.discover_movie) or (self.library.is_show and discover_final in tmdb.discover_tv): - if discover_final == "language": - if re.compile("([a-z]{2})-([A-Z]{2})").match(str(discover_data)): - new_dictionary[discover_final] = str(discover_data) - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} must match pattern ([a-z]{{2}})-([A-Z]{{2}}) e.g. en-US") - elif discover_final == "region": - if re.compile("^[A-Z]{2}$").match(str(discover_data)): - new_dictionary[discover_final] = str(discover_data) - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} must match pattern ^[A-Z]{{2}}$ e.g. US") - elif discover_final == "sort_by": - if (self.library.is_movie and discover_data in tmdb.discover_movie_sort) or (self.library.is_show and discover_data in tmdb.discover_tv_sort): - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: {discover_data} is invalid") - elif discover_final == "certification_country": - if "certification" in dict_data or "certification.lte" in dict_data or "certification.gte" in dict_data: - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be used with either certification, certification.lte, or certification.gte") - elif discover_final in ["certification", "certification.lte", "certification.gte"]: - if "certification_country" in dict_data: - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be used with certification_country") - elif discover_final in ["include_adult", "include_null_first_air_dates", "screened_theatrically"]: - if discover_data is True: - new_dictionary[discover_final] = discover_data - elif discover_final in tmdb.discover_dates: - new_dictionary[discover_final] = util.validate_date(discover_data, f"{method_name} attribute {discover_final}", return_as="%m/%d/%Y") - elif discover_final in ["primary_release_year", "year", "first_air_date_year"]: - new_dictionary[discover_final] = util.parse(discover_final, discover_data, datatype="int", parent=method_name, minimum=1800, maximum=self.current_year + 1) - elif discover_final in ["vote_count.gte", "vote_count.lte", "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte"]: - new_dictionary[discover_final] = util.parse(discover_final, discover_data, datatype="int", parent=method_name) - elif discover_final in ["with_cast", "with_crew", "with_people", "with_companies", "with_networks", "with_genres", "without_genres", "with_keywords", "without_keywords", "with_original_language", "timezone"]: - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final} not supported") - elif discover_final == "limit": - if isinstance(discover_data, int) and discover_data > 0: - new_dictionary[discover_final] = discover_data - else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final}: must be a valid number greater then 0") + new_dictionary = {"limit": util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=100, parent=method_name)} + for discover_method, discover_data in dict_data.items(): + discover_attr, modifier, discover_final = self._split(discover_method) + if discover_data is None: + raise Failed(f"Collection Error: {method_name} {discover_final} attribute is blank") + elif discover_final not in tmdb.discover_all: + raise Failed(f"Collection Error: {method_name} {discover_final} attribute not supported") + elif self.library.is_movie and discover_attr in tmdb.discover_tv_only: + raise Failed(f"Collection Error: {method_name} {discover_final} attribute only works for show libraries") + elif self.library.is_show and discover_attr in tmdb.discover_movie_only: + raise Failed(f"Collection Error: {method_name} {discover_final} attribute only works for movie libraries") + elif discover_attr in ["language", "region"]: + regex = ("([a-z]{2})-([A-Z]{2})", "en-US") if discover_attr == "language" else ("^[A-Z]{2}$", "US") + new_dictionary[discover_attr] = util.parse(discover_attr, discover_data, parent=method_name, regex=regex) + elif discover_attr == "sort_by" and self.library.is_movie: + options = tmdb.discover_movie_sort if self.library.is_movie else tmdb.discover_tv_sort + new_dictionary[discover_attr] = util.parse(discover_attr, discover_data, parent=method_name, options=options) + elif discover_attr == "certification_country": + if "certification" in dict_data or "certification.lte" in dict_data or "certification.gte" in dict_data: + new_dictionary[discover_attr] = discover_data else: - raise Failed(f"Collection Error: {method_name} attribute {discover_final} not supported") - else: - raise Failed(f"Collection Error: {method_name} parameter {discover_final} is blank") + raise Failed(f"Collection Error: {method_name} {discover_attr} attribute: must be used with either certification, certification.lte, or certification.gte") + elif discover_attr == "certification": + if "certification_country" in dict_data: + new_dictionary[discover_final] = discover_data + else: + raise Failed(f"Collection Error: {method_name} {discover_final} attribute: must be used with certification_country") + elif discover_attr in ["include_adult", "include_null_first_air_dates", "screened_theatrically"]: + new_dictionary[discover_attr] = util.parse(discover_attr, discover_data, datatype="bool", parent=method_name) + elif discover_final in tmdb.discover_dates: + new_dictionary[discover_final] = util.validate_date(discover_data, f"{method_name} {discover_final} attribute", return_as="%m/%d/%Y") + elif discover_attr in ["primary_release_year", "year", "first_air_date_year"]: + new_dictionary[discover_attr] = util.parse(discover_attr, discover_data, datatype="int", parent=method_name, minimum=1800, maximum=self.current_year + 1) + elif discover_attr in ["vote_count", "vote_average", "with_runtime"]: + new_dictionary[discover_final] = util.parse(discover_final, discover_data, datatype="int", parent=method_name) + elif discover_final in ["with_cast", "with_crew", "with_people", "with_companies", "with_networks", "with_genres", "without_genres", "with_keywords", "without_keywords", "with_original_language", "timezone"]: + new_dictionary[discover_final] = discover_data + elif discover_attr != "limit": + raise Failed(f"Collection Error: {method_name} {discover_final} attribute not supported") if len(new_dictionary) > 1: self.builders.append((method_name, new_dictionary)) else: @@ -1191,7 +1213,7 @@ class CollectionBuilder: if attr in string_filters and modifier in ["", ".not"]: mod_s = "does not contain" if modifier == ".not" else "contains" elif mod_s is None: - mod_s = plex.mod_displays[modifier] + mod_s = util.mod_displays[modifier] param_s = plex.search_display[attr] if attr in plex.search_display else attr.title().replace('_', ' ') display_line = f"{indent}{param_s} {mod_s} {arg_s}" return f"{arg_key}{mod}={arg}&", display_line diff --git a/modules/mal.py b/modules/mal.py index b86f6f13..06c4ee9f 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -208,7 +208,7 @@ class MyAnimeList: logger.info(f"Processing {mal_ranked_pretty[method]} ID: {data['producer_id']}") mal_ids = self._producer(data["producer_id"], data["limit"]) elif method == "mal_season": - logger.info(f"Processing MyAnimeList Season: {data['limit']} Anime from {util.pretty_seasons[data['season']]} {data['year']} sorted by {pretty_names[data['sort_by']]}") + logger.info(f"Processing MyAnimeList Season: {data['limit']} Anime from {data['season'].title()} {data['year']} sorted by {pretty_names[data['sort_by']]}") mal_ids = self._season(data["season"], data["year"], data["sort_by"], data["limit"]) elif method == "mal_suggested": logger.info(f"Processing MyAnimeList Suggested: {data} Anime") diff --git a/modules/plex.py b/modules/plex.py index 192c99d9..ba502322 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -152,10 +152,6 @@ sorts = { "added.asc": "addedAt:asc", "added.desc": "addedAt:desc" } modifiers = {".not": "!", ".begins": "<", ".ends": ">", ".before": "<<", ".after": ">>", ".gt": ">>", ".gte": "__gte", ".lt": "<<", ".lte": "__lte"} -mod_displays = { - "": "is", ".not": "is not", ".begins": "begins with", ".ends": "ends with", ".before": "is before", ".after": "is after", - ".gt": "is greater than", ".gte": "is greater than or equal", ".lt": "is less than", ".lte": "is less than or equal" -} tags = [ "actor", "audio_language", "collection", "content_rating", "country", "director", "genre", "label", "network", "producer", "resolution", "studio", "subtitle_language", "writer" diff --git a/modules/tmdb.py b/modules/tmdb.py index 9b56e222..6bc9b28d 100644 --- a/modules/tmdb.py +++ b/modules/tmdb.py @@ -21,19 +21,23 @@ type_map = { "tmdb_network": "Network", "tmdb_person": "Person", "tmdb_producer": "Person", "tmdb_producer_details": "Person", "tmdb_show": "Show", "tmdb_show_details": "Show", "tmdb_writer": "Person", "tmdb_writer_details": "Person" } -discover_movie = [ +discover_all = [ "language", "with_original_language", "region", "sort_by", "with_cast", "with_crew", "with_people", "certification_country", "certification", "certification.lte", "certification.gte", "year", "primary_release_year", "primary_release_date.gte", "primary_release_date.lte", "release_date.gte", "release_date.lte", "vote_count.gte", "vote_count.lte", "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte", - "with_companies", "with_genres", "without_genres", "with_keywords", "without_keywords", "include_adult" + "with_companies", "with_genres", "without_genres", "with_keywords", "without_keywords", "include_adult", + "timezone", "screened_theatrically", "include_null_first_air_dates", "limit", + "air_date.gte", "air_date.lte", "first_air_date.gte", "first_air_date.lte", "first_air_date_year", "with_networks" ] -discover_tv = [ - "language", "with_original_language", "timezone", "sort_by", "screened_theatrically", "include_null_first_air_dates", - "air_date.gte", "air_date.lte", "first_air_date.gte", "first_air_date.lte", "first_air_date_year", - "vote_count.gte", "vote_count.lte", "vote_average.gte", "vote_average.lte", "with_runtime.gte", "with_runtime.lte", - "with_genres", "without_genres", "with_keywords", "without_keywords", "with_networks", "with_companies" +discover_movie_only = [ + "region", "with_cast", "with_crew", "with_people", "certification_country", "certification", + "year", "primary_release_year", "primary_release_date", "release_date", "include_adult" +] +discover_tv_only = [ + "timezone", "screened_theatrically", "include_null_first_air_dates", + "air_date", "first_air_date", "first_air_date_year", "with_networks", ] discover_dates = [ "primary_release_date.gte", "primary_release_date.lte", "release_date.gte", "release_date.lte", diff --git a/modules/util.py b/modules/util.py index 597edfcc..8b55a3cb 100644 --- a/modules/util.py +++ b/modules/util.py @@ -48,12 +48,16 @@ days_alias = { "saturday": 5, "sat": 5, "s": 5, "sunday": 6, "sun": 6, "su": 6, "u": 6 } +mod_displays = { + "": "is", ".not": "is not", ".begins": "begins with", ".ends": "ends with", ".before": "is before", ".after": "is after", + ".gt": "is greater than", ".gte": "is greater than or equal", ".lt": "is less than", ".lte": "is less than or equal" +} pretty_days = {0: "Monday", 1: "Tuesday", 2: "Wednesday", 3: "Thursday", 4: "Friday", 5: "Saturday", 6: "Sunday"} pretty_months = { 1: "January", 2: "February", 3: "March", 4: "April", 5: "May", 6: "June", 7: "July", 8: "August", 9: "September", 10: "October", 11: "November", 12: "December" } -pretty_seasons = {"winter": "Winter", "spring": "Spring", "summer": "Summer", "fall": "Fall"} +seasons = ["winter", "spring", "summer", "fall"] pretty_ids = {"anidbid": "AniDB", "imdbid": "IMDb", "mal_id": "MyAnimeList", "themoviedb_id": "TMDb", "thetvdb_id": "TVDb", "tvdbid": "TVDb"} def tab_new_lines(data): @@ -283,7 +287,7 @@ def is_string_filter(values, modifier, data): if jailbreak: break return (jailbreak and modifier == ".not") or (not jailbreak and modifier in ["", ".begins", ".ends", ".regex"]) -def parse(attribute, data, datatype=None, methods=None, parent=None, default=None, options=None, translation=None, minimum=1, maximum=None): +def parse(attribute, data, datatype=None, methods=None, parent=None, default=None, options=None, translation=None, minimum=1, maximum=None, regex=None): display = f"{parent + ' ' if parent else ''}{attribute} attribute" if options is None and translation is not None: options = [o for o in translation] @@ -305,6 +309,12 @@ def parse(attribute, data, datatype=None, methods=None, parent=None, default=Non message = f"{display} not found" elif value is None: message = f"{display} is blank" + elif regex is not None: + regex_str, example = regex + if re.compile(regex_str).match(str(value)): + return str(value) + else: + message = f"{display}: {value} must match pattern {regex_str} e.g. {example}" elif datatype == "bool": if isinstance(value, bool): return value @@ -330,7 +340,7 @@ def parse(attribute, data, datatype=None, methods=None, parent=None, default=Non message = f"{pre} between {minimum} and {maximum}" elif (translation is not None and str(value).lower() not in translation) or \ (options is not None and translation is None and str(value).lower() not in options): - message = f"{display} {value} must be in {options}" + message = f"{display} {value} must be in {', '.join([str(o) for o in options])}" else: return translation[value] if translation is not None else value From ae34662ad73accca23e0c579d158a62395e72b55 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Sat, 14 Aug 2021 21:30:03 -0400 Subject: [PATCH 94/95] small fix --- modules/cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/cache.py b/modules/cache.py index 1c0e0fcd..25e3f193 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -97,7 +97,7 @@ class Cache: for row in cursor.fetchall(): if row["type"] == "poster": final_table = table_name if row["type"] == "poster" else f"{table_name}_backgrounds" - self.update_image_map(row["rating_key"], final_table, row["location"], row["compare"], row["overlay"]) + self.update_image_map(row["rating_key"], final_table, row["location"], row["compare"], overlay=row["overlay"]) cursor.execute("DROP TABLE IF EXISTS image_map") def query_guid_map(self, plex_guid): @@ -318,9 +318,9 @@ class Cache: return row["location"], row["compare"] return None, None - def update_image_map(self, rating_key, table_name, location, compare): + def update_image_map(self, rating_key, table_name, location, compare, overlay=""): with sqlite3.connect(self.cache_path) as connection: connection.row_factory = sqlite3.Row with closing(connection.cursor()) as cursor: cursor.execute(f"INSERT OR IGNORE INTO {table_name}(rating_key) VALUES(?)", (rating_key,)) - cursor.execute(f"UPDATE {table_name} SET location = ?, compare = ?, overlay = ? WHERE rating_key = ?", (location, compare, "", rating_key)) + cursor.execute(f"UPDATE {table_name} SET location = ?, compare = ?, overlay = ? WHERE rating_key = ?", (location, compare, overlay, rating_key)) From 1e2fc349110535763df9515397d0a590ab01a2af Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 16 Aug 2021 01:41:04 -0400 Subject: [PATCH 95/95] final cleanup for v1.12.0 --- config/config.yml.template | 4 +- modules/anilist.py | 14 ++- modules/builder.py | 205 ++++++++++++++++++------------------- modules/config.py | 8 +- modules/convert.py | 2 +- modules/mal.py | 28 +++-- modules/plex.py | 2 +- modules/radarr.py | 1 + modules/sonarr.py | 1 + plex_meta_manager.py | 10 +- requirements.txt | 3 +- 11 files changed, 146 insertions(+), 132 deletions(-) diff --git a/config/config.yml.template b/config/config.yml.template index 9a2bb7d6..d6b0d925 100644 --- a/config/config.yml.template +++ b/config/config.yml.template @@ -89,6 +89,6 @@ mal: token_type: expires_in: refresh_token: -anidb: - username: ###### - optional +anidb: # Optional + username: ###### password: ###### \ No newline at end of file diff --git a/modules/anilist.py b/modules/anilist.py index d73ecc76..6e82d86c 100644 --- a/modules/anilist.py +++ b/modules/anilist.py @@ -4,10 +4,7 @@ from modules.util import Failed logger = logging.getLogger("Plex Meta Manager") -builders = [ - "anilist_genre", "anilist_id", "anilist_popular", "anilist_relations", - "anilist_season", "anilist_studio", "anilist_tag", "anilist_top_rated", "anilist_search" -] +builders = ["anilist_id", "anilist_popular", "anilist_relations", "anilist_studio", "anilist_top_rated", "anilist_search"] pretty_names = {"score": "Average Score", "popular": "Popularity"} attr_translation = {"year": "seasonYear", "adult": "isAdult", "start": "startDate", "end": "endDate", "tag_category": "tagCategory", "score": "averageScore", "min_tag_percent": "minimumTagRank"} mod_translation = {"": "in", "not": "not_in", "before": "greater", "after": "lesser", "gt": "greater", "gte": "greater", "lt": "lesser", "lte": "lesser"} @@ -17,12 +14,13 @@ mod_searches = [ "episodes.gt", "episodes.gte", "episodes.lt", "episodes.lte", "duration.gt", "duration.gte", "duration.lt", "duration.lte", "score.gt", "score.gte", "score.lt", "score.lte", "popularity.gt", "popularity.gte", "popularity.lt", "popularity.lte" ] -no_mod_searches = ["season", "year", "adult", "min_tag_percent"] +no_mod_searches = ["search", "season", "year", "adult", "min_tag_percent"] searches = mod_searches + no_mod_searches search_types = { - "season": "MediaSeason", "seasonYear": "Int", "isAdult": "Boolean", "startDate": "FuzzyDateInt", "endDate": "FuzzyDateInt", - "format": "[MediaFormat]", "status": "[MediaStatus]", "genre": "[String]", "tag": "[String]", "tagCategory": "[String]", - "episodes": "Int", "duration": "Int", "averageScore": "Int", "popularity": "Int", "minimumTagRank": "Int" + "search": "String", "season": "MediaSeason", "seasonYear": "Int", "isAdult": "Boolean", "minimumTagRank": "Int", + "startDate": "FuzzyDateInt", "endDate": "FuzzyDateInt", "format": "[MediaFormat]", "status": "[MediaStatus]", + "genre": "[String]", "tag": "[String]", "tagCategory": "[String]", + "episodes": "Int", "duration": "Int", "averageScore": "Int", "popularity": "Int" } media_season = {"winter": "WINTER", "spring": "SPRING", "summer": "SUMMER", "fall": "FALL"} media_format = {"tv": "TV", "short": "TV_SHORT", "movie": "MOVIE", "special": "SPECIAL", "ova": "OVA", "ona": "ONA", "music": "MUSIC"} diff --git a/modules/builder.py b/modules/builder.py index 950650c9..235cce03 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -39,7 +39,8 @@ method_alias = { "years": "year", "show_year": "year", "show_years": "year", "show_title": "title", "seasonyear": "year", "isadult": "adult", "startdate": "start", "enddate": "end", "averagescore": "score", - "minimum_tag_percentage": "min_tag_percent", "minimumtagrank": "min_tag_percent", "minimum_tag_rank": "min_tag_percent" + "minimum_tag_percentage": "min_tag_percent", "minimumtagrank": "min_tag_percent", "minimum_tag_rank": "min_tag_percent", + "anilist_tag": "anilist_search", "anilist_genre": "anilist_search", "anilist_season": "anilist_search" } filter_translation = { "actor": "actors", @@ -74,15 +75,15 @@ summary_details = [ ] poster_details = ["url_poster", "tmdb_poster", "tmdb_profile", "tvdb_poster", "file_poster"] background_details = ["url_background", "tmdb_background", "tvdb_background", "file_background"] -boolean_details = ["visible_library", "visible_home", "visible_shared", "show_filtered", "show_missing", "save_missing", "item_assets", "create_asset_folders", "released_missing_only"] +boolean_details = ["visible_library", "visible_home", "visible_shared", "show_filtered", "show_missing", "save_missing", "item_assets", "missing_only_released"] string_details = ["sort_title", "content_rating", "name_mapping"] ignored_details = ["smart_filter", "smart_label", "smart_url", "run_again", "schedule", "sync_mode", "template", "test", "tmdb_person", "build_collection", "collection_order", "validate_builders"] details = ["collection_mode", "collection_order", "label"] + boolean_details + string_details collectionless_details = ["collection_order", "plex_collectionless", "label", "label_sync_mode", "test"] + \ poster_details + background_details + summary_details + string_details item_details = ["item_label", "item_radarr_tag", "item_sonarr_tag", "item_overlay"] + list(plex.item_advance_keys.keys()) -radarr_details = ["radarr_add", "radarr_folder", "radarr_monitor", "radarr_search", "radarr_availability", "radarr_quality", "radarr_tag"] -sonarr_details = ["sonarr_add", "sonarr_folder", "sonarr_monitor", "sonarr_language", "sonarr_series", "sonarr_quality", "sonarr_season", "sonarr_search", "sonarr_cutoff_search", "sonarr_tag"] +radarr_details = ["radarr_add", "radarr_add_existing", "radarr_folder", "radarr_monitor", "radarr_search", "radarr_availability", "radarr_quality", "radarr_tag"] +sonarr_details = ["sonarr_add", "sonarr_add_existing", "sonarr_folder", "sonarr_monitor", "sonarr_language", "sonarr_series", "sonarr_quality", "sonarr_season", "sonarr_search", "sonarr_cutoff_search", "sonarr_tag"] all_filters = [ "actor", "actor.not", "audio_language", "audio_language.not", @@ -133,35 +134,34 @@ smart_url_invalid = ["filters", "run_again", "sync_mode", "show_filtered", "show custom_sort_builders = [ "tmdb_list", "tmdb_popular", "tmdb_now_playing", "tmdb_top_rated", "tmdb_trending_daily", "tmdb_trending_weekly", "tmdb_discover", - "tvdb_list", - "imdb_list", + "tvdb_list", "imdb_list", "stevenlu_popular", "anidb_popular", "trakt_list", "trakt_trending", "trakt_popular", "trakt_recommended", "trakt_watched", "trakt_collected", "tautulli_popular", "tautulli_watched", "letterboxd_list", "icheckmovies_list", - "anidb_popular", - "anilist_top_rated", "anilist_popular", "anilist_season", "anilist_studio", "anilist_genre", "anilist_tag", + "anilist_top_rated", "anilist_popular", "anilist_season", "anilist_studio", "anilist_genre", "anilist_tag", "anilist_search", "mal_all", "mal_airing", "mal_upcoming", "mal_tv", "mal_movie", "mal_ova", "mal_special", "mal_popular", "mal_favorite", "mal_suggested", "mal_userlist", "mal_season", "mal_genre", "mal_producer" ] class CollectionBuilder: - def __init__(self, config, library, metadata, name, data): + def __init__(self, config, library, metadata, name, no_missing, data): self.config = config self.library = library self.metadata = metadata self.name = name + self.no_missing = no_missing self.data = data self.language = self.library.Plex.language self.details = { "show_filtered": self.library.show_filtered, "show_missing": self.library.show_missing, "save_missing": self.library.save_missing, - "released_missing_only": self.library.released_missing_only, + "missing_only_released": self.library.missing_only_released, "create_asset_folders": self.library.create_asset_folders, "item_assets": False } self.item_details = {} - self.radarr_options = {} - self.sonarr_options = {} + self.radarr_details = {} + self.sonarr_details = {} self.missing_movies = [] self.missing_shows = [] self.builders = [] @@ -175,8 +175,6 @@ class CollectionBuilder: self.backgrounds = {} self.summaries = {} self.schedule = "" - self.add_to_radarr = None - self.add_to_sonarr = None self.current_time = datetime.now() self.current_year = self.current_time.year @@ -538,21 +536,30 @@ class CollectionBuilder: if self.custom_sort and self.builders[0][0] not in custom_sort_builders: raise Failed(f"Collection Error: collection_order: custom cannot be used with {self.builders[0][0]}") - if self.add_to_radarr is None: - self.add_to_radarr = self.library.Radarr.add if self.library.Radarr else False - if self.add_to_sonarr is None: - self.add_to_sonarr = self.library.Sonarr.add if self.library.Sonarr else False + if "add" not in self.radarr_details: + self.radarr_details["add"] = self.library.Radarr.add if self.library.Radarr else False + if "add_existing" not in self.radarr_details: + self.radarr_details["add_existing"] = self.library.Radarr.add_existing if self.library.Radarr else False + + if "add" not in self.sonarr_details: + self.sonarr_details["add"] = self.library.Sonarr.add if self.library.Sonarr else False + if "add_existing" not in self.sonarr_details: + self.sonarr_details["add_existing"] = self.library.Sonarr.add_existing if self.library.Sonarr else False - if self.smart_url: - self.add_to_radarr = False - self.add_to_sonarr = False + if self.smart_url or self.collectionless: + self.radarr_details["add"] = False + self.radarr_details["add_existing"] = False + self.sonarr_details["add"] = False + self.sonarr_details["add_existing"] = False if self.collectionless: - self.add_to_radarr = False - self.add_to_sonarr = False self.details["collection_mode"] = "hide" self.sync = True + self.do_missing = not self.no_missing and (self.details["show_missing"] or self.details["save_missing"] + or (self.library.Radarr and self.radarr_details["add"]) + or (self.library.Sonarr and self.sonarr_details["add"])) + if self.build_collection: try: self.obj = self.library.get_collection(self.name) @@ -684,48 +691,44 @@ class CollectionBuilder: self.item_details[method_name] = str(method_data).lower() def _radarr(self, method_name, method_data): - if method_name == "radarr_add": - self.add_to_radarr = util.parse(method_name, method_data, datatype="bool") + if method_name in ["radarr_add", "radarr_add_existing", "radarr_monitor", "radarr_search"]: + self.radarr_details[method_name[7:]] = util.parse(method_name, method_data, datatype="bool") elif method_name == "radarr_folder": - self.radarr_options["folder"] = method_data - elif method_name in ["radarr_monitor", "radarr_search"]: - self.radarr_options[method_name[7:]] = util.parse(method_name, method_data, datatype="bool") + self.radarr_details["folder"] = method_data elif method_name == "radarr_availability": if str(method_data).lower() in radarr.availability_translation: - self.radarr_options["availability"] = str(method_data).lower() + self.radarr_details["availability"] = str(method_data).lower() else: raise Failed(f"Collection Error: {method_name} attribute must be either announced, cinemas, released or db") elif method_name == "radarr_quality": self.library.Radarr.get_profile_id(method_data) - self.radarr_options["quality"] = method_data + self.radarr_details["quality"] = method_data elif method_name == "radarr_tag": - self.radarr_options["tag"] = util.get_list(method_data) + self.radarr_details["tag"] = util.get_list(method_data) def _sonarr(self, method_name, method_data): - if method_name == "sonarr_add": - self.add_to_sonarr = util.parse(method_name, method_data, datatype="bool") + if method_name in ["sonarr_add", "sonarr_add_existing", "sonarr_season", "sonarr_search", "sonarr_cutoff_search"]: + self.sonarr_details[method_name[7:]] = util.parse(method_name, method_data, datatype="bool") elif method_name == "sonarr_folder": - self.sonarr_options["folder"] = method_data + self.sonarr_details["folder"] = method_data elif method_name == "sonarr_monitor": if str(method_data).lower() in sonarr.monitor_translation: - self.sonarr_options["monitor"] = str(method_data).lower() + self.sonarr_details["monitor"] = str(method_data).lower() else: raise Failed(f"Collection Error: {method_name} attribute must be either all, future, missing, existing, pilot, first, latest or none") elif method_name == "sonarr_quality": self.library.Sonarr.get_profile_id(method_data, "quality_profile") - self.sonarr_options["quality"] = method_data + self.sonarr_details["quality"] = method_data elif method_name == "sonarr_language": self.library.Sonarr.get_profile_id(method_data, "language_profile") - self.sonarr_options["language"] = method_data + self.sonarr_details["language"] = method_data elif method_name == "sonarr_series": if str(method_data).lower() in sonarr.series_type: - self.sonarr_options["series"] = str(method_data).lower() + self.sonarr_details["series"] = str(method_data).lower() else: raise Failed(f"Collection Error: {method_name} attribute must be either standard, daily, or anime") - elif method_name in ["sonarr_season", "sonarr_search", "sonarr_cutoff_search"]: - self.sonarr_options[method_name[7:]] = util.parse(method_name, method_data, datatype="bool") elif method_name == "sonarr_tag": - self.sonarr_options["tag"] = util.get_list(method_data) + self.sonarr_details["tag"] = util.get_list(method_data) def _anidb(self, method_name, method_data): if method_name == "anidb_popular": @@ -751,55 +754,47 @@ class CollectionBuilder: self.builders.append((method_name, anilist_id)) elif method_name in ["anilist_popular", "anilist_top_rated"]: self.builders.append((method_name, util.parse(method_name, method_data, datatype="int", default=10))) - elif method_name in ["anilist_season", "anilist_genre", "anilist_tag"]: + elif method_name == "anilist_search": + if self.current_time.month in [12, 1, 2]: current_season = "winter" + elif self.current_time.month in [3, 4, 5]: current_season = "spring" + elif self.current_time.month in [6, 7, 8]: current_season = "summer" + else: current_season = "fall" for dict_data, dict_methods in util.parse(method_name, method_data, datatype="dictlist"): new_dictionary = {} - if method_name == "anilist_season": - if self.current_time.month in [12, 1, 2]: new_dictionary["season"] = "winter" - elif self.current_time.month in [3, 4, 5]: new_dictionary["season"] = "spring" - elif self.current_time.month in [6, 7, 8]: new_dictionary["season"] = "summer" - elif self.current_time.month in [9, 10, 11]: new_dictionary["season"] = "fall" - new_dictionary["season"] = util.parse("season", dict_data, methods=dict_methods, parent=method_name, default=new_dictionary["season"], options=util.seasons) - new_dictionary["year"] = util.parse("year", dict_data, datatype="int", methods=dict_methods, default=self.current_year, parent=method_name, minimum=1917, maximum=self.current_year + 1) - elif method_name == "anilist_genre": - new_dictionary["genre"] = self.config.AniList.validate("Genre", util.parse("genre", dict_data, methods=dict_methods, parent=method_name)) - elif method_name == "anilist_tag": - new_dictionary["tag"] = self.config.AniList.validate("Tag", util.parse("tag", dict_data, methods=dict_methods, parent=method_name)) - elif method_name == "anilist_search": - for search_method, search_data in dict_data.items(): - search_attr, modifier, search_final = self._split(search_method) - if search_data is None: - raise Failed(f"Collection Error: {method_name} {search_final} attribute is blank") - elif search_final not in anilist.searches: - raise Failed(f"Collection Error: {method_name} {search_final} attribute not supported") - elif search_attr == "season": - if self.current_time.month in [12, 1, 2]: new_dictionary["season"] = "winter" - elif self.current_time.month in [3, 4, 5]: new_dictionary["season"] = "spring" - elif self.current_time.month in [6, 7, 8]: new_dictionary["season"] = "summer" - elif self.current_time.month in [9, 10, 11]: new_dictionary["season"] = "fall" - new_dictionary["season"] = util.parse("season", dict_data, parent=method_name, default=new_dictionary["season"], options=util.seasons) - if "year" not in dict_methods: - logger.warning(f"Collection Warning: {method_name} {search_final} attribute must be used with the year attribute using this year by default") - elif search_attr == "year": - if "season" not in dict_methods: - raise Failed(f"Collection Error: {method_name} {search_final} attribute must be used with the season attribute") - new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="int", parent=method_name, default=self.current_year, minimum=1917, maximum=self.current_year + 1) - elif search_attr == "adult": - new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="bool", parent=method_name) - elif search_attr in ["episodes", "duration", "score", "popularity"]: - new_dictionary[search_final] = util.parse(search_final, search_data, datatype="int", parent=method_name) - elif search_attr in ["format", "status", "genre", "tag", "tag_category"]: - new_dictionary[search_final] = self.config.AniList.validate(search_attr.replace("_", " ").title(), util.parse(search_final, search_data)) - elif search_attr in ["start", "end"]: - new_dictionary[search_final] = util.validate_date(search_data, f"{method_name} {search_final} attribute", return_as="%m/%d/%Y") - elif search_attr == "min_tag_percent": - new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="int", parent=method_name, minimum=0, maximum=100) - elif search_final not in ["sort_by", "limit"]: - raise Failed(f"Collection Error: {method_name} {search_final} attribute not supported") - if len(new_dictionary) > 0: - raise Failed(f"Collection Error: {method_name} must have at least one valid search option") + for search_method, search_data in dict_data.items(): + search_attr, modifier, search_final = self._split(search_method) + if search_data is None: + raise Failed(f"Collection Error: {method_name} {search_final} attribute is blank") + elif search_final not in anilist.searches: + raise Failed(f"Collection Error: {method_name} {search_final} attribute not supported") + elif search_attr == "season": + new_dictionary[search_attr] = util.parse(search_attr, search_data, parent=method_name, default=current_season, options=util.seasons) + if "year" not in dict_methods: + logger.warning(f"Collection Warning: {method_name} year attribute not found using this year: {self.current_year} by default") + new_dictionary["year"] = self.current_year + elif search_attr == "year": + new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="int", parent=method_name, default=self.current_year, minimum=1917, maximum=self.current_year + 1) + if "season" not in dict_methods: + logger.warning(f"Collection Warning: {method_name} season attribute not found using this season: {current_season} by default") + new_dictionary["season"] = current_season + elif search_attr == "adult": + new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="bool", parent=method_name) + elif search_attr in ["episodes", "duration", "score", "popularity"]: + new_dictionary[search_final] = util.parse(search_final, search_data, datatype="int", parent=method_name) + elif search_attr in ["format", "status", "genre", "tag", "tag_category"]: + new_dictionary[search_final] = self.config.AniList.validate(search_attr.replace("_", " ").title(), util.parse(search_final, search_data)) + elif search_attr in ["start", "end"]: + new_dictionary[search_final] = util.validate_date(search_data, f"{method_name} {search_final} attribute", return_as="%m/%d/%Y") + elif search_attr == "min_tag_percent": + new_dictionary[search_attr] = util.parse(search_attr, search_data, datatype="int", parent=method_name, minimum=0, maximum=100) + elif search_attr == "search": + new_dictionary[search_attr] = str(search_data) + elif search_final not in ["sort_by", "limit"]: + raise Failed(f"Collection Error: {method_name} {search_final} attribute not supported") + if len(new_dictionary) > 0: + raise Failed(f"Collection Error: {method_name} must have at least one valid search option") new_dictionary["sort_by"] = util.parse("sort_by", dict_data, methods=dict_methods, parent=method_name, default="score", options=["score", "popular"]) - new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=0, parent=method_name, maximum=500) + new_dictionary["limit"] = util.parse("limit", dict_data, datatype="int", methods=dict_methods, default=0, parent=method_name) self.builders.append((method_name, new_dictionary)) def _icheckmovies(self, method_name, method_data): @@ -1029,7 +1024,7 @@ class CollectionBuilder: else: logger.error(message) - def find_rating_keys(self, no_missing): + def find_rating_keys(self): for method, value in self.builders: ids = [] rating_keys = [] @@ -1094,9 +1089,7 @@ class CollectionBuilder: if input_id in self.library.imdb_map: rating_keys.append(self.library.imdb_map[input_id][0]) else: - if (self.details["show_missing"] or self.details["save_missing"] - or (self.library.Radarr and self.add_to_radarr) - or (self.library.Sonarr and self.add_to_sonarr)) and not no_missing: + if self.do_missing: try: tmdb_id, tmdb_type = self.config.Convert.imdb_to_tmdb(input_id) if tmdb_type == "movie": @@ -1553,7 +1546,7 @@ class CollectionBuilder: logger.error(e) continue current_title = f"{movie.title} ({util.validate_date(movie.release_date, 'test').year})" if movie.release_date else movie.title - if self.check_tmdb_filter(missing_id, True, item=movie, check_released=self.details["released_missing_only"]): + if self.check_tmdb_filter(missing_id, True, item=movie, check_released=self.details["missing_only_released"]): missing_movies_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: logger.info(f"{self.name} Collection | ? | {current_title} (TMDb: {missing_id})") @@ -1565,12 +1558,12 @@ class CollectionBuilder: if len(missing_movies_with_names) > 0: if self.details["save_missing"] is True: self.library.add_missing(self.name, missing_movies_with_names, True) - if self.run_again or (self.library.Radarr and (self.add_to_radarr or "item_radarr_tag" in self.item_details)): + if self.run_again or (self.library.Radarr and (self.radarr_details["add"] or "item_radarr_tag" in self.item_details)): missing_tmdb_ids = [missing_id for title, missing_id in missing_movies_with_names] if self.library.Radarr: - if self.add_to_radarr: + if self.radarr_details["add"]: try: - self.library.Radarr.add_tmdb(missing_tmdb_ids, **self.radarr_options) + self.library.Radarr.add_tmdb(missing_tmdb_ids, **self.radarr_details) except Failed as e: logger.error(e) if "item_radarr_tag" in self.item_details: @@ -1589,7 +1582,7 @@ class CollectionBuilder: logger.error(e) continue current_title = str(show.title.encode("ascii", "replace").decode()) - if self.check_tmdb_filter(missing_id, False, check_released=self.details["released_missing_only"]): + if self.check_tmdb_filter(missing_id, False, check_released=self.details["missing_only_released"]): missing_shows_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: logger.info(f"{self.name} Collection | ? | {current_title} (TVDB: {missing_id})") @@ -1601,12 +1594,12 @@ class CollectionBuilder: if len(missing_shows_with_names) > 0: if self.details["save_missing"] is True: self.library.add_missing(self.name, missing_shows_with_names, False) - if self.run_again or (self.library.Sonarr and (self.add_to_sonarr or "item_sonarr_tag" in self.item_details)): + if self.run_again or (self.library.Sonarr and (self.sonarr_details["add"] or "item_sonarr_tag" in self.item_details)): missing_tvdb_ids = [missing_id for title, missing_id in missing_shows_with_names] if self.library.Sonarr: - if self.add_to_sonarr: + if self.sonarr_details["add"]: try: - self.library.Sonarr.add_tvdb(missing_tvdb_ids, **self.sonarr_options) + self.library.Sonarr.add_tvdb(missing_tvdb_ids, **self.sonarr_details) except Failed as e: logger.error(e) if "item_sonarr_tag" in self.item_details: @@ -1693,9 +1686,9 @@ class CollectionBuilder: except Failed as e: logger.error(e) self.library.edit_tags("label", item, add_tags=add_tags, remove_tags=remove_tags, sync_tags=sync_tags) - if "item_radarr_tag" in self.item_details and item.ratingKey in self.library.movie_rating_key_map: + if item.ratingKey in self.library.movie_rating_key_map: tmdb_ids.append(self.library.movie_rating_key_map[item.ratingKey]) - if "item_sonarr_tag" in self.item_details and item.ratingKey in self.library.show_rating_key_map: + if item.ratingKey in self.library.show_rating_key_map: tvdb_ids.append(self.library.show_rating_key_map[item.ratingKey]) advance_edits = {} for method_name, method_data in self.item_details.items(): @@ -1706,10 +1699,16 @@ class CollectionBuilder: self.library.edit_item(item, item.title, "Movie" if self.library.is_movie else "Show", advance_edits, advanced=True) if len(tmdb_ids) > 0: - self.library.Radarr.edit_tags(tmdb_ids, self.item_details["item_radarr_tag"], self.item_details["apply_tags"]) + if "item_radarr_tag" in self.item_details: + self.library.Radarr.edit_tags(tmdb_ids, self.item_details["item_radarr_tag"], self.item_details["apply_tags"]) + if self.radarr_details["add_existing"]: + self.library.Radarr.add_tmdb(tmdb_ids, **self.radarr_details) if len(tvdb_ids) > 0: - self.library.Sonarr.edit_tags(tvdb_ids, self.item_details["item_sonarr_tag"], self.item_details["apply_tags"]) + if "item_sonarr_tag" in self.item_details: + self.library.Sonarr.edit_tags(tvdb_ids, self.item_details["item_sonarr_tag"], self.item_details["apply_tags"]) + if self.sonarr_details["add_existing"]: + self.library.Sonarr.add_tvdb(tvdb_ids, **self.sonarr_details) for rating_key in rating_keys: try: diff --git a/modules/config.py b/modules/config.py index 4386d4be..efad2b29 100644 --- a/modules/config.py +++ b/modules/config.py @@ -183,7 +183,7 @@ class Config: "show_filtered": check_for_attribute(self.data, "show_filtered", parent="settings", var_type="bool", default=False), "show_missing": check_for_attribute(self.data, "show_missing", parent="settings", var_type="bool", default=True), "save_missing": check_for_attribute(self.data, "save_missing", parent="settings", var_type="bool", default=True), - "released_missing_only": check_for_attribute(self.data, "released_missing_only", parent="settings", var_type="bool", default=False), + "missing_only_released": check_for_attribute(self.data, "missing_only_released", parent="settings", var_type="bool", default=False), "create_asset_folders": check_for_attribute(self.data, "create_asset_folders", parent="settings", var_type="bool", default=False) } if self.general["cache"]: @@ -295,6 +295,7 @@ class Config: "url": check_for_attribute(self.data, "url", parent="radarr", var_type="url", default_is_none=True), "token": check_for_attribute(self.data, "token", parent="radarr", default_is_none=True), "add": check_for_attribute(self.data, "add", parent="radarr", var_type="bool", default=False), + "add_existing": check_for_attribute(self.data, "add_existing", parent="radarr", var_type="bool", default=False), "root_folder_path": check_for_attribute(self.data, "root_folder_path", parent="radarr", default_is_none=True), "monitor": check_for_attribute(self.data, "monitor", parent="radarr", var_type="bool", default=True), "availability": check_for_attribute(self.data, "availability", parent="radarr", test_list=radarr.availability_descriptions, default="announced"), @@ -306,6 +307,7 @@ class Config: "url": check_for_attribute(self.data, "url", parent="sonarr", var_type="url", default_is_none=True), "token": check_for_attribute(self.data, "token", parent="sonarr", default_is_none=True), "add": check_for_attribute(self.data, "add", parent="sonarr", var_type="bool", default=False), + "add_existing": check_for_attribute(self.data, "add_existing", parent="sonarr", var_type="bool", default=False), "root_folder_path": check_for_attribute(self.data, "root_folder_path", parent="sonarr", default_is_none=True), "monitor": check_for_attribute(self.data, "monitor", parent="sonarr", test_list=sonarr.monitor_descriptions, default="all"), "quality_profile": check_for_attribute(self.data, "quality_profile", parent="sonarr", default_is_none=True), @@ -349,7 +351,7 @@ class Config: params["show_filtered"] = check_for_attribute(lib, "show_filtered", parent="settings", var_type="bool", default=self.general["show_filtered"], do_print=False, save=False) params["show_missing"] = check_for_attribute(lib, "show_missing", parent="settings", var_type="bool", default=self.general["show_missing"], do_print=False, save=False) params["save_missing"] = check_for_attribute(lib, "save_missing", parent="settings", var_type="bool", default=self.general["save_missing"], do_print=False, save=False) - params["released_missing_only"] = check_for_attribute(lib, "released_missing_only", parent="settings", var_type="bool", default=self.general["released_missing_only"], do_print=False, save=False) + params["missing_only_released"] = check_for_attribute(lib, "missing_only_released", parent="settings", var_type="bool", default=self.general["missing_only_released"], do_print=False, save=False) params["create_asset_folders"] = check_for_attribute(lib, "create_asset_folders", parent="settings", var_type="bool", default=self.general["create_asset_folders"], do_print=False, save=False) params["mass_genre_update"] = check_for_attribute(lib, "mass_genre_update", test_list=mass_update_options, default_is_none=True, save=False, do_print=lib and "mass_genre_update" in lib) @@ -427,6 +429,7 @@ class Config: "url": check_for_attribute(lib, "url", parent="radarr", var_type="url", default=self.general["radarr"]["url"], req_default=True, save=False), "token": check_for_attribute(lib, "token", parent="radarr", default=self.general["radarr"]["token"], req_default=True, save=False), "add": check_for_attribute(lib, "add", parent="radarr", var_type="bool", default=self.general["radarr"]["add"], save=False), + "add_existing": check_for_attribute(lib, "add_existing", parent="radarr", var_type="bool", default=self.general["radarr"]["add_existing"], save=False), "root_folder_path": check_for_attribute(lib, "root_folder_path", parent="radarr", default=self.general["radarr"]["root_folder_path"], req_default=True, save=False), "monitor": check_for_attribute(lib, "monitor", parent="radarr", var_type="bool", default=self.general["radarr"]["monitor"], save=False), "availability": check_for_attribute(lib, "availability", parent="radarr", test_list=radarr.availability_descriptions, default=self.general["radarr"]["availability"], save=False), @@ -451,6 +454,7 @@ class Config: "url": check_for_attribute(lib, "url", parent="sonarr", var_type="url", default=self.general["sonarr"]["url"], req_default=True, save=False), "token": check_for_attribute(lib, "token", parent="sonarr", default=self.general["sonarr"]["token"], req_default=True, save=False), "add": check_for_attribute(lib, "add", parent="sonarr", var_type="bool", default=self.general["sonarr"]["add"], save=False), + "add_existing": check_for_attribute(lib, "add_existing", parent="sonarr", var_type="bool", default=self.general["sonarr"]["add_existing"], save=False), "root_folder_path": check_for_attribute(lib, "root_folder_path", parent="sonarr", default=self.general["sonarr"]["root_folder_path"], req_default=True, save=False), "monitor": check_for_attribute(lib, "monitor", parent="sonarr", test_list=sonarr.monitor_descriptions, default=self.general["sonarr"]["monitor"], save=False), "quality_profile": check_for_attribute(lib, "quality_profile", parent="sonarr", default=self.general["sonarr"]["quality_profile"], req_default=True, save=False), diff --git a/modules/convert.py b/modules/convert.py index 015d0411..5391f50d 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -85,7 +85,7 @@ class Convert: if tvdb_id: ids.append((tvdb_id, "tvdb")) if tmdb_ids: - ids.extend((tmdb_ids, "tmdb")) + ids.extend([(t, "tmdb") for t in tmdb_ids]) except Failed as e: logger.error(e) return ids diff --git a/modules/mal.py b/modules/mal.py index 06c4ee9f..02416400 100644 --- a/modules/mal.py +++ b/modules/mal.py @@ -1,4 +1,4 @@ -import logging, re, secrets, time, webbrowser +import logging, math, re, secrets, time, webbrowser from modules import util from modules.util import Failed, TimeoutExpired from ruamel import yaml @@ -169,12 +169,26 @@ class MyAnimeList: if total_items < limit or limit <= 0: limit = total_items mal_ids = [] - for i in range(1, int(((total_items - 1) / 100) + 2)): - if i > 1: - data = self._jiken_request(f"/genre/anime/{genre_id}/{i}") - mal_ids.extend([anime["mal_id"] for anime in data["anime"]]) - if len(mal_ids) > limit: - return mal_ids[:limit] + num_of_pages = math.ceil(int(limit) / 100) + current_page = 1 + chances = 0 + while current_page <= num_of_pages: + if chances > 6: + logger.debug(data) + raise Failed("AniList Error: Connection Failed") + start_num = (current_page - 1) * 100 + 1 + util.print_return(f"Parsing Page {current_page}/{num_of_pages} {start_num}-{limit if current_page == num_of_pages else current_page * 100}") + if current_page > 1: + data = self._jiken_request(f"/genre/anime/{genre_id}/{current_page}") + if "anime" in data: + chances = 0 + mal_ids.extend([anime["mal_id"] for anime in data["anime"]]) + if len(mal_ids) > limit: + return mal_ids[:limit] + current_page += 1 + else: + chances += 1 + util.print_end() return mal_ids def _producer(self, producer_id, limit): diff --git a/modules/plex.py b/modules/plex.py index ba502322..e7e9f113 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -288,7 +288,7 @@ class Plex: self.show_filtered = params["show_filtered"] self.show_missing = params["show_missing"] self.save_missing = params["save_missing"] - self.released_missing_only = params["released_missing_only"] + self.missing_only_released = params["missing_only_released"] self.create_asset_folders = params["create_asset_folders"] self.mass_genre_update = params["mass_genre_update"] self.mass_audience_rating_update = params["mass_audience_rating_update"] diff --git a/modules/radarr.py b/modules/radarr.py index 8bb0909a..5d37bc1a 100644 --- a/modules/radarr.py +++ b/modules/radarr.py @@ -20,6 +20,7 @@ class Radarr: except ArrException as e: raise Failed(e) self.add = params["add"] + self.add_existing = params["add_existing"] self.root_folder_path = params["root_folder_path"] self.monitor = params["monitor"] self.availability = params["availability"] diff --git a/modules/sonarr.py b/modules/sonarr.py index bba35783..7c428c29 100644 --- a/modules/sonarr.py +++ b/modules/sonarr.py @@ -38,6 +38,7 @@ class Sonarr: except ArrException as e: raise Failed(e) self.add = params["add"] + self.add_existing = params["add_existing"] self.root_folder_path = params["root_folder_path"] self.monitor = params["monitor"] self.quality_profile = params["quality_profile"] diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 19292c21..92ca3858 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -108,7 +108,7 @@ def start(config_path, is_test=False, time_scheduled=None, requested_collections logger.info(util.centered("| __/| | __/> < | | | | __/ || (_| | | | | | (_| | | | | (_| | (_| | __/ | ")) logger.info(util.centered("|_| |_|\\___/_/\\_\\ |_| |_|\\___|\\__\\__,_| |_| |_|\\__,_|_| |_|\\__,_|\\__, |\\___|_| ")) logger.info(util.centered(" |___/ ")) - logger.info(util.centered(" Version: 1.11.3-beta5 ")) + logger.info(util.centered(" Version: 1.12.0 ")) if time_scheduled: start_type = f"{time_scheduled} " elif is_test: start_type = "Test " elif requested_collections: start_type = "Collections " @@ -474,7 +474,7 @@ def run_collection(config, library, metadata, requested_collections): util.separator(f"Validating {mapping_name} Attributes", space=False, border=False) - builder = CollectionBuilder(config, library, metadata, mapping_name, collection_attrs) + builder = CollectionBuilder(config, library, metadata, mapping_name, no_missing, collection_attrs) logger.info("") util.separator(f"Building {mapping_name} Collection", space=False, border=False) @@ -495,16 +495,14 @@ def run_collection(config, library, metadata, requested_collections): for filter_key, filter_value in builder.filters: logger.info(f"Collection Filter {filter_key}: {filter_value}") - builder.find_rating_keys(no_missing) + builder.find_rating_keys() if len(builder.rating_keys) > 0 and builder.build_collection: logger.info("") util.separator(f"Adding to {mapping_name} Collection", space=False, border=False) logger.info("") builder.add_to_collection() - if (builder.details["show_missing"] or builder.details["save_missing"] - or (library.Radarr and builder.add_to_radarr) or (library.Sonarr and builder.add_to_sonarr)) \ - and (len(builder.missing_movies) > 0 or len(builder.missing_shows) > 0) and not no_missing: + if builder.do_missing and (len(builder.missing_movies) > 0 or len(builder.missing_shows) > 0): if builder.details["show_missing"] is True: logger.info("") util.separator(f"Missing from Library", space=False, border=False) diff --git a/requirements.txt b/requirements.txt index b8f6cf3f..847af976 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,4 @@ ruamel.yaml==0.17.10 schedule==1.1.0 retrying==1.3.3 pathvalidate==2.4.1 -pillow==8.3.1 -python-slugify==5.0.2 \ No newline at end of file +pillow==8.3.1 \ No newline at end of file