From b76b54ade5a94ea411eaa57e4a43d5e523bfc3e3 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 5 Aug 2021 10:59:45 -0400 Subject: [PATCH] fixed missing filters --- modules/builder.py | 178 +++++++++++++++++++++---------------------- modules/cache.py | 2 +- modules/convert.py | 1 + modules/meta.py | 8 +- modules/plex.py | 7 +- plex_meta_manager.py | 2 +- 6 files changed, 97 insertions(+), 101 deletions(-) diff --git a/modules/builder.py b/modules/builder.py index 4c29c871..c30981d6 100644 --- a/modules/builder.py +++ b/modules/builder.py @@ -110,6 +110,7 @@ all_filters = [ "writer", "writer.not", "year", "year.gt", "year.gte", "year.lt", "year.lte", "year.not" ] +tmdb_filters = ("original_language", "tmdb_vote_count", "year") movie_only_filters = [ "audio_language", "audio_language.not", "audio_track_title", "audio_track_title.not", "audio_track_title.begins", "audio_track_title.ends", "audio_track_title.regex", @@ -158,10 +159,9 @@ class CollectionBuilder: self.sonarr_options = {} self.missing_movies = [] self.missing_shows = [] - self.filtered_missing_movies = [] - self.filtered_missing_shows = [] self.builders = [] self.filters = [] + self.tmdb_filters = [] self.rating_keys = [] self.run_again_movies = [] self.run_again_shows = [] @@ -177,8 +177,8 @@ class CollectionBuilder: methods = {m.lower(): m for m in self.data} if "template" in methods: - logger.info("") - logger.info("Validating Method: template") + logger.debug("") + logger.debug("Validating Method: template") if not self.metadata.templates: raise Failed("Collection Error: No templates found") elif not self.data[methods["template"]]: @@ -290,8 +290,8 @@ class CollectionBuilder: continue if "schedule" in methods: - logger.info("") - logger.info("Validating Method: schedule") + logger.debug("") + logger.debug("Validating Method: schedule") if not self.data[methods["schedule"]]: raise Failed("Collection Error: schedule attribute is blank") else: @@ -359,29 +359,29 @@ class CollectionBuilder: self.validate_builders = True if "validate_builders" in methods: - logger.info("") - logger.info("Validating Method: validate_builders") - logger.info(f"Value: {data[methods['validate_builders']]}") + logger.debug("") + logger.debug("Validating Method: validate_builders") + logger.debug(f"Value: {data[methods['validate_builders']]}") self.validate_builders = util.parse("validate_builders", self.data, datatype="bool", methods=methods, default=True) self.run_again = False if "run_again" in methods: - logger.info("") - logger.info("Validating Method: run_again") - logger.info(f"Value: {data[methods['run_again']]}") + logger.debug("") + logger.debug("Validating Method: run_again") + logger.debug(f"Value: {data[methods['run_again']]}") self.run_again = util.parse("run_again", self.data, datatype="bool", methods=methods, default=False) self.build_collection = True if "build_collection" in methods: - logger.info("") - logger.info("Validating Method: build_collection") - logger.info(f"Value: {data[methods['build_collection']]}") + logger.debug("") + logger.debug("Validating Method: build_collection") + logger.debug(f"Value: {data[methods['build_collection']]}") self.build_collection = util.parse("build_collection", self.data, datatype="bool", methods=methods, default=True) self.sync = self.library.sync_mode == "sync" if "sync_mode" in methods: - logger.info("") - logger.info("Validating Method: sync_mode") + logger.debug("") + logger.debug("Validating Method: sync_mode") if not self.data[methods["sync_mode"]]: logger.warning(f"Collection Warning: sync_mode attribute is blank using general: {self.library.sync_mode}") else: @@ -393,20 +393,22 @@ class CollectionBuilder: self.custom_sort = False if "collection_order" in methods: - logger.info("") - logger.info("Validating Method: collection_order") + logger.debug("") + logger.debug("Validating Method: collection_order") if self.data[methods["collection_order"]] is None: raise Failed(f"Collection Warning: collection_order attribute is blank") - elif self.data[methods["collection_order"]].lower() in plex.collection_order_options: - self.details["collection_order"] = self.data[methods["collection_order"]].lower() - if self.data[methods["collection_order"]].lower() == "custom" and self.build_collection: - self.custom_sort = True else: - raise Failed(f"Collection Error: {self.data[methods['collection_order']]} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)\n\tcustom (Custom Order Collection)") + logger.debug(f"Value: {self.data[methods['collection_order']]}") + if self.data[methods["collection_order"]].lower() in plex.collection_order_options: + self.details["collection_order"] = self.data[methods["collection_order"]].lower() + if self.data[methods["collection_order"]].lower() == "custom" and self.build_collection: + self.custom_sort = True + else: + raise Failed(f"Collection Error: {self.data[methods['collection_order']]} collection_order invalid\n\trelease (Order Collection by release dates)\n\talpha (Order Collection Alphabetically)\n\tcustom (Custom Order Collection)") if "tmdb_person" in methods: - logger.info("") - logger.info("Validating Method: tmdb_person") + logger.debug("") + logger.debug("Validating Method: tmdb_person") if not self.data[methods["tmdb_person"]]: raise Failed("Collection Error: tmdb_person attribute is blank") else: @@ -427,8 +429,8 @@ class CollectionBuilder: self.smart_sort = "random" self.smart_label_collection = False if "smart_label" in methods: - logger.info("") - logger.info("Validating Method: smart_label") + logger.debug("") + logger.debug("Validating Method: smart_label") self.smart_label_collection = True if not self.data[methods["smart_label"]]: logger.warning("Collection Error: smart_label attribute is blank defaulting to random") @@ -444,8 +446,8 @@ class CollectionBuilder: self.smart_type_key = None self.smart_filter_details = "" if "smart_url" in methods: - logger.info("") - logger.info("Validating Method: smart_url") + logger.debug("") + logger.debug("Validating Method: smart_url") if not self.data[methods["smart_url"]]: raise Failed("Collection Error: smart_url attribute is blank") else: @@ -524,6 +526,8 @@ class CollectionBuilder: else: logger.error(e) + self.tmdb_filters = [(fm, fd) for fm, fd in self.filters if fm.startswith(tmdb_filters)] + if self.custom_sort and len(self.builders) > 1: raise Failed("Collection Error: collection_order: custom can only be used with a single builder per collection") @@ -969,16 +973,22 @@ class CollectionBuilder: validate = dict_data["validate"] for filter_method, filter_data in dict_data.items(): filter_attr, modifier, filter_final = self._split(filter_method) + message = None if filter_final not in all_filters: - raise Failed(f"Collection Error: {filter_final} is not a valid filter attribute") + message = f"Collection Error: {filter_final} is not a valid filter attribute" elif filter_final in movie_only_filters and self.library.is_show: - raise Failed(f"Collection Error: {filter_final} filter attribute only works for movie libraries") + message = f"Collection Error: {filter_final} filter attribute only works for movie libraries" elif filter_final in show_only_filters and self.library.is_movie: - raise Failed(f"Collection Error: {filter_final} filter attribute only works for show libraries") + message = f"Collection Error: {filter_final} filter attribute only works for show libraries" elif filter_final is None: - raise Failed(f"Collection Error: {filter_final} filter attribute is blank") + message = f"Collection Error: {filter_final} filter attribute is blank" else: self.filters.append((filter_final, self.validate_attribute(filter_attr, modifier, f"{filter_final} filter", filter_data, validate))) + if message: + if validate: + raise Failed(message) + else: + logger.error(message) def collect_rating_keys(self): filtered_keys = {} @@ -1018,37 +1028,15 @@ class CollectionBuilder: for movie_id in movie_ids: if movie_id in self.library.movie_map: add_rating_keys(self.library.movie_map[movie_id]) - elif movie_id not in self.missing_movies and movie_id not in self.filtered_missing_movies: - filter_missing = False - if self.details["released_missing_only"]: - try: - movie = self.config.TMDb.get_movie(movie_id) - if util.validate_date(movie.release_date, "") > self.current_time: - filter_missing = True - except Failed: - pass - if filter_missing: - self.filtered_missing_movies.append(movie_id) - else: - self.missing_movies.append(movie_id) + elif movie_id not in self.missing_movies: + self.missing_movies.append(movie_id) if len(show_ids) > 0: items_found_inside += len(show_ids) for show_id in show_ids: if show_id in self.library.show_map: add_rating_keys(self.library.show_map[show_id]) - elif show_id not in self.missing_shows and show_id not in self.filtered_missing_shows: - filter_missing = False - if self.details["released_missing_only"]: - try: - show = self.config.TMDb.get_show(show_id) - if util.validate_date(show.first_air_date, "") > self.current_time: - filter_missing = True - except Failed: - pass - if filter_missing: - self.filtered_missing_shows.append(show_id) - else: - self.missing_shows.append(show_id) + elif show_id not in self.missing_shows: + self.missing_shows.append(show_id) return items_found_inside for method, value in self.builders: logger.debug("") @@ -1068,6 +1056,31 @@ class CollectionBuilder: elif "trakt" in method: check_map(self.config.Trakt.get_items(method, value, self.library.is_movie)) else: logger.error(f"Collection Error: {method} method not supported") + def tmdb_filter(self, item_id, is_movie, item=None): + filter_missing = False + if self.tmdb_filters or self.details["released_missing_only"]: + try: + if item is None: + item = self.config.TMDb.get_movie(item_id) if is_movie else self.config.TMDb.get_movie(self.config.Convert.tvdb_to_tmdb(item_id)) + if self.details["released_missing_only"]: + if util.validate_date(item.release_date if is_movie else item.first_air_date, "") > self.current_time: + return True + for filter_method, filter_data in self.tmdb_filters: + if (filter_method == "original_language" and item.original_language not in filter_data) \ + or (filter_method == "original_language.not" and item.original_language in filter_data) \ + or (filter_method == "tmdb_vote_count.gt" and item.vote_count <= filter_data) \ + or (filter_method == "tmdb_vote_count.gte" and item.vote_count < filter_data) \ + or (filter_method == "tmdb_vote_count.lt" and item.vote_count >= filter_data) \ + or (filter_method == "tmdb_vote_count.lte" and item.vote_count > filter_data) \ + or (filter_method == "year.gt" and item.year <= filter_data) \ + or (filter_method == "year.gte" and item.year < filter_data) \ + or (filter_method == "year.lt" and item.year >= filter_data) \ + or (filter_method == "year.lte" and item.year > filter_data): + return True + except Failed: + return True + return filter_missing + def build_filter(self, method, plex_filter, smart=False): if smart: logger.info("") @@ -1500,10 +1513,6 @@ class CollectionBuilder: return True def run_missing(self): - arr_filters = [] - for filter_method, filter_data in self.filters: - if (filter_method.startswith("original_language") and self.library.is_movie) or filter_method.startswith("tmdb_vote_count"): - arr_filters.append((filter_method, filter_data)) if len(self.missing_movies) > 0: missing_movies_with_names = [] for missing_id in self.missing_movies: @@ -1512,23 +1521,14 @@ class CollectionBuilder: except Failed as e: logger.error(e) continue - match = True - for filter_method, filter_data in arr_filters: - if (filter_method == "original_language" and movie.original_language not in filter_data) \ - or (filter_method == "original_language.not" and movie.original_language in filter_data) \ - or (filter_method == "tmdb_vote_count.gt" and movie.vote_count <= filter_data) \ - or (filter_method == "tmdb_vote_count.gte" and movie.vote_count < filter_data) \ - or (filter_method == "tmdb_vote_count.lt" and movie.vote_count >= filter_data) \ - or (filter_method == "tmdb_vote_count.lte" and movie.vote_count > filter_data): - match = False - break current_title = f"{movie.title} ({util.validate_date(movie.release_date, 'test').year})" if movie.release_date else movie.title - if match: + if self.tmdb_filter(missing_id, True, item=movie): + if self.details["show_filtered"] is True and self.details["show_missing"] is True: + logger.info(f"{self.name} Collection | X | {current_title} (TMDb: {missing_id})") + else: missing_movies_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: logger.info(f"{self.name} Collection | ? | {current_title} (TMDb: {missing_id})") - elif self.details["show_filtered"] is True: - logger.info(f"{self.name} Collection | X | {current_title} (TMDb: {missing_id})") logger.info("") logger.info(f"{len(missing_movies_with_names)} Movie{'s' if len(missing_movies_with_names) > 1 else ''} Missing") if self.details["save_missing"] is True: @@ -1546,26 +1546,18 @@ class CollectionBuilder: missing_shows_with_names = [] for missing_id in self.missing_shows: try: - title = str(self.config.TVDb.get_series(self.language, missing_id).title.encode("ascii", "replace").decode()) + show = self.config.TVDb.get_series(self.language, missing_id) except Failed as e: logger.error(e) continue - match = True - if arr_filters: - show = self.config.TMDb.get_show(self.config.Convert.tvdb_to_tmdb(missing_id)) - for filter_method, filter_data in arr_filters: - if (filter_method == "tmdb_vote_count.gt" and show.vote_count <= filter_data) \ - or (filter_method == "tmdb_vote_count.gte" and show.vote_count < filter_data) \ - or (filter_method == "tmdb_vote_count.lt" and show.vote_count >= filter_data) \ - or (filter_method == "tmdb_vote_count.lte" and show.vote_count > filter_data): - match = False - break - if match: - missing_shows_with_names.append((title, missing_id)) + current_title = str(show.title.encode("ascii", "replace").decode()) + if self.tmdb_filter(missing_id, False): + if self.details["show_filtered"] is True and self.details["show_missing"] is True: + logger.info(f"{self.name} Collection | X | {current_title} (TVDb: {missing_id})") + else: + missing_shows_with_names.append((current_title, missing_id)) if self.details["show_missing"] is True: - logger.info(f"{self.name} Collection | ? | {title} (TVDB: {missing_id})") - elif self.details["show_filtered"] is True: - logger.info(f"{self.name} Collection | X | {title} (TVDb: {missing_id})") + logger.info(f"{self.name} Collection | ? | {current_title} (TVDB: {missing_id})") logger.info("") logger.info(f"{len(missing_shows_with_names)} Show{'s' if len(missing_shows_with_names) > 1 else ''} Missing") if self.details["save_missing"] is True: diff --git a/modules/cache.py b/modules/cache.py index ecf1865c..562d1a39 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -108,7 +108,7 @@ class Cache: row = cursor.fetchone() if row: time_between_insertion = datetime.now() - datetime.strptime(row["expiration_date"], "%Y-%m-%d") - id_to_return = row["t_id"] + id_to_return = int(row["t_id"]) media_type = row["media_type"] expired = time_between_insertion.days > self.expiration return id_to_return, media_type, expired diff --git a/modules/convert.py b/modules/convert.py index 2982455c..578f0c88 100644 --- a/modules/convert.py +++ b/modules/convert.py @@ -315,6 +315,7 @@ class Convert: update_cache(tmdb_id, "TMDb", "show_movie") return "movie", tmdb_id else: + logger.debug(f"TMDb: {tmdb_id}, IMDb: {imdb_id}, TVDb: {tvdb_id}") raise Failed(f"No ID to convert") except Failed as e: logger.info(util.adjust_space(f"Mapping Error | {item.guid:<46} | {e} for {item.title}")) diff --git a/modules/meta.py b/modules/meta.py index b3afccc2..d15ecf8e 100644 --- a/modules/meta.py +++ b/modules/meta.py @@ -67,7 +67,7 @@ class Metadata: else: return self.collections - def update_metadata(self, TMDb, test): + def update_metadata(self): if not self.metadata: return None logger.info("") @@ -75,7 +75,7 @@ class Metadata: logger.info("") for mapping_name, meta in self.metadata.items(): methods = {mm.lower(): mm for mm in meta} - if test and ("test" not in methods or meta[methods["test"]] is not True): + if self.config.test and ("test" not in methods or meta[methods["test"]] is not True): continue updated = False @@ -212,13 +212,13 @@ class Metadata: logger.error("Metadata Error: tmdb_show attribute is blank") else: tmdb_is_movie = False - tmdb_item = TMDb.get_show(util.regex_first_int(data, "Show")) + tmdb_item = self.config.TMDb.get_show(util.regex_first_int(data, "Show")) elif "tmdb_movie" in methods: if meta[methods["tmdb_movie"]] is None: logger.error("Metadata Error: tmdb_movie attribute is blank") else: tmdb_is_movie = True - tmdb_item = TMDb.get_movie(util.regex_first_int(meta[methods["tmdb_movie"]], "Movie")) + tmdb_item = self.config.TMDb.get_movie(util.regex_first_int(meta[methods["tmdb_movie"]], "Movie")) except Failed as e: logger.error(e) diff --git a/modules/plex.py b/modules/plex.py index 09f04c4f..ecea22e1 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -132,14 +132,17 @@ show_only_searches = [ "episode_user_rating.gt", "episode_user_rating.gte", "episode_user_rating.lt", "episode_user_rating.lte", "episode_year", "episode_year.not", "episode_year.gt", "episode_year.gte", "episode_year.lt", "episode_year.lte" ] -number_attributes = ["plays", "episode_plays", "added", "episode_added", "release", "episode_air_date", "duration", "tmdb_vote_count"] +number_attributes = [ + "plays", "episode_plays", "duration", "tmdb_vote_count", "last_episode_aired" + "added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played" +] float_attributes = ["user_rating", "episode_user_rating", "critic_rating", "audience_rating"] boolean_attributes = [ "hdr", "unmatched", "duplicate", "unplayed", "progress", "trash", "unplayed_episodes", "episode_unplayed", "episode_duplicate", "episode_progress", "episode_unmatched", ] tmdb_attributes = ["actor", "director", "producer", "writer"] -date_attributes = ["added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played"] +date_attributes = ["added", "episode_added", "release", "episode_air_date", "last_played", "episode_last_played", "last_episode_aired"] search_display = {"added": "Date Added", "release": "Release Date", "hdr": "HDR", "progress": "In Progress", "episode_progress": "Episode In Progress"} sorts = { None: None, diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 62fb0c3c..c5a346e0 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -152,7 +152,7 @@ def update_libraries(config): util.separator(f"Running Metadata File\n{metadata.path}") if not config.test_mode and not config.resume_from and not collection_only: try: - metadata.update_metadata(config.TMDb, config.test_mode) + metadata.update_metadata() except Failed as e: logger.error(e) collections_to_run = metadata.get_collections(config.requested_collections)