From d27cb1896a4f7b4bf1b417b22250f451c6ba138b Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Fri, 7 May 2021 22:30:28 -0400 Subject: [PATCH] moved main methods out of config --- modules/config.py | 293 +---------------------------------------- plex_meta_manager.py | 306 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 300 insertions(+), 299 deletions(-) diff --git a/modules/config.py b/modules/config.py index f9f67514..76aea432 100644 --- a/modules/config.py +++ b/modules/config.py @@ -1,9 +1,8 @@ -import logging, os, re, requests, time +import logging, os from modules import util from modules.anidb import AniDBAPI from modules.anilist import AniListAPI from modules.arms import ArmsAPI -from modules.builder import CollectionBuilder from modules.cache import Cache from modules.imdb import IMDbAPI from modules.letterboxd import LetterboxdAPI @@ -17,8 +16,6 @@ from modules.tmdb import TMDbAPI from modules.trakttv import TraktAPI from modules.tvdb import TVDbAPI from modules.util import Failed -from plexapi.exceptions import BadRequest -from retrying import retry from ruamel import yaml logger = logging.getLogger("Plex Meta Manager") @@ -485,291 +482,3 @@ class Config: util.separator() - def update_libraries(self, test, requested_collections, resume_from): - for library in self.libraries: - os.environ["PLEXAPI_PLEXAPI_TIMEOUT"] = str(library.timeout) - logger.info("") - util.separator(f"{library.name} Library") - logger.info("") - util.separator(f"Mapping {library.name} Library") - logger.info("") - movie_map, show_map = self.map_guids(library) - if not test and not resume_from and library.mass_update: - self.mass_metadata(library, movie_map, show_map) - for metadata in library.metadata_files: - logger.info("") - util.separator(f"Running Metadata File\n{metadata.path}") - if not test and not resume_from: - try: metadata.update_metadata(self.TMDb, test) - except Failed as e: logger.error(e) - logger.info("") - util.separator(f"{'Test ' if test else ''}Collections") - collections = metadata.get_collections(requested_collections) - if resume_from and resume_from not in collections: - logger.warning(f"Collection: {resume_from} not in Metadata File: {metadata.path}") - continue - if collections: - resume_from = self.run_collection(library, metadata, collections, test, resume_from, movie_map, show_map) - - if library.show_unmanaged is True and not test and not requested_collections: - logger.info("") - util.separator(f"Unmanaged Collections in {library.name} Library") - logger.info("") - unmanaged_count = 0 - collections_in_plex = [str(plex_col) for plex_col in library.collections] - for col in library.get_all_collections(): - if col.title not in collections_in_plex: - logger.info(col.title) - unmanaged_count += 1 - logger.info("{} Unmanaged Collections".format(unmanaged_count)) - - if library.assets_for_all is True and not test and not requested_collections: - logger.info("") - util.separator(f"All {'Movies' if library.is_movie else 'Shows'} Assets Check for {library.name} Library") - logger.info("") - for item in library.get_all(): - library.update_item_from_assets(item) - - has_run_again = False - for library in self.libraries: - if library.run_again: - has_run_again = True - break - - if has_run_again: - logger.info("") - util.separator("Run Again") - logger.info("") - length = 0 - for x in range(1, self.general["run_again_delay"] + 1): - length = util.print_return(length, f"Waiting to run again in {self.general['run_again_delay'] - x + 1} minutes") - for y in range(60): - time.sleep(1) - util.print_end(length) - for library in self.libraries: - if library.run_again: - os.environ["PLEXAPI_PLEXAPI_TIMEOUT"] = str(library.timeout) - logger.info("") - util.separator(f"{library.name} Library Run Again") - logger.info("") - movie_map, show_map = self.map_guids(library) - for builder in library.run_again: - logger.info("") - util.separator(f"{builder.name} Collection") - logger.info("") - try: - collection_obj = library.get_collection(builder.name) - except Failed as e: - util.print_multiline(e, error=True) - continue - builder.run_collections_again(collection_obj, movie_map, show_map) - - def run_collection(self, library, metadata, collections, test, resume_from, movie_map, show_map): - for mapping_name, collection_attrs in collections.items(): - if test and ("test" not in collection_attrs or collection_attrs["test"] is not True): - no_template_test = True - if "template" in collection_attrs and collection_attrs["template"]: - for data_template in util.get_list(collection_attrs["template"], split=False): - if "name" in data_template \ - and data_template["name"] \ - and metadata.templates \ - and data_template["name"] in metadata.templates \ - and metadata.templates[data_template["name"]] \ - and "test" in metadata.templates[data_template["name"]] \ - and metadata.templates[data_template["name"]]["test"] is True: - no_template_test = False - if no_template_test: - continue - try: - if resume_from and resume_from != mapping_name: - continue - elif resume_from == mapping_name: - resume_from = None - logger.info("") - util.separator(f"Resuming Collections") - - logger.info("") - util.separator(f"{mapping_name} Collection") - logger.info("") - - try: - builder = CollectionBuilder(self, library, metadata, mapping_name, collection_attrs) - except Failed as f: - util.print_stacktrace() - util.print_multiline(f, error=True) - continue - except Exception as e: - util.print_stacktrace() - logger.error(e) - continue - - try: - collection_obj = library.get_collection(mapping_name) - collection_name = collection_obj.title - collection_smart = library.smart(collection_obj) - if (builder.smart and not collection_smart) or (not builder.smart and collection_smart): - logger.info("") - logger.error(f"Collection Error: Converting {collection_obj.title} to a {'smart' if builder.smart else 'normal'} collection") - library.query(collection_obj.delete) - collection_obj = None - except Failed: - collection_obj = None - collection_name = mapping_name - - if len(builder.schedule) > 0: - util.print_multiline(builder.schedule, info=True) - - rating_key_map = {} - logger.info("") - if builder.sync: - logger.info("Sync Mode: sync") - if collection_obj: - for item in library.get_collection_items(collection_obj, builder.smart_label_collection): - rating_key_map[item.ratingKey] = item - else: - logger.info("Sync Mode: append") - - for i, f in enumerate(builder.filters): - if i == 0: - logger.info("") - logger.info(f"Collection Filter {f[0]}: {f[1]}") - - if not builder.smart_url: - builder.run_methods(collection_obj, collection_name, rating_key_map, movie_map, show_map) - - try: - if not collection_obj and builder.smart_url: - library.create_smart_collection(collection_name, builder.smart_type_key, builder.smart_url) - elif not collection_obj and builder.smart_label_collection: - library.create_smart_labels(collection_name, sort=builder.smart_sort) - plex_collection = library.get_collection(collection_name) - except Failed as e: - util.print_stacktrace() - logger.error(e) - continue - - builder.update_details(plex_collection) - - if builder.run_again and (len(builder.missing_movies) > 0 or len(builder.missing_shows) > 0): - library.run_again.append(builder) - - except Exception as e: - util.print_stacktrace() - logger.error(f"Unknown Error: {e}") - return resume_from - - def mass_metadata(self, library, movie_map, show_map): - length = 0 - logger.info("") - util.separator(f"Mass Editing {'Movie' if library.is_movie else 'Show'} Library: {library.name}") - logger.info("") - items = library.Plex.all() - for i, item in enumerate(items, 1): - length = util.print_return(length, f"Processing: {i}/{len(items)} {item.title}") - ids = {} - if self.Cache: - ids, expired = self.Cache.get_ids("movie" if library.is_movie else "show", plex_guid=item.guid) - elif library.is_movie: - for tmdb, rating_keys in movie_map.items(): - if item.ratingKey in rating_keys: - ids["tmdb"] = tmdb - break - else: - for tvdb, rating_keys in show_map.items(): - if item.ratingKey in rating_keys: - ids["tvdb"] = tvdb - break - - if library.mass_genre_update: - if library.mass_genre_update == "tmdb": - if "tmdb" not in ids: - util.print_end(length, f"{item.title[:25]:<25} | No TMDb for Guid: {item.guid}") - continue - try: - tmdb_item = self.TMDb.get_movie(ids["tmdb"]) if library.is_movie else self.TMDb.get_show(ids["tmdb"]) - except Failed as e: - util.print_end(length, str(e)) - continue - new_genres = [genre.name for genre in tmdb_item.genres] - elif library.mass_genre_update in ["omdb", "imdb"]: - if self.OMDb.limit is True: - break - if "imdb" not in ids: - util.print_end(length, f"{item.title[:25]:<25} | No IMDb for Guid: {item.guid}") - continue - try: - omdb_item = self.OMDb.get_omdb(ids["imdb"]) - except Failed as e: - util.print_end(length, str(e)) - continue - new_genres = omdb_item.genres - else: - raise Failed - item_genres = [genre.tag for genre in item.genres] - display_str = "" - for genre in (g for g in item_genres if g not in new_genres): - library.query_data(item.removeGenre, genre) - display_str += f"{', ' if len(display_str) > 0 else ''}-{genre}" - for genre in (g for g in new_genres if g not in item_genres): - library.query_data(item.addGenre, genre) - display_str += f"{', ' if len(display_str) > 0 else ''}+{genre}" - if len(display_str) > 0: - util.print_end(length, f"{item.title[:25]:<25} | Genres | {display_str}") - if library.mass_audience_rating_update: - if library.mass_audience_rating_update == "tmdb": - if "tmdb" not in ids: - util.print_end(length, f"{item.title[:25]:<25} | No TMDb for Guid: {item.guid}") - continue - try: - tmdb_item = self.TMDb.get_movie(ids["tmdb"]) if library.is_movie else self.TMDb.get_show(ids["tmdb"]) - except Failed as e: - util.print_end(length, str(e)) - continue - new_rating = tmdb_item.vote_average - elif library.mass_audience_rating_update in ["omdb", "imdb"]: - if self.OMDb.limit is True: - break - if "imdb" not in ids: - util.print_end(length, f"{item.title[:25]:<25} | No IMDb for Guid: {item.guid}") - continue - try: - omdb_item = self.OMDb.get_omdb(ids["imdb"]) - except Failed as e: - util.print_end(length, str(e)) - continue - new_rating = omdb_item.imdb_rating - else: - raise Failed - if new_rating is None: - util.print_end(length, f"{item.title[:25]:<25} | No Rating Found") - elif str(item.audienceRating) != str(new_rating): - library.edit_query(item, {"audienceRating.value": new_rating, "audienceRating.locked": 1}) - util.print_end(length, f"{item.title[:25]:<25} | Audience Rating | {new_rating}") - - def map_guids(self, library): - movie_map = {} - show_map = {} - length = 0 - logger.info(f"Mapping {'Movie' if library.is_movie else 'Show'} Library: {library.name}") - items = library.Plex.all() - for i, item in enumerate(items, 1): - length = util.print_return(length, f"Processing: {i}/{len(items)} {item.title}") - try: - id_type, main_id = self.Arms.get_id(item, library, length) - except BadRequest: - util.print_stacktrace() - util.print_end(length, f"{'Cache | ! |' if self.Cache else 'Mapping Error:'} | {item.guid} for {item.title} not found") - continue - if not isinstance(main_id, list): - main_id = [main_id] - if id_type == "movie": - for m in main_id: - if m in movie_map: movie_map[m].append(item.ratingKey) - else: movie_map[m] = [item.ratingKey] - elif id_type == "show": - for m in main_id: - if m in show_map: show_map[m].append(item.ratingKey) - else: show_map[m] = [item.ratingKey] - util.print_end(length, f"Processed {len(items)} {'Movies' if library.is_movie else 'Shows'}") - return movie_map, show_map - diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 8607ef9f..e224e4c4 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -3,7 +3,10 @@ from datetime import datetime try: import schedule from modules import tests, util + from modules.builder import CollectionBuilder from modules.config import Config + from modules.util import Failed + from plexapi.exceptions import BadRequest except ModuleNotFoundError: print("Error: Requirements are not installed") sys.exit(0) @@ -98,28 +101,317 @@ if my_tests: tests.run_tests(default_dir) sys.exit(0) -def start(config_path, is_test, daily, collections_to_run, libraries_to_run, resume_from): +def start(config_path, is_test, daily, requested_collections, requested_libraries, resume_from): if daily: start_type = "Daily " elif is_test: start_type = "Test " - elif collections_to_run: start_type = "Collections " - elif libraries_to_run: start_type = "Libraries " + elif requested_collections: start_type = "Collections " + elif requested_libraries: start_type = "Libraries " else: start_type = "" start_time = datetime.now() util.separator(f"Starting {start_type}Run") try: - config = Config(default_dir, config_path, libraries_to_run) - config.update_libraries(is_test, collections_to_run, resume_from) + config = Config(default_dir, config_path, requested_libraries) + update_libraries(config, is_test, requested_collections, resume_from) except Exception as e: util.print_stacktrace() logger.critical(e) logger.info("") util.separator(f"Finished {start_type}Run\nRun Time: {str(datetime.now() - start_time).split('.')[0]}") +def update_libraries(config, is_test, requested_collections, resume_from): + for library in config.libraries: + os.environ["PLEXAPI_PLEXAPI_TIMEOUT"] = str(library.timeout) + logger.info("") + util.separator(f"{library.name} Library") + logger.info("") + util.separator(f"Mapping {library.name} Library") + logger.info("") + movie_map, show_map = map_guids(config, library) + if not is_test and not resume_from and library.mass_update: + mass_metadata(config, library, movie_map, show_map) + for metadata in library.metadata_files: + logger.info("") + util.separator(f"Running Metadata File\n{metadata.path}") + if not is_test and not resume_from: + try: + metadata.update_metadata(config.TMDb, is_test) + except Failed as e: + logger.error(e) + logger.info("") + util.separator(f"{'Test ' if is_test else ''}Collections") + collections_to_run = metadata.get_collections(requested_collections) + if resume_from and resume_from not in collections_to_run: + logger.warning(f"Collection: {resume_from} not in Metadata File: {metadata.path}") + continue + if collections_to_run: + resume_from = run_collection(config, library, metadata, collections_to_run, is_test, resume_from, movie_map, show_map) + + if library.show_unmanaged is True and not is_test and not requested_collections: + logger.info("") + util.separator(f"Unmanaged Collections in {library.name} Library") + logger.info("") + unmanaged_count = 0 + collections_in_plex = [str(plex_col) for plex_col in library.collections] + for col in library.get_all_collections(): + if col.title not in collections_in_plex: + logger.info(col.title) + unmanaged_count += 1 + logger.info("{} Unmanaged Collections".format(unmanaged_count)) + + if library.assets_for_all is True and not is_test and not requested_collections: + logger.info("") + util.separator(f"All {'Movies' if library.is_movie else 'Shows'} Assets Check for {library.name} Library") + logger.info("") + for item in library.get_all(): + library.update_item_from_assets(item) + has_run_again = False + for library in config.libraries: + if library.run_again: + has_run_again = True + break + + if has_run_again: + logger.info("") + util.separator("Run Again") + logger.info("") + length = 0 + for x in range(1, config.general["run_again_delay"] + 1): + length = util.print_return(length, f"Waiting to run again in {config.general['run_again_delay'] - x + 1} minutes") + for y in range(60): + time.sleep(1) + util.print_end(length) + for library in config.libraries: + if library.run_again: + os.environ["PLEXAPI_PLEXAPI_TIMEOUT"] = str(library.timeout) + logger.info("") + util.separator(f"{library.name} Library Run Again") + logger.info("") + movie_map, show_map = map_guids(config, library) + for builder in library.run_again: + logger.info("") + util.separator(f"{builder.name} Collection") + logger.info("") + try: + collection_obj = library.get_collection(builder.name) + except Failed as e: + util.print_multiline(e, error=True) + continue + builder.run_collections_again(collection_obj, movie_map, show_map) + +def run_collection(config, library, metadata, requested_collections, is_test, resume_from, movie_map, show_map): + for mapping_name, collection_attrs in requested_collections.items(): + if is_test and ("test" not in collection_attrs or collection_attrs["test"] is not True): + no_template_test = True + if "template" in collection_attrs and collection_attrs["template"]: + for data_template in util.get_list(collection_attrs["template"], split=False): + if "name" in data_template \ + and data_template["name"] \ + and metadata.templates \ + and data_template["name"] in metadata.templates \ + and metadata.templates[data_template["name"]] \ + and "test" in metadata.templates[data_template["name"]] \ + and metadata.templates[data_template["name"]]["test"] is True: + no_template_test = False + if no_template_test: + continue + try: + if resume_from and resume_from != mapping_name: + continue + elif resume_from == mapping_name: + resume_from = None + logger.info("") + util.separator(f"Resuming Collections") + + logger.info("") + util.separator(f"{mapping_name} Collection") + logger.info("") + + try: + builder = CollectionBuilder(config, library, metadata, mapping_name, collection_attrs) + except Failed as f: + util.print_stacktrace() + util.print_multiline(f, error=True) + continue + except Exception as e: + util.print_stacktrace() + logger.error(e) + continue + + try: + collection_obj = library.get_collection(mapping_name) + collection_name = collection_obj.title + collection_smart = library.smart(collection_obj) + if (builder.smart and not collection_smart) or (not builder.smart and collection_smart): + logger.info("") + logger.error(f"Collection Error: Converting {collection_obj.title} to a {'smart' if builder.smart else 'normal'} collection") + library.query(collection_obj.delete) + collection_obj = None + except Failed: + collection_obj = None + collection_name = mapping_name + + if len(builder.schedule) > 0: + util.print_multiline(builder.schedule, info=True) + + rating_key_map = {} + logger.info("") + if builder.sync: + logger.info("Sync Mode: sync") + if collection_obj: + for item in library.get_collection_items(collection_obj, builder.smart_label_collection): + rating_key_map[item.ratingKey] = item + else: + logger.info("Sync Mode: append") + + for i, f in enumerate(builder.filters): + if i == 0: + logger.info("") + logger.info(f"Collection Filter {f[0]}: {f[1]}") + + if not builder.smart_url: + builder.run_methods(collection_obj, collection_name, rating_key_map, movie_map, show_map) + + try: + if not collection_obj and builder.smart_url: + library.create_smart_collection(collection_name, builder.smart_type_key, builder.smart_url) + elif not collection_obj and builder.smart_label_collection: + library.create_smart_labels(collection_name, sort=builder.smart_sort) + plex_collection = library.get_collection(collection_name) + except Failed as e: + util.print_stacktrace() + logger.error(e) + continue + + builder.update_details(plex_collection) + + if builder.run_again and (len(builder.missing_movies) > 0 or len(builder.missing_shows) > 0): + library.run_again.append(builder) + + except Exception as e: + util.print_stacktrace() + logger.error(f"Unknown Error: {e}") + return resume_from + +def mass_metadata(config, library, movie_map, show_map): + length = 0 + logger.info("") + util.separator(f"Mass Editing {'Movie' if library.is_movie else 'Show'} Library: {library.name}") + logger.info("") + items = library.Plex.all() + for i, item in enumerate(items, 1): + length = util.print_return(length, f"Processing: {i}/{len(items)} {item.title}") + ids = {} + if config.Cache: + ids, expired = config.Cache.get_ids("movie" if library.is_movie else "show", plex_guid=item.guid) + elif library.is_movie: + for tmdb, rating_keys in movie_map.items(): + if item.ratingKey in rating_keys: + ids["tmdb"] = tmdb + break + else: + for tvdb, rating_keys in show_map.items(): + if item.ratingKey in rating_keys: + ids["tvdb"] = tvdb + break + + if library.mass_genre_update: + if library.mass_genre_update == "tmdb": + if "tmdb" not in ids: + util.print_end(length, f"{item.title[:25]:<25} | No TMDb for Guid: {item.guid}") + continue + try: + tmdb_item = config.TMDb.get_movie(ids["tmdb"]) if library.is_movie else config.TMDb.get_show(ids["tmdb"]) + except Failed as e: + util.print_end(length, str(e)) + continue + new_genres = [genre.name for genre in tmdb_item.genres] + elif library.mass_genre_update in ["omdb", "imdb"]: + if config.OMDb.limit is True: + break + if "imdb" not in ids: + util.print_end(length, f"{item.title[:25]:<25} | No IMDb for Guid: {item.guid}") + continue + try: + omdb_item = config.OMDb.get_omdb(ids["imdb"]) + except Failed as e: + util.print_end(length, str(e)) + continue + new_genres = omdb_item.genres + else: + raise Failed + item_genres = [genre.tag for genre in item.genres] + display_str = "" + for genre in (g for g in item_genres if g not in new_genres): + library.query_data(item.removeGenre, genre) + display_str += f"{', ' if len(display_str) > 0 else ''}-{genre}" + for genre in (g for g in new_genres if g not in item_genres): + library.query_data(item.addGenre, genre) + display_str += f"{', ' if len(display_str) > 0 else ''}+{genre}" + if len(display_str) > 0: + util.print_end(length, f"{item.title[:25]:<25} | Genres | {display_str}") + if library.mass_audience_rating_update: + if library.mass_audience_rating_update == "tmdb": + if "tmdb" not in ids: + util.print_end(length, f"{item.title[:25]:<25} | No TMDb for Guid: {item.guid}") + continue + try: + tmdb_item = config.TMDb.get_movie(ids["tmdb"]) if library.is_movie else config.TMDb.get_show(ids["tmdb"]) + except Failed as e: + util.print_end(length, str(e)) + continue + new_rating = tmdb_item.vote_average + elif library.mass_audience_rating_update in ["omdb", "imdb"]: + if config.OMDb.limit is True: + break + if "imdb" not in ids: + util.print_end(length, f"{item.title[:25]:<25} | No IMDb for Guid: {item.guid}") + continue + try: + omdb_item = config.OMDb.get_omdb(ids["imdb"]) + except Failed as e: + util.print_end(length, str(e)) + continue + new_rating = omdb_item.imdb_rating + else: + raise Failed + if new_rating is None: + util.print_end(length, f"{item.title[:25]:<25} | No Rating Found") + elif str(item.audienceRating) != str(new_rating): + library.edit_query(item, {"audienceRating.value": new_rating, "audienceRating.locked": 1}) + util.print_end(length, f"{item.title[:25]:<25} | Audience Rating | {new_rating}") + +def map_guids(config, library): + movie_map = {} + show_map = {} + length = 0 + logger.info(f"Mapping {'Movie' if library.is_movie else 'Show'} Library: {library.name}") + items = library.Plex.all() + for i, item in enumerate(items, 1): + length = util.print_return(length, f"Processing: {i}/{len(items)} {item.title}") + try: + id_type, main_id = config.Arms.get_id(item, library, length) + except BadRequest: + util.print_stacktrace() + util.print_end(length, f"{'Cache | ! |' if config.Cache else 'Mapping Error:'} | {item.guid} for {item.title} not found") + continue + if not isinstance(main_id, list): + main_id = [main_id] + if id_type == "movie": + for m in main_id: + if m in movie_map: movie_map[m].append(item.ratingKey) + else: movie_map[m] = [item.ratingKey] + elif id_type == "show": + for m in main_id: + if m in show_map: show_map[m].append(item.ratingKey) + else: show_map[m] = [item.ratingKey] + util.print_end(length, f"Processed {len(items)} {'Movies' if library.is_movie else 'Shows'}") + return movie_map, show_map + try: if run or test or collections or libraries or resume: start(config_file, test, False, collections, libraries, resume) else: - length = 0 + time_length = 0 schedule.every().day.at(time_to_run).do(start, config_file, False, True, None, None, None) while True: schedule.run_pending() @@ -132,7 +424,7 @@ try: time_str = f"{hours} Hour{'s' if hours > 1 else ''} and " if hours > 0 else "" time_str += f"{minutes} Minute{'s' if minutes > 1 else ''}" - length = util.print_return(length, f"Current Time: {current} | {time_str} until the daily run at {time_to_run}") + time_length = util.print_return(time_length, f"Current Time: {current} | {time_str} until the daily run at {time_to_run}") time.sleep(1) except KeyboardInterrupt: util.separator("Exiting Plex Meta Manager")