[34] added run_order setting

VERSION

@@ -1 +1 @@
-1.19.1-develop33
+1.19.1-develop34

config/config.yml.template

@@ -33,6 +33,7 @@ playlist_files:
       libraries: Movies, TV Shows   # list of libraries that you want the PMM Defaults playlists to look at
   # see the wiki for how to use local files, folders, URLs, or files from git
 settings:
+  run_order: metadata, overlays, operations
   cache: true
   cache_expiration: 60
   asset_directory: config/assets

modules/config.py

@@ -33,6 +33,11 @@ from retrying import retry
 logger = util.logger
 
 mediastingers_url = "https://raw.githubusercontent.com/meisnate12/PMM-Mediastingers/master/stingers.yml"
+run_order_options = {
+    "metadata": "Represents Collection and Metadata Updates",
+    "overlays": "Represents Overlay Updates",
+    "operations": "Represents Operations Updates"
+}
 sync_modes = {"append": "Only Add Items to the Collection or Playlist", "sync": "Add & Remove Items from the Collection or Playlist"}
 imdb_label_options = {
     "none": "Add IMDb Parental Labels for None, Mild, Moderate, or Severe",
@@ -153,6 +158,7 @@ class ConfigFile:
         self.collection_only = attrs["collection_only"] if "collection_only" in attrs else False
         self.operations_only = attrs["operations_only"] if "operations_only" in attrs else False
         self.overlays_only = attrs["overlays_only"] if "overlays_only" in attrs else False
+        self.libraries_first = attrs["libraries_first"] if "libraries_first" in attrs else False
         self.env_plex_url = attrs["plex_url"] if "plex_url" in attrs else ""
         self.env_plex_token = attrs["plex_token"] if "plex_token" in attrs else ""
         current_time = datetime.now()
@@ -341,9 +347,13 @@ class ConfigFile:
             elif var_type == "path":
                 if os.path.exists(os.path.abspath(data[attribute])):                return data[attribute]
                 else:                                                               message = f"Path {os.path.abspath(data[attribute])} does not exist"
-            elif var_type == "list":                                            return util.get_list(data[attribute], split=False)
+            elif var_type in ["list", "comma_list", "int_list"]:
-            elif var_type == "comma_list":                                      return util.get_list(data[attribute])
+                output_list = list(set(util.get_list(data[attribute], lower=var_type != "int_list", split=var_type != "list", int_list=var_type == "int_list")))
-            elif var_type == "int_list":                                        return util.get_list(data[attribute], int_list=True)
+                failed_items = [o for o in output_list if o not in test_list] if test_list else []
+                if failed_items:
+                    message = f"{text}: {', '.join(failed_items)} is an invalid input"
+                else:
+                    return output_list
             elif var_type == "list_path":
                 temp_list = []
                 warning_message = ""
@@ -390,7 +400,10 @@ class ConfigFile:
                     logger.warning(options)
             return default
 
+        default_run = ["overlays", "operations", "metadata"] if self.libraries_first else ["metadata", "overlays", "operations"]
+
         self.general = {
+            "run_order": check_for_attribute(self.data, "run_order", parent="settings", var_type="comma_list", test_list=run_order_options, default=default_run),
             "cache": check_for_attribute(self.data, "cache", parent="settings", var_type="bool", default=True),
             "cache_expiration": check_for_attribute(self.data, "cache_expiration", parent="settings", var_type="int", default=60, int_min=1),
             "asset_directory": check_for_attribute(self.data, "asset_directory", parent="settings", var_type="list_path", default_is_none=True),
@@ -421,7 +434,7 @@ class ConfigFile:
             "save_report": check_for_attribute(self.data, "save_report", parent="settings", var_type="bool", default=False),
             "tvdb_language": check_for_attribute(self.data, "tvdb_language", parent="settings", default="default"),
             "ignore_ids": check_for_attribute(self.data, "ignore_ids", parent="settings", var_type="int_list", default_is_none=True),
-            "ignore_imdb_ids": check_for_attribute(self.data, "ignore_imdb_ids", parent="settings", var_type="list", default_is_none=True),
+            "ignore_imdb_ids": check_for_attribute(self.data, "ignore_imdb_ids", parent="settings", var_type="comma_list", default_is_none=True),
             "playlist_sync_to_users": check_for_attribute(self.data, "playlist_sync_to_users", parent="settings", default="all", default_is_none=True),
             "playlist_exclude_users": check_for_attribute(self.data, "playlist_exclude_users", parent="settings", default_is_none=True),
             "playlist_report": check_for_attribute(self.data, "playlist_report", parent="settings", var_type="bool", default=True),
@@ -722,7 +735,8 @@ class ConfigFile:
                 logger.info("")
                 logger.info(f"Connecting to {display_name} Library...")
 
-                params["asset_directory"] = check_for_attribute(lib, "asset_directory", parent="settings", var_type="list_path", default=self.general["asset_directory"], default_is_none=True, save=False)
+                params["run_order"] = check_for_attribute(lib, "run_order", parent="settings", var_type="comma_list", default=self.general["run_order"], do_print=False, save=False)
+                params["asset_directory"] = check_for_attribute(lib, "asset_directory", parent="settings", var_type="list_path", default=self.general["asset_directory"], default_is_none=True, do_print=False, save=False)
                 params["asset_folders"] = check_for_attribute(lib, "asset_folders", parent="settings", var_type="bool", default=self.general["asset_folders"], do_print=False, save=False)
                 params["asset_depth"] = check_for_attribute(lib, "asset_depth", parent="settings", var_type="int", default=self.general["asset_depth"], do_print=False, save=False)
                 params["sync_mode"] = check_for_attribute(lib, "sync_mode", parent="settings", test_list=sync_modes, default=self.general["sync_mode"], do_print=False, save=False)
@@ -749,7 +763,7 @@ class ConfigFile:
                 params["delete_not_scheduled"] = check_for_attribute(lib, "delete_not_scheduled", parent="settings", var_type="bool", default=self.general["delete_not_scheduled"], do_print=False, save=False)
                 params["ignore_ids"] = check_for_attribute(lib, "ignore_ids", parent="settings", var_type="int_list", default_is_none=True, do_print=False, save=False)
                 params["ignore_ids"].extend([i for i in self.general["ignore_ids"] if i not in params["ignore_ids"]])
-                params["ignore_imdb_ids"] = check_for_attribute(lib, "ignore_imdb_ids", parent="settings", var_type="list", default_is_none=True, do_print=False, save=False)
+                params["ignore_imdb_ids"] = check_for_attribute(lib, "ignore_imdb_ids", parent="settings", var_type="comma_list", default_is_none=True, do_print=False, save=False)
                 params["ignore_imdb_ids"].extend([i for i in self.general["ignore_imdb_ids"] if i not in params["ignore_imdb_ids"]])
                 params["changes_webhooks"] = check_for_attribute(lib, "changes", parent="webhooks", var_type="list", default=self.webhooks["changes"], do_print=False, save=False, default_is_none=True)
                 params["report_path"] = None

modules/library.py

@@ -54,6 +54,7 @@ class Library(ABC):
         self.overlay_backup = os.path.join(self.overlay_folder, f"{self.mapping_name} Original Posters")
         self.report_path = params["report_path"] if params["report_path"] else os.path.join(self.default_dir, f"{self.mapping_name}_report.yml")
         self.report_data = {}
+        self.run_order = params["run_order"]
         self.asset_folders = params["asset_folders"]
         self.create_asset_folders = params["create_asset_folders"]
         self.dimensional_asset_rename = params["dimensional_asset_rename"]

plex_meta_manager.py

@@ -38,7 +38,7 @@ arguments = {
     "run-collections": {"args": ["rc", "cl", "collection", "collections", "run-collection"], "type": "str", "help": "Process only specified collections (pipe-separated list '|')"},
     "run-libraries": {"args": ["rl", "l", "library", "libraries", "run-library"], "type": "str", "help": "Process only specified libraries (pipe-separated list '|')"},
     "run-metadata-files": {"args": ["rm", "m", "metadata", "metadata-files"], "type": "str", "help": "Process only specified Metadata files (pipe-separated list '|')"},
-    "libraries-first": {"args": ["lf", "library-first"], "type": "bool", "help": "Run library operations before collections"},
+    "libraries-first": {"args": ["lf", "library-first"], "type": "bool", "help": argparse.SUPPRESS},
     "ignore-schedules": {"args": "is", "type": "bool", "help": "Run ignoring collection schedules"},
     "ignore-ghost": {"args": "ig", "type": "bool", "help": "Run ignoring ghost logging"},
     "delete-collections": {"args": ["dc", "delete", "delete-collection"], "type": "bool", "help": "Deletes all Collections in the Plex Library before running"},
@@ -269,13 +269,15 @@ def start(attrs):
     attrs["playlist_only"] = run_args["playlists-only"]
     attrs["operations_only"] = run_args["operations-only"]
     attrs["overlays_only"] = run_args["overlays-only"]
+    attrs["libraries_first"] = run_args["libraries-first"]
     attrs["plex_url"] = plex_url
     attrs["plex_token"] = plex_token
     logger.separator(debug=True)
     logger.debug(f"Run Command: {run_arg}")
     for akey, adata in arguments.items():
-        ext = '"' if adata["type"] == "str" and run_args[akey] not in [None, "None"] else ""
+        if isinstance(adata["help"], str):
-        logger.debug(f"--{akey} (PMM_{akey.upper()}): {ext}{run_args[akey]}{ext}")
+            ext = '"' if adata["type"] == "str" and run_args[akey] not in [None, "None"] else ""
+            logger.debug(f"--{akey} (PMM_{akey.upper()}): {ext}{run_args[akey]}{ext}")
     logger.debug("")
     if secret_args:
         logger.debug("PMM Secrets Read:")
@@ -501,6 +503,7 @@ def run_libraries(config):
 
             logger.debug("")
             logger.debug(f"Library Name: {library.name}")
+            logger.debug(f"Run Order: {', '.join(library.run_order)}")
             logger.debug(f"Folder Name: {library.mapping_name}")
             for ad in library.asset_directory:
                 logger.debug(f"Asset Directory: {ad}")
@@ -561,9 +564,8 @@ def run_libraries(config):
             time_start = datetime.now()
             temp_items = None
             list_key = None
-            expired = None
             if config.Cache:
-                list_key, expired = config.Cache.query_list_cache("library", library.mapping_name, 1)
+                list_key, _ = config.Cache.query_list_cache("library", library.mapping_name, 1)
 
             if not temp_items:
                 temp_items = library.cache_items()
@@ -576,65 +578,60 @@ def run_libraries(config):
                 library.map_guids(temp_items)
             library_status[library.name]["Library Loading and Mapping"] = str(datetime.now() - time_start).split('.')[0]
 
-            def run_operations_and_overlays():
+            for run_type in library.run_order:
-                if not run_args["tests"] and not run_args["collections-only"] and not run_args["playlists-only"] and not config.requested_metadata_files:
+                if run_type == "metadata":
-                    if not run_args["overlays-only"] and library.library_operation:
+                    if not run_args["operations-only"] and not run_args["overlays-only"] and not run_args["playlists-only"]:
+                        time_start = datetime.now()
+                        for images in library.images_files:
+                            images_name = images.get_file_name()
+                            if config.requested_metadata_files and images_name not in config.requested_metadata_files:
+                                logger.info("")
+                                logger.separator(f"Skipping {images_name} Images File")
+                                continue
+                            logger.info("")
+                            logger.separator(f"Running {images_name} Images File\n{images.path}")
+                            if not run_args["tests"] and not run_args["resume"] and not run_args["collections-only"]:
+                                try:
+                                    images.update_metadata()
+                                except Failed as e:
+                                    library.notify(e)
+                                    logger.error(e)
+                        library_status[library.name]["Library Images Files"] = str(datetime.now() - time_start).split('.')[0]
+
+                        time_start = datetime.now()
+                        for metadata in library.metadata_files:
+                            metadata_name = metadata.get_file_name()
+                            if config.requested_metadata_files and metadata_name not in config.requested_metadata_files:
+                                logger.info("")
+                                logger.separator(f"Skipping {metadata_name} Metadata File")
+                                continue
+                            logger.info("")
+                            logger.separator(f"Running {metadata_name} Metadata File\n{metadata.path}")
+                            if not run_args["tests"] and not run_args["resume"] and not run_args["collections-only"]:
+                                try:
+                                    metadata.update_metadata()
+                                except Failed as e:
+                                    library.notify(e)
+                                    logger.error(e)
+                            collections_to_run = metadata.get_collections(config.requested_collections)
+                            if run_args["resume"] and run_args["resume"] not in collections_to_run:
+                                logger.info("")
+                                logger.warning(f"Collection: {run_args['resume']} not in Metadata File: {metadata.path}")
+                                continue
+                            if collections_to_run:
+                                logger.info("")
+                                logger.separator(f"{'Test ' if run_args['tests'] else ''}Collections")
+                                # logger.remove_library_handler(library.mapping_name)
+                                run_collection(config, library, metadata, collections_to_run)
+                                # logger.re_add_library_handler(library.mapping_name)
+                        library_status[library.name]["Library Metadata Files"] = str(datetime.now() - time_start).split('.')[0]
+                elif run_type == "overlays":
+                    if not run_args["tests"] and not run_args["collections-only"] and not run_args["playlists-only"] and not config.requested_metadata_files and not run_args["overlays-only"] and library.library_operation:
                         library_status[library.name]["Library Operations"] = library.Operations.run_operations()
-                    if not run_args["operations-only"] and (library.overlay_files or library.remove_overlays):
+                elif run_type == "operations":
+                    if not run_args["tests"] and not run_args["collections-only"] and not run_args["playlists-only"] and not config.requested_metadata_files and not run_args["operations-only"] and (library.overlay_files or library.remove_overlays):
                         library_status[library.name]["Library Overlays"] = library.Overlays.run_overlays()
 
-            if run_args["libraries-first"]:
-                run_operations_and_overlays()
-
-            if not run_args["operations-only"] and not run_args["overlays-only"] and not run_args["playlists-only"]:
-                time_start = datetime.now()
-                for images in library.images_files:
-                    images_name = images.get_file_name()
-                    if config.requested_metadata_files and images_name not in config.requested_metadata_files:
-                        logger.info("")
-                        logger.separator(f"Skipping {images_name} Images File")
-                        continue
-                    logger.info("")
-                    logger.separator(f"Running {images_name} Images File\n{images.path}")
-                    if not run_args["tests"] and not run_args["resume"] and not run_args["collections-only"]:
-                        try:
-                            images.update_metadata()
-                        except Failed as e:
-                            library.notify(e)
-                            logger.error(e)
-                library_status[library.name]["Library Images Files"] = str(datetime.now() - time_start).split('.')[0]
-
-                time_start = datetime.now()
-                for metadata in library.metadata_files:
-                    metadata_name = metadata.get_file_name()
-                    if config.requested_metadata_files and metadata_name not in config.requested_metadata_files:
-                        logger.info("")
-                        logger.separator(f"Skipping {metadata_name} Metadata File")
-                        continue
-                    logger.info("")
-                    logger.separator(f"Running {metadata_name} Metadata File\n{metadata.path}")
-                    if not run_args["tests"] and not run_args["resume"] and not run_args["collections-only"]:
-                        try:
-                            metadata.update_metadata()
-                        except Failed as e:
-                            library.notify(e)
-                            logger.error(e)
-                    collections_to_run = metadata.get_collections(config.requested_collections)
-                    if run_args["resume"] and run_args["resume"] not in collections_to_run:
-                        logger.info("")
-                        logger.warning(f"Collection: {run_args['resume']} not in Metadata File: {metadata.path}")
-                        continue
-                    if collections_to_run:
-                        logger.info("")
-                        logger.separator(f"{'Test ' if run_args['tests'] else ''}Collections")
-                        #logger.remove_library_handler(library.mapping_name)
-                        run_collection(config, library, metadata, collections_to_run)
-                        #logger.re_add_library_handler(library.mapping_name)
-                library_status[library.name]["Library Metadata Files"] = str(datetime.now() - time_start).split('.')[0]
-
-            if not run_args["libraries-first"]:
-                run_operations_and_overlays()
-
             #logger.remove_library_handler(library.mapping_name)
         except Exception as e:
             library.notify(e)