From 4a323e7705cef2d18c7a2ebbafcfdff4140fde8d Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 21 Jan 2021 16:42:31 -0500 Subject: [PATCH] fix requests --- README.md | 2 +- modules/anidb.py | 8 ++++---- modules/config.py | 7 +++---- modules/imdb.py | 7 +++---- modules/plex.py | 24 +++++++++--------------- modules/trakt.py | 5 ++--- modules/tvdb.py | 15 +++++++-------- requirements.txt | 2 -- 8 files changed, 29 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 315e0840..727d4f51 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Plex Meta Manager -The original concept for Plex Meta Manager is [Plex Auto Collections](https://github.com/mza921/Plex-Auto-Collections), but this is rewritten from the ground up to be able to include a scheduler, metadata edits, multiple libraries, and logging. Plex Meta Manager is a Python 3 script that can be continuously run using YMAL configuration files to update on a schedule the metadata of the movies, shows, and collections in your libraries as well as automatically build collections based on various methods all detailed in the wiki. Some collection examples that the script can automatically build and update daily include Plex Based Searches like actor, genre, or studio collections or Collections based on TMDb, IMDb, Trakt, TVDb, AniDB, or MyAnimeList lists and various other services. +The original concept for Plex Meta Manager is [Plex Auto Collections](https://github.com/mza921/Plex-Auto-Collections), but this is rewritten from the ground up to be able to include a scheduler, metadata edits, multiple libraries, and logging. Plex Meta Manager is a Python 3 script that can be continuously run using YAML configuration files to update on a schedule the metadata of the movies, shows, and collections in your libraries as well as automatically build collections based on various methods all detailed in the wiki. Some collection examples that the script can automatically build and update daily include Plex Based Searches like actor, genre, or studio collections or Collections based on TMDb, IMDb, Trakt, TVDb, AniDB, or MyAnimeList lists and various other services. The script can update many metadata fields for movies, shows, collections, seasons, and episodes and can act as a backup if your plex DB goes down. It can even update metadata the plex UI can't like Season Names. If the time is put into the metadata configuration file you can have a way to recreate your library and all its metadata changes with the click of a button. diff --git a/modules/anidb.py b/modules/anidb.py index 30bde9b1..318f442f 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -31,21 +31,21 @@ class AniDBAPI: @retry(stop_max_attempt_number=6, wait_fixed=10000) def send_request(self, url, language): - return requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content + return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content) def get_popular(self, language): - response = html.fromstring(self.send_request(self.urls["popular"], language)) + response = self.send_request(self.urls["popular"], language) return util.get_int_list(response.xpath("//td[@class='name anime']/a/@href"), "AniDB ID") def validate_anidb_id(self, anidb_id, language): - response = html.fromstring(self.send_request("{}/{}".format(self.urls["anime"], anidb_id), language)) + response = self.send_request("{}/{}".format(self.urls["anime"], anidb_id), language) ids = response.xpath("//*[text()='a{}']/text()".format(anidb_id)) if len(ids) > 0: return util.regex_first_int(ids[0], "AniDB ID") raise Failed("AniDB Error: AniDB ID: {} not found".format(anidb_id)) def get_anidb_relations(self, anidb_id, language): - response = html.fromstring(self.send_request("{}/{}{}".format(self.urls["anime"], anidb_id, self.urls["relation"]), language)) + response = self.send_request("{}/{}{}".format(self.urls["anime"], anidb_id, self.urls["relation"]), language) return util.get_int_list(response.xpath("//area/@href"), "AniDB ID") def validate_anidb_list(self, anidb_list, language): diff --git a/modules/config.py b/modules/config.py index 5ffda651..317aa11b 100644 --- a/modules/config.py +++ b/modules/config.py @@ -1,4 +1,4 @@ -import glob, json, logging, os, re +import glob, json, logging, os, re, requests from datetime import datetime, timedelta from modules import util from modules.anidb import AniDBAPI @@ -12,7 +12,6 @@ from modules.trakt import TraktAPI from modules.tvdb import TVDbAPI from modules.util import Failed from ruamel import yaml -from urllib.parse import urlparse logger = logging.getLogger("Plex Meta Manager") @@ -662,7 +661,7 @@ class Config: if len(prefix_list) == 0 and len(exact_list) == 0: raise Failed("Collection Error: you must have at least one exclusion") details["add_to_arr"] = False - details["collection_mode"] = "showItems" + details["collection_mode"] = "hide" new_dictionary["exclude_prefix"] = prefix_list new_dictionary["exclude"] = exact_list methods.append((method_name, [new_dictionary])) @@ -1065,7 +1064,7 @@ class Config: tmdb_id, expired = self.Cache.get_tmdb_id("show", plex_guid=item.guid) anidb_id, expired = self.Cache.get_anidb_id("show", plex_guid=item.guid) if expired or (not tmdb_id and library.is_movie) or (not tvdb_id and not tmdb_id and library.is_show): - guid = urlparse(item.guid) + guid = requests.utils.urlparse(item.guid) item_type = guid.scheme.split(".")[-1] check_id = guid.netloc diff --git a/modules/imdb.py b/modules/imdb.py index 9866ba02..d4bc13a2 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -29,8 +29,7 @@ class IMDbAPI: header = {"Accept-Language": language} length = 0 imdb_ids = [] - response = self.send_request(current_url, header) - try: results = html.fromstring(response).xpath("//div[@class='desc']/span/text()")[0].replace(",", "") + try: results = self.send_request(current_url, header).xpath("//div[@class='desc']/span/text()")[0].replace(",", "") except IndexError: raise Failed("IMDb Error: Failed to parse URL: {}".format(imdb_url)) try: total = int(re.findall("(\\d+) title", results)[0]) except IndexError: raise Failed("IMDb Error: No Results at URL: {}".format(imdb_url)) @@ -44,14 +43,14 @@ class IMDbAPI: start_num = (i - 1) * 250 + 1 length = util.print_return(length, "Parsing Page {}/{} {}-{}".format(i, num_of_pages, start_num, limit if i == num_of_pages else i * 250)) response = self.send_request("{}&count={}&start={}".format(current_url, remainder if i == num_of_pages else 250, start_num), header) - imdb_ids.extend(html.fromstring(response).xpath("//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst")) + imdb_ids.extend(response.xpath("//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst")) util.print_end(length) if imdb_ids: return imdb_ids else: raise Failed("IMDb Error: No Movies Found at {}".format(imdb_url)) @retry(stop_max_attempt_number=6, wait_fixed=10000) def send_request(self, url, header): - return requests.get(url, headers=header).content + return html.fromstring(requests.get(url, headers=header).content) def get_items(self, method, data, language, status_message=True): pretty = util.pretty_names[method] if method in util.pretty_names else method diff --git a/modules/plex.py b/modules/plex.py index e22e7007..1dd21ad5 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -1,5 +1,4 @@ import logging, os, requests -from bs4 import BeautifulSoup from modules import util from modules.radarr import RadarrAPI from modules.sonarr import SonarrAPI @@ -11,15 +10,12 @@ from plexapi.server import PlexServer from plexapi.video import Movie, Show from retrying import retry from ruamel import yaml -from urllib.parse import urlparse -from urllib.request import Request -from urllib.request import urlopen logger = logging.getLogger("Plex Meta Manager") class PlexAPI: def __init__(self, params): - try: self.PlexServer = PlexServer(params["plex"]["url"], params["plex"]["token"], timeout=60) + try: self.PlexServer = PlexServer(params["plex"]["url"], params["plex"]["token"], timeout=600) except Unauthorized: raise Failed("Plex Error: Plex token is invalid") except ValueError as e: raise Failed("Plex Error: {}".format(e)) except requests.exceptions.ConnectionError as e: @@ -140,20 +136,18 @@ class PlexAPI: raise Failed("Plex Error: Actor: {} not found".format(data)) def get_ids(self, movie): - req = Request("{}{}".format(self.url, movie.key)) - req.add_header("X-Plex-Token", self.token) - req.add_header("User-Agent", "Mozilla/5.0") - with urlopen(req) as response: - contents = response.read() tmdb_id = None imdb_id = None - for guid_tag in BeautifulSoup(contents, "lxml").find_all("guid"): - agent = urlparse(guid_tag["id"]).scheme - guid = urlparse(guid_tag["id"]).netloc - if agent == "tmdb": tmdb_id = guid - elif agent == "imdb": imdb_id = guid + for guid_tag in self.send_request("{}{}".format(self.plex["url"], movie.key)).xpath("//guid/@id"): + parsed_url = requests.utils.urlparse(guid_tag) + if parsed_url.scheme == "tmdb": tmdb_id = parsed_url.netloc + elif parsed_url.scheme == "imdb": imdb_id = parsed_url.netloc return tmdb_id, imdb_id + @retry(stop_max_attempt_number=6, wait_fixed=10000) + def send_request(self, url): + return html.fromstring(requests.get(url, headers={"X-Plex-Token": self.token, "User-Agent": "Mozilla/5.0 x64"}).content) + def del_collection_if_empty(self, collection): missing_data = {} if not os.path.exists(self.missing_path): diff --git a/modules/trakt.py b/modules/trakt.py index 26368ea2..31d5f595 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -1,4 +1,4 @@ -import logging, webbrowser +import logging, requests, webbrowser from modules import util from modules.util import Failed, TimeoutExpired from retrying import retry @@ -8,7 +8,6 @@ from trakt.objects.episode import Episode from trakt.objects.movie import Movie from trakt.objects.season import Season from trakt.objects.show import Show -from urllib.parse import urlparse logger = logging.getLogger("Plex Meta Manager") @@ -106,7 +105,7 @@ class TraktAPI: @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed) def standard_list(self, data): - try: items = Trakt[urlparse(data).path].items() + try: items = Trakt[requests.utils.urlparse(data).path].items() except AttributeError: items = None if items is None: raise Failed("Trakt Error: No List found") else: return items diff --git a/modules/tvdb.py b/modules/tvdb.py index 76c330dc..d907135d 100644 --- a/modules/tvdb.py +++ b/modules/tvdb.py @@ -17,29 +17,29 @@ class TVDbObj: raise Failed("TVDb Error: {} must begin with {}".format(tvdb_url, TVDb.movies_url if is_movie else TVDb.series_url)) response = TVDb.send_request(tvdb_url, language) - results = html.fromstring(response).xpath("//*[text()='TheTVDB.com {} ID']/parent::node()/span/text()".format(self.media_type)) + results = response.xpath("//*[text()='TheTVDB.com {} ID']/parent::node()/span/text()".format(self.media_type)) if len(results) > 0: self.id = int(results[0]) else: raise Failed("TVDb Error: Could not find a TVDb {} ID at the URL {}".format(self.media_type, tvdb_url)) - results = html.fromstring(response).xpath("//div[@class='change_translation_text' and @data-language='eng']/@data-title") + results = response.xpath("//div[@class='change_translation_text' and @data-language='eng']/@data-title") if len(results) > 0 and len(results[0]) > 0: self.title = results[0] else: raise Failed("TVDb Error: Name not found from TVDb URL: {}".format(tvdb_url)) - results = html.fromstring(response).xpath("//div[@class='row hidden-xs hidden-sm']/div/img/@src") + results = response.xpath("//div[@class='row hidden-xs hidden-sm']/div/img/@src") self.poster_path = results[0] if len(results) > 0 and len(results[0]) > 0 else None tmdb_id = None if is_movie: - results = html.fromstring(response).xpath("//*[text()='TheMovieDB.com']/@href") + results = response.xpath("//*[text()='TheMovieDB.com']/@href") if len(results) > 0: try: tmdb_id = util.regex_first_int(results[0], "TMDb ID") except Failed as e: logger.error(e) if not tmdb_id: - results = html.fromstring(response).xpath("//*[text()='IMDB']/@href") + results = response.xpath("//*[text()='IMDB']/@href") if len(results) > 0: try: tmdb_id = TVDb.convert_from_imdb(util.get_id_from_imdb_url(results[0]), language) except Failed as e: logger.error(e) @@ -85,8 +85,7 @@ class TVDbAPI: tvdb_url = tvdb_url.strip() if tvdb_url.startswith((self.list_url, self.alt_list_url)): try: - response = self.send_request(tvdb_url, language) - items = html.fromstring(response).xpath("//div[@class='col-xs-12 col-sm-12 col-md-8 col-lg-8 col-md-pull-4']/div[@class='row']") + items = self.send_request(tvdb_url, language).xpath("//div[@class='col-xs-12 col-sm-12 col-md-8 col-lg-8 col-md-pull-4']/div[@class='row']") for item in items: title = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/text()")[0] item_url = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/@href")[0] @@ -113,7 +112,7 @@ class TVDbAPI: @retry(stop_max_attempt_number=6, wait_fixed=10000) def send_request(self, url, language): - return requests.get(url, headers={"Accept-Language": language}).content + return html.fromstring(requests.get(url, headers={"Accept-Language": language}).content) def get_items(self, method, data, language, status_message=True): pretty = util.pretty_names[method] if method in util.pretty_names else method diff --git a/requirements.txt b/requirements.txt index 4135ff03..646555eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,10 +4,8 @@ PlexAPI==4.2.0 tmdbv3api==1.7.3 trakt.py==4.2.0 # More common, flexible -bs4 lxml requests>=2.4.2 ruamel.yaml schedule retrying -mutagen