fix requests

README.md

@@ -1,6 +1,6 @@
 # Plex Meta Manager
 
-The original concept for Plex Meta Manager is [Plex Auto Collections](https://github.com/mza921/Plex-Auto-Collections), but this is rewritten from the ground up to be able to include a scheduler, metadata edits, multiple libraries, and logging. Plex Meta Manager is a Python 3 script that can be continuously run using YMAL configuration files to update on a schedule the metadata of the movies, shows, and collections in your libraries as well as automatically build collections based on various methods all detailed in the wiki. Some collection examples that the script can automatically build and update daily include Plex Based Searches like actor, genre, or studio collections or Collections based on TMDb, IMDb, Trakt, TVDb, AniDB, or MyAnimeList lists and various other services.
+The original concept for Plex Meta Manager is [Plex Auto Collections](https://github.com/mza921/Plex-Auto-Collections), but this is rewritten from the ground up to be able to include a scheduler, metadata edits, multiple libraries, and logging. Plex Meta Manager is a Python 3 script that can be continuously run using YAML configuration files to update on a schedule the metadata of the movies, shows, and collections in your libraries as well as automatically build collections based on various methods all detailed in the wiki. Some collection examples that the script can automatically build and update daily include Plex Based Searches like actor, genre, or studio collections or Collections based on TMDb, IMDb, Trakt, TVDb, AniDB, or MyAnimeList lists and various other services.
 
 The script can update many metadata fields for movies, shows, collections, seasons, and episodes and can act as a backup if your plex DB goes down. It can even update metadata the plex UI can't like Season Names. If the time is put into the metadata configuration file you can have a way to recreate your library and all its metadata changes with the click of a button.
 

modules/anidb.py

@@ -31,21 +31,21 @@ class AniDBAPI:
 
     @retry(stop_max_attempt_number=6, wait_fixed=10000)
     def send_request(self, url, language):
-        return requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content
+        return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content)
 
     def get_popular(self, language):
-        response = html.fromstring(self.send_request(self.urls["popular"], language))
+        response = self.send_request(self.urls["popular"], language)
         return util.get_int_list(response.xpath("//td[@class='name anime']/a/@href"), "AniDB ID")
 
     def validate_anidb_id(self, anidb_id, language):
-        response = html.fromstring(self.send_request("{}/{}".format(self.urls["anime"], anidb_id), language))
+        response = self.send_request("{}/{}".format(self.urls["anime"], anidb_id), language)
         ids = response.xpath("//*[text()='a{}']/text()".format(anidb_id))
         if len(ids) > 0:
             return util.regex_first_int(ids[0], "AniDB ID")
         raise Failed("AniDB Error: AniDB ID: {} not found".format(anidb_id))
 
     def get_anidb_relations(self, anidb_id, language):
-        response = html.fromstring(self.send_request("{}/{}{}".format(self.urls["anime"], anidb_id, self.urls["relation"]), language))
+        response = self.send_request("{}/{}{}".format(self.urls["anime"], anidb_id, self.urls["relation"]), language)
         return util.get_int_list(response.xpath("//area/@href"), "AniDB ID")
 
     def validate_anidb_list(self, anidb_list, language):

modules/config.py

@@ -1,4 +1,4 @@
-import glob, json, logging, os, re
+import glob, json, logging, os, re, requests
 from datetime import datetime, timedelta
 from modules import util
 from modules.anidb import AniDBAPI
@@ -12,7 +12,6 @@ from modules.trakt import TraktAPI
 from modules.tvdb import TVDbAPI
 from modules.util import Failed
 from ruamel import yaml
-from urllib.parse import urlparse
 
 logger = logging.getLogger("Plex Meta Manager")
 
@@ -662,7 +661,7 @@ class Config:
                                                 if len(prefix_list) == 0 and len(exact_list) == 0:
                                                     raise Failed("Collection Error: you must have at least one exclusion")
                                                 details["add_to_arr"] = False
-                                                details["collection_mode"] = "showItems"
+                                                details["collection_mode"] = "hide"
                                                 new_dictionary["exclude_prefix"] = prefix_list
                                                 new_dictionary["exclude"] = exact_list
                                                 methods.append((method_name, [new_dictionary]))
@@ -1065,7 +1064,7 @@ class Config:
                 tmdb_id, expired = self.Cache.get_tmdb_id("show", plex_guid=item.guid)
                 anidb_id, expired = self.Cache.get_anidb_id("show", plex_guid=item.guid)
         if expired or (not tmdb_id and library.is_movie) or (not tvdb_id and not tmdb_id and library.is_show):
-            guid = urlparse(item.guid)
+            guid = requests.utils.urlparse(item.guid)
             item_type = guid.scheme.split(".")[-1]
             check_id = guid.netloc
 

modules/imdb.py

@@ -29,8 +29,7 @@ class IMDbAPI:
         header = {"Accept-Language": language}
         length = 0
         imdb_ids = []
-        response = self.send_request(current_url, header)
-        try:                                results = html.fromstring(response).xpath("//div[@class='desc']/span/text()")[0].replace(",", "")
+        try:                                results = self.send_request(current_url, header).xpath("//div[@class='desc']/span/text()")[0].replace(",", "")
         except IndexError:                  raise Failed("IMDb Error: Failed to parse URL: {}".format(imdb_url))
         try:                                total = int(re.findall("(\\d+) title", results)[0])
         except IndexError:                  raise Failed("IMDb Error: No Results at URL: {}".format(imdb_url))
@@ -44,14 +43,14 @@ class IMDbAPI:
             start_num = (i - 1) * 250 + 1
             length = util.print_return(length, "Parsing Page {}/{} {}-{}".format(i, num_of_pages, start_num, limit if i == num_of_pages else i * 250))
             response = self.send_request("{}&count={}&start={}".format(current_url, remainder if i == num_of_pages else 250, start_num), header)
-            imdb_ids.extend(html.fromstring(response).xpath("//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst"))
+            imdb_ids.extend(response.xpath("//div[contains(@class, 'lister-item-image')]//a/img//@data-tconst"))
         util.print_end(length)
         if imdb_ids:                        return imdb_ids
         else:                               raise Failed("IMDb Error: No Movies Found at {}".format(imdb_url))
 
     @retry(stop_max_attempt_number=6, wait_fixed=10000)
     def send_request(self, url, header):
-        return requests.get(url, headers=header).content
+        return html.fromstring(requests.get(url, headers=header).content)
 
     def get_items(self, method, data, language, status_message=True):
         pretty = util.pretty_names[method] if method in util.pretty_names else method

modules/plex.py

@@ -1,5 +1,4 @@
 import logging, os, requests
-from bs4 import BeautifulSoup
 from modules import util
 from modules.radarr import RadarrAPI
 from modules.sonarr import SonarrAPI
@@ -11,15 +10,12 @@ from plexapi.server import PlexServer
 from plexapi.video import Movie, Show
 from retrying import retry
 from ruamel import yaml
-from urllib.parse import urlparse
-from urllib.request import Request
-from urllib.request import urlopen
 
 logger = logging.getLogger("Plex Meta Manager")
 
 class PlexAPI:
     def __init__(self, params):
-        try:                                                                    self.PlexServer = PlexServer(params["plex"]["url"], params["plex"]["token"], timeout=60)
+        try:                                                                    self.PlexServer = PlexServer(params["plex"]["url"], params["plex"]["token"], timeout=600)
         except Unauthorized:                                                    raise Failed("Plex Error: Plex token is invalid")
         except ValueError as e:                                                 raise Failed("Plex Error: {}".format(e))
         except requests.exceptions.ConnectionError as e:
@@ -140,20 +136,18 @@ class PlexAPI:
         raise Failed("Plex Error: Actor: {} not found".format(data))
 
     def get_ids(self, movie):
-        req = Request("{}{}".format(self.url, movie.key))
-        req.add_header("X-Plex-Token", self.token)
-        req.add_header("User-Agent", "Mozilla/5.0")
-        with urlopen(req) as response:
-            contents = response.read()
         tmdb_id = None
         imdb_id = None
-        for guid_tag in BeautifulSoup(contents, "lxml").find_all("guid"):
-            agent = urlparse(guid_tag["id"]).scheme
-            guid = urlparse(guid_tag["id"]).netloc
-            if agent == "tmdb":                 tmdb_id = guid
-            elif agent == "imdb":               imdb_id = guid
+        for guid_tag in self.send_request("{}{}".format(self.plex["url"], movie.key)).xpath("//guid/@id"):
+            parsed_url = requests.utils.urlparse(guid_tag)
+            if parsed_url.scheme == "tmdb":                 tmdb_id = parsed_url.netloc
+            elif parsed_url.scheme == "imdb":               imdb_id = parsed_url.netloc
         return tmdb_id, imdb_id
 
+    @retry(stop_max_attempt_number=6, wait_fixed=10000)
+    def send_request(self, url):
+        return html.fromstring(requests.get(url, headers={"X-Plex-Token": self.token, "User-Agent": "Mozilla/5.0 x64"}).content)
+
     def del_collection_if_empty(self, collection):
         missing_data = {}
         if not os.path.exists(self.missing_path):

modules/trakt.py

@@ -1,4 +1,4 @@
-import logging, webbrowser
+import logging, requests, webbrowser
 from modules import util
 from modules.util import Failed, TimeoutExpired
 from retrying import retry
@@ -8,7 +8,6 @@ from trakt.objects.episode import Episode
 from trakt.objects.movie import Movie
 from trakt.objects.season import Season
 from trakt.objects.show import Show
-from urllib.parse import urlparse
 
 logger = logging.getLogger("Plex Meta Manager")
 
@@ -106,7 +105,7 @@ class TraktAPI:
 
     @retry(stop_max_attempt_number=6, wait_fixed=10000, retry_on_exception=util.retry_if_not_failed)
     def standard_list(self, data):
-        try:                                items = Trakt[urlparse(data).path].items()
+        try:                                items = Trakt[requests.utils.urlparse(data).path].items()
         except AttributeError:              items = None
         if items is None:                   raise Failed("Trakt Error: No List found")
         else:                               return items

modules/tvdb.py

@@ -17,29 +17,29 @@ class TVDbObj:
             raise Failed("TVDb Error: {} must begin with {}".format(tvdb_url, TVDb.movies_url if is_movie else TVDb.series_url))
 
         response = TVDb.send_request(tvdb_url, language)
-        results = html.fromstring(response).xpath("//*[text()='TheTVDB.com {} ID']/parent::node()/span/text()".format(self.media_type))
+        results = response.xpath("//*[text()='TheTVDB.com {} ID']/parent::node()/span/text()".format(self.media_type))
         if len(results) > 0:
             self.id = int(results[0])
         else:
             raise Failed("TVDb Error: Could not find a TVDb {} ID at the URL {}".format(self.media_type, tvdb_url))
 
-        results = html.fromstring(response).xpath("//div[@class='change_translation_text' and @data-language='eng']/@data-title")
+        results = response.xpath("//div[@class='change_translation_text' and @data-language='eng']/@data-title")
         if len(results) > 0 and len(results[0]) > 0:
             self.title = results[0]
         else:
             raise Failed("TVDb Error: Name not found from TVDb URL: {}".format(tvdb_url))
 
-        results = html.fromstring(response).xpath("//div[@class='row hidden-xs hidden-sm']/div/img/@src")
+        results = response.xpath("//div[@class='row hidden-xs hidden-sm']/div/img/@src")
         self.poster_path = results[0] if len(results) > 0 and len(results[0]) > 0 else None
 
         tmdb_id = None
         if is_movie:
-            results = html.fromstring(response).xpath("//*[text()='TheMovieDB.com']/@href")
+            results = response.xpath("//*[text()='TheMovieDB.com']/@href")
             if len(results) > 0:
                 try:                                                    tmdb_id = util.regex_first_int(results[0], "TMDb ID")
                 except Failed as e:                                     logger.error(e)
             if not tmdb_id:
-                results = html.fromstring(response).xpath("//*[text()='IMDB']/@href")
+                results = response.xpath("//*[text()='IMDB']/@href")
                 if len(results) > 0:
                     try:                                                tmdb_id = TVDb.convert_from_imdb(util.get_id_from_imdb_url(results[0]), language)
                     except Failed as e:                                 logger.error(e)
@@ -85,8 +85,7 @@ class TVDbAPI:
         tvdb_url = tvdb_url.strip()
         if tvdb_url.startswith((self.list_url, self.alt_list_url)):
             try:
-                response = self.send_request(tvdb_url, language)
-                items = html.fromstring(response).xpath("//div[@class='col-xs-12 col-sm-12 col-md-8 col-lg-8 col-md-pull-4']/div[@class='row']")
+                items = self.send_request(tvdb_url, language).xpath("//div[@class='col-xs-12 col-sm-12 col-md-8 col-lg-8 col-md-pull-4']/div[@class='row']")
                 for item in items:
                     title = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/text()")[0]
                     item_url = item.xpath(".//div[@class='col-xs-12 col-sm-9 mt-2']//a/@href")[0]
@@ -113,7 +112,7 @@ class TVDbAPI:
 
     @retry(stop_max_attempt_number=6, wait_fixed=10000)
     def send_request(self, url, language):
-        return requests.get(url, headers={"Accept-Language": language}).content
+        return html.fromstring(requests.get(url, headers={"Accept-Language": language}).content)
 
     def get_items(self, method, data, language, status_message=True):
         pretty = util.pretty_names[method] if method in util.pretty_names else method

requirements.txt

@@ -4,10 +4,8 @@ PlexAPI==4.2.0
 tmdbv3api==1.7.3
 trakt.py==4.2.0
 # More common, flexible
-bs4
 lxml
 requests>=2.4.2
 ruamel.yaml
 schedule
 retrying
-mutagen