|
|
|
@ -1,4 +1,4 @@
|
|
|
|
|
import time
|
|
|
|
|
import re, time
|
|
|
|
|
from modules import util
|
|
|
|
|
from modules.util import Failed
|
|
|
|
|
|
|
|
|
@ -20,6 +20,16 @@ class Letterboxd:
|
|
|
|
|
for letterboxd_id in letterboxd_ids:
|
|
|
|
|
slugs = response.xpath(f"//div[@data-film-id='{letterboxd_id}']/@data-film-slug")
|
|
|
|
|
items.append((letterboxd_id, slugs[0]))
|
|
|
|
|
slugs = response.xpath(f"//div[@data-film-id='{letterboxd_id}']/@data-film-slug")
|
|
|
|
|
notes = response.xpath(f"//div[@data-film-id='{letterboxd_id}']/parent::li/div[@class='film-detail-content']/div/p/text()")
|
|
|
|
|
ratings = response.xpath(f"//div[@data-film-id='{letterboxd_id}']/parent::li/div[@class='film-detail-content']//span[contains(@class, 'rating')]/@class")
|
|
|
|
|
years = response.xpath(f"//div[@data-film-id='{letterboxd_id}']/parent::li/div[@class='film-detail-content']/h2/small/a/text()")
|
|
|
|
|
rating = None
|
|
|
|
|
if ratings:
|
|
|
|
|
match = re.search("rated-(\\d+)", ratings[0])
|
|
|
|
|
if match:
|
|
|
|
|
rating = int(match.group(1))
|
|
|
|
|
items.append((letterboxd_id, slugs[0], int(years[0]) if years else None, notes[0] if notes else None, rating))
|
|
|
|
|
next_url = response.xpath("//a[@class='next']/@href")
|
|
|
|
|
if len(next_url) > 0:
|
|
|
|
|
time.sleep(2)
|
|
|
|
@ -44,27 +54,50 @@ class Letterboxd:
|
|
|
|
|
descriptions = response.xpath("//meta[@property='og:description']/@content")
|
|
|
|
|
return descriptions[0] if len(descriptions) > 0 and len(descriptions[0]) > 0 else None
|
|
|
|
|
|
|
|
|
|
def validate_letterboxd_lists(self, letterboxd_lists, language):
|
|
|
|
|
def validate_letterboxd_lists(self, err_type, letterboxd_lists, language):
|
|
|
|
|
valid_lists = []
|
|
|
|
|
for letterboxd_list in util.get_list(letterboxd_lists, split=False):
|
|
|
|
|
list_url = letterboxd_list.strip()
|
|
|
|
|
if not list_url.startswith(base_url):
|
|
|
|
|
raise Failed(f"Letterboxd Error: {list_url} must begin with: {base_url}")
|
|
|
|
|
elif len(self._parse_list(list_url, language)) > 0:
|
|
|
|
|
valid_lists.append(list_url)
|
|
|
|
|
else:
|
|
|
|
|
raise Failed(f"Letterboxd Error: {list_url} failed to parse")
|
|
|
|
|
for letterboxd_dict in util.get_list(letterboxd_lists, split=False):
|
|
|
|
|
if not isinstance(letterboxd_dict, dict):
|
|
|
|
|
letterboxd_dict = {"url": letterboxd_dict}
|
|
|
|
|
dict_methods = {dm.lower(): dm for dm in letterboxd_dict}
|
|
|
|
|
final = {
|
|
|
|
|
"url": util.parse(err_type, "url", letterboxd_dict, methods=dict_methods, parent="letterboxd_list").strip(),
|
|
|
|
|
"note": util.parse(err_type, "note", letterboxd_dict, methods=dict_methods, parent="letterboxd_list") if "note" in dict_methods else None,
|
|
|
|
|
"rating": util.parse(err_type, "rating", letterboxd_dict, methods=dict_methods, datatype="int", parent="letterboxd_list", maximum=100, range_split="-") if "rating" in dict_methods else None,
|
|
|
|
|
"year": util.parse(err_type, "year", letterboxd_dict, methods=dict_methods, datatype="int", parent="letterboxd_list", minimum=1000, maximum=3000, range_split="-") if "year" in dict_methods else None
|
|
|
|
|
}
|
|
|
|
|
if not final["url"].startswith(base_url):
|
|
|
|
|
raise Failed(f"{err_type} Error: {final['url']} must begin with: {base_url}")
|
|
|
|
|
elif not self._parse_list(final["url"], language):
|
|
|
|
|
raise Failed(f"{err_type} Error: {final['url']} failed to parse")
|
|
|
|
|
valid_lists.append(final)
|
|
|
|
|
return valid_lists
|
|
|
|
|
|
|
|
|
|
def get_tmdb_ids(self, method, data, language):
|
|
|
|
|
if method == "letterboxd_list":
|
|
|
|
|
logger.info(f"Processing Letterboxd List: {data}")
|
|
|
|
|
items = self._parse_list(data, language)
|
|
|
|
|
items = self._parse_list(data["url"], language)
|
|
|
|
|
total_items = len(items)
|
|
|
|
|
if total_items > 0:
|
|
|
|
|
ids = []
|
|
|
|
|
filtered_ids = []
|
|
|
|
|
for i, item in enumerate(items, 1):
|
|
|
|
|
letterboxd_id, slug = item
|
|
|
|
|
letterboxd_id, slug, year, note, rating = item
|
|
|
|
|
filtered = False
|
|
|
|
|
if data["year"]:
|
|
|
|
|
start_year, end_year = data["year"].split("-")
|
|
|
|
|
if not year or int(end_year) < year or year < int(start_year):
|
|
|
|
|
filtered = True
|
|
|
|
|
if data["rating"]:
|
|
|
|
|
start_rating, end_rating = data["rating"].split("-")
|
|
|
|
|
if not rating or int(end_rating) < rating or rating < int(start_rating):
|
|
|
|
|
filtered = True
|
|
|
|
|
if data["note"]:
|
|
|
|
|
if not note or data["note"] not in note:
|
|
|
|
|
filtered = True
|
|
|
|
|
if filtered:
|
|
|
|
|
filtered_ids.append(slug)
|
|
|
|
|
continue
|
|
|
|
|
logger.ghost(f"Finding TMDb ID {i}/{total_items}")
|
|
|
|
|
tmdb_id = None
|
|
|
|
|
expired = None
|
|
|
|
@ -80,6 +113,8 @@ class Letterboxd:
|
|
|
|
|
self.config.Cache.update_letterboxd_map(expired, letterboxd_id, tmdb_id)
|
|
|
|
|
ids.append((tmdb_id, "tmdb"))
|
|
|
|
|
logger.info(f"Processed {total_items} TMDb IDs")
|
|
|
|
|
if filtered_ids:
|
|
|
|
|
logger.info(f"Filtered: {filtered_ids}")
|
|
|
|
|
return ids
|
|
|
|
|
else:
|
|
|
|
|
raise Failed(f"Letterboxd Error: No List Items found in {data}")
|
|
|
|
|