increase tolerance for errors

pull/11/head
Anson Lai 3 years ago
parent 650d61a317
commit 5821e23e13

@ -115,6 +115,7 @@ class Webdriver:
open("docs/tds-fonts/3.x/woff2/GothamSSm-Medium_web.woff2", 'wb').write(r.content)
def get_html(self):
# Loop to get all the html pages, and store information about images to be downloaded later.
error_count = 0
while upcoming_urls:
for url in upcoming_urls:
if len(visited_urls) % 50 == 0:
@ -128,6 +129,8 @@ class Webdriver:
source = self.driver.find_element_by_css_selector("html").get_attribute('outerHTML')
if not check_source_validity(source):
error_count += 1
if error_count > 10:
self.restart_scrape()
with open('docs/' + url, 'w', encoding='utf-8') as f:

Loading…
Cancel
Save