Bug fix for infinite loop

pull/11/head
Anson Lai 3 years ago
parent 5ca5a6f445
commit 650d61a317

@ -140,11 +140,12 @@ class Webdriver:
upcoming_urls.remove(url) upcoming_urls.remove(url)
print("visited: " + str(len(visited_urls))) print("visited: " + str(len(visited_urls)))
print("upcoming: " + str(len(upcoming_urls))) print("upcoming: " + str(len(upcoming_urls)))
print("images: " + str(len(set(img_urls)))) print("images to be downloaded: " + str(len(set(img_urls))))
append_upcoming_and_img_urls(source) append_upcoming_and_img_urls(source)
if len(visited_urls) % 151 == 0: if len(visited_urls) % 150 == 0:
save_session()
self.restart_scrape() self.restart_scrape()
def get_imgs(self): def get_imgs(self):

Loading…
Cancel
Save