pull/11/head
Anson Lai 2 years ago
parent fd895c74f8
commit 5ca5a6f445

@ -119,6 +119,7 @@ class Webdriver:
for url in upcoming_urls:
if len(visited_urls) % 50 == 0:
save_session()
if url.startswith('GUID') and url.endswith('.html'):
self.driver.get(base_url + url)
else:
@ -142,7 +143,8 @@ class Webdriver:
print("images: " + str(len(set(img_urls))))
append_upcoming_and_img_urls(source)
if len(visited_urls) % 150 == 0:
if len(visited_urls) % 151 == 0:
self.restart_scrape()
def get_imgs(self):

Loading…
Cancel
Save