|
|
@ -293,13 +293,10 @@ class base_html_playwright(Fetcher):
|
|
|
|
# Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
|
|
|
|
# Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
|
|
|
|
# This seemed to solve nearly all 'TimeoutErrors'
|
|
|
|
# This seemed to solve nearly all 'TimeoutErrors'
|
|
|
|
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
|
|
|
|
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
|
|
|
|
p=time.time()
|
|
|
|
|
|
|
|
page.wait_for_timeout(extra_wait * 1000)
|
|
|
|
page.wait_for_timeout(extra_wait * 1000)
|
|
|
|
except playwright._impl._api_types.TimeoutError as e:
|
|
|
|
except playwright._impl._api_types.TimeoutError as e:
|
|
|
|
raise EmptyReply(url=url, status_code=None)
|
|
|
|
raise EmptyReply(url=url, status_code=None)
|
|
|
|
|
|
|
|
|
|
|
|
r=time.time()-p
|
|
|
|
|
|
|
|
x=1
|
|
|
|
|
|
|
|
if response is None:
|
|
|
|
if response is None:
|
|
|
|
raise EmptyReply(url=url, status_code=None)
|
|
|
|
raise EmptyReply(url=url, status_code=None)
|
|
|
|
|
|
|
|
|
|
|
@ -308,8 +305,6 @@ class base_html_playwright(Fetcher):
|
|
|
|
|
|
|
|
|
|
|
|
# Bug 2(?) Set the viewport size AFTER loading the page
|
|
|
|
# Bug 2(?) Set the viewport size AFTER loading the page
|
|
|
|
page.set_viewport_size({"width": 1280, "height": 1024})
|
|
|
|
page.set_viewport_size({"width": 1280, "height": 1024})
|
|
|
|
# Bugish - Let the page redraw/reflow
|
|
|
|
|
|
|
|
page.set_viewport_size({"width": 1280, "height": 1024})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
self.status_code = response.status
|
|
|
|
self.status_code = response.status
|
|
|
|
self.content = page.content()
|
|
|
|
self.content = page.content()
|
|
|
|