|
|
|
@ -91,19 +91,20 @@ class ReplyWithContentButNoText(Exception):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Fetcher():
|
|
|
|
|
browser_connection_is_custom = None
|
|
|
|
|
browser_connection_url = None
|
|
|
|
|
browser_steps = None
|
|
|
|
|
browser_steps_screenshot_path = None
|
|
|
|
|
content = None
|
|
|
|
|
error = None
|
|
|
|
|
fetcher_description = "No description"
|
|
|
|
|
browser_connection_url = None
|
|
|
|
|
headers = {}
|
|
|
|
|
instock_data = None
|
|
|
|
|
instock_data_js = ""
|
|
|
|
|
status_code = None
|
|
|
|
|
webdriver_js_execute_code = None
|
|
|
|
|
xpath_data = None
|
|
|
|
|
xpath_element_js = ""
|
|
|
|
|
instock_data = None
|
|
|
|
|
instock_data_js = ""
|
|
|
|
|
|
|
|
|
|
# Will be needed in the future by the VisualSelector, always get this where possible.
|
|
|
|
|
screenshot = False
|
|
|
|
@ -252,16 +253,19 @@ class base_html_playwright(Fetcher):
|
|
|
|
|
|
|
|
|
|
proxy = None
|
|
|
|
|
|
|
|
|
|
def __init__(self, proxy_override=None, browser_connection_url=None):
|
|
|
|
|
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
|
|
|
|
|
super().__init__()
|
|
|
|
|
|
|
|
|
|
self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
|
|
|
|
|
|
|
|
|
|
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
|
|
|
|
|
if not browser_connection_url:
|
|
|
|
|
self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
|
|
|
|
|
if custom_browser_connection_url:
|
|
|
|
|
self.browser_connection_is_custom = True
|
|
|
|
|
self.browser_connection_url = custom_browser_connection_url
|
|
|
|
|
else:
|
|
|
|
|
self.browser_connection_url = browser_connection_url
|
|
|
|
|
# Fallback to fetching from system
|
|
|
|
|
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
|
|
|
|
|
self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# If any proxy settings are enabled, then we should setup the proxy object
|
|
|
|
|
proxy_args = {}
|
|
|
|
@ -421,8 +425,10 @@ class base_html_playwright(Fetcher):
|
|
|
|
|
current_include_filters=None,
|
|
|
|
|
is_binary=False):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
|
|
|
|
|
if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
|
|
|
|
|
# browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
|
|
|
|
|
if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
|
|
|
|
|
if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
|
|
|
|
|
# Temporary backup solution until we rewrite the playwright code
|
|
|
|
|
return self.run_fetch_browserless_puppeteer(
|
|
|
|
@ -569,15 +575,16 @@ class base_html_webdriver(Fetcher):
|
|
|
|
|
'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
|
|
|
|
|
proxy = None
|
|
|
|
|
|
|
|
|
|
def __init__(self, proxy_override=None, browser_connection_url=None):
|
|
|
|
|
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
|
|
|
|
|
super().__init__()
|
|
|
|
|
from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
|
|
|
|
|
|
|
|
|
|
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
|
|
|
|
|
if not browser_connection_url:
|
|
|
|
|
if not custom_browser_connection_url:
|
|
|
|
|
self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
|
|
|
|
|
else:
|
|
|
|
|
self.browser_connection_url = browser_connection_url
|
|
|
|
|
self.browser_connection_is_custom = True
|
|
|
|
|
self.browser_connection_url = custom_browser_connection_url
|
|
|
|
|
|
|
|
|
|
# If any proxy settings are enabled, then we should setup the proxy object
|
|
|
|
|
proxy_args = {}
|
|
|
|
@ -674,7 +681,7 @@ class base_html_webdriver(Fetcher):
|
|
|
|
|
class html_requests(Fetcher):
|
|
|
|
|
fetcher_description = "Basic fast Plaintext/HTTP Client"
|
|
|
|
|
|
|
|
|
|
def __init__(self, proxy_override=None, browser_connection_url=None):
|
|
|
|
|
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
|
|
|
|
|
super().__init__()
|
|
|
|
|
self.proxy_override = proxy_override
|
|
|
|
|
# browser_connection_url is none because its always 'launched locally'
|
|
|
|
|