|
|
|
@ -6,6 +6,7 @@ import time
|
|
|
|
|
import urllib3.exceptions
|
|
|
|
|
import sys
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class EmptyReply(Exception):
|
|
|
|
|
def __init__(self, status_code, url):
|
|
|
|
|
# Set this so we can use it in other parts of the app
|
|
|
|
@ -15,6 +16,7 @@ class EmptyReply(Exception):
|
|
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Fetcher():
|
|
|
|
|
error = None
|
|
|
|
|
status_code = None
|
|
|
|
@ -56,11 +58,11 @@ class Fetcher():
|
|
|
|
|
def is_ready(self):
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Maybe for the future, each fetcher provides its own diff output, could be used for text, image
|
|
|
|
|
# the current one would return javascript output (as we use JS to generate the diff)
|
|
|
|
|
#
|
|
|
|
|
def available_fetchers():
|
|
|
|
|
|
|
|
|
|
# See the if statement at the bottom of this file for how we switch between playwright and webdriver
|
|
|
|
|
import inspect
|
|
|
|
|
p = []
|
|
|
|
@ -72,9 +74,9 @@ def available_fetchers():
|
|
|
|
|
t = tuple([name, obj.fetcher_description])
|
|
|
|
|
p.append(t)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return p
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class base_html_playwright(Fetcher):
|
|
|
|
|
fetcher_description = "Playwright {}/Javascript".format(
|
|
|
|
|
os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
|
|
|
|
@ -166,9 +168,6 @@ class base_html_webdriver(Fetcher):
|
|
|
|
|
selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
|
|
|
|
|
'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
|
|
|
|
|
'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proxy = None
|
|
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
@ -177,7 +176,6 @@ class base_html_webdriver(Fetcher):
|
|
|
|
|
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
|
|
|
|
|
self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# If any proxy settings are enabled, then we should setup the proxy object
|
|
|
|
|
proxy_args = {}
|
|
|
|
|
for k in self.selenium_proxy_settings_mappings:
|
|
|
|
@ -247,6 +245,7 @@ class base_html_webdriver(Fetcher):
|
|
|
|
|
except Exception as e:
|
|
|
|
|
print("Exception in chrome shutdown/quit" + str(e))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# "html_requests" is listed as the default fetcher in store.py!
|
|
|
|
|
class html_requests(Fetcher):
|
|
|
|
|
fetcher_description = "Basic fast Plaintext/HTTP Client"
|
|
|
|
@ -292,4 +291,3 @@ if use_playwright_as_chrome_fetcher:
|
|
|
|
|
html_webdriver = base_html_playwright
|
|
|
|
|
else:
|
|
|
|
|
html_webdriver = base_html_webdriver
|
|
|
|
|
|
|
|
|
|