diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py
index 83599fa1..707de4f8 100644
--- a/changedetectionio/__init__.py
+++ b/changedetectionio/__init__.py
@@ -518,10 +518,31 @@ def changedetection_app(config=None, datastore_o=None):
if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])
+ # Defaults for proxy choice
+ if datastore.proxy_list is not None: # When enabled
+ system_proxy = datastore.data['settings']['requests']['proxy']
+ if default['proxy'] is None:
+ default['proxy'] = system_proxy
+ else:
+ # Does the chosen one exist?
+ if not any(default['proxy'] in tup for tup in datastore.proxy_list):
+ default['proxy'] = datastore.proxy_list[0][0]
+
+ # Used by the form handler to keep or remove the proxy settings
+ default['proxy_list'] = datastore.proxy_list
+
+ # proxy_override set to the json/text list of the items
form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
- data=default
- )
+ data=default,
+ )
+ if datastore.proxy_list is None:
+ # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+ del form.proxy
+ else:
+ form.proxy.choices = datastore.proxy_list
+ if default['proxy'] is None:
+ form.proxy.default='http://hello'
if request.method == 'POST' and form.validate():
extra_update_obj = {}
@@ -601,10 +622,28 @@ def changedetection_app(config=None, datastore_o=None):
def settings_page():
from changedetectionio import content_fetcher, forms
+ default = deepcopy(datastore.data['settings'])
+ if datastore.proxy_list is not None:
+ # When enabled
+ system_proxy = datastore.data['settings']['requests']['proxy']
+ # In the case it doesnt exist anymore
+ if not any([system_proxy in tup for tup in datastore.proxy_list]):
+ system_proxy = None
+
+ default['requests']['proxy'] = system_proxy if system_proxy is not None else datastore.proxy_list[0][0]
+ # Used by the form handler to keep or remove the proxy settings
+ default['proxy_list'] = datastore.proxy_list
+
+
# Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
- data=datastore.data['settings']
+ data=default
)
+ if datastore.proxy_list is None:
+ # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+ del form.requests.form.proxy
+ else:
+ form.requests.form.proxy.choices = datastore.proxy_list
if request.method == 'POST':
# Password unset is a GET, but we can lock the session to a salted env password to always need the password
diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py
index ff86c9af..04276e67 100644
--- a/changedetectionio/content_fetcher.py
+++ b/changedetectionio/content_fetcher.py
@@ -91,7 +91,8 @@ class base_html_playwright(Fetcher):
proxy = None
- def __init__(self):
+ def __init__(self, proxy_override=None):
+
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
self.command_executor = os.getenv(
@@ -109,6 +110,10 @@ class base_html_playwright(Fetcher):
if proxy_args:
self.proxy = proxy_args
+ # allow per-watch proxy selection override
+ if proxy_override:
+ self.proxy = {'server': proxy_override}
+
def run(self,
url,
timeout,
@@ -177,7 +182,7 @@ class base_html_webdriver(Fetcher):
'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
proxy = None
- def __init__(self):
+ def __init__(self, proxy_override=None):
from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
@@ -196,6 +201,10 @@ class base_html_webdriver(Fetcher):
if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
proxy_args['httpsProxy'] = self.system_https_proxy
+ # Allows override the proxy on a per-request basis
+ if proxy_override is not None:
+ proxy_args['httpProxy'] = proxy_override
+
if proxy_args:
self.proxy = SeleniumProxy(raw=proxy_args)
@@ -263,6 +272,9 @@ class base_html_webdriver(Fetcher):
class html_requests(Fetcher):
fetcher_description = "Basic fast Plaintext/HTTP Client"
+ def __init__(self, proxy_override=None):
+ self.proxy_override = proxy_override
+
def run(self,
url,
timeout,
@@ -271,12 +283,16 @@ class html_requests(Fetcher):
request_method,
ignore_status_codes=False):
- # Map back standard HTTP_ and HTTPS_PROXY to requests http/https proxy
proxies={}
- if self.system_http_proxy:
- proxies['http'] = self.system_http_proxy
- if self.system_https_proxy:
- proxies['https'] = self.system_https_proxy
+
+ # Allows override the proxy on a per-request basis
+ if self.proxy_override:
+ proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
+ else:
+ if self.system_http_proxy:
+ proxies['http'] = self.system_http_proxy
+ if self.system_https_proxy:
+ proxies['https'] = self.system_https_proxy
r = requests.request(method=request_method,
data=request_body,
diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py
index 71415dc1..36ead8ec 100644
--- a/changedetectionio/fetch_site_status.py
+++ b/changedetectionio/fetch_site_status.py
@@ -16,6 +16,34 @@ class perform_site_check():
super().__init__(*args, **kwargs)
self.datastore = datastore
+ # If there was a proxy list enabled, figure out what proxy_args/which proxy to use
+ # if watch.proxy use that
+ # fetcher.proxy_override = watch.proxy or main config proxy
+ # Allows override the proxy on a per-request basis
+ # ALWAYS use the first one is nothing selected
+
+ def set_proxy_from_list(self, watch):
+ proxy_args = None
+ if self.datastore.proxy_list is None:
+ return None
+
+ # If its a valid one
+ if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
+ proxy_args = watch['proxy']
+
+ # not valid (including None), try the system one
+ else:
+ system_proxy = self.datastore.data['settings']['requests']['proxy']
+ # Is not None and exists
+ if any([system_proxy in p for p in self.datastore.proxy_list]):
+ proxy_args = system_proxy
+
+ # Fallback - Did not resolve anything, use the first available
+ if proxy_args is None:
+ proxy_args = self.datastore.proxy_list[0][0]
+
+ return proxy_args
+
def run(self, uuid):
timestamp = int(time.time()) # used for storage etc too
@@ -66,7 +94,10 @@ class perform_site_check():
# If the klass doesnt exist, just use a default
klass = getattr(content_fetcher, "html_requests")
- fetcher = klass()
+ proxy_args = self.set_proxy_from_list(watch)
+ fetcher = klass(proxy_override=proxy_args)
+
+ # Proxy List support
fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
# Fetching complete, now filters
diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py
index 72dae639..6d12267e 100644
--- a/changedetectionio/forms.py
+++ b/changedetectionio/forms.py
@@ -337,9 +337,9 @@ class watchForm(commonSettingsForm):
method = SelectField('Request method', choices=valid_method, default=default_method)
ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
-
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
+ proxy = RadioField('Proxy')
def validate(self, **kwargs):
if not super().validate():
@@ -358,6 +358,7 @@ class watchForm(commonSettingsForm):
# datastore.data['settings']['requests']..
class globalSettingsRequestForm(Form):
time_between_check = FormField(TimeBetweenCheckForm)
+ proxy = RadioField('Proxy')
# datastore.data['settings']['application']..
@@ -382,4 +383,3 @@ class globalSettingsForm(Form):
requests = FormField(globalSettingsRequestForm)
application = FormField(globalSettingsApplicationForm)
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py
index ebd5731a..21d53f7d 100644
--- a/changedetectionio/model/App.py
+++ b/changedetectionio/model/App.py
@@ -23,7 +23,8 @@ class model(dict):
'requests': {
'timeout': 15, # Default 15 seconds
'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
- 'workers': 10 # Number of threads, lower is better for slow connections
+ 'workers': 10, # Number of threads, lower is better for slow connections
+ 'proxy': None # Preferred proxy connection
},
'application': {
'password': False,
diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py
index c0313868..43d6b979 100644
--- a/changedetectionio/model/Watch.py
+++ b/changedetectionio/model/Watch.py
@@ -39,6 +39,7 @@ class model(dict):
'trigger_text': [], # List of text or regex to wait for until a change is detected
'fetch_backend': None,
'extract_title_as_title': False,
+ 'proxy': None, # Preferred proxy connection
# Re #110, so then if this is set to None, we know to use the default value instead
# Requires setting to None on submit if it's the same as the default
# Should be all None by default, so we use the system default in this case.
diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css
index 71ff2f5e..73609e47 100644
--- a/changedetectionio/static/styles/styles.css
+++ b/changedetectionio/static/styles/styles.css
@@ -309,10 +309,10 @@ footer {
font-weight: bold; }
.pure-form textarea {
width: 100%; }
- .pure-form ul.fetch-backend {
+ .pure-form .inline-radio ul {
margin: 0px;
list-style: none; }
- .pure-form ul.fetch-backend li > * {
+ .pure-form .inline-radio ul li > * {
display: inline-block; }
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss
index a79c051b..e2f7d1cf 100644
--- a/changedetectionio/static/styles/styles.scss
+++ b/changedetectionio/static/styles/styles.scss
@@ -418,14 +418,16 @@ footer {
textarea {
width: 100%;
}
- ul.fetch-backend {
- margin: 0px;
- list-style: none;
- li {
- > * {
- display: inline-block;
+ .inline-radio {
+ ul {
+ margin: 0px;
+ list-style: none;
+ li {
+ > * {
+ display: inline-block;
+ }
}
- }
+ }
}
}
diff --git a/changedetectionio/store.py b/changedetectionio/store.py
index ed4a915a..a8ff3568 100644
--- a/changedetectionio/store.py
+++ b/changedetectionio/store.py
@@ -33,6 +33,7 @@ class ChangeDetectionStore:
self.needs_write = False
self.datastore_path = datastore_path
self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+ self.proxy_list = None
self.stop_thread = False
self.__data = App.model()
@@ -111,6 +112,14 @@ class ChangeDetectionStore:
secret = secrets.token_hex(16)
self.__data['settings']['application']['rss_access_token'] = secret
+
+ # Proxy list support - available as a selection in settings when text file is imported
+ # CSV list
+ # "name, address", or just "name"
+ proxy_list_file = "{}/proxies.txt".format(self.datastore_path)
+ if path.isfile(proxy_list_file):
+ self.import_proxy_list(proxy_list_file)
+
# Bump the update version by running updates
self.run_updates()
@@ -427,6 +436,21 @@ class ChangeDetectionStore:
print ("Removing",item)
unlink(item)
+ def import_proxy_list(self, filename):
+ import csv
+ with open(filename, newline='') as f:
+ reader = csv.reader(f, skipinitialspace=True)
+ # @todo This loop can could be improved
+ l = []
+ for row in reader:
+ if len(row):
+ if len(row)>=2:
+ l.append(tuple(row[:2]))
+ else:
+ l.append(tuple([row[0], row[0]]))
+ self.proxy_list = l if len(l) else None
+
+
# Run all updates
# IMPORTANT - Each update could be run even when they have a new install and the schema is correct
# So therefor - each `update_n` should be very careful about checking if it needs to actually run
diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja
index 30ada5c0..961ff1db 100644
--- a/changedetectionio/templates/_common_fields.jinja
+++ b/changedetectionio/templates/_common_fields.jinja
@@ -2,7 +2,6 @@
{% from '_helpers.jinja' import render_field %}
{% macro render_common_settings_form(form, current_base_url, emailprefix) %}
-