Ability to specify a list of proxies to choose from, always using the first one by default, See wiki (#591)

changedetectionio/__init__.py

@@ -518,10 +518,31 @@ def changedetection_app(config=None, datastore_o=None):
         if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
             default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])
 
+        # Defaults for proxy choice
+        if datastore.proxy_list is not None:  # When enabled
+            system_proxy = datastore.data['settings']['requests']['proxy']
+            if default['proxy'] is None:
+                default['proxy'] = system_proxy
+            else:
+                # Does the chosen one exist?
+                if not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                    default['proxy'] = datastore.proxy_list[0][0]
+
+            # Used by the form handler to keep or remove the proxy settings
+            default['proxy_list'] = datastore.proxy_list
+
+        # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=default
+                               data=default,
-                                        )
+                               )
 
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.proxy
+        else:
+            form.proxy.choices = datastore.proxy_list
+            if default['proxy'] is None:
+                form.proxy.default='http://hello'
 
         if request.method == 'POST' and form.validate():
             extra_update_obj = {}
@@ -601,10 +622,28 @@ def changedetection_app(config=None, datastore_o=None):
     def settings_page():
         from changedetectionio import content_fetcher, forms
 
+        default = deepcopy(datastore.data['settings'])
+        if datastore.proxy_list is not None:
+            # When enabled
+            system_proxy = datastore.data['settings']['requests']['proxy']
+            # In the case it doesnt exist anymore
+            if not any([system_proxy in tup for tup in datastore.proxy_list]):
+                system_proxy = None
+
+            default['requests']['proxy'] = system_proxy if system_proxy is not None else datastore.proxy_list[0][0]
+            # Used by the form handler to keep or remove the proxy settings
+            default['proxy_list'] = datastore.proxy_list
+
+
         # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
         form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=datastore.data['settings']
+                                        data=default
                                         )
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.requests.form.proxy
+        else:
+            form.requests.form.proxy.choices = datastore.proxy_list
 
         if request.method == 'POST':
             # Password unset is a GET, but we can lock the session to a salted env password to always need the password

changedetectionio/content_fetcher.py

@@ -91,7 +91,8 @@ class base_html_playwright(Fetcher):
 
     proxy = None
 
-    def __init__(self):
+    def __init__(self, proxy_override=None):
+
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
         self.command_executor = os.getenv(
@@ -109,6 +110,10 @@ class base_html_playwright(Fetcher):
         if proxy_args:
             self.proxy = proxy_args
 
+        # allow per-watch proxy selection override
+        if proxy_override:
+            self.proxy = {'server': proxy_override}
+
     def run(self,
             url,
             timeout,
@@ -177,7 +182,7 @@ class base_html_webdriver(Fetcher):
                                         'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
     proxy = None
 
-    def __init__(self):
+    def __init__(self, proxy_override=None):
         from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
@@ -196,6 +201,10 @@ class base_html_webdriver(Fetcher):
         if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
             proxy_args['httpsProxy'] = self.system_https_proxy
 
+        # Allows override the proxy on a per-request basis
+        if proxy_override is not None:
+            proxy_args['httpProxy'] = proxy_override
+
         if proxy_args:
             self.proxy = SeleniumProxy(raw=proxy_args)
 
@@ -263,6 +272,9 @@ class base_html_webdriver(Fetcher):
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
 
+    def __init__(self, proxy_override=None):
+        self.proxy_override = proxy_override
+
     def run(self,
             url,
             timeout,
@@ -271,12 +283,16 @@ class html_requests(Fetcher):
             request_method,
             ignore_status_codes=False):
 
-        # Map back standard HTTP_ and HTTPS_PROXY to requests http/https proxy
         proxies={}
-        if self.system_http_proxy:
+
-            proxies['http'] = self.system_http_proxy
+        # Allows override the proxy on a per-request basis
-        if self.system_https_proxy:
+        if self.proxy_override:
-            proxies['https'] = self.system_https_proxy
+            proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
+        else:
+            if self.system_http_proxy:
+                proxies['http'] = self.system_http_proxy
+            if self.system_https_proxy:
+                proxies['https'] = self.system_https_proxy
 
         r = requests.request(method=request_method,
                              data=request_body,

changedetectionio/fetch_site_status.py

@@ -16,6 +16,34 @@ class perform_site_check():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
+    # If there was a proxy list enabled, figure out what proxy_args/which proxy to use
+        # if watch.proxy use that
+        # fetcher.proxy_override = watch.proxy or main config proxy
+        # Allows override the proxy on a per-request basis
+        # ALWAYS use the first one is nothing selected
+
+    def set_proxy_from_list(self, watch):
+        proxy_args = None
+        if self.datastore.proxy_list is None:
+            return None
+
+        # If its a valid one
+        if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
+            proxy_args = watch['proxy']
+
+        # not valid (including None), try the system one
+        else:
+            system_proxy = self.datastore.data['settings']['requests']['proxy']
+            # Is not None and exists
+            if any([system_proxy in p for p in self.datastore.proxy_list]):
+                proxy_args = system_proxy
+
+        # Fallback - Did not resolve anything, use the first available
+        if proxy_args is None:
+            proxy_args = self.datastore.proxy_list[0][0]
+
+        return proxy_args
+
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
 
@@ -66,7 +94,10 @@ class perform_site_check():
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
-        fetcher = klass()
+        proxy_args = self.set_proxy_from_list(watch)
+        fetcher = klass(proxy_override=proxy_args)
+
+        # Proxy List support
         fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
 
         # Fetching complete, now filters

changedetectionio/forms.py

@@ -337,9 +337,9 @@ class watchForm(commonSettingsForm):
     method = SelectField('Request method', choices=valid_method, default=default_method)
     ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
-
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
     save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
+    proxy = RadioField('Proxy')
 
     def validate(self, **kwargs):
         if not super().validate():
@@ -358,6 +358,7 @@ class watchForm(commonSettingsForm):
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
+    proxy = RadioField('Proxy')
 
 
 # datastore.data['settings']['application']..
@@ -382,4 +383,3 @@ class globalSettingsForm(Form):
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-

changedetectionio/model/App.py

@@ -23,7 +23,8 @@ class model(dict):
                 'requests': {
                     'timeout': 15,  # Default 15 seconds
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
-                    'workers': 10  # Number of threads, lower is better for slow connections
+                    'workers': 10,  # Number of threads, lower is better for slow connections
+                    'proxy': None # Preferred proxy connection
                 },
                 'application': {
                     'password': False,

changedetectionio/model/Watch.py

@@ -39,6 +39,7 @@ class model(dict):
             'trigger_text': [],  # List of text or regex to wait for until a change is detected
             'fetch_backend': None,
             'extract_title_as_title': False,
+            'proxy': None, # Preferred proxy connection
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.

changedetectionio/static/styles/styles.css

@@ -309,10 +309,10 @@ footer {
     font-weight: bold; }
   .pure-form textarea {
     width: 100%; }
-  .pure-form ul.fetch-backend {
+  .pure-form .inline-radio ul {
     margin: 0px;
     list-style: none; }
-    .pure-form ul.fetch-backend li > * {
+    .pure-form .inline-radio ul li > * {
       display: inline-block; }
 
 @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {

changedetectionio/static/styles/styles.scss

@@ -418,14 +418,16 @@ footer {
   textarea {
     width: 100%;
   }
-  ul.fetch-backend {
+  .inline-radio {
-    margin: 0px;
+      ul {
-    list-style: none;
+        margin: 0px;
-    li {
+        list-style: none;
-        > * {
+        li {
-            display: inline-block;
+            > * {
+                display: inline-block;
+            }
         }
-    }
+      }
   }
 }
 

changedetectionio/store.py

@@ -33,6 +33,7 @@ class ChangeDetectionStore:
         self.needs_write = False
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        self.proxy_list = None
         self.stop_thread = False
 
         self.__data = App.model()
@@ -111,6 +112,14 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
+
+        # Proxy list support - available as a selection in settings when text file is imported
+        # CSV list
+        # "name, address", or just "name"
+        proxy_list_file = "{}/proxies.txt".format(self.datastore_path)
+        if path.isfile(proxy_list_file):
+            self.import_proxy_list(proxy_list_file)
+
         # Bump the update version by running updates
         self.run_updates()
 
@@ -427,6 +436,21 @@ class ChangeDetectionStore:
                     print ("Removing",item)
                     unlink(item)
 
+    def import_proxy_list(self, filename):
+        import csv
+        with open(filename, newline='') as f:
+            reader = csv.reader(f, skipinitialspace=True)
+            # @todo This loop can could be improved
+            l = []
+            for row in reader:
+                if len(row):
+                    if len(row)>=2:
+                        l.append(tuple(row[:2]))
+                    else:
+                        l.append(tuple([row[0], row[0]]))
+            self.proxy_list = l if len(l) else None
+
+
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
     #             So therefor - each `update_n` should be very careful about checking if it needs to actually run

changedetectionio/templates/_common_fields.jinja

@@ -2,7 +2,6 @@
 {% from '_helpers.jinja' import render_field %}
 
 {% macro render_common_settings_form(form, current_base_url, emailprefix) %}
-
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
     Gitter - gitter://token/room

changedetectionio/templates/edit.html

@@ -58,14 +58,21 @@
             </div>
 
             <div class="tab-pane-inner" id="request">
-                    <div class="pure-control-group">
+                    <div class="pure-control-group inline-radio">
                         {{ render_field(form.fetch_backend, class="fetch-backend") }}
                         <span class="pure-form-message-inline">
                             <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
                             <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                         </span>
                     </div>
-
+                {% if form.proxy %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">
+                        Choose a proxy for this watch
+                        </span>
+                    </div>
+                {% endif %}
                 <fieldset class="pure-group" id="requests-override-options">
                     <div class="pure-form-message-inline">
                         <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>

changedetectionio/templates/settings.html

@@ -60,7 +60,14 @@
                         {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }}
                         <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
                     </div>
-
+                {% if form.requests.proxy %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">
+                        Choose a default proxy for all watches
+                        </span>
+                    </div>
+                {% endif %}
                 </fieldset>
             </div>
 
@@ -73,7 +80,7 @@
             </div>
 
             <div class="tab-pane-inner" id="fetching">
-                <div class="pure-control-group">
+                <div class="pure-control-group inline-radio">
                     {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
                     <span class="pure-form-message-inline">
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>