Ability to select "No proxy" for a watch when you have proxy's configured

changedetectionio/run_proxy_tests.sh

@@ -2,6 +2,8 @@
 
 # exit when any command fails
 set -e
+# enable debug
+set -x
 
 # Test proxy list handling, starting two squids on different ports
 # Each squid adds a different header to the response, which is the main thing we test for.
@@ -19,7 +21,6 @@ docker run --network changedet-network -d \
 
 
 ## 2nd test actually choose the preferred proxy from proxies.json
-
 docker run --network changedet-network \
   -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
   test-changedetectionio \
@@ -44,7 +45,6 @@ fi
 
 
 # Test the UI configurable proxies
-
 docker run --network changedet-network \
   test-changedetectionio \
   bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
@@ -58,4 +58,25 @@ then
   exit 1
 fi
 
+# Test "no-proxy" option
+docker run --network changedet-network \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py'
+
+# We need to handle grep returning 1
+set +e
+# Check request was never seen in any container
+for c in $(echo "squid-one squid-two squid-custom"); do
+  echo Checking $c
+  docker logs $c &> $c.txt
+  grep noproxy $c.txt
+  if [ $? -ne 1 ]
+  then
+    echo "Saw request for noproxy in $c container"
+    cat $c.txt
+    exit 1
+  fi
+done
+
+
 docker kill squid-one squid-two squid-custom

changedetectionio/store.py

@@ -1,3 +1,5 @@
+from distutils.util import strtobool
+
 from flask import (
     flash
 )
@@ -468,6 +470,8 @@ class ChangeDetectionStore:
                     k = "ui-" + str(i) + proxy.get('proxy_name')
                     proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}
 
+        if proxy_list and strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')):
+            proxy_list["no-proxy"] = {'label': "No proxy", 'url': ''}
 
         return proxy_list if len(proxy_list) else None
 
@@ -485,6 +489,9 @@ class ChangeDetectionStore:
         # If it's a valid one
         watch = self.data['watching'].get(uuid)
 
+        if strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')) and watch.get('proxy') == "no-proxy":
+            return None
+
         if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
             return watch.get('proxy')
 

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+
+
+def test_noproxy_option(client, live_server):
+    live_server_setup(live_server)
+    # Run by run_proxy_tests.sh
+    # Call this URL then scan the containers that it never went through them
+    url = "http://noproxy.changedetection.io"
+
+    # Should only be available when a proxy is setup
+    res = client.get(
+        url_for("edit_page", uuid="first", unpause_on_save=1))
+    assert b'No proxy' not in res.data
+
+    # Setup a proxy
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-ignore_whitespace": "y",
+            "application-fetch_backend": "html_requests",
+            "requests-extra_proxies-0-proxy_name": "custom-one-proxy",
+            "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-one:3128",
+            "requests-extra_proxies-1-proxy_name": "custom-two-proxy",
+            "requests-extra_proxies-1-proxy_url": "http://test:awesome@squid-two:3128",
+            "requests-extra_proxies-2-proxy_name": "custom-proxy",
+            "requests-extra_proxies-2-proxy_url": "http://test:awesome@squid-custom:3128",
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    # Should be available as an option
+    res = client.get(
+        url_for("settings_page", unpause_on_save=1))
+    assert b'No proxy' in res.data
+
+
+    # This will add it paused
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+    uuid = extract_UUID_from_client(client)
+    res = client.get(
+        url_for("edit_page", uuid=uuid, unpause_on_save=1))
+    assert b'No proxy' in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid=uuid, unpause_on_save=1),
+        data={
+                "include_filters": "",
+                "fetch_backend": "html_requests",
+                "headers": "",
+                "proxy": "no-proxy",
+                "tags": "",
+                "url": url,
+              },
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+    wait_for_all_checks(client)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    # Now the request should NOT appear in the second-squid logs (handled by the run_test_proxies.sh script)
+
+    # Prove that it actually checked
+
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != 0
+