From 22b20682080c95a906e334cf28a7cda3287f3f9e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 8 Sep 2023 14:14:47 +0200 Subject: [PATCH] Ability to select "No proxy" for a watch when you have proxy's configured --- changedetectionio/run_proxy_tests.sh | 25 +++++- changedetectionio/store.py | 7 ++ .../tests/proxy_list/test_noproxy.py | 77 +++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 changedetectionio/tests/proxy_list/test_noproxy.py diff --git a/changedetectionio/run_proxy_tests.sh b/changedetectionio/run_proxy_tests.sh index 1761bae1..47af3b09 100755 --- a/changedetectionio/run_proxy_tests.sh +++ b/changedetectionio/run_proxy_tests.sh @@ -2,6 +2,8 @@ # exit when any command fails set -e +# enable debug +set -x # Test proxy list handling, starting two squids on different ports # Each squid adds a different header to the response, which is the main thing we test for. @@ -19,7 +21,6 @@ docker run --network changedet-network -d \ ## 2nd test actually choose the preferred proxy from proxies.json - docker run --network changedet-network \ -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \ test-changedetectionio \ @@ -44,7 +45,6 @@ fi # Test the UI configurable proxies - docker run --network changedet-network \ test-changedetectionio \ bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py' @@ -58,4 +58,25 @@ then exit 1 fi +# Test "no-proxy" option +docker run --network changedet-network \ + test-changedetectionio \ + bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py' + +# We need to handle grep returning 1 +set +e +# Check request was never seen in any container +for c in $(echo "squid-one squid-two squid-custom"); do + echo Checking $c + docker logs $c &> $c.txt + grep noproxy $c.txt + if [ $? -ne 1 ] + then + echo "Saw request for noproxy in $c container" + cat $c.txt + exit 1 + fi +done + + docker kill squid-one squid-two squid-custom diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 82777067..388756d7 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -1,3 +1,5 @@ +from distutils.util import strtobool + from flask import ( flash ) @@ -468,6 +470,8 @@ class ChangeDetectionStore: k = "ui-" + str(i) + proxy.get('proxy_name') proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')} + if proxy_list and strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')): + proxy_list["no-proxy"] = {'label': "No proxy", 'url': ''} return proxy_list if len(proxy_list) else None @@ -485,6 +489,9 @@ class ChangeDetectionStore: # If it's a valid one watch = self.data['watching'].get(uuid) + if strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')) and watch.get('proxy') == "no-proxy": + return None + if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()): return watch.get('proxy') diff --git a/changedetectionio/tests/proxy_list/test_noproxy.py b/changedetectionio/tests/proxy_list/test_noproxy.py new file mode 100644 index 00000000..5a3cbce2 --- /dev/null +++ b/changedetectionio/tests/proxy_list/test_noproxy.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 + +import time +from flask import url_for +from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client + + +def test_noproxy_option(client, live_server): + live_server_setup(live_server) + # Run by run_proxy_tests.sh + # Call this URL then scan the containers that it never went through them + url = "http://noproxy.changedetection.io" + + # Should only be available when a proxy is setup + res = client.get( + url_for("edit_page", uuid="first", unpause_on_save=1)) + assert b'No proxy' not in res.data + + # Setup a proxy + res = client.post( + url_for("settings_page"), + data={ + "requests-time_between_check-minutes": 180, + "application-ignore_whitespace": "y", + "application-fetch_backend": "html_requests", + "requests-extra_proxies-0-proxy_name": "custom-one-proxy", + "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-one:3128", + "requests-extra_proxies-1-proxy_name": "custom-two-proxy", + "requests-extra_proxies-1-proxy_url": "http://test:awesome@squid-two:3128", + "requests-extra_proxies-2-proxy_name": "custom-proxy", + "requests-extra_proxies-2-proxy_url": "http://test:awesome@squid-custom:3128", + }, + follow_redirects=True + ) + + assert b"Settings updated." in res.data + + # Should be available as an option + res = client.get( + url_for("settings_page", unpause_on_save=1)) + assert b'No proxy' in res.data + + + # This will add it paused + res = client.post( + url_for("form_quick_watch_add"), + data={"url": url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'}, + follow_redirects=True + ) + assert b"Watch added in Paused state, saving will unpause" in res.data + uuid = extract_UUID_from_client(client) + res = client.get( + url_for("edit_page", uuid=uuid, unpause_on_save=1)) + assert b'No proxy' in res.data + + res = client.post( + url_for("edit_page", uuid=uuid, unpause_on_save=1), + data={ + "include_filters": "", + "fetch_backend": "html_requests", + "headers": "", + "proxy": "no-proxy", + "tags": "", + "url": url, + }, + follow_redirects=True + ) + assert b"unpaused" in res.data + wait_for_all_checks(client) + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + # Now the request should NOT appear in the second-squid logs (handled by the run_test_proxies.sh script) + + # Prove that it actually checked + + assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != 0 +