From 1987e109e8d51da7095f4b0614a50d7200f587d8 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 10 Jul 2023 16:08:45 +0200 Subject: [PATCH] New feature - Helper button to trigger a scan/access test of all proxies for a particular watch (#1685) --- MANIFEST.in | 3 + changedetectionio/__init__.py | 4 + .../blueprint/check_proxies/__init__.py | 105 ++++++++++++++++++ changedetectionio/processors/__init__.py | 2 +- .../processors/text_json_diff.py | 8 +- changedetectionio/static/js/recheck-proxy.js | 87 +++++++++++++++ .../styles/scss/parts/_extra_proxies.scss | 28 +++++ changedetectionio/static/styles/styles.css | 27 ++++- changedetectionio/templates/_helpers.jinja | 15 +-- changedetectionio/templates/edit.html | 22 ++-- 10 files changed, 271 insertions(+), 30 deletions(-) create mode 100644 changedetectionio/blueprint/check_proxies/__init__.py create mode 100644 changedetectionio/static/js/recheck-proxy.js diff --git a/MANIFEST.in b/MANIFEST.in index 501f7320..41ed620d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,3 +13,6 @@ include changedetection.py global-exclude *.pyc global-exclude node_modules global-exclude venv + +global-exclude test-datastore +global-exclude changedetection.io*dist-info diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 5a36c593..8557d5a2 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -1439,6 +1439,10 @@ def changedetection_app(config=None, datastore_o=None): import changedetectionio.blueprint.tags as tags app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags') + import changedetectionio.blueprint.check_proxies as check_proxies + app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy') + + # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() threading.Thread(target=notification_runner).start() diff --git a/changedetectionio/blueprint/check_proxies/__init__.py b/changedetectionio/blueprint/check_proxies/__init__.py new file mode 100644 index 00000000..50389dee --- /dev/null +++ b/changedetectionio/blueprint/check_proxies/__init__.py @@ -0,0 +1,105 @@ +from concurrent.futures import ThreadPoolExecutor + +from functools import wraps + +from flask import Blueprint +from flask_login import login_required + +from changedetectionio.processors import text_json_diff +from changedetectionio.store import ChangeDetectionStore + + +STATUS_CHECKING = 0 +STATUS_FAILED = 1 +STATUS_OK = 2 +THREADPOOL_MAX_WORKERS = 3 +_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS) + + +# Maybe use fetch-time if its >5 to show some expected load time? +def threadpool(f, executor=None): + @wraps(f) + def wrap(*args, **kwargs): + return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs) + + return wrap + + +def construct_blueprint(datastore: ChangeDetectionStore): + check_proxies_blueprint = Blueprint('check_proxies', __name__) + checks_in_progress = {} + + @threadpool + def long_task(uuid, preferred_proxy): + import time + from changedetectionio import content_fetcher + + status = {'status': '', 'length': 0, 'text': ''} + from jinja2 import Environment, BaseLoader + + contents = '' + now = time.time() + try: + update_handler = text_json_diff.perform_site_check(datastore=datastore) + changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False) + # title, size is len contents not len xfer + except content_fetcher.Non200ErrorCodeReceived as e: + if e.status_code == 404: + status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"}) + elif e.status_code == 403: + status.update({'status': 'ERROR', 'length': len(contents), 'text': f"403 - Access denied"}) + else: + status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"}) + except content_fetcher.EmptyReply as e: + status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': "Empty reply, needs chrome?"}) + except Exception as e: + status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+str(e)}) + else: + status.update({'status': 'OK', 'length': len(contents), 'text': ''}) + + if status.get('text'): + status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']}) + + status['time'] = "{:.2f}s".format(time.time() - now) + + return status + + def _recalc_check_status(uuid): + + results = {} + for k, v in checks_in_progress.get(uuid, {}).items(): + try: + r_1 = v.result(timeout=0.05) + except Exception as e: + # If timeout error? + results[k] = {'status': 'RUNNING'} + + else: + results[k] = r_1 + + return results + + @login_required + @check_proxies_blueprint.route("//status", methods=['GET']) + def get_recheck_status(uuid): + results = _recalc_check_status(uuid=uuid) + return results + + @login_required + @check_proxies_blueprint.route("//start", methods=['GET']) + def start_check(uuid): + + if not datastore.proxy_list: + return + + # @todo - Cancel any existing runs + checks_in_progress[uuid] = {} + + for k, v in datastore.proxy_list.items(): + if not checks_in_progress[uuid].get(k): + checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k) + + results = _recalc_check_status(uuid=uuid) + return results + + return check_proxies_blueprint diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index 0d90e4c9..225e9c8a 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -9,7 +9,7 @@ class difference_detection_processor(): super().__init__(*args, **kwargs) @abstractmethod - def run(self, uuid, skip_when_checksum_same=True): + def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None): update_obj = {'last_notification_error': False, 'last_error': False} some_data = 'xxxxx' update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest() diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index 63d6e64b..fb810f0c 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -50,7 +50,7 @@ class perform_site_check(difference_detection_processor): return regex - def run(self, uuid, skip_when_checksum_same=True): + def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None): changed_detected = False screenshot = False # as bytes stripped_text_from_html = "" @@ -105,7 +105,11 @@ class perform_site_check(difference_detection_processor): # If the klass doesnt exist, just use a default klass = getattr(content_fetcher, "html_requests") - proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid) + if preferred_proxy: + proxy_id = preferred_proxy + else: + proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid) + proxy_url = None if proxy_id: proxy_url = self.datastore.proxy_list.get(proxy_id).get('url') diff --git a/changedetectionio/static/js/recheck-proxy.js b/changedetectionio/static/js/recheck-proxy.js new file mode 100644 index 00000000..e42d2036 --- /dev/null +++ b/changedetectionio/static/js/recheck-proxy.js @@ -0,0 +1,87 @@ +$(function () { + /* add container before each proxy location to show status */ + + var option_li = $('.fetch-backend-proxy li').filter(function() { + return $("input",this)[0].value.length >0; + }); + + //var option_li = $('.fetch-backend-proxy li'); + var isActive = false; + $(option_li).prepend('
'); + $(option_li).append('
'); + + function set_proxy_check_status(proxy_key, state) { + // select input by value name + const proxy_li = $("input[value=" + proxy_key + "]").parent(); + if (state['status'] === 'RUNNING') { + $('.proxy-status', proxy_li).html(''); + } + if (state['status'] === 'OK') { + $('.proxy-status', proxy_li).html('OK'); + $('.proxy-check-details', proxy_li).html(state['text']); + } + if (state['status'] === 'ERROR' || state['status'] === 'ERROR OTHER') { + $('.proxy-status', proxy_li).html('X'); + $('.proxy-check-details', proxy_li).html(state['text']); + } + $('.proxy-timing', proxy_li).html(state['time']); + } + + + function pollServer() { + if (isActive) { + window.setTimeout(function () { + $.ajax({ + url: proxy_recheck_status_url, + success: function (data) { + var all_done = true; + $.each(data, function (proxy_key, state) { + set_proxy_check_status(proxy_key, state); + if (state['status'] === 'RUNNING') { + all_done = false; + } + }); + + if (all_done) { + console.log("Shutting down poller, all done.") + isActive = false; + } else { + pollServer(); + } + }, + error: function () { + //ERROR HANDLING + pollServer(); + } + }); + }, 2000); + } + } + + $('#check-all-proxies').click(function (e) { + e.preventDefault() + $('body').addClass('proxy-check-active'); + $('.proxy-check-details').html(''); + $('.proxy-status').html('').fadeIn(); + $('.proxy-timing').html(''); + + // Request start, needs CSRF? + $.ajax({ + type: "GET", + url: recheck_proxy_start_url, + }).done(function (data) { + $.each(data, function (proxy_key, state) { + set_proxy_check_status(proxy_key, state['status']) + }); + isActive = true; + pollServer(); + + }).fail(function (data) { + console.log(data); + alert('There was an error communicating with the server.'); + }); + + }); + +}); + diff --git a/changedetectionio/static/styles/scss/parts/_extra_proxies.scss b/changedetectionio/static/styles/scss/parts/_extra_proxies.scss index 83993dc6..789431a2 100644 --- a/changedetectionio/static/styles/scss/parts/_extra_proxies.scss +++ b/changedetectionio/static/styles/scss/parts/_extra_proxies.scss @@ -7,6 +7,7 @@ ul#requests-extra_proxies { } } + /* each proxy entry is a `table` */ table { tr { @@ -15,3 +16,30 @@ ul#requests-extra_proxies { } } +#request { + /* Auto proxy scan/checker */ + label[for=proxy] { + display: inline-block; + } +} + +body.proxy-check-active { + #request { + .proxy-status { + width: 2em; + } + + .proxy-check-details { + font-size: 80%; + color: #555; + display: block; + padding-left: 4em; + } + + .proxy-timing { + font-size: 80%; + padding-left: 1rem; + color: var(--color-link); + } + } +} \ No newline at end of file diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index e4c5636c..d4b97ea8 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -95,6 +95,25 @@ ul#requests-extra_proxies { ul#requests-extra_proxies table tr { display: inline; } +#request { + /* Auto proxy scan/checker */ } + #request label[for=proxy] { + display: inline-block; } + +body.proxy-check-active #request .proxy-status { + width: 2em; } + +body.proxy-check-active #request .proxy-check-details { + font-size: 80%; + color: #555; + display: block; + padding-left: 4em; } + +body.proxy-check-active #request .proxy-timing { + font-size: 80%; + padding-left: 1rem; + color: var(--color-link); } + .pagination-page-info { color: #fff; font-size: 0.85rem; @@ -283,10 +302,6 @@ html[data-darkmode="true"] { --color-icon-github-hover: var(--color-grey-700); --color-watch-table-error: var(--color-light-red); --color-watch-table-row-text: var(--color-grey-800); } - html[data-darkmode="true"] #toggle-light-mode .icon-light { - display: none; } - html[data-darkmode="true"] #toggle-light-mode .icon-dark { - display: block; } html[data-darkmode="true"] .icon-spread { filter: hue-rotate(-10deg) brightness(1.5); } html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after, @@ -339,6 +354,10 @@ a.github-link { width: 3rem; } #toggle-light-mode .icon-dark { display: none; } + #toggle-light-mode.dark .icon-light { + display: none; } + #toggle-light-mode.dark .icon-dark { + display: block; } #toggle-search { width: 2rem; } diff --git a/changedetectionio/templates/_helpers.jinja b/changedetectionio/templates/_helpers.jinja index 2b0d2829..878a6196 100644 --- a/changedetectionio/templates/_helpers.jinja +++ b/changedetectionio/templates/_helpers.jinja @@ -1,7 +1,6 @@ {% macro render_field(field) %} -
{{ field(**kwargs)|safe }}
{{ field.label }}
- +
{{ field(**kwargs)|safe }} {% if field.errors %}
    {% for error in field.errors %} @@ -25,18 +24,6 @@
{% endmacro %} -{% macro render_field(field) %} -
{{ field.label }}
-
{{ field(**kwargs)|safe }} - {% if field.errors %} -
    - {% for error in field.errors %} -
  • {{ error }}
    • - {% endfor %} -
- {% endif %} -
-{% endmacro %} {% macro render_simple_field(field) %} {{ field.label }} diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 81e102ce..a6302b30 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -4,18 +4,19 @@ {% from '_common_fields.jinja' import render_common_settings_form %} @@ -27,6 +28,8 @@ {% endif %} + +
@@ -111,7 +114,8 @@
{% if form.proxy %}
- {{ render_field(form.proxy, class="fetch-backend-proxy") }} +
{{ form.proxy.label }} Check/Scan all
+
{{ form.proxy(class="fetch-backend-proxy") }}
Choose a proxy for this watch