From 1b625dc18a6189f0b08e7efa7b94de7399d2d510 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 28 Sep 2024 10:40:26 +0200 Subject: [PATCH] UI - "Filters & Triggers" - Live preview of text filters (Preview the output of the filters section in realtime) (#2612) --- changedetectionio/flask_app.py | 55 ++++++++++++++++ changedetectionio/processors/__init__.py | 7 +- .../processors/restock_diff/processor.py | 2 +- .../processors/text_json_diff/processor.py | 4 +- changedetectionio/static/js/watch-settings.js | 66 +++++++++++++++++++ .../scss/parts/_preview_text_filter.scss | 45 +++++++++++++ .../static/styles/scss/styles.scss | 1 + changedetectionio/static/styles/styles.css | 41 ++++++++++++ changedetectionio/templates/base.html | 2 +- changedetectionio/templates/edit.html | 25 +++++-- .../tests/test_filter_failure_notification.py | 4 +- changedetectionio/tests/util.py | 1 + changedetectionio/update_worker.py | 2 +- 13 files changed, 240 insertions(+), 15 deletions(-) create mode 100644 changedetectionio/static/styles/scss/parts/_preview_text_filter.scss diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index eb9d7799..14a7570c 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import datetime +import importlib + import flask_login import locale import os @@ -10,7 +12,9 @@ import threading import time import timeago +from .content_fetchers.exceptions import ReplyWithContentButNoText from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor +from .processors.text_json_diff.processor import FilterNotFoundInResponse from .safe_jinja import render as jinja_render from changedetectionio.strtobool import strtobool from copy import deepcopy @@ -1396,6 +1400,57 @@ def changedetection_app(config=None, datastore_o=None): # Return a 500 error abort(500) + @app.route("/edit//preview-rendered", methods=['POST']) + @login_optionally_required + def watch_get_preview_rendered(uuid): + '''For when viewing the "preview" of the rendered text from inside of Edit''' + now = time.time() + import brotli + from . import forms + + text_after_filter = '' + tmp_watch = deepcopy(datastore.data['watching'].get(uuid)) + + if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir): + # Splice in the temporary stuff from the form + form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None, + data=request.form + ) + # Only update vars that came in via the AJAX post + p = {k: v for k, v in form.data.items() if k in request.form.keys()} + tmp_watch.update(p) + + latest_filename = next(reversed(tmp_watch.history)) + html_fname = os.path.join(tmp_watch.watch_data_dir, f"{latest_filename}.html.br") + with open(html_fname, 'rb') as f: + decompressed_data = brotli.decompress(f.read()).decode('utf-8') if html_fname.endswith('.br') else f.read().decode('utf-8') + + # Just like a normal change detection except provide a fake "watch" object and dont call .call_browser() + processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor") + update_handler = processor_module.perform_site_check(datastore=datastore, + watch_uuid=uuid # probably not needed anymore anyway? + ) + # Use the last loaded HTML as the input + update_handler.fetcher.content = decompressed_data + try: + changed_detected, update_obj, contents, text_after_filter = update_handler.run_changedetection( + watch=tmp_watch, + skip_when_checksum_same=False, + ) + except FilterNotFoundInResponse as e: + text_after_filter = f"Filter not found in HTML: {str(e)}" + except ReplyWithContentButNoText as e: + text_after_filter = f"Filter found but no text (empty result)" + except Exception as e: + text_after_filter = f"Error: {str(e)}" + + if not text_after_filter.strip(): + text_after_filter = 'Empty content' + + logger.trace(f"Parsed in {time.time()-now:.3f}s") + return text_after_filter.strip() + + @app.route("/form/add/quickwatch", methods=['POST']) @login_optionally_required def form_quick_watch_add(): diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index 529f57da..5821e6ff 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -1,4 +1,6 @@ from abc import abstractmethod + +from changedetectionio.content_fetchers.base import Fetcher from changedetectionio.strtobool import strtobool from copy import deepcopy @@ -23,10 +25,11 @@ class difference_detection_processor(): super().__init__(*args, **kwargs) self.datastore = datastore self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid)) + # Generic fetcher that should be extended (requests, playwright etc) + self.fetcher = Fetcher() def call_browser(self): from requests.structures import CaseInsensitiveDict - from changedetectionio.content_fetchers.exceptions import EmptyReply # Protect against file:// access if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE): @@ -159,7 +162,7 @@ class difference_detection_processor(): some_data = 'xxxxx' update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest() changed_detected = False - return changed_detected, update_obj, ''.encode('utf-8') + return changed_detected, update_obj, ''.encode('utf-8'), b'' def find_sub_packages(package_name): diff --git a/changedetectionio/processors/restock_diff/processor.py b/changedetectionio/processors/restock_diff/processor.py index eaa48cc5..e14b07b6 100644 --- a/changedetectionio/processors/restock_diff/processor.py +++ b/changedetectionio/processors/restock_diff/processor.py @@ -298,4 +298,4 @@ class perform_site_check(difference_detection_processor): # Always record the new checksum update_obj["previous_md5"] = fetched_md5 - return changed_detected, update_obj, snapshot_content.encode('utf-8').strip() + return changed_detected, update_obj, snapshot_content.encode('utf-8').strip(), b'' diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py index 6ab815bb..8ad1a0ef 100644 --- a/changedetectionio/processors/text_json_diff/processor.py +++ b/changedetectionio/processors/text_json_diff/processor.py @@ -242,7 +242,7 @@ class perform_site_check(difference_detection_processor): # We had some content, but no differences were found # Store our new file as the MD5 so it will trigger in the future c = hashlib.md5(text_content_before_ignored_filter.translate(None, b'\r\n\t ')).hexdigest() - return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8') + return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8'), stripped_text_from_html.encode('utf-8') else: stripped_text_from_html = rendered_diff @@ -365,4 +365,4 @@ class perform_site_check(difference_detection_processor): if not watch.get('previous_md5'): watch['previous_md5'] = fetched_md5 - return changed_detected, update_obj, text_content_before_ignored_filter + return changed_detected, update_obj, text_content_before_ignored_filter, stripped_text_from_html diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index a55d2813..0a6c96c4 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -12,6 +12,54 @@ function toggleOpacity(checkboxSelector, fieldSelector, inverted) { checkbox.addEventListener('change', updateOpacity); } +(function($) { + // Object to store ongoing requests by namespace + const requests = {}; + + $.abortiveSingularAjax = function(options) { + const namespace = options.namespace || 'default'; + + // Abort the current request in this namespace if it's still ongoing + if (requests[namespace]) { + requests[namespace].abort(); + } + + // Start a new AJAX request and store its reference in the correct namespace + requests[namespace] = $.ajax(options); + + // Return the current request in case it's needed + return requests[namespace]; + }; +})(jQuery); + +function request_textpreview_update() { + if (!$('body').hasClass('preview-text-enabled')) { + return + } + + const data = {}; + $('textarea:visible, input:visible').each(function () { + const $element = $(this); // Cache the jQuery object for the current element + const name = $element.attr('name'); // Get the name attribute of the element + data[name] = $element.is(':checkbox') ? ($element.is(':checked') ? $element.val() : undefined) : $element.val(); + }); + + $.abortiveSingularAjax({ + type: "POST", + url: preview_text_edit_filters_url, + data: data, + namespace: 'watchEdit' + }).done(function (data) { + $('#filters-and-triggers #text-preview-inner').text(data); + }).fail(function (error) { + if (error.statusText === 'abort') { + console.log('Request was aborted due to a new request being fired.'); + } else { + $('#filters-and-triggers #text-preview-inner').text('There was an error communicating with the server.'); + } + }) +} + $(document).ready(function () { $('#notification-setting-reset-to-default').click(function (e) { $('#notification_title').val(''); @@ -27,5 +75,23 @@ $(document).ready(function () { toggleOpacity('#time_between_check_use_default', '#time_between_check', false); + const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0); + $("#text-preview-inner").css('max-height', (vh-300)+"px"); + var debounced_request_textpreview_update = request_textpreview_update.debounce(100); + + $("#activate-text-preview").click(function (e) { + $(this).fadeOut(); + $('body').toggleClass('preview-text-enabled') + + request_textpreview_update(); + + $("#text-preview-refresh").click(function (e) { + request_textpreview_update(); + }); + $('textarea:visible').on('keyup blur', debounced_request_textpreview_update); + $('input:visible').on('keyup blur change', debounced_request_textpreview_update); + $("#filters-and-triggers-tab").on('click', debounced_request_textpreview_update); + }); + }); diff --git a/changedetectionio/static/styles/scss/parts/_preview_text_filter.scss b/changedetectionio/static/styles/scss/parts/_preview_text_filter.scss new file mode 100644 index 00000000..1e18c586 --- /dev/null +++ b/changedetectionio/static/styles/scss/parts/_preview_text_filter.scss @@ -0,0 +1,45 @@ +body.preview-text-enabled { + #filters-and-triggers > div { + display: flex; /* Establishes Flexbox layout */ + gap: 20px; /* Adds space between the columns */ + position: relative; /* Ensures the sticky positioning is relative to this parent */ + } + + /* layout of the page */ + #edit-text-filter, #text-preview { + flex: 1; /* Each column takes an equal amount of available space */ + align-self: flex-start; /* Aligns the right column to the start, allowing it to maintain its content height */ + } + + #edit-text-filter { + #pro-tips { + display: none; + } + } + + #text-preview { + position: sticky; + top: 25px; + display: block !important; + } + + /* actual preview area */ + #text-preview-inner { + background: var(--color-grey-900); + border: 1px solid var(--color-grey-600); + padding: 1rem; + color: #333; + font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */ + font-size: 12px; + overflow-x: scroll; + white-space: pre-wrap; /* Preserves whitespace and line breaks like 
 */
+    overflow-wrap: break-word; /* Allows long words to break and wrap to the next line */
+  }
+}
+
+#activate-text-preview {
+  right: 0;
+  position: absolute;
+  z-index: 0;
+  box-shadow: 1px 1px 4px var(--color-shadow-jump);
+}
diff --git a/changedetectionio/static/styles/scss/styles.scss b/changedetectionio/static/styles/scss/styles.scss
index b720b6d4..0c3fd6cd 100644
--- a/changedetectionio/static/styles/scss/styles.scss
+++ b/changedetectionio/static/styles/scss/styles.scss
@@ -12,6 +12,7 @@
 @import "parts/_darkmode";
 @import "parts/_menu";
 @import "parts/_love";
+@import "parts/preview_text_filter";
 
 body {
   color: var(--color-text);
diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css
index b9a3c240..7bc04082 100644
--- a/changedetectionio/static/styles/styles.css
+++ b/changedetectionio/static/styles/styles.css
@@ -428,6 +428,47 @@ html[data-darkmode="true"] #toggle-light-mode .icon-dark {
     fill: #ff0000 !important;
     transition: all ease 0.3s !important; }
 
+body.preview-text-enabled {
+  /* layout of the page */
+  /* actual preview area */ }
+  body.preview-text-enabled #filters-and-triggers > div {
+    display: flex;
+    /* Establishes Flexbox layout */
+    gap: 20px;
+    /* Adds space between the columns */
+    position: relative;
+    /* Ensures the sticky positioning is relative to this parent */ }
+  body.preview-text-enabled #edit-text-filter, body.preview-text-enabled #text-preview {
+    flex: 1;
+    /* Each column takes an equal amount of available space */
+    align-self: flex-start;
+    /* Aligns the right column to the start, allowing it to maintain its content height */ }
+  body.preview-text-enabled #edit-text-filter #pro-tips {
+    display: none; }
+  body.preview-text-enabled #text-preview {
+    position: sticky;
+    top: 25px;
+    display: block !important; }
+  body.preview-text-enabled #text-preview-inner {
+    background: var(--color-grey-900);
+    border: 1px solid var(--color-grey-600);
+    padding: 1rem;
+    color: #333;
+    font-family: "Courier New", Courier, monospace;
+    /* Sets the font to a monospace type */
+    font-size: 12px;
+    overflow-x: scroll;
+    white-space: pre-wrap;
+    /* Preserves whitespace and line breaks like 
 */
+    overflow-wrap: break-word;
+    /* Allows long words to break and wrap to the next line */ }
+
+#activate-text-preview {
+  right: 0;
+  position: absolute;
+  z-index: 0;
+  box-shadow: 1px 1px 4px var(--color-shadow-jump); }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page);
diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html
index 27ce8419..7f10d3bd 100644
--- a/changedetectionio/templates/base.html
+++ b/changedetectionio/templates/base.html
@@ -33,7 +33,7 @@
     
   
 
-  
+