live preview of text filters

changedetectionio/flask_app.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
 import datetime
+import importlib
+
 import flask_login
 import locale
 import os
@@ -10,7 +12,9 @@ import threading
 import time
 import timeago
 
+from .content_fetchers.exceptions import ReplyWithContentButNoText
 from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor
+from .processors.text_json_diff.processor import FilterNotFoundInResponse
 from .safe_jinja import render as jinja_render
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
@@ -1395,6 +1399,57 @@ def changedetection_app(config=None, datastore_o=None):
         # Return a 500 error
         abort(500)
 
+    @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])
+    @login_optionally_required
+    def watch_get_preview_rendered(uuid):
+        '''For when viewing the "preview" of the rendered text from inside of Edit'''
+        now = time.time()
+        import brotli
+        from . import forms
+
+        text_after_filter = ''
+        tmp_watch = deepcopy(datastore.data['watching'].get(uuid))
+
+        if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
+            # Splice in the temporary stuff from the form
+            form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
+                                                       data=request.form
+                                                       )
+            # Only update vars that came in via the AJAX post
+            p = {k: v for k, v in form.data.items() if k in request.form.keys()}
+            tmp_watch.update(p)
+
+            latest_filename = next(reversed(tmp_watch.history))
+            html_fname = os.path.join(tmp_watch.watch_data_dir, f"{latest_filename}.html.br")
+            with open(html_fname, 'rb') as f:
+                decompressed_data = brotli.decompress(f.read()).decode('utf-8') if html_fname.endswith('.br') else f.read().decode('utf-8')
+
+                # Just like a normal change detection except provide a fake "watch" object and dont call .call_browser()
+                processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
+                update_handler = processor_module.perform_site_check(datastore=datastore,
+                                                                     watch_uuid=uuid # probably not needed anymore anyway?
+                                                                     )
+                # Use the last loaded HTML as the input
+                update_handler.fetcher.content = decompressed_data
+                try:
+                    changed_detected, update_obj, contents, text_after_filter = update_handler.run_changedetection(
+                        watch=tmp_watch,
+                        skip_when_checksum_same=False,
+                    )
+                except FilterNotFoundInResponse as e:
+                    text_after_filter = f"Filter not found in HTML: {str(e)}"
+                except ReplyWithContentButNoText as e:
+                    text_after_filter = f"Filter found but no text (empty result)"
+                except Exception as e:
+                    text_after_filter = f"Error: {str(e)}"
+
+            if not text_after_filter.strip():
+                text_after_filter = 'Empty content'
+
+        logger.trace(f"Parsed in {time.time()-now:.3f}s")
+        return text_after_filter.strip()
+
+
     @app.route("/form/add/quickwatch", methods=['POST'])
     @login_optionally_required
     def form_quick_watch_add():

changedetectionio/processors/__init__.py

@@ -1,4 +1,6 @@
 from abc import abstractmethod
+
+from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.strtobool import strtobool
 
 from copy import deepcopy
@@ -23,10 +25,11 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
+        # Generic fetcher that should be extended (requests, playwright etc)
+        self.fetcher = Fetcher()
 
     def call_browser(self):
         from requests.structures import CaseInsensitiveDict
-        from changedetectionio.content_fetchers.exceptions import EmptyReply
 
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
@@ -159,7 +162,7 @@ class difference_detection_processor():
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
         changed_detected = False
-        return changed_detected, update_obj, ''.encode('utf-8')
+        return changed_detected, update_obj, ''.encode('utf-8'), b''
 
 
 def find_sub_packages(package_name):

changedetectionio/processors/restock_diff/processor.py

@@ -263,4 +263,4 @@ class perform_site_check(difference_detection_processor):
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5
 
-        return changed_detected, update_obj, snapshot_content.encode('utf-8').strip()
+        return changed_detected, update_obj, snapshot_content.encode('utf-8').strip(), b''

changedetectionio/processors/text_json_diff/processor.py

@@ -357,4 +357,4 @@ class perform_site_check(difference_detection_processor):
         if not watch.get('previous_md5'):
             watch['previous_md5'] = fetched_md5
 
-        return changed_detected, update_obj, text_content_before_ignored_filter
+        return changed_detected, update_obj, text_content_before_ignored_filter, stripped_text_from_html

changedetectionio/static/js/watch-settings.js

@@ -12,6 +12,26 @@ function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
     checkbox.addEventListener('change', updateOpacity);
 }
 
+function request_textpreview_update() {
+    const data = {};
+    $('textarea:visible, input:visible').each(function () {
+        const $element = $(this); // Cache the jQuery object for the current element
+        const name = $element.attr('name'); // Get the name attribute of the element
+        data[name] = $element.is(':checkbox') ? ($element.is(':checked') ? $element.val() : undefined) : $element.val();
+    });
+
+    $.ajax({
+        type: "POST",
+        url: preview_text_edit_filters_url,
+        data: data
+    }).done(function (data) {
+        $('#filters-and-triggers #text-preview-inner').text(data);
+    }).fail(function (data) {
+        console.log(data);
+        $('#filters-and-triggers #text-preview-inner').text('There was an error communicating with the server.');
+    })
+}
+
 $(document).ready(function () {
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
@@ -27,5 +47,22 @@ $(document).ready(function () {
 
     toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
 
+    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+    $("#text-preview-inner").css('max-height', (vh-300)+"px");
+
+    $("#activate-text-preview").click(function (e) {
+        $(this).fadeOut();
+        $('body').toggleClass('preview-text-enabled')
+
+        request_textpreview_update();
+        $("#text-preview-refresh").click(function (e) {
+            request_textpreview_update();
+        });
+        $('textarea:visible, input:visible').on('keyup keypress blur change click', function (e) {
+            request_textpreview_update();
+        });
+
+    });
+
 });
 

changedetectionio/static/styles/scss/parts/_preview_text_filter.scss

@@ -0,0 +1,39 @@
+body.preview-text-enabled {
+  #filters-and-triggers > div {
+    display: flex; /* Establishes Flexbox layout */
+    gap: 20px; /* Adds space between the columns */
+    position: relative; /* Ensures the sticky positioning is relative to this parent */
+  }
+
+  /* layout of the page */
+  #edit-text-filter, #text-preview {
+    flex: 1; /* Each column takes an equal amount of available space */
+    align-self: flex-start; /* Aligns the right column to the start, allowing it to maintain its content height */
+  }
+
+  #edit-text-filter {
+    #pro-tips {
+      display: none;
+    }
+  }
+
+  #text-preview {
+    position: sticky;
+    top: 25px;
+    display: block !important;
+  }
+
+  /* actual preview area */
+  #text-preview-inner {
+    background: var(--color-grey-900);
+    border: 1px solid var(--color-grey-600);
+    padding: 1rem;
+    color: #333;
+    font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */
+    font-size: 12px;
+    overflow-x: scroll;
+    white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */
+    overflow-wrap: break-word; /* Allows long words to break and wrap to the next line */
+  }
+
+}

changedetectionio/static/styles/scss/styles.scss

@@ -12,6 +12,7 @@
 @import "parts/_darkmode";
 @import "parts/_menu";
 @import "parts/_love";
+@import "parts/preview_text_filter";
 
 body {
   color: var(--color-text);

changedetectionio/static/styles/styles.css

@@ -411,6 +411,41 @@ html[data-darkmode="true"] #toggle-light-mode .icon-dark {
     fill: #ff0000 !important;
     transition: all ease 0.3s !important; }
 
+body.preview-text-enabled {
+  /* layout of the page */
+  /* actual preview area */ }
+  body.preview-text-enabled #filters-and-triggers > div {
+    display: flex;
+    /* Establishes Flexbox layout */
+    gap: 20px;
+    /* Adds space between the columns */
+    position: relative;
+    /* Ensures the sticky positioning is relative to this parent */ }
+  body.preview-text-enabled #edit-text-filter, body.preview-text-enabled #text-preview {
+    flex: 1;
+    /* Each column takes an equal amount of available space */
+    align-self: flex-start;
+    /* Aligns the right column to the start, allowing it to maintain its content height */ }
+  body.preview-text-enabled #edit-text-filter #pro-tips {
+    display: none; }
+  body.preview-text-enabled #text-preview {
+    position: sticky;
+    top: 25px;
+    display: block !important; }
+  body.preview-text-enabled #text-preview-inner {
+    background: var(--color-grey-900);
+    border: 1px solid var(--color-grey-600);
+    padding: 1rem;
+    color: #333;
+    font-family: "Courier New", Courier, monospace;
+    /* Sets the font to a monospace type */
+    font-size: 12px;
+    overflow-x: scroll;
+    white-space: pre-wrap;
+    /* Preserves whitespace and line breaks like <pre> */
+    overflow-wrap: break-word;
+    /* Allows long words to break and wrap to the next line */ }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page);
@@ -1194,11 +1229,9 @@ ul {
   color: #fff;
   opacity: 0.7; }
 
-
 .restock-label svg {
   vertical-align: middle; }
 
-
 #chrome-extension-link {
   padding: 9px;
   border: 1px solid var(--color-grey-800);

changedetectionio/templates/base.html

@@ -33,7 +33,7 @@
     <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
   </head>
 
-  <body>
+  <body class="">
     <div class="header">
       <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}

changedetectionio/templates/edit.html

@@ -254,7 +254,10 @@ User-Agent: wonderbra 1.0") }}
 
             {% if watch['processor'] == 'text_json_diff' %}
             <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
+            <span id="activate-text-preview" style="right: 0;position: absolute;     z-index: 100;" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
+              <div>
+                <div id="edit-text-filter">
+                    <div class="pure-control-group" id="pro-tips">
                             <strong>Pro-tips:</strong><br>
                             <ul>
                                 <li>
@@ -407,7 +410,19 @@ Unavailable") }}
                 </fieldset>
                 </div>
             </div>
-            {% endif %}
+            <div id="text-preview" style="display: none;" >
+                    <script>
+                        const preview_text_edit_filters_url="{{url_for('watch_get_preview_rendered', uuid=uuid)}}";
+                    </script>
+                    <span><strong>Preview of the text that is used for changedetection after all filters run.</strong></span><br>
+                    {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
+                <p>
+                    <div id="text-preview-inner"></div>
+                </p>
+            </div>
+          </div>
+        </div>
+        {% endif %}
         {# rendered sub Template #}
         {% if extra_form_content %}
             <div class="tab-pane-inner" id="extras_tab">

changedetectionio/update_worker.py

@@ -276,7 +276,7 @@ class update_worker(threading.Thread):
 
                         update_handler.call_browser()
 
-                        changed_detected, update_obj, contents = update_handler.run_changedetection(
+                        changed_detected, update_obj, contents, content_after_filters = update_handler.run_changedetection(
                             watch=watch,
                             skip_when_checksum_same=skip_when_same_checksum,
                         )