diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 0da33bc8..32c2b316 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1158,8 +1158,6 @@ def changedetection_app(config=None, datastore_o=None): @login_optionally_required def preview_page(uuid): content = [] - ignored_line_numbers = [] - trigger_line_numbers = [] versions = [] timestamp = None @@ -1176,11 +1174,10 @@ def changedetection_app(config=None, datastore_o=None): system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')] - is_html_webdriver = False if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): is_html_webdriver = True - + triggered_line_numbers = [] if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()): flash("Preview unavailable - No fetch/check completed or triggers not reached", "error") else: @@ -1193,31 +1190,12 @@ def changedetection_app(config=None, datastore_o=None): try: versions = list(watch.history.keys()) - tmp = watch.get_history_snapshot(timestamp).splitlines() - - # Get what needs to be highlighted - ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] - - # .readlines will keep the \n, but we will parse it here again, in the future tidy this up - ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), - wordlist=ignore_rules, - mode='line numbers' - ) - - trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), - wordlist=watch['trigger_text'], - mode='line numbers' - ) - # Prepare the classes and lines used in the template - i=0 - for l in tmp: - classes=[] - i+=1 - if i in ignored_line_numbers: - classes.append('ignored') - if i in trigger_line_numbers: - classes.append('triggered') - content.append({'line': l, 'classes': ' '.join(classes)}) + content = watch.get_history_snapshot(timestamp) + + triggered_line_numbers = html_tools.strip_ignore_text(content=content, + wordlist=watch['trigger_text'], + mode='line numbers' + ) except Exception as e: content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) @@ -1228,8 +1206,7 @@ def changedetection_app(config=None, datastore_o=None): history_n=watch.history_n, extra_stylesheets=extra_stylesheets, extra_title=f" - Diff - {watch.label} @ {timestamp}", - ignored_line_numbers=ignored_line_numbers, - triggered_line_numbers=trigger_line_numbers, + triggered_line_numbers=triggered_line_numbers, current_diff_url=watch['url'], screenshot=watch.get_screenshot(), watch=watch, @@ -1400,9 +1377,11 @@ def changedetection_app(config=None, datastore_o=None): # Return a 500 error abort(500) + # Ajax callback @app.route("/edit//preview-rendered", methods=['POST']) @login_optionally_required def watch_get_preview_rendered(uuid): + from flask import jsonify '''For when viewing the "preview" of the rendered text from inside of Edit''' now = time.time() import brotli @@ -1434,7 +1413,7 @@ def changedetection_app(config=None, datastore_o=None): update_handler.fetcher.content = decompressed_data update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type') try: - changed_detected, update_obj, contents, text_after_filter = update_handler.run_changedetection( + changed_detected, update_obj, text_after_filter = update_handler.run_changedetection( watch=tmp_watch, skip_when_checksum_same=False, ) @@ -1448,8 +1427,32 @@ def changedetection_app(config=None, datastore_o=None): if not text_after_filter.strip(): text_after_filter = 'Empty content' - logger.trace(f"Parsed in {time.time()-now:.3f}s") - return text_after_filter.strip() + # because run_changedetection always returns bytes due to saving the snapshots etc + text_after_filter = text_after_filter.decode('utf-8') if isinstance(text_after_filter, bytes) else text_after_filter + + do_anchor = datastore.data["settings"]["application"].get("render_anchor_tag_content", False) + + trigger_line_numbers = [] + try: + text_before_filter = html_tools.html_to_text(html_content=decompressed_data, + render_anchor_tag_content=do_anchor) + + trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter, + wordlist=tmp_watch['trigger_text'], + mode='line numbers' + ) + except Exception as e: + text_before_filter = f"Error: {str(e)}" + + logger.trace(f"Parsed in {time.time() - now:.3f}s") + + return jsonify( + { + 'after_filter': text_after_filter, + 'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter, + 'trigger_line_numbers': trigger_line_numbers + } + ) @app.route("/form/add/quickwatch", methods=['POST']) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index d3ac30fc..e389acfb 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -475,7 +475,7 @@ class processor_text_json_diff_form(commonSettingsForm): title = StringField('Title', default='') - ignore_text = StringListField('Ignore text', [ValidateListRegex()]) + ignore_text = StringListField('Remove lines containing', [ValidateListRegex()]) headers = StringDictKeyValue('Request headers') body = TextAreaField('Request body', [validators.Optional()]) method = SelectField('Request method', choices=valid_method, default=default_method) diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index 5821e6ff..54ffcea7 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -1,16 +1,14 @@ from abc import abstractmethod - from changedetectionio.content_fetchers.base import Fetcher from changedetectionio.strtobool import strtobool - from copy import deepcopy from loguru import logger import hashlib -import os -import re import importlib -import pkgutil import inspect +import os +import pkgutil +import re class difference_detection_processor(): @@ -157,12 +155,12 @@ class difference_detection_processor(): # After init, call run_changedetection() which will do the actual change-detection @abstractmethod - def run_changedetection(self, watch, skip_when_checksum_same=True): + def run_changedetection(self, watch, skip_when_checksum_same: bool = True): update_obj = {'last_notification_error': False, 'last_error': False} some_data = 'xxxxx' update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest() changed_detected = False - return changed_detected, update_obj, ''.encode('utf-8'), b'' + return changed_detected, update_obj, ''.encode('utf-8') def find_sub_packages(package_name): diff --git a/changedetectionio/processors/restock_diff/processor.py b/changedetectionio/processors/restock_diff/processor.py index e14b07b6..eaa48cc5 100644 --- a/changedetectionio/processors/restock_diff/processor.py +++ b/changedetectionio/processors/restock_diff/processor.py @@ -298,4 +298,4 @@ class perform_site_check(difference_detection_processor): # Always record the new checksum update_obj["previous_md5"] = fetched_md5 - return changed_detected, update_obj, snapshot_content.encode('utf-8').strip(), b'' + return changed_detected, update_obj, snapshot_content.encode('utf-8').strip() diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py index 23c9d7e9..a35724b5 100644 --- a/changedetectionio/processors/text_json_diff/processor.py +++ b/changedetectionio/processors/text_json_diff/processor.py @@ -202,7 +202,6 @@ class perform_site_check(difference_detection_processor): render_anchor_tag_content=do_anchor, is_rss=is_rss) # 1874 activate the { + const { color, lines: lineNumbers } = config; + lineNumbers.forEach(lineNumber => { + lineStyles[lineNumber] = color; + }); + }); + + // Function to escape HTML characters + function escapeHtml(text) { + return text.replace(/[&<>"'`=\/]/g, function(s) { + return "&#" + s.charCodeAt(0) + ";"; + }); + } + + // Process each line + const processedLines = lines.map((line, index) => { + const lineNumber = index + 1; // Line numbers start at 1 + const escapedLine = escapeHtml(line); + const color = lineStyles[lineNumber]; + + if (color) { + // Wrap the line in a span with inline style + return `<span style="background-color: ${color}">${escapedLine}</span>`; + } else { + return escapedLine; + } + }); + + // Join the lines back together + const newContent = processedLines.join('\n'); + + // Set the new content as HTML + $pre.html(newContent); + }); + }; + $.fn.miniTabs = function(tabsConfig, options) { + const settings = { + tabClass: 'minitab', + tabsContainerClass: 'minitabs', + activeClass: 'active', + ...(options || {}) + }; + + return this.each(function() { + const $wrapper = $(this); + const $contents = $wrapper.find('div[id]').hide(); + const $tabsContainer = $('<div>', { class: settings.tabsContainerClass }).prependTo($wrapper); + + // Generate tabs + Object.entries(tabsConfig).forEach(([tabTitle, contentSelector], index) => { + const $content = $wrapper.find(contentSelector); + if (index === 0) $content.show(); // Show first content by default + + $('<a>', { + class: `${settings.tabClass}${index === 0 ? ` ${settings.activeClass}` : ''}`, + text: tabTitle, + 'data-target': contentSelector + }).appendTo($tabsContainer); + }); + + // Tab click event + $tabsContainer.on('click', `.${settings.tabClass}`, function(e) { + e.preventDefault(); + const $tab = $(this); + const target = $tab.data('target'); + + // Update active tab + $tabsContainer.find(`.${settings.tabClass}`).removeClass(settings.activeClass); + $tab.addClass(settings.activeClass); + + // Show/hide content + $contents.hide(); + $wrapper.find(target).show(); + }); + }); + }; + + // Object to store ongoing requests by namespace + const requests = {}; + + $.abortiveSingularAjax = function(options) { + const namespace = options.namespace || 'default'; + + // Abort the current request in this namespace if it's still ongoing + if (requests[namespace]) { + requests[namespace].abort(); + } + + // Start a new AJAX request and store its reference in the correct namespace + requests[namespace] = $.ajax(options); + + // Return the current request in case it's needed + return requests[namespace]; + }; +})(jQuery); \ No newline at end of file diff --git a/changedetectionio/static/js/preview.js b/changedetectionio/static/js/preview.js index d85dd9fd..ea6588e9 100644 --- a/changedetectionio/static/js/preview.js +++ b/changedetectionio/static/js/preview.js @@ -1,53 +1,63 @@ -function redirect_to_version(version) { - var currentUrl = window.location.href; - var baseUrl = currentUrl.split('?')[0]; // Base URL without query parameters +function redirectToVersion(version) { + var currentUrl = window.location.href.split('?')[0]; // Base URL without query parameters var anchor = ''; // Check if there is an anchor - if (baseUrl.indexOf('#') !== -1) { - anchor = baseUrl.substring(baseUrl.indexOf('#')); - baseUrl = baseUrl.substring(0, baseUrl.indexOf('#')); + if (currentUrl.indexOf('#') !== -1) { + anchor = currentUrl.substring(currentUrl.indexOf('#')); + currentUrl = currentUrl.substring(0, currentUrl.indexOf('#')); } - window.location.href = baseUrl + '?version=' + version + anchor; + + window.location.href = currentUrl + '?version=' + version + anchor; } -document.addEventListener('keydown', function (event) { - var selectElement = document.getElementById('preview-version'); - if (selectElement) { - var selectedOption = selectElement.querySelector('option:checked'); - if (selectedOption) { - if (event.key === 'ArrowLeft') { - if (selectedOption.previousElementSibling) { - redirect_to_version(selectedOption.previousElementSibling.value); - } - } else if (event.key === 'ArrowRight') { - if (selectedOption.nextElementSibling) { - redirect_to_version(selectedOption.nextElementSibling.value); - } +function setupDateWidget() { + $(document).on('keydown', function (event) { + var $selectElement = $('#preview-version'); + var $selectedOption = $selectElement.find('option:selected'); + + if ($selectedOption.length) { + if (event.key === 'ArrowLeft' && $selectedOption.prev().length) { + redirectToVersion($selectedOption.prev().val()); + } else if (event.key === 'ArrowRight' && $selectedOption.next().length) { + redirectToVersion($selectedOption.next().val()); } } - } -}); + }); + $('#preview-version').on('change', function () { + redirectToVersion($(this).val()); + }); -document.getElementById('preview-version').addEventListener('change', function () { - redirect_to_version(this.value); -}); + var $selectedOption = $('#preview-version option:selected'); + + if ($selectedOption.length) { + var $prevOption = $selectedOption.prev(); + var $nextOption = $selectedOption.next(); -var selectElement = document.getElementById('preview-version'); -if (selectElement) { - var selectedOption = selectElement.querySelector('option:checked'); - if (selectedOption) { - if (selectedOption.previousElementSibling) { - document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value; + if ($prevOption.length) { + $('#btn-previous').attr('href', '?version=' + $prevOption.val()); } else { - document.getElementById('btn-previous').remove() + $('#btn-previous').remove(); } - if (selectedOption.nextElementSibling) { - document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value; + + if ($nextOption.length) { + $('#btn-next').attr('href', '?version=' + $nextOption.val()); } else { - document.getElementById('btn-next').remove() + $('#btn-next').remove(); } - } } + +$(document).ready(function () { + if ($('#preview-version').length) { + setupDateWidget(); + } + + $('#diff-col > pre').highlightLines([ + { + 'color': '#ee0000', + 'lines': triggered_line_numbers + } + ]); +}); diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index 24840520..90ff30e3 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -12,25 +12,6 @@ function toggleOpacity(checkboxSelector, fieldSelector, inverted) { checkbox.addEventListener('change', updateOpacity); } -(function($) { - // Object to store ongoing requests by namespace - const requests = {}; - - $.abortiveSingularAjax = function(options) { - const namespace = options.namespace || 'default'; - - // Abort the current request in this namespace if it's still ongoing - if (requests[namespace]) { - requests[namespace].abort(); - } - - // Start a new AJAX request and store its reference in the correct namespace - requests[namespace] = $.ajax(options); - - // Return the current request in case it's needed - return requests[namespace]; - }; -})(jQuery); function request_textpreview_update() { if (!$('body').hasClass('preview-text-enabled')) { @@ -51,7 +32,19 @@ function request_textpreview_update() { data: data, namespace: 'watchEdit' }).done(function (data) { - $('#filters-and-triggers #text-preview-inner').text(data); + $('#filters-and-triggers #text-preview-before-inner').text(data['before_filter']); + + $('#filters-and-triggers #text-preview-inner') + .text(data['after_filter']) + .highlightLines([ + { + 'color': '#ee0000', + 'lines': data['trigger_line_numbers'] + } + ]); + + + }).fail(function (error) { if (error.statusText === 'abort') { console.log('Request was aborted due to a new request being fired.'); @@ -78,6 +71,7 @@ $(document).ready(function () { const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0); $("#text-preview-inner").css('max-height', (vh-300)+"px"); + $("#text-preview-before-inner").css('max-height', (vh-300)+"px"); // Realtime preview of 'Filters & Text' setup var debounced_request_textpreview_update = request_textpreview_update.debounce(100); @@ -92,6 +86,9 @@ $(document).ready(function () { $('input:visible')[method]('keyup blur change', debounced_request_textpreview_update); $("#filters-and-triggers-tab")[method]('click', debounced_request_textpreview_update); }); - + $('.minitabs-wrapper').miniTabs({ + "Content after filters": "#text-preview-inner", + "Content raw/before filters": "#text-preview-before-inner" + }); }); diff --git a/changedetectionio/static/styles/scss/parts/_minitabs.scss b/changedetectionio/static/styles/scss/parts/_minitabs.scss new file mode 100644 index 00000000..db73acc1 --- /dev/null +++ b/changedetectionio/static/styles/scss/parts/_minitabs.scss @@ -0,0 +1,37 @@ +.minitabs-wrapper { + width: 100%; + + > div[id] { + padding: 20px; + border: 1px solid #ccc; + border-top: none; + } + + .minitabs { + display: flex; + border-bottom: 1px solid #ccc; + } + + .minitab { + flex: 1; + text-align: center; + padding: 12px 0; + text-decoration: none; + color: #333; + background-color: #f1f1f1; + border: 1px solid #ccc; + border-bottom: none; + cursor: pointer; + transition: background-color 0.3s; + } + + .minitab:hover { + background-color: #ddd; + } + + .minitab.active { + background-color: #fff; + font-weight: bold; + } + +} diff --git a/changedetectionio/static/styles/scss/parts/_preview_text_filter.scss b/changedetectionio/static/styles/scss/parts/_preview_text_filter.scss index 1e18c586..4fd30d47 100644 --- a/changedetectionio/static/styles/scss/parts/_preview_text_filter.scss +++ b/changedetectionio/static/styles/scss/parts/_preview_text_filter.scss @@ -1,3 +1,5 @@ +@import "minitabs"; + body.preview-text-enabled { #filters-and-triggers > div { display: flex; /* Establishes Flexbox layout */ @@ -19,18 +21,22 @@ body.preview-text-enabled { #text-preview { position: sticky; - top: 25px; + padding-top: 1rem; display: block !important; } + #activate-text-preview { + background-color: var(--color-grey-500); + } + /* actual preview area */ - #text-preview-inner { + .monospace-preview { background: var(--color-grey-900); border: 1px solid var(--color-grey-600); padding: 1rem; - color: #333; + color: var(--color-grey-100); font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */ - font-size: 12px; + font-size: 70%; overflow-x: scroll; white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */ overflow-wrap: break-word; /* Allows long words to break and wrap to the next line */ @@ -40,6 +46,6 @@ body.preview-text-enabled { #activate-text-preview { right: 0; position: absolute; - z-index: 0; + z-index: 3; box-shadow: 1px 1px 4px var(--color-shadow-jump); } diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 296d8bb7..a254c932 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -428,6 +428,32 @@ html[data-darkmode="true"] #toggle-light-mode .icon-dark { fill: #ff0000 !important; transition: all ease 0.3s !important; } +.minitabs-wrapper { + width: 100%; } + .minitabs-wrapper > div[id] { + padding: 20px; + border: 1px solid #ccc; + border-top: none; } + .minitabs-wrapper .minitabs { + display: flex; + border-bottom: 1px solid #ccc; } + .minitabs-wrapper .minitab { + flex: 1; + text-align: center; + padding: 12px 0; + text-decoration: none; + color: #333; + background-color: #f1f1f1; + border: 1px solid #ccc; + border-bottom: none; + cursor: pointer; + transition: background-color 0.3s; } + .minitabs-wrapper .minitab:hover { + background-color: #ddd; } + .minitabs-wrapper .minitab.active { + background-color: #fff; + font-weight: bold; } + body.preview-text-enabled { /* layout of the page */ /* actual preview area */ } @@ -447,16 +473,18 @@ body.preview-text-enabled { display: none; } body.preview-text-enabled #text-preview { position: sticky; - top: 25px; + padding-top: 1rem; display: block !important; } - body.preview-text-enabled #text-preview-inner { + body.preview-text-enabled #activate-text-preview { + background-color: var(--color-grey-500); } + body.preview-text-enabled .monospace-preview { background: var(--color-grey-900); border: 1px solid var(--color-grey-600); padding: 1rem; - color: #333; + color: var(--color-grey-100); font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */ - font-size: 12px; + font-size: 70%; overflow-x: scroll; white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */ @@ -466,7 +494,7 @@ body.preview-text-enabled { #activate-text-preview { right: 0; position: absolute; - z-index: 0; + z-index: 3; box-shadow: 1px 1px 4px var(--color-shadow-jump); } body { diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 2d0014b9..81ed8bb5 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -24,7 +24,7 @@ const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}"; const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}"; </script> - +<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script> <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script> <script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script> <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script> @@ -371,10 +371,10 @@ nav ") }} <span class="pure-form-message-inline"> <ul> + <li>Matching text will be <strong>removed</strong> from the text snapshot</li> <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li> <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li> <li>Changing this will affect the comparison checksum which may trigger an alert</li> - <li>Use the preview/show current tab to see ignores</li> </ul> </span> @@ -422,14 +422,21 @@ Unavailable") }} <script> const preview_text_edit_filters_url="{{url_for('watch_get_preview_rendered', uuid=uuid)}}"; </script> - <span><strong>Preview of the text that is used for changedetection after all filters run.</strong></span><br> + <br> {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#} - <p> - <div id="text-preview-inner"></div> - </p> + + <div class="minitabs-wrapper"> + <div id="text-preview-inner" class="monospace-preview"> + <p>Loading...</p> + </div> + <div id="text-preview-before-inner" style="display: none;" class="monospace-preview"> + <p>Loading...</p> + </div> + </div> </div> </div> </div> + {% endif %} {# rendered sub Template #} {% if extra_form_content %} diff --git a/changedetectionio/templates/preview.html b/changedetectionio/templates/preview.html index 28431fe9..6915da33 100644 --- a/changedetectionio/templates/preview.html +++ b/changedetectionio/templates/preview.html @@ -3,11 +3,13 @@ {% block content %} <script> const screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid)}}"; + const triggered_line_numbers = {{ triggered_line_numbers|tojson }}; {% if last_error_screenshot %} const error_screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}"; {% endif %} const highlight_submit_ignore_url = "{{url_for('highlight_submit_ignore_url', uuid=uuid)}}"; </script> + <script src="{{url_for('static_content', group='js', filename='plugins.js')}}"></script> <script src="{{ url_for('static_content', group='js', filename='diff-overview.js') }}" defer></script> <script src="{{ url_for('static_content', group='js', filename='preview.js') }}" defer></script> <script src="{{ url_for('static_content', group='js', filename='tabs.js') }}" defer></script> @@ -67,16 +69,15 @@ <div class="tab-pane-inner" id="text"> <div class="snapshot-age">{{ current_version|format_timestamp_timeago }}</div> - <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span> <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span> <table> <tbody> <tr> <td id="diff-col" class="highlightable-filter"> - {% for row in content %} - <div class="{{ row.classes }}">{{ row.line }}</div> - {% endfor %} + <pre style="border-left: 2px solid #ddd;"> +{{ content }} + </pre> </td> </tr> </tbody> diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index e9911770..ad41e7b6 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -172,11 +172,11 @@ nav <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br> <span class="pure-form-message-inline"> <ul> + <li>Matching text will be <strong>removed</strong> from the text snapshot</li> <li>Note: This is applied globally in addition to the per-watch rules.</li> <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li> <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li> <li>Changing this will affect the comparison checksum which may trigger an alert</li> - <li>Use the preview/show current tab to see ignores</li> </ul> </span> </fieldset> diff --git a/changedetectionio/tests/proxy_list/test_select_custom_proxy.py b/changedetectionio/tests/proxy_list/test_select_custom_proxy.py index 1ae7ac69..266c46e1 100644 --- a/changedetectionio/tests/proxy_list/test_select_custom_proxy.py +++ b/changedetectionio/tests/proxy_list/test_select_custom_proxy.py @@ -44,7 +44,7 @@ def test_select_custom(client, live_server, measure_memory_usage): follow_redirects=True ) # We should see something via proxy - assert b'<div class=""> - 0.' in res.data + assert b' - 0.' in res.data # # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default diff --git a/changedetectionio/tests/test_add_replace_remove_filter.py b/changedetectionio/tests/test_add_replace_remove_filter.py index 86c67c05..7746c871 100644 --- a/changedetectionio/tests/test_add_replace_remove_filter.py +++ b/changedetectionio/tests/test_add_replace_remove_filter.py @@ -39,9 +39,8 @@ def test_setup(client, live_server, measure_memory_usage): live_server_setup(live_server) def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage): - + #live_server_setup(live_server) # Give the endpoint time to spin up - time.sleep(1) set_original() # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -152,7 +151,9 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa # A line thats not the trigger should not trigger anything res = client.get(url_for("form_watch_checknow"), follow_redirects=True) + assert b'1 watches queued for rechecking.' in res.data + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data diff --git a/changedetectionio/tests/test_extract_regex.py b/changedetectionio/tests/test_extract_regex.py index da52da33..522cff0c 100644 --- a/changedetectionio/tests/test_extract_regex.py +++ b/changedetectionio/tests/test_extract_regex.py @@ -115,9 +115,9 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage): # Plaintext that doesnt look like a regex should match also assert b'and this should be' in res.data - assert b'<div class="">Something' in res.data - assert b'<div class="">across 6 billion multiple' in res.data - assert b'<div class="">lines' in res.data + assert b'Something' in res.data + assert b'across 6 billion multiple' in res.data + assert b'lines' in res.data # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking) assert b'aaand something lines' not in res.data @@ -183,20 +183,19 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag follow_redirects=True ) - # Class will be blank for now because the frontend didnt apply the diff - assert b'<div class="">1000 online' in res.data + assert b'1000 online' in res.data # All regex matching should be here - assert b'<div class="">2000 online' in res.data + assert b'2000 online' in res.data # Both regexs should be here - assert b'<div class="">80 guests' in res.data + assert b'80 guests' in res.data # Regex with flag handling should be here - assert b'<div class="">SomeCase insensitive 3456' in res.data + assert b'SomeCase insensitive 3456' in res.data # Singular group from /somecase insensitive (345\d)/i - assert b'<div class="">3456' in res.data + assert b'3456' in res.data # Regex with multiline flag handling should be here diff --git a/changedetectionio/tests/test_ignorehighlighter.py b/changedetectionio/tests/test_ignore.py similarity index 89% rename from changedetectionio/tests/test_ignorehighlighter.py rename to changedetectionio/tests/test_ignore.py index 58ecfeb4..c710d7d4 100644 --- a/changedetectionio/tests/test_ignorehighlighter.py +++ b/changedetectionio/tests/test_ignore.py @@ -23,7 +23,7 @@ def set_original_ignore_response(): f.write(test_return_data) -def test_highlight_ignore(client, live_server, measure_memory_usage): +def test_ignore(client, live_server, measure_memory_usage): live_server_setup(live_server) set_original_ignore_response() test_url = url_for('test_endpoint', _external=True) @@ -51,9 +51,9 @@ def test_highlight_ignore(client, live_server, measure_memory_usage): # Should return a link assert b'href' in res.data - # And it should register in the preview page + # It should not be in the preview anymore res = client.get(url_for("preview_page", uuid=uuid)) - assert b'<div class="ignored">oh yeah 456' in res.data + assert b'<div class="ignored">oh yeah 456' not in res.data # Should be in base.html assert b'csrftoken' in res.data diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index 60a2f3a2..37d21d1b 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -79,14 +79,14 @@ def set_modified_ignore_response(): f.write(test_return_data) +# Ignore text now just removes it entirely, is a LOT more simpler code this way + def test_check_ignore_text_functionality(client, live_server, measure_memory_usage): # Use a mix of case in ZzZ to prove it works case-insensitive. ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff" set_original_ignore_response() - # Give the endpoint time to spin up - time.sleep(1) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -151,12 +151,10 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa res = client.get(url_for("index")) assert b'unviewed' in res.data - # Check the preview/highlighter, we should be able to see what we ignored, but it should be highlighted - # We only introduce the "modified" content that includes what we ignore so we can prove the newest version also displays - # at /preview res = client.get(url_for("preview_page", uuid="first")) - # We should be able to see what we ignored - assert b'<div class="ignored">new ignore stuff' in res.data + + # Should no longer be in the preview + assert b'new ignore stuff' not in res.data res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index d139e1cf..40f2a29b 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -499,7 +499,7 @@ def test_correct_header_detect(client, live_server, measure_memory_usage): ) assert b'"hello": 123,' in res.data - assert b'"world": 123</div>' in res.data + assert b'"world": 123' in res.data res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_trigger.py b/changedetectionio/tests/test_trigger.py index d13fe79e..a47c1fed 100644 --- a/changedetectionio/tests/test_trigger.py +++ b/changedetectionio/tests/test_trigger.py @@ -2,7 +2,7 @@ import time from flask import url_for -from . util import live_server_setup +from .util import live_server_setup, wait_for_all_checks def set_original_ignore_response(): @@ -59,12 +59,9 @@ def test_trigger_functionality(client, live_server, measure_memory_usage): live_server_setup(live_server) - sleep_time_for_fetch_thread = 3 trigger_text = "Add to cart" set_original_ignore_response() - # Give the endpoint time to spin up - time.sleep(1) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -89,14 +86,14 @@ def test_trigger_functionality(client, live_server, measure_memory_usage): ) assert b"Updated watch." in res.data + wait_for_all_checks(client) # Check it saved res = client.get( url_for("edit_page", uuid="first"), ) assert bytes(trigger_text.encode('utf-8')) in res.data - # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + # so that we set the state to 'unviewed' after all the edits client.get(url_for("diff_history_page", uuid="first")) @@ -104,8 +101,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) - # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) @@ -117,19 +113,17 @@ def test_trigger_functionality(client, live_server, measure_memory_usage): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) - # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) assert b'unviewed' not in res.data # Now set the content which contains the trigger text - time.sleep(sleep_time_for_fetch_thread) set_modified_with_trigger_text_response() client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data @@ -142,4 +136,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage): res = client.get(url_for("preview_page", uuid="first")) # We should be able to see what we triggered on - assert b'<div class="triggered">Add to cart' in res.data + # The JS highlighter should tell us which lines (also used in the live-preview) + assert b'const triggered_line_numbers = [6]' in res.data + assert b'Add to cart' in res.data + diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index e8b5d855..4f50ad0d 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -161,8 +161,8 @@ def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usag follow_redirects=True ) - assert b'<div class="">Stock Alert (UK): RPi CM4' in res.data - assert b'<div class="">Stock Alert (UK): Big monitor' in res.data + assert b'Stock Alert (UK): RPi CM4' in res.data + assert b'Stock Alert (UK): Big monitor' in res.data res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 1bd82989..06daf337 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -278,7 +278,7 @@ class update_worker(threading.Thread): update_handler.call_browser() - changed_detected, update_obj, contents, content_after_filters = update_handler.run_changedetection( + changed_detected, update_obj, contents = update_handler.run_changedetection( watch=watch, skip_when_checksum_same=skip_when_same_checksum, )