diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 14a7570c..0da33bc8 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1432,6 +1432,7 @@ def changedetection_app(config=None, datastore_o=None): ) # Use the last loaded HTML as the input update_handler.fetcher.content = decompressed_data + update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type') try: changed_detected, update_obj, contents, text_after_filter = update_handler.run_changedetection( watch=tmp_watch, diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py index 3b95c91c..36d12384 100644 --- a/changedetectionio/model/__init__.py +++ b/changedetectionio/model/__init__.py @@ -18,6 +18,7 @@ class watch_base(dict): 'check_count': 0, 'check_unique_lines': False, # On change-detected, compare against all history if its something new 'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine. + 'content-type': None, 'date_created': None, 'extract_text': [], # Extract text by regex after filters 'extract_title_as_title': False, diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py index 8ad1a0ef..23c9d7e9 100644 --- a/changedetectionio/processors/text_json_diff/processor.py +++ b/changedetectionio/processors/text_json_diff/processor.py @@ -207,7 +207,7 @@ class perform_site_check(difference_detection_processor): stripped_text_from_html = '\n'.join(line.strip() for line in stripped_text_from_html.replace("\n\n", "\n").splitlines()) if watch.get('remove_duplicate_lines'): - stripped_text_from_html = '\n'.join(dict.fromkeys(line.strip() for line in stripped_text_from_html.replace("\n\n", "\n").splitlines())) + stripped_text_from_html = '\n'.join(dict.fromkeys(line for line in stripped_text_from_html.replace("\n\n", "\n").splitlines())) if watch.get('sort_text_alphabetically'): # Note: Because a

something

will add an extra line feed to signify the paragraph gap diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index 0a6c96c4..24840520 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -34,6 +34,7 @@ function toggleOpacity(checkboxSelector, fieldSelector, inverted) { function request_textpreview_update() { if (!$('body').hasClass('preview-text-enabled')) { + console.error("Preview text was requested but body tag was not setup") return } @@ -77,20 +78,19 @@ $(document).ready(function () { const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0); $("#text-preview-inner").css('max-height', (vh-300)+"px"); + + // Realtime preview of 'Filters & Text' setup var debounced_request_textpreview_update = request_textpreview_update.debounce(100); $("#activate-text-preview").click(function (e) { - $(this).fadeOut(); $('body').toggleClass('preview-text-enabled') - request_textpreview_update(); - $("#text-preview-refresh").click(function (e) { - request_textpreview_update(); - }); - $('textarea:visible').on('keyup blur', debounced_request_textpreview_update); - $('input:visible').on('keyup blur change', debounced_request_textpreview_update); - $("#filters-and-triggers-tab").on('click', debounced_request_textpreview_update); + const method = $('body').hasClass('preview-text-enabled') ? 'on' : 'off'; + $("#text-preview-refresh")[method]('click', debounced_request_textpreview_update); + $('textarea:visible')[method]('keyup blur', debounced_request_textpreview_update); + $('input:visible')[method]('keyup blur change', debounced_request_textpreview_update); + $("#filters-and-triggers-tab")[method]('click', debounced_request_textpreview_update); }); }); diff --git a/changedetectionio/tests/test_encoding.py b/changedetectionio/tests/test_encoding.py index 08bdf170..6d0aa1ca 100644 --- a/changedetectionio/tests/test_encoding.py +++ b/changedetectionio/tests/test_encoding.py @@ -3,7 +3,7 @@ import time from flask import url_for -from .util import live_server_setup, wait_for_all_checks +from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client import pytest @@ -38,6 +38,11 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage): # Give the thread time to pick it up wait_for_all_checks(client) + + # Content type recording worked + uuid = extract_UUID_from_client(client) + assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html" + res = client.get( url_for("preview_page", uuid="first"), follow_redirects=True diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 95d02e2b..1bd82989 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -491,6 +491,8 @@ class update_worker(threading.Thread): if not self.datastore.data['watching'].get(uuid): continue + update_obj['content-type'] = update_handler.fetcher.get_all_headers().get('content-type', '').lower() + # Mark that we never had any failures if not watch.get('ignore_status_codes'): update_obj['consecutive_filter_failures'] = 0