From d5f574ca17251ae7bb610b93d4b3f468af805a2a Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 21 Mar 2023 19:16:13 +0100 Subject: [PATCH] Notifications - Include triggered text token as `{{triggered_text}}` in notifications, so you can send just the content that matches. (#1485) --- changedetectionio/html_tools.py | 15 ++++ changedetectionio/notification.py | 28 ++++--- .../templates/_common_fields.jinja | 4 + .../tests/test_add_replace_remove_filter.py | 83 ++++++++++++++++++- changedetectionio/update_worker.py | 19 ++++- 5 files changed, 129 insertions(+), 20 deletions(-) diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 49e95663..63848030 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -287,3 +287,18 @@ def workarounds_for_obfuscations(content): content = re.sub('', '', content) return content + + +def get_triggered_text(content, trigger_text): + triggered_text = [] + result = strip_ignore_text(content=content, + wordlist=trigger_text, + mode="line numbers") + + i = 1 + for p in content.splitlines(): + if i in result: + triggered_text.append(p) + i += 1 + + return triggered_text diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 15cc15f1..44fec5c6 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -5,17 +5,18 @@ import json valid_tokens = { 'base_url': '', - 'watch_url': '', - 'watch_uuid': '', - 'watch_title': '', - 'watch_tag': '', + 'current_snapshot': '', 'diff': '', 'diff_added': '', - 'diff_removed': '', 'diff_full': '', + 'diff_removed': '', 'diff_url': '', 'preview_url': '', - 'current_snapshot': '' + 'triggered_text': '', + 'watch_tag': '', + 'watch_title': '', + 'watch_url': '', + 'watch_uuid': '', } default_notification_format_for_watch = 'System default' @@ -211,17 +212,18 @@ def create_notification_parameters(n_object, datastore): tokens.update( { 'base_url': base_url if base_url is not None else '', - 'watch_url': watch_url, - 'watch_uuid': uuid, - 'watch_title': watch_title if watch_title is not None else '', - 'watch_tag': watch_tag if watch_tag is not None else '', - 'diff_url': diff_url, + 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '', 'diff': n_object.get('diff', ''), # Null default in the case we use a test 'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test - 'diff_removed': n_object.get('diff_removed', ''), # Null default in the case we use a test 'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test + 'diff_removed': n_object.get('diff_removed', ''), # Null default in the case we use a test + 'diff_url': diff_url, 'preview_url': preview_url, - 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '' + 'triggered_text': n_object.get('triggered_text', ''), + 'watch_tag': watch_tag if watch_tag is not None else '', + 'watch_title': watch_title if watch_title is not None else '', + 'watch_url': watch_url, + 'watch_uuid': uuid, }) return tokens diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index 29214edb..96128592 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -104,6 +104,10 @@ The current snapshot value, useful when combined with JSON or CSS filters + + {{ '{{triggered_text}}' }} + Text that tripped the trigger from filters +
diff --git a/changedetectionio/tests/test_add_replace_remove_filter.py b/changedetectionio/tests/test_add_replace_remove_filter.py index aac41f92..fa74fe96 100644 --- a/changedetectionio/tests/test_add_replace_remove_filter.py +++ b/changedetectionio/tests/test_add_replace_remove_filter.py @@ -6,7 +6,7 @@ from .util import live_server_setup from changedetectionio import html_tools -def set_original(excluding=None): +def set_original(excluding=None, add_line=None): test_return_data = """

Some initial text

@@ -19,6 +19,11 @@ def set_original(excluding=None): """ + if add_line: + c=test_return_data.splitlines() + c.insert(5, add_line) + test_return_data = "\n".join(c) + if excluding: output = "" for i in test_return_data.splitlines(): @@ -30,10 +35,10 @@ def set_original(excluding=None): with open("test-datastore/endpoint-content.txt", "w") as f: f.write(test_return_data) - -def test_check_removed_line_contains_trigger(client, live_server): +def test_setup(client, live_server): live_server_setup(live_server) +def test_check_removed_line_contains_trigger(client, live_server): sleep_time_for_fetch_thread = 3 # Give the endpoint time to spin up @@ -97,3 +102,75 @@ def test_check_removed_line_contains_trigger(client, live_server): res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data + + +def test_check_add_line_contains_trigger(client, live_server): + + sleep_time_for_fetch_thread = 3 + + # Give the endpoint time to spin up + time.sleep(1) + test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}" + + res = client.post( + url_for("settings_page"), + data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}", + "application-notification_body": 'triggered text was -{{triggered_text}}-', + # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation + "application-notification_urls": test_notification_url, + "application-minutes_between_check": 180, + "application-fetch_backend": "html_requests" + }, + follow_redirects=True + ) + assert b'Settings updated' in res.data + + set_original() + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Give the thread time to pick it up + time.sleep(sleep_time_for_fetch_thread) + + # Goto the edit page, add our ignore text + # Add our URL to the import page + res = client.post( + url_for("edit_page", uuid="first"), + data={"trigger_text": 'Oh yes please', + "url": test_url, + 'fetch_backend': "html_requests", + 'filter_text_removed': '', + 'filter_text_added': 'y'}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + time.sleep(sleep_time_for_fetch_thread) + set_original(excluding='Something irrelevant') + + # A line thats not the trigger should not trigger anything + res = client.get(url_for("form_watch_checknow"), follow_redirects=True) + assert b'1 watches queued for rechecking.' in res.data + time.sleep(sleep_time_for_fetch_thread) + res = client.get(url_for("index")) + assert b'unviewed' not in res.data + + # The trigger line is ADDED, this should trigger + set_original(add_line='

Oh yes please

') + client.get(url_for("form_watch_checknow"), follow_redirects=True) + time.sleep(sleep_time_for_fetch_thread) + res = client.get(url_for("index")) + assert b'unviewed' in res.data + + with open("test-datastore/notification.txt", 'r') as f: + response= f.read() + assert '-Oh yes please-' in response + + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 85bb0ff6..9fb49c45 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -69,17 +69,28 @@ class update_worker(threading.Thread): else: line_feed_sep = "\n" + # Add text that was triggered snapshot_contents = watch.get_history_snapshot(dates[-1]) + trigger_text = watch.get('trigger_text', []) + triggered_text = '' + + if len(trigger_text): + from . import html_tools + triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text) + if triggered_text: + triggered_text = line_feed_sep.join(triggered_text) + n_object.update({ - 'watch_url': watch['url'], - 'uuid': watch_uuid, - 'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None, 'current_snapshot': snapshot_contents, 'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep), 'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep), + 'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep), 'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep), - 'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep) + 'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None, + 'triggered_text': triggered_text, + 'uuid': watch_uuid, + 'watch_url': watch['url'], }) logging.info (">> SENDING NOTIFICATION") self.notification_q.put(n_object)