From 5f150c4f03c36aaa99cd87ef08d90fcf2af605de Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 27 Jun 2023 17:05:32 +0200 Subject: [PATCH 1/4] Bug - Fix watch clone (#1647) --- changedetectionio/api/api_v1.py | 8 ++- changedetectionio/blueprint/tags/__init__.py | 10 +++ changedetectionio/importer.py | 8 +-- changedetectionio/store.py | 28 ++++---- changedetectionio/tests/test_api.py | 3 +- changedetectionio/tests/test_group.py | 70 ++++++++++++++++++-- changedetectionio/tests/test_import.py | 1 + 7 files changed, 98 insertions(+), 30 deletions(-) diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py index 07e4022e..e0c7125c 100644 --- a/changedetectionio/api/api_v1.py +++ b/changedetectionio/api/api_v1.py @@ -219,13 +219,15 @@ class CreateWatch(Resource): extras = copy.deepcopy(json_data) - # Because we renamed 'tag' to 'tags' but dont want to change the API (can do this in v2 of the API) + # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API) + tags = None if extras.get('tag'): - extras['tags'] = extras.get('tag') + tags = extras.get('tag') + del extras['tag'] del extras['url'] - new_uuid = self.datastore.add_watch(url=url, extras=extras) + new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags) if new_uuid: self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True})) return {'uuid': new_uuid}, 201 diff --git a/changedetectionio/blueprint/tags/__init__.py b/changedetectionio/blueprint/tags/__init__.py index 3e1c732a..0ab3d477 100644 --- a/changedetectionio/blueprint/tags/__init__.py +++ b/changedetectionio/blueprint/tags/__init__.py @@ -76,6 +76,16 @@ def construct_blueprint(datastore: ChangeDetectionStore): flash(f"Tag unlinked removed from {unlinked} watches") return redirect(url_for('tags.tags_overview_page')) + @tags_blueprint.route("/delete_all", methods=['GET']) + @login_optionally_required + def delete_all(): + for watch_uuid, watch in datastore.data['watching'].items(): + watch['tags'] = [] + datastore.data['settings']['application']['tags'] = {} + + flash(f"All tags deleted") + return redirect(url_for('tags.tags_overview_page')) + @tags_blueprint.route("/edit/", methods=['GET']) @login_optionally_required def form_tag_edit(uuid): diff --git a/changedetectionio/importer.py b/changedetectionio/importer.py index a420ea1e..f046c895 100644 --- a/changedetectionio/importer.py +++ b/changedetectionio/importer.py @@ -85,7 +85,8 @@ class import_distill_io_json(Importer): now = time.time() self.new_uuids=[] - + # @todo Use JSONSchema like in the API to validate here. + try: data = json.loads(data.strip()) except json.decoder.JSONDecodeError: @@ -120,11 +121,8 @@ class import_distill_io_json(Importer): except IndexError: pass -# Does this need to be here anymore? - if d.get('tags', False): - extras['tags'] = ", ".join(d['tags']) - new_uuid = datastore.add_watch(url=d['uri'].strip(), + tag=",".join(d.get('tags', [])), extras=extras, write_to_disk_now=False) diff --git a/changedetectionio/store.py b/changedetectionio/store.py index d7d66b4d..f0fb3a72 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -205,10 +205,9 @@ class ChangeDetectionStore: # Clone a watch by UUID def clone(self, uuid): - url = self.data['watching'][uuid]['url'] - tag = self.data['watching'][uuid].get('tags',[]) + url = self.data['watching'][uuid].get('url') extras = self.data['watching'][uuid] - new_uuid = self.add_watch(url=url, tag_uuids=tag, extras=extras) + new_uuid = self.add_watch(url=url, extras=extras) return new_uuid def url_exists(self, url): @@ -248,12 +247,9 @@ class ChangeDetectionStore: if extras is None: extras = {} - # should always be str - if tag is None or not tag: - tag = '' - # Incase these are copied across, assume it's a reference and deepcopy() apply_extras = deepcopy(extras) + apply_extras['tags'] = [] if not apply_extras.get('tags') else apply_extras.get('tags') # Was it a share link? try to fetch the data if (url.startswith("https://changedetection.io/share/")): @@ -303,20 +299,22 @@ class ChangeDetectionStore: flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error') return None - - # #Re 569 - # Could be in 'tags', var or extras, smash them together and strip - apply_extras['tags'] = [] - if tag or extras.get('tags'): - tags = list(filter(None, list(set().union(tag.split(','), extras.get('tags', '').split(','))))) - for t in list(map(str.strip, tags)): + if tag and type(tag) == str: + # Then it's probably a string of the actual tag by name, split and add it + for t in tag.split(','): # for each stripped tag, add tag as UUID - apply_extras['tags'].append(self.add_tag(t)) + for a_t in t.split(','): + tag_uuid = self.add_tag(a_t) + apply_extras['tags'].append(tag_uuid) # Or if UUIDs given directly if tag_uuids: apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids)) + # Make any uuids unique + if apply_extras.get('tags'): + apply_extras['tags'] = list(set(apply_extras.get('tags'))) + new_watch = Watch.model(datastore_path=self.datastore_path, url=url) new_uuid = new_watch.get('uuid') diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index c19812ab..23051454 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -267,7 +267,7 @@ def test_api_watch_PUT_update(client, live_server): #live_server_setup(live_server) api_key = extract_api_key_from_UI(client) - time.sleep(1) + # Create a watch set_original_response() test_url = url_for('test_endpoint', _external=True, @@ -283,7 +283,6 @@ def test_api_watch_PUT_update(client, live_server): assert res.status_code == 201 - time.sleep(1) # Get a listing, it will be the first one res = client.get( diff --git a/changedetectionio/tests/test_group.py b/changedetectionio/tests/test_group.py index 1518356c..2bac9710 100644 --- a/changedetectionio/tests/test_group.py +++ b/changedetectionio/tests/test_group.py @@ -2,7 +2,7 @@ import time from flask import url_for -from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name +from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name, extract_UUID_from_client import os @@ -154,6 +154,10 @@ def test_tag_add_in_ui(client, live_server): ) assert b"Tag added" in res.data assert b"new-test-tag" in res.data + + res = client.get(url_for("tags.delete_all"), follow_redirects=True) + assert b'All tags deleted' in res.data + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -219,12 +223,10 @@ def test_group_tag_notification(client, live_server): assert "test-tag" in notification_submission assert "other-tag" in notification_submission - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) - assert b'Deleted' in res.data - #@todo Test that multiple notifications fired #@todo Test that each of multiple notifications with different settings - + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data def test_limit_tag_ui(client, live_server): #live_server_setup(live_server) @@ -260,3 +262,61 @@ def test_limit_tag_ui(client, live_server): assert b'test-tag' in res.data assert res.data.count(b'processor-text_json_diff') == 20 assert b"object at" not in res.data + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + res = client.get(url_for("tags.delete_all"), follow_redirects=True) + assert b'All tags deleted' in res.data +def test_clone_tag_on_import(client, live_server): + #live_server_setup(live_server) + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url + " test-tag, another-tag\r\n"}, + follow_redirects=True + ) + + assert b"1 Imported" in res.data + + res = client.get(url_for("index")) + assert b'test-tag' in res.data + assert b'another-tag' in res.data + + watch_uuid = extract_UUID_from_client(client) + res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True) + + assert b'Cloned' in res.data + # 2 times plus the top link to tag + assert res.data.count(b'test-tag') == 3 + assert res.data.count(b'another-tag') == 3 + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + +def test_clone_tag_on_quickwatchform_add(client, live_server): + #live_server_setup(live_server) + + test_url = url_for('test_endpoint', _external=True) + + res = client.post( + url_for("form_quick_watch_add"), + data={"url": test_url, "tags": ' test-tag, another-tag '}, + follow_redirects=True + ) + + assert b"Watch added" in res.data + + res = client.get(url_for("index")) + assert b'test-tag' in res.data + assert b'another-tag' in res.data + + watch_uuid = extract_UUID_from_client(client) + res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True) + + assert b'Cloned' in res.data + # 2 times plus the top link to tag + assert res.data.count(b'test-tag') == 3 + assert res.data.count(b'another-tag') == 3 + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + + res = client.get(url_for("tags.delete_all"), follow_redirects=True) + assert b'All tags deleted' in res.data \ No newline at end of file diff --git a/changedetectionio/tests/test_import.py b/changedetectionio/tests/test_import.py index 88f541b4..a044ff79 100644 --- a/changedetectionio/tests/test_import.py +++ b/changedetectionio/tests/test_import.py @@ -112,6 +112,7 @@ def test_import_distillio(client, live_server): # did the tags work? res = client.get( url_for("index")) + # check tags assert b"nice stuff" in res.data assert b"nerd-news" in res.data From 572f71299fe049ef823d86c8949ca133e8363f1a Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 27 Jun 2023 18:27:33 +0200 Subject: [PATCH 2/4] Bug fix - Notification settings were not cascading from global -> tags -> watch correctly in some cases (#1654) --- changedetectionio/store.py | 2 +- changedetectionio/tests/test_notification.py | 9 +- changedetectionio/update_worker.py | 93 +++++++++----------- 3 files changed, 47 insertions(+), 57 deletions(-) diff --git a/changedetectionio/store.py b/changedetectionio/store.py index f0fb3a72..48443b53 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -566,7 +566,7 @@ class ChangeDetectionStore: def add_tag(self, name): # If name exists, return that n = name.strip().lower() - print (f">>> Adding new tag - '{n}") + print (f">>> Adding new tag - '{n}'") if not n: return False diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index a94909ef..cc5e6588 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -24,9 +24,6 @@ def test_check_notification(client, live_server): #live_server_setup(live_server) set_original_response() - # Give the endpoint time to spin up - time.sleep(1) - # Re 360 - new install should have defaults set res = client.get(url_for("settings_page")) notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json') @@ -142,8 +139,7 @@ def test_check_notification(client, live_server): # Did we see the URL that had a change, in the notification? # Diff was correctly executed - assert test_url in notification_submission - assert ':-)' in notification_submission + assert "Diff Full: Some initial text" in notification_submission assert "Diff: (changed) Which is across multiple lines" in notification_submission assert "(into) which has this one new line" in notification_submission @@ -156,7 +152,8 @@ def test_check_notification(client, live_server): assert "preview/" in notification_submission assert ":-)" in notification_submission assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission - + assert test_url in notification_submission + assert ':-)' in notification_submission # Check the attachment was added, and that it is a JPEG from the original PNG notification_submission_object = json.loads(notification_submission) # We keep PNG screenshots for now diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 5287e68b..8c43a148 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -65,15 +65,45 @@ class update_worker(threading.Thread): logging.info (">> SENDING NOTIFICATION") self.notification_q.put(n_object) - - def send_content_changed_notification(self, watch_uuid): + # Prefer - Individual watch settings > Tag settings > Global settings (in that order) + def _check_cascading_vars(self, var_name, watch): from changedetectionio.notification import ( - default_notification_format_for_watch + default_notification_format_for_watch, + default_notification_body, + default_notification_title, ) + + # Would be better if this was some kind of Object where Watch can reference the parent datastore etc + v = watch.get(var_name) + if v and not watch.get('notification_muted'): + return v + + tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid')) + if tags: + for tag_uuid, tag in tags.items(): + v = tag.get(var_name) + if v and not tag.get('notification_muted'): + return v + + if self.datastore.data['settings']['application'].get(var_name): + return self.datastore.data['settings']['application'].get(var_name) + + # Otherwise could be defaults + if var_name == 'notification_format': + return default_notification_format_for_watch + if var_name == 'notification_body': + return default_notification_body + if var_name == 'notification_title': + return default_notification_title + + return None + + def send_content_changed_notification(self, watch_uuid): + n_object = {} - watch = self.datastore.data['watching'].get(watch_uuid, False) + watch = self.datastore.data['watching'].get(watch_uuid) if not watch: return @@ -87,57 +117,20 @@ class update_worker(threading.Thread): ) # Should be a better parent getter in the model object - # Prefer - Individual watch settings > Tag settings > Global settings (in that order) - n_object['notification_urls'] = watch.get('notification_urls') - n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \ - self.datastore.data['settings']['application']['notification_title'] - - n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \ - self.datastore.data['settings']['application']['notification_body'] - - n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \ - self.datastore.data['settings']['application']['notification_format'] + # Prefer - Individual watch settings > Tag settings > Global settings (in that order) + n_object['notification_urls'] = self._check_cascading_vars('notification_urls', watch) + n_object['notification_title'] = self._check_cascading_vars('notification_title', watch) + n_object['notification_body'] = self._check_cascading_vars('notification_body', watch) + n_object['notification_format'] = self._check_cascading_vars('notification_format', watch) # (Individual watch) Only prepare to notify if the rules above matched - sent = False - if 'notification_urls' in n_object and n_object['notification_urls']: - sent = True + queued = False + if n_object and n_object.get('notification_urls'): + queued = True self.queue_notification_for_watch(n_object, watch) - # (Group tags) try by group tag - if not sent: - # Else, Try by tag, and use system default vars for format, body etc as fallback - tags = self.datastore.get_all_tags_for_watch(uuid=watch_uuid) - for tag_uuid, tag in tags.items(): - n_object = {} - n_object['notification_urls'] = tag.get('notification_urls') - - n_object['notification_title'] = tag.get('notification_title') if tag.get('notification_title') else \ - self.datastore.data['settings']['application']['notification_title'] - - n_object['notification_body'] = tag.get('notification_body') if tag.get('notification_body') else \ - self.datastore.data['settings']['application']['notification_body'] - - n_object['notification_format'] = tag.get('notification_format') if tag.get('notification_format') != default_notification_format_for_watch else \ - self.datastore.data['settings']['application']['notification_format'] - - if 'notification_urls' in n_object and n_object.get('notification_urls') and not tag.get('notification_muted'): - sent = True - self.queue_notification_for_watch(n_object, watch) - - # (Group tags) try by global - if not sent: - # leave this as is, but repeat in a loop for each tag also - n_object['notification_urls'] = self.datastore.data['settings']['application'].get('notification_urls') - n_object['notification_title'] = self.datastore.data['settings']['application'].get('notification_title') - n_object['notification_body'] = self.datastore.data['settings']['application'].get('notification_body') - n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format') - if n_object.get('notification_urls') and n_object.get('notification_body') and n_object.get('notification_title'): - sent = True - self.queue_notification_for_watch(n_object, watch) - - return sent + return queued def send_filter_failure_notification(self, watch_uuid): From e70920195594c7183342ba9e7e9ef79feb583de3 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 27 Jun 2023 18:28:18 +0200 Subject: [PATCH 3/4] 0.43.1 --- changedetectionio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index fa6e82a5..77f896ab 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter from changedetectionio import html_tools from changedetectionio.api import api_v1 -__version__ = '0.43' +__version__ = '0.43.1' datastore = None From 2f777ea3bb688bdee8c46af90f1f6a0c2b31a089 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 28 Jun 2023 00:03:02 +0200 Subject: [PATCH 4/4] Fix - Watches werent falling back to global default formats correctly when required (#1656) --- changedetectionio/notification.py | 6 ++++++ changedetectionio/update_worker.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index a9b31702..ca5ea21d 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -92,6 +92,12 @@ def process_notification(n_object, datastore): n_object.get('notification_format', default_notification_format), valid_notification_formats[default_notification_format], ) + + # If we arrived with 'System default' then look it up + if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch: + # Initially text or whatever + n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]) + # https://github.com/caronc/apprise/wiki/Development_LogCapture # Anything higher than or equal to WARNING (which covers things like Connection errors) diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 8c43a148..7c2c5792 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -71,7 +71,7 @@ class update_worker(threading.Thread): from changedetectionio.notification import ( default_notification_format_for_watch, default_notification_body, - default_notification_title, + default_notification_title )