diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 30306208..4abcfc4e 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -1,16 +1,5 @@ #!/usr/bin/python3 - -# @todo logging -# @todo extra options for url like , verify=False etc. -# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option? -# @todo option for interval day/6 hour/etc -# @todo on change detected, config for calling some API -# @todo fetch title into json -# https://distill.io/features -# proxy per check -# - flask_cors, itsdangerous,MarkupSafe - import datetime import os import queue @@ -552,10 +541,6 @@ def changedetection_app(config=None, datastore_o=None): # be sure we update with a copy instead of accidently editing the live object by reference default = deepcopy(datastore.data['watching'][uuid]) - # Show system wide default if nothing configured - if datastore.data['watching'][uuid]['fetch_backend'] is None: - default['fetch_backend'] = datastore.data['settings']['application']['fetch_backend'] - # Show system wide default if nothing configured if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()): default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check']) @@ -598,10 +583,8 @@ def changedetection_app(config=None, datastore_o=None): if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']: extra_update_obj['fetch_backend'] = None - # Notification URLs - datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data - # Ignore text + # Ignore text form_ignore_text = form.ignore_text.data datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text @@ -655,9 +638,11 @@ def changedetection_app(config=None, datastore_o=None): watch=datastore.data['watching'][uuid], form=form, has_empty_checktime=using_default_check_time, + has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, using_global_webdriver_wait=default['webdriver_delay'] is None, current_base_url=datastore.data['settings']['application']['base_url'], emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + settings_application=datastore.data['settings']['application'], visualselector_data_is_ready=visualselector_data_is_ready, visualselector_enabled=visualselector_enabled, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False) @@ -687,6 +672,10 @@ def changedetection_app(config=None, datastore_o=None): form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None, data=default ) + + # Remove the last option 'System default' + form.application.form.notification_format.choices.pop() + if datastore.proxy_list is None: # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead del form.requests.form.proxy @@ -732,7 +721,8 @@ def changedetection_app(config=None, datastore_o=None): current_base_url = datastore.data['settings']['application']['base_url'], hide_remove_pass=os.getenv("SALTED_PASS", False), api_key=datastore.data['settings']['application'].get('api_access_token'), - emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)) + emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + settings_application=datastore.data['settings']['application']) return output diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 13d576a4..4f1bbd7e 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -315,7 +315,7 @@ class quickWatchForm(Form): # Common to a single watch and the global settings class commonSettingsForm(Form): - notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()]) + notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateAppRiseServers()]) notification_title = StringField('Notification title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()]) notification_body = TextAreaField('Notification body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()]) notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format) @@ -355,7 +355,7 @@ class watchForm(commonSettingsForm): filter_failure_notification_send = BooleanField( 'Send a notification when the filter can no longer be found on the page', default=False) - notification_use_default = BooleanField('Use default/system notification settings', default=True) + notification_muted = BooleanField('Notifications Muted / Off', default=False) def validate(self, **kwargs): if not super().validate(): diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index b8e48196..b7aaca86 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -6,9 +6,7 @@ minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60) mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7} from changedetectionio.notification import ( - default_notification_body, - default_notification_format, - default_notification_title, + default_notification_format_for_watch ) @@ -32,10 +30,9 @@ class model(dict): 'ignore_text': [], # List of text to ignore when calculating the comparison checksum # Custom notification content 'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise) - 'notification_title': default_notification_title, - 'notification_body': default_notification_body, - 'notification_format': default_notification_format, - 'notification_use_default': True, # Use default for new + 'notification_title': None, + 'notification_body': None, + 'notification_format': default_notification_format_for_watch, 'notification_muted': False, 'css_filter': '', 'last_error': False, diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index ba27c94a..55364fcd 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -14,16 +14,19 @@ valid_tokens = { 'current_snapshot': '' } +default_notification_format_for_watch = 'System default' +default_notification_format = 'Text' +default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n' +default_notification_title = 'ChangeDetection.io Notification - {watch_url}' + valid_notification_formats = { 'Text': NotifyFormat.TEXT, 'Markdown': NotifyFormat.MARKDOWN, 'HTML': NotifyFormat.HTML, + # Used only for editing a watch (not for global) + default_notification_format_for_watch: default_notification_format_for_watch } -default_notification_format = 'Text' -default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n' -default_notification_title = 'ChangeDetection.io Notification - {watch_url}' - def process_notification(n_object, datastore): # Get the notification body from datastore diff --git a/changedetectionio/static/images/notice.svg b/changedetectionio/static/images/notice.svg new file mode 100644 index 00000000..8a7060b2 --- /dev/null +++ b/changedetectionio/static/images/notice.svg @@ -0,0 +1,51 @@ + + + + + + + + + diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index 249f2e4c..b45ca95d 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -1,7 +1,7 @@ -$(document).ready(function () { - function toggle_fetch_backend() { +$(document).ready(function() { + function toggle() { if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') { - if (playwright_enabled) { + if(playwright_enabled) { // playwright supports headers, so hide everything else // See #664 $('#requests-override-options #request-method').hide(); @@ -13,8 +13,12 @@ $(document).ready(function () { // selenium/webdriver doesnt support anything afaik, hide it all $('#requests-override-options').hide(); } + + $('#webdriver-override-options').show(); + } else { + $('#requests-override-options').show(); $('#requests-override-options *:hidden').show(); $('#webdriver-override-options').hide(); @@ -22,27 +26,8 @@ $(document).ready(function () { } $('input[name="fetch_backend"]').click(function (e) { - toggle_fetch_backend(); + toggle(); }); - toggle_fetch_backend(); + toggle(); - function toggle_default_notifications() { - var n=$('#notification_urls, #notification_title, #notification_body, #notification_format'); - if ($('#notification_use_default').is(':checked')) { - $('#notification-field-group').fadeOut(); - $(n).each(function (e) { - $(this).attr('readonly', true); - }); - } else { - $('#notification-field-group').show(); - $(n).each(function (e) { - $(this).attr('readonly', false); - }); - } - } - - $('#notification_use_default').click(function (e) { - toggle_default_notifications(); - }); - toggle_default_notifications(); }); diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 0444da7a..9835c9b0 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -565,3 +565,16 @@ ul { .checkbox-uuid > * { vertical-align: middle; } + +.inline-warning { + border: 1px solid #ff3300; + padding: 0.5rem; + border-radius: 5px; + color: #ff3300; } + .inline-warning > span { + display: inline-block; + vertical-align: middle; } + .inline-warning img.inline-warning-icon { + display: inline; + height: 26px; + vertical-align: middle; } diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss index 4839aec1..e8b4a6ab 100644 --- a/changedetectionio/static/styles/styles.scss +++ b/changedetectionio/static/styles/styles.scss @@ -786,3 +786,21 @@ ul { vertical-align: middle; } } + +.inline-warning { + > span { + display: inline-block; + vertical-align: middle; + } + + img.inline-warning-icon { + display: inline; + height: 26px; + vertical-align: middle; + } + + border: 1px solid #ff3300; + padding: 0.5rem; + border-radius: 5px; + color: #ff3300; +} \ No newline at end of file diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 01b40b4d..d6f74146 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -536,27 +536,3 @@ class ChangeDetectionStore: except: continue return - - - def update_5(self): - - from changedetectionio.notification import ( - default_notification_body, - default_notification_format, - default_notification_title, - ) - - for uuid, watch in self.data['watching'].items(): - try: - # If it's all the same to the system settings, then prefer system notification settings - # include \r\n -> \n incase they already hit submit and the browser put \r in - if watch.get('notification_body').replace('\r\n', '\n') == default_notification_body.replace('\r\n', '\n') and \ - watch.get('notification_format') == default_notification_format and \ - watch.get('notification_title').replace('\r\n', '\n') == default_notification_title.replace('\r\n', '\n') and \ - watch.get('notification_urls') == self.__data['settings']['application']['notification_urls']: - watch['notification_use_default'] = True - else: - watch['notification_use_default'] = False - except: - continue - return \ No newline at end of file diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index e8f1bd5a..a12f6dff 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -1,13 +1,14 @@ {% from '_helpers.jinja' import render_field %} -{% macro render_common_settings_form(form, current_base_url, emailprefix) %} +{% macro render_common_settings_form(form, emailprefix, settings_application) %}
{{ render_field(form.notification_urls, rows=5, placeholder="Examples: Gitter - gitter://token/room Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo - SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com", class="notification-urls") + SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com", + class="notification-urls" ) }}
- {{ render_field(form.notification_title, class="m-d notification-title") }} + {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }} Title for all notifications
- {{ render_field(form.notification_body , rows=5, class="notification-body") }} + {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }} Body for all notifications
- {{ render_field(form.notification_format , rows=5, class="notification-format") }} + + {{ render_field(form.notification_format , class="notification-format") }} Format for all notifications
@@ -94,7 +96,7 @@
URLs generated by changedetection.io (such as {diff_url}) require the BASE_URL environment variable set.
- Your BASE_URL var is currently "{{current_base_url}}" + Your BASE_URL var is currently "{{settings_application['current_base_url']}}"
diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 47cb7815..a52579f5 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -137,10 +137,16 @@ User-Agent: wonderbra 1.0") }}
- {{ render_checkbox_field(form.notification_use_default) }} + {{ render_checkbox_field(form.notification_muted) }}
- {{ render_common_settings_form(form, current_base_url, emailprefix) }} + {% if has_default_notification_urls %} +
+ Look out! + There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications. +
+ {% endif %} + {{ render_common_settings_form(form, emailprefix, settings_application) }}
diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 0518eb9b..2db8c8b6 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -60,7 +60,7 @@ {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/", class="m-d") }} - Base URL used for the {base_url} token in notifications and RSS links.
Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"), + Base URL used for the {base_url} token in notifications and RSS links.
Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"), read more here.
@@ -87,7 +87,7 @@
- {{ render_common_settings_form(form.application.form, current_base_url, emailprefix) }} + {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 2a620e50..6b534f62 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -4,7 +4,13 @@ import re from flask import url_for from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup import logging -from changedetectionio.notification import default_notification_body, default_notification_title + +from changedetectionio.notification import ( + default_notification_body, + default_notification_format, + default_notification_title, + valid_notification_formats, +) def test_setup(live_server): live_server_setup(live_server) @@ -20,9 +26,26 @@ def test_check_notification(client, live_server): # Re 360 - new install should have defaults set res = client.get(url_for("settings_page")) + notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json') + assert default_notification_body.encode() in res.data assert default_notification_title.encode() in res.data + ##################### + # Set this up for when we remove the notification from the watch, it should fallback with these details + res = client.post( + url_for("settings_page"), + data={"application-notification_urls": notification_url, + "application-notification_title": "fallback-title "+default_notification_title, + "application-notification_body": "fallback-body "+default_notification_body, + "application-notification_format": default_notification_format, + "requests-time_between_check-minutes": 180, + 'application-fetch_backend': "html_requests"}, + follow_redirects=True + ) + + assert b"Settings updated." in res.data + # When test mode is in BASE_URL env mode, we should see this already configured env_base_url = os.getenv('BASE_URL', '').strip() if len(env_base_url): @@ -47,8 +70,6 @@ def test_check_notification(client, live_server): # Goto the edit page, add our ignore text # Add our URL to the import page - url = url_for('test_notification_endpoint', _external=True) - notification_url = url.replace('http', 'json') print (">>>> Notification URL: "+notification_url) @@ -71,7 +92,6 @@ def test_check_notification(client, live_server): "url": test_url, "tag": "my tag", "title": "my title", - # No 'notification_use_default' here, so it's effectively False/off "headers": "", "fetch_backend": "html_requests"}) @@ -159,6 +179,30 @@ def test_check_notification(client, live_server): # be sure we see it in the output log assert b'New ChangeDetection.io Notification - ' + test_url.encode('utf-8') in res.data + set_original_response() + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "my tag", + "title": "my title", + "notification_urls": '', + "notification_title": '', + "notification_body": '', + "notification_format": default_notification_format, + "fetch_backend": "html_requests"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + time.sleep(2) + + # Verify what was sent as a notification, this file should exist + with open("test-datastore/notification.txt", "r") as f: + notification_submission = f.read() + assert "fallback-title" in notification_submission + assert "fallback-body" in notification_submission + # cleanup for the next client.get( url_for("form_delete", uuid="all"), @@ -181,20 +225,20 @@ def test_notification_validation(client, live_server): assert b"Watch added" in res.data # Re #360 some validation - res = client.post( - url_for("edit_page", uuid="first"), - data={"notification_urls": 'json://localhost/foobar', - "notification_title": "", - "notification_body": "", - "notification_format": "Text", - "url": test_url, - "tag": "my tag", - "title": "my title", - "headers": "", - "fetch_backend": "html_requests"}, - follow_redirects=True - ) - assert b"Notification Body and Title is required when a Notification URL is used" in res.data +# res = client.post( +# url_for("edit_page", uuid="first"), +# data={"notification_urls": 'json://localhost/foobar', +# "notification_title": "", +# "notification_body": "", +# "notification_format": "Text", +# "url": test_url, +# "tag": "my tag", +# "title": "my title", +# "headers": "", +# "fetch_backend": "html_requests"}, +# follow_redirects=True +# ) +# assert b"Notification Body and Title is required when a Notification URL is used" in res.data # Now adding a wrong token should give us an error res = client.post( @@ -217,81 +261,4 @@ def test_notification_validation(client, live_server): follow_redirects=True ) -# Check that the default VS watch specific notification is hit -def test_check_notification_use_default(client, live_server): - set_original_response() - notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json') - test_url = url_for('test_endpoint', _external=True) - res = client.post( - url_for("form_quick_watch_add"), - data={"url": test_url, "tag": ''}, - follow_redirects=True - ) - assert b"Watch added" in res.data - - ## Setup the local one and enable it - res = client.post( - url_for("edit_page", uuid="first"), - data={"notification_urls": notification_url, - "notification_title": "watch-notification", - "notification_body": "watch-body", - 'notification_use_default': "True", - "notification_format": "Text", - "url": test_url, - "tag": "my tag", - "title": "my title", - "headers": "", - "fetch_backend": "html_requests"}, - follow_redirects=True - ) - - res = client.post( - url_for("settings_page"), - data={"application-notification_title": "global-notifications-title", - "application-notification_body": "global-notifications-body\n", - "application-notification_format": "Text", - "application-notification_urls": notification_url, - "requests-time_between_check-minutes": 180, - "fetch_backend": "html_requests" - }, - follow_redirects=True - ) - - # A change should by default trigger a notification of the global-notifications - time.sleep(1) - set_modified_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(2) - with open("test-datastore/notification.txt", "r") as f: - assert 'global-notifications-title' in f.read() - - ## Setup the local one and enable it - res = client.post( - url_for("edit_page", uuid="first"), - data={"notification_urls": notification_url, - "notification_title": "watch-notification", - "notification_body": "watch-body", - # No 'notification_use_default' here, so it's effectively False/off = "dont use default, use this one" - "notification_format": "Text", - "url": test_url, - "tag": "my tag", - "title": "my title", - "headers": "", - "fetch_backend": "html_requests"}, - follow_redirects=True - ) - set_original_response() - - client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(2) - assert os.path.isfile("test-datastore/notification.txt") - with open("test-datastore/notification.txt", "r") as f: - assert 'watch-notification' in f.read() - - - # cleanup for the next - client.get( - url_for("form_delete", uuid="all"), - follow_redirects=True - ) \ No newline at end of file diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 1617e61f..8f1c0f28 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -11,11 +11,14 @@ from changedetectionio.html_tools import FilterNotFoundInResponse # Requests for checking on a single site(watch) from a queue of watches # (another process inserts watches into the queue that are time-ready for checking) +import logging +import sys class update_worker(threading.Thread): current_uuid = None def __init__(self, q, notification_q, app, datastore, *args, **kwargs): + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) self.q = q self.app = app self.notification_q = notification_q @@ -26,6 +29,10 @@ class update_worker(threading.Thread): from changedetectionio import diff + from changedetectionio.notification import ( + default_notification_format_for_watch + ) + n_object = {} watch = self.datastore.data['watching'].get(watch_uuid, False) if not watch: @@ -40,33 +47,27 @@ class update_worker(threading.Thread): "History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?" ) - # Did it have any notification alerts to hit? - if not watch.get('notification_use_default') and len(watch['notification_urls']): - print(">>> Notifications queued for UUID from watch {}".format(watch_uuid)) - n_object['notification_urls'] = watch['notification_urls'] - n_object['notification_title'] = watch['notification_title'] - n_object['notification_body'] = watch['notification_body'] - n_object['notification_format'] = watch['notification_format'] + n_object['notification_urls'] = watch['notification_urls'] if len(watch['notification_urls']) else \ + self.datastore.data['settings']['application']['notification_urls'] + + n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \ + self.datastore.data['settings']['application']['notification_title'] + + n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \ + self.datastore.data['settings']['application']['notification_body'] + + n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \ + self.datastore.data['settings']['application']['notification_format'] - # No? maybe theres a global setting, queue them all - elif watch.get('notification_use_default') and len(self.datastore.data['settings']['application']['notification_urls']): - print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(watch_uuid)) - n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls'] - n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title'] - n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body'] - n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format'] - else: - print(">>> NO notifications queued, watch and global notification URLs were empty.") # Only prepare to notify if the rules above matched - if 'notification_urls' in n_object: + if 'notification_urls' in n_object and n_object['notification_urls']: # HTML needs linebreak, but MarkDown and Text can use a linefeed if n_object['notification_format'] == 'HTML': line_feed_sep = "
" else: line_feed_sep = "\n" - snapshot_contents = '' with open(watch_history[dates[-1]], 'rb') as f: snapshot_contents = f.read() @@ -77,8 +78,10 @@ class update_worker(threading.Thread): 'diff': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], line_feed_sep=line_feed_sep), 'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], True, line_feed_sep=line_feed_sep) }) - + logging.info (">> SENDING NOTIFICATION") self.notification_q.put(n_object) + else: + logging.info (">> NO Notification sent, notification_url was empty in both watch and system") def send_filter_failure_notification(self, watch_uuid):