Send diff in notifications (#296)

pull/316/head^2
dgtlmoon 3 years ago committed by GitHub
parent a7c09c8990
commit 5a10acfd09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -14,6 +14,9 @@ jobs:
with:
python-version: 3.9
- name: Show env vars
run: set
- name: Install dependencies
run: |
python -m pip install --upgrade pip
@ -27,12 +30,15 @@ jobs:
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Unit tests
run: |
python3 -m unittest changedetectionio.tests.unit.test_notification_diff
- name: Test with pytest
run: |
# Each test is totally isolated and performs its own cleanup/reset
cd changedetectionio; ./run_all_tests.sh
# https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
# https://github.com/docker/buildx/issues/59 ? Needs to be one platform?

@ -464,7 +464,6 @@ def changedetection_app(config=None, datastore_o=None):
else:
flash('No notification URLs set, cannot send test.', 'error')
# Diff page [edit] link should go back to diff page
if request.args.get("next") and request.args.get("next") == 'diff':
return redirect(url_for('diff_history_page', uuid=uuid))
@ -621,6 +620,7 @@ def changedetection_app(config=None, datastore_o=None):
dates = list(watch['history'].keys())
# Convert to int, sort and back to str again
# @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
dates = [str(i) for i in dates]
@ -631,13 +631,11 @@ def changedetection_app(config=None, datastore_o=None):
# Save the current newest history as the most recently viewed
datastore.set_last_viewed(uuid, dates[0])
newest_file = watch['history'][dates[0]]
with open(newest_file, 'r') as f:
newest_version_file_contents = f.read()
previous_version = request.args.get('previous_version')
try:
previous_file = watch['history'][previous_version]
except KeyError:
@ -843,8 +841,10 @@ def changedetection_app(config=None, datastore_o=None):
threading.Thread(target=notification_runner).start()
# Check for new release version
threading.Thread(target=check_for_new_version).start()
# Check for new release version, but not when running in test/build
if not os.getenv("GITHUB_REF", False):
threading.Thread(target=check_for_new_version).start()
return app
@ -893,8 +893,6 @@ def notification_runner():
except Exception as e:
print("Watch URL: {} Error {}".format(n_object['watch_url'], e))
# Thread runner to check every minute, look for new watches to feed into the Queue.
def ticker_thread_check_time_launch_checks():
from changedetectionio import update_worker

@ -0,0 +1,43 @@
# used for the notifications, the front-end is using a JS library
import difflib
# like .compare but a little different output
def customSequenceMatcher(before, after, include_equal=False):
cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
if include_equal and tag == 'equal':
g = before[alo:ahi]
yield g
elif tag == 'delete':
g = "(removed) {}".format(before[alo])
yield g
elif tag == 'replace':
g = ["(changed) {}".format(before[alo]), "(-> into) {}".format(after[blo])]
yield g
elif tag == 'insert':
g = "(added) {}".format(after[blo])
yield g
# only_differences - only return info about the differences, no context
# line_feed_sep could be "<br/>" or "<li>" or "\n" etc
def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
with open(newest_file, 'r') as f:
newest_version_file_contents = f.read()
newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
if previous_file:
with open(previous_file, 'r') as f:
previous_version_file_contents = f.read()
previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
else:
previous_version_file_contents = ""
rendered_diff = customSequenceMatcher(previous_version_file_contents,
newest_version_file_contents,
include_equal)
# Recursively join lists
f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
return f(rendered_diff)

@ -6,7 +6,7 @@ from wtforms.fields import html5
from changedetectionio import content_fetcher
import re
from changedetectionio.notification import default_notification_format, valid_notification_formats
from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title
class StringListField(StringField):
widget = widgets.TextArea()
@ -203,8 +203,8 @@ class quickWatchForm(Form):
class commonSettingsForm(Form):
notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {watch_url}', validators=[validators.Optional(), ValidateTokensList()])
notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()])
notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format)
trigger_check = BooleanField('Send test notification on save')
fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])

@ -1,4 +1,3 @@
import os
import apprise
from apprise import NotifyFormat
@ -8,6 +7,8 @@ valid_tokens = {
'watch_uuid': '',
'watch_title': '',
'watch_tag': '',
'diff': '',
'diff_full': '',
'diff_url': '',
'preview_url': '',
'current_snapshot': ''
@ -20,6 +21,8 @@ valid_notification_formats = {
}
default_notification_format = 'Text'
default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
def process_notification(n_object, datastore):
import logging
@ -33,8 +36,8 @@ def process_notification(n_object, datastore):
apobj.add(url)
# Get the notification body from datastore
n_body = n_object['notification_body']
n_title = n_object['notification_title']
n_body = n_object.get('notification_body', default_notification_body)
n_title = n_object.get('notification_title', default_notification_title)
n_format = valid_notification_formats.get(
n_object['notification_format'],
valid_notification_formats[default_notification_format],
@ -88,15 +91,17 @@ def create_notification_parameters(n_object, datastore):
# Valid_tokens also used as a field validator
tokens.update(
{
'base_url': base_url if base_url is not None else '',
'watch_url': watch_url,
'watch_uuid': uuid,
'watch_title': watch_title if watch_title is not None else '',
'watch_tag': watch_tag if watch_tag is not None else '',
'diff_url': diff_url,
'preview_url': preview_url,
'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
})
{
'base_url': base_url if base_url is not None else '',
'watch_url': watch_url,
'watch_uuid': uuid,
'watch_title': watch_title if watch_title is not None else '',
'watch_tag': watch_tag if watch_tag is not None else '',
'diff_url': diff_url,
'diff': n_object.get('diff', ''), # Null default in the case we use a test
'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test
'preview_url': preview_url,
'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
})
return tokens

@ -9,15 +9,16 @@
# exit when any command fails
set -e
find tests/test_*py -type f|while read test_name
do
echo "TEST RUNNING $test_name"
pytest $test_name
done
echo "RUNNING WITH BASE_URL SET"
# Now re-run some tests with BASE_URL enabled
# Re #65 - Ability to include a link back to the installation, in the notification.
export BASE_URL="https://really-unique-domain.io"
pytest tests/test_notification.py

@ -9,6 +9,8 @@ import time
import threading
import os
from changedetectionio.notification import default_notification_format, default_notification_body, default_notification_title
# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
# Open a github issue if you know something :)
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@ -157,6 +159,7 @@ class ChangeDetectionStore:
dates = list(self.__data['watching'][uuid]['history'].keys())
# Convert to int, sort and back to str again
# @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
if len(dates):

@ -65,6 +65,14 @@
<td><code>{preview_url}</code></td>
<td>The URL of the preview page generated by changedetection.io.</td>
</tr>
<tr>
<td><code>{diff}</code></td>
<td>The diff output - differences only</td>
</tr>
<tr>
<td><code>{diff_full}</code></td>
<td>The diff output - full difference output</td>
</tr>
<tr>
<td><code>{diff_url}</code></td>
<td>The URL of the diff page generated by changedetection.io.</td>

@ -22,7 +22,6 @@ def cleanup(datastore_path):
for file in files:
try:
os.unlink("{}/{}".format(datastore_path, file))
x = 1
except FileNotFoundError:
pass

@ -55,6 +55,8 @@ def test_check_notification(client, live_server):
"Preview: {preview_url}\n"
"Diff URL: {diff_url}\n"
"Snapshot: {current_snapshot}\n"
"Diff: {diff}\n"
"Diff Full: {diff_full}\n"
":-)",
"notification_format": "Text",
"url": test_url,
@ -114,6 +116,11 @@ def test_check_notification(client, live_server):
assert test_url in notification_submission
# Diff was correctly executed
assert "Diff Full: (changed) Which is across multiple lines" in notification_submission
assert "(-> into) which has this one new line" in notification_submission
if env_base_url:
# Re #65 - did we see our BASE_URl ?
logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)

@ -0,0 +1 @@
"""Unit tests for the app."""

@ -0,0 +1,5 @@
# What is this?
This is test content for the python diff engine, we use the JS interface for the front end, because you can explore
differences in words etc, but we use (at the moment) the python difflib engine.
This content `before.txt` and `after.txt` is for unit testing

@ -0,0 +1,6 @@
After twenty years, as cursed as I may be
for having learned computerese,
I continue to examine bits, bytes and words
xok
and insure that I'm one of those computer nerds.
and something new

@ -0,0 +1,5 @@
After twenty years, as cursed as I may be
for having learned computerese,
I continue to examine bits, bytes and words
ok
and insure that I'm one of those computer nerds.

@ -0,0 +1,25 @@
#!/usr/bin/python3
# run from dir above changedetectionio/ dir
# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
import unittest
import os
from changedetectionio import diff
# mostly
class TestDiffBuilder(unittest.TestCase):
def test_expected_diff_output(self):
base_dir=os.path.dirname(__file__)
output = diff.render_diff(base_dir+"/test-content/before.txt", base_dir+"/test-content/after.txt")
output = output.split("\n")
self.assertIn("(changed) ok", output)
self.assertIn("(-> into) xok", output)
self.assertIn("(added) and something new", output)
# @todo test blocks of changed, blocks of added, blocks of removed
if __name__ == '__main__':
unittest.main()

@ -56,9 +56,8 @@ class update_worker(threading.Thread):
try:
self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
if changed_detected:
n_object = {}
# A change was detected
newest_version_file_contents = ""
fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
# Update history with the stripped text for future reference, this will also mean we save the first
@ -69,37 +68,55 @@ class update_worker(threading.Thread):
print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
# Get the newest snapshot data to be possibily used in a notification
newest_key = self.datastore.get_newest_history_key(uuid)
if newest_key:
with open(watch['history'][newest_key], 'r') as f:
newest_version_file_contents = f.read().strip()
n_object = {
'watch_url': watch['url'],
'uuid': uuid,
'current_snapshot': newest_version_file_contents
}
# Did it have any notification alerts to hit?
if len(watch['notification_urls']):
print(">>> Notifications queued for UUID from watch {}".format(uuid))
n_object['notification_urls'] = watch['notification_urls']
n_object['notification_title'] = watch['notification_title']
n_object['notification_body'] = watch['notification_body']
n_object['notification_format'] = watch['notification_format']
self.notification_q.put(n_object)
# No? maybe theres a global setting, queue them all
elif len(self.datastore.data['settings']['application']['notification_urls']):
print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
self.notification_q.put(n_object)
else:
print(">>> NO notifications queued, watch and global notification URLs were empty.")
# Notifications should only trigger on the second time (first time, we gather the initial snapshot)
if len(watch['history']) > 1:
dates = list(watch['history'].keys())
# Convert to int, sort and back to str again
# @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
dates = [str(i) for i in dates]
prev_fname = watch['history'][dates[1]]
# Did it have any notification alerts to hit?
if len(watch['notification_urls']):
print(">>> Notifications queued for UUID from watch {}".format(uuid))
n_object['notification_urls'] = watch['notification_urls']
n_object['notification_title'] = watch['notification_title']
n_object['notification_body'] = watch['notification_body']
n_object['notification_format'] = watch['notification_format']
# No? maybe theres a global setting, queue them all
elif len(self.datastore.data['settings']['application']['notification_urls']):
print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
else:
print(">>> NO notifications queued, watch and global notification URLs were empty.")
# Only prepare to notify if the rules above matched
if 'notification_urls' in n_object:
# HTML needs linebreak, but MarkDown and Text can use a linefeed
if n_object['notification_format'] == 'HTML':
line_feed_sep = "</br>"
else:
line_feed_sep = "\n"
from changedetectionio import diff
n_object.update({
'watch_url': watch['url'],
'uuid': uuid,
'current_snapshot': str(contents),
'diff_full': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep),
'diff': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep)
})
self.notification_q.put(n_object)
except Exception as e:
print("!!!! Exception in update_worker !!!\n", e)

Loading…
Cancel
Save