diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml
index 7543e9d8..baf1d178 100644
--- a/.github/workflows/test-only.yml
+++ b/.github/workflows/test-only.yml
@@ -14,6 +14,9 @@ jobs:
with:
python-version: 3.9
+ - name: Show env vars
+ run: set
+
- name: Install dependencies
run: |
python -m pip install --upgrade pip
@@ -27,12 +30,15 @@ jobs:
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+ - name: Unit tests
+ run: |
+ python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
- name: Test with pytest
run: |
# Each test is totally isolated and performs its own cleanup/reset
cd changedetectionio; ./run_all_tests.sh
-
# https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
# https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py
index 718731f8..88341426 100644
--- a/changedetectionio/__init__.py
+++ b/changedetectionio/__init__.py
@@ -464,7 +464,6 @@ def changedetection_app(config=None, datastore_o=None):
else:
flash('No notification URLs set, cannot send test.', 'error')
-
# Diff page [edit] link should go back to diff page
if request.args.get("next") and request.args.get("next") == 'diff':
return redirect(url_for('diff_history_page', uuid=uuid))
@@ -621,6 +620,7 @@ def changedetection_app(config=None, datastore_o=None):
dates = list(watch['history'].keys())
# Convert to int, sort and back to str again
+ # @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
dates = [str(i) for i in dates]
@@ -631,13 +631,11 @@ def changedetection_app(config=None, datastore_o=None):
# Save the current newest history as the most recently viewed
datastore.set_last_viewed(uuid, dates[0])
-
newest_file = watch['history'][dates[0]]
with open(newest_file, 'r') as f:
newest_version_file_contents = f.read()
previous_version = request.args.get('previous_version')
-
try:
previous_file = watch['history'][previous_version]
except KeyError:
@@ -843,8 +841,10 @@ def changedetection_app(config=None, datastore_o=None):
threading.Thread(target=notification_runner).start()
- # Check for new release version
- threading.Thread(target=check_for_new_version).start()
+ # Check for new release version, but not when running in test/build
+ if not os.getenv("GITHUB_REF", False):
+ threading.Thread(target=check_for_new_version).start()
+
return app
@@ -893,8 +893,6 @@ def notification_runner():
except Exception as e:
print("Watch URL: {} Error {}".format(n_object['watch_url'], e))
-
-
# Thread runner to check every minute, look for new watches to feed into the Queue.
def ticker_thread_check_time_launch_checks():
from changedetectionio import update_worker
diff --git a/changedetectionio/diff.py b/changedetectionio/diff.py
new file mode 100644
index 00000000..d2c2ab8a
--- /dev/null
+++ b/changedetectionio/diff.py
@@ -0,0 +1,43 @@
+# used for the notifications, the front-end is using a JS library
+
+import difflib
+
+# like .compare but a little different output
+def customSequenceMatcher(before, after, include_equal=False):
+ cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
+
+ for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
+ if include_equal and tag == 'equal':
+ g = before[alo:ahi]
+ yield g
+ elif tag == 'delete':
+ g = "(removed) {}".format(before[alo])
+ yield g
+ elif tag == 'replace':
+ g = ["(changed) {}".format(before[alo]), "(-> into) {}".format(after[blo])]
+ yield g
+ elif tag == 'insert':
+ g = "(added) {}".format(after[blo])
+ yield g
+
+# only_differences - only return info about the differences, no context
+# line_feed_sep could be "
" or "
" or "\n" etc
+def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
+ with open(newest_file, 'r') as f:
+ newest_version_file_contents = f.read()
+ newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+
+ if previous_file:
+ with open(previous_file, 'r') as f:
+ previous_version_file_contents = f.read()
+ previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+ else:
+ previous_version_file_contents = ""
+
+ rendered_diff = customSequenceMatcher(previous_version_file_contents,
+ newest_version_file_contents,
+ include_equal)
+
+ # Recursively join lists
+ f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
+ return f(rendered_diff)
diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py
index 418bd5af..bc650165 100644
--- a/changedetectionio/forms.py
+++ b/changedetectionio/forms.py
@@ -6,7 +6,7 @@ from wtforms.fields import html5
from changedetectionio import content_fetcher
import re
-from changedetectionio.notification import default_notification_format, valid_notification_formats
+from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title
class StringListField(StringField):
widget = widgets.TextArea()
@@ -203,8 +203,8 @@ class quickWatchForm(Form):
class commonSettingsForm(Form):
notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
- notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {watch_url}', validators=[validators.Optional(), ValidateTokensList()])
- notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()])
+ notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
+ notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format)
trigger_check = BooleanField('Send test notification on save')
fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py
index 96f18a90..5c5a1fb1 100644
--- a/changedetectionio/notification.py
+++ b/changedetectionio/notification.py
@@ -1,4 +1,3 @@
-import os
import apprise
from apprise import NotifyFormat
@@ -8,6 +7,8 @@ valid_tokens = {
'watch_uuid': '',
'watch_title': '',
'watch_tag': '',
+ 'diff': '',
+ 'diff_full': '',
'diff_url': '',
'preview_url': '',
'current_snapshot': ''
@@ -20,6 +21,8 @@ valid_notification_formats = {
}
default_notification_format = 'Text'
+default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
def process_notification(n_object, datastore):
import logging
@@ -33,8 +36,8 @@ def process_notification(n_object, datastore):
apobj.add(url)
# Get the notification body from datastore
- n_body = n_object['notification_body']
- n_title = n_object['notification_title']
+ n_body = n_object.get('notification_body', default_notification_body)
+ n_title = n_object.get('notification_title', default_notification_title)
n_format = valid_notification_formats.get(
n_object['notification_format'],
valid_notification_formats[default_notification_format],
@@ -88,15 +91,17 @@ def create_notification_parameters(n_object, datastore):
# Valid_tokens also used as a field validator
tokens.update(
- {
- 'base_url': base_url if base_url is not None else '',
- 'watch_url': watch_url,
- 'watch_uuid': uuid,
- 'watch_title': watch_title if watch_title is not None else '',
- 'watch_tag': watch_tag if watch_tag is not None else '',
- 'diff_url': diff_url,
- 'preview_url': preview_url,
- 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
- })
+ {
+ 'base_url': base_url if base_url is not None else '',
+ 'watch_url': watch_url,
+ 'watch_uuid': uuid,
+ 'watch_title': watch_title if watch_title is not None else '',
+ 'watch_tag': watch_tag if watch_tag is not None else '',
+ 'diff_url': diff_url,
+ 'diff': n_object.get('diff', ''), # Null default in the case we use a test
+ 'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test
+ 'preview_url': preview_url,
+ 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
+ })
return tokens
diff --git a/changedetectionio/run_all_tests.sh b/changedetectionio/run_all_tests.sh
index b9f8e229..82b603f3 100755
--- a/changedetectionio/run_all_tests.sh
+++ b/changedetectionio/run_all_tests.sh
@@ -9,15 +9,16 @@
# exit when any command fails
set -e
-
find tests/test_*py -type f|while read test_name
do
echo "TEST RUNNING $test_name"
pytest $test_name
done
+echo "RUNNING WITH BASE_URL SET"
# Now re-run some tests with BASE_URL enabled
# Re #65 - Ability to include a link back to the installation, in the notification.
export BASE_URL="https://really-unique-domain.io"
pytest tests/test_notification.py
+
diff --git a/changedetectionio/store.py b/changedetectionio/store.py
index b4ebbc2e..bb70672f 100644
--- a/changedetectionio/store.py
+++ b/changedetectionio/store.py
@@ -9,6 +9,8 @@ import time
import threading
import os
+from changedetectionio.notification import default_notification_format, default_notification_body, default_notification_title
+
# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
# Open a github issue if you know something :)
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -157,6 +159,7 @@ class ChangeDetectionStore:
dates = list(self.__data['watching'][uuid]['history'].keys())
# Convert to int, sort and back to str again
+ # @todo replace datastore getter that does this automatically
dates = [int(i) for i in dates]
dates.sort(reverse=True)
if len(dates):
diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja
index 58b997fc..0719dc19 100644
--- a/changedetectionio/templates/_common_fields.jinja
+++ b/changedetectionio/templates/_common_fields.jinja
@@ -65,6 +65,14 @@
{preview_url} |
The URL of the preview page generated by changedetection.io. |
+
+ {diff} |
+ The diff output - differences only |
+
+
+ {diff_full} |
+ The diff output - full difference output |
+
{diff_url} |
The URL of the diff page generated by changedetection.io. |
diff --git a/changedetectionio/tests/conftest.py b/changedetectionio/tests/conftest.py
index fd2806d8..f34ed5bb 100644
--- a/changedetectionio/tests/conftest.py
+++ b/changedetectionio/tests/conftest.py
@@ -22,7 +22,6 @@ def cleanup(datastore_path):
for file in files:
try:
os.unlink("{}/{}".format(datastore_path, file))
- x = 1
except FileNotFoundError:
pass
diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py
index 1cf4819c..3cfeecf9 100644
--- a/changedetectionio/tests/test_notification.py
+++ b/changedetectionio/tests/test_notification.py
@@ -55,6 +55,8 @@ def test_check_notification(client, live_server):
"Preview: {preview_url}\n"
"Diff URL: {diff_url}\n"
"Snapshot: {current_snapshot}\n"
+ "Diff: {diff}\n"
+ "Diff Full: {diff_full}\n"
":-)",
"notification_format": "Text",
"url": test_url,
@@ -114,6 +116,11 @@ def test_check_notification(client, live_server):
assert test_url in notification_submission
+ # Diff was correctly executed
+ assert "Diff Full: (changed) Which is across multiple lines" in notification_submission
+ assert "(-> into) which has this one new line" in notification_submission
+
+
if env_base_url:
# Re #65 - did we see our BASE_URl ?
logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)
diff --git a/changedetectionio/tests/unit/__init__.py b/changedetectionio/tests/unit/__init__.py
new file mode 100644
index 00000000..368609ff
--- /dev/null
+++ b/changedetectionio/tests/unit/__init__.py
@@ -0,0 +1 @@
+"""Unit tests for the app."""
diff --git a/changedetectionio/tests/unit/test-content/README.md b/changedetectionio/tests/unit/test-content/README.md
new file mode 100644
index 00000000..a1e55fbc
--- /dev/null
+++ b/changedetectionio/tests/unit/test-content/README.md
@@ -0,0 +1,5 @@
+# What is this?
+This is test content for the python diff engine, we use the JS interface for the front end, because you can explore
+differences in words etc, but we use (at the moment) the python difflib engine.
+
+This content `before.txt` and `after.txt` is for unit testing
diff --git a/changedetectionio/tests/unit/test-content/after.txt b/changedetectionio/tests/unit/test-content/after.txt
new file mode 100644
index 00000000..ffb0e3a5
--- /dev/null
+++ b/changedetectionio/tests/unit/test-content/after.txt
@@ -0,0 +1,6 @@
+After twenty years, as cursed as I may be
+for having learned computerese,
+I continue to examine bits, bytes and words
+xok
+and insure that I'm one of those computer nerds.
+and something new
\ No newline at end of file
diff --git a/changedetectionio/tests/unit/test-content/before.txt b/changedetectionio/tests/unit/test-content/before.txt
new file mode 100644
index 00000000..8ca997df
--- /dev/null
+++ b/changedetectionio/tests/unit/test-content/before.txt
@@ -0,0 +1,5 @@
+After twenty years, as cursed as I may be
+for having learned computerese,
+I continue to examine bits, bytes and words
+ok
+and insure that I'm one of those computer nerds.
diff --git a/changedetectionio/tests/unit/test_notification_diff.py b/changedetectionio/tests/unit/test_notification_diff.py
new file mode 100755
index 00000000..a3ef3e72
--- /dev/null
+++ b/changedetectionio/tests/unit/test_notification_diff.py
@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
+import unittest
+import os
+
+from changedetectionio import diff
+
+# mostly
+class TestDiffBuilder(unittest.TestCase):
+
+ def test_expected_diff_output(self):
+ base_dir=os.path.dirname(__file__)
+ output = diff.render_diff(base_dir+"/test-content/before.txt", base_dir+"/test-content/after.txt")
+ output = output.split("\n")
+ self.assertIn("(changed) ok", output)
+ self.assertIn("(-> into) xok", output)
+ self.assertIn("(added) and something new", output)
+
+ # @todo test blocks of changed, blocks of added, blocks of removed
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py
index ef31756e..4ab1d806 100644
--- a/changedetectionio/update_worker.py
+++ b/changedetectionio/update_worker.py
@@ -56,9 +56,8 @@ class update_worker(threading.Thread):
try:
self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
if changed_detected:
-
+ n_object = {}
# A change was detected
- newest_version_file_contents = ""
fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
# Update history with the stripped text for future reference, this will also mean we save the first
@@ -69,37 +68,55 @@ class update_worker(threading.Thread):
print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
- # Get the newest snapshot data to be possibily used in a notification
- newest_key = self.datastore.get_newest_history_key(uuid)
- if newest_key:
- with open(watch['history'][newest_key], 'r') as f:
- newest_version_file_contents = f.read().strip()
-
- n_object = {
- 'watch_url': watch['url'],
- 'uuid': uuid,
- 'current_snapshot': newest_version_file_contents
- }
-
- # Did it have any notification alerts to hit?
- if len(watch['notification_urls']):
- print(">>> Notifications queued for UUID from watch {}".format(uuid))
- n_object['notification_urls'] = watch['notification_urls']
- n_object['notification_title'] = watch['notification_title']
- n_object['notification_body'] = watch['notification_body']
- n_object['notification_format'] = watch['notification_format']
- self.notification_q.put(n_object)
-
- # No? maybe theres a global setting, queue them all
- elif len(self.datastore.data['settings']['application']['notification_urls']):
- print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
- n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
- n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
- n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
- n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
- self.notification_q.put(n_object)
- else:
- print(">>> NO notifications queued, watch and global notification URLs were empty.")
+ # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
+ if len(watch['history']) > 1:
+
+ dates = list(watch['history'].keys())
+ # Convert to int, sort and back to str again
+ # @todo replace datastore getter that does this automatically
+ dates = [int(i) for i in dates]
+ dates.sort(reverse=True)
+ dates = [str(i) for i in dates]
+
+ prev_fname = watch['history'][dates[1]]
+
+
+ # Did it have any notification alerts to hit?
+ if len(watch['notification_urls']):
+ print(">>> Notifications queued for UUID from watch {}".format(uuid))
+ n_object['notification_urls'] = watch['notification_urls']
+ n_object['notification_title'] = watch['notification_title']
+ n_object['notification_body'] = watch['notification_body']
+ n_object['notification_format'] = watch['notification_format']
+
+ # No? maybe theres a global setting, queue them all
+ elif len(self.datastore.data['settings']['application']['notification_urls']):
+ print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
+ n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+ n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
+ n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
+ n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
+ else:
+ print(">>> NO notifications queued, watch and global notification URLs were empty.")
+
+ # Only prepare to notify if the rules above matched
+ if 'notification_urls' in n_object:
+ # HTML needs linebreak, but MarkDown and Text can use a linefeed
+ if n_object['notification_format'] == 'HTML':
+ line_feed_sep = ""
+ else:
+ line_feed_sep = "\n"
+
+ from changedetectionio import diff
+ n_object.update({
+ 'watch_url': watch['url'],
+ 'uuid': uuid,
+ 'current_snapshot': str(contents),
+ 'diff_full': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep),
+ 'diff': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep)
+ })
+
+ self.notification_q.put(n_object)
except Exception as e:
print("!!!! Exception in update_worker !!!\n", e)