diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index 7543e9d8..baf1d178 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -14,6 +14,9 @@ jobs: with: python-version: 3.9 + - name: Show env vars + run: set + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -27,12 +30,15 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Unit tests + run: | + python3 -m unittest changedetectionio.tests.unit.test_notification_diff + - name: Test with pytest run: | # Each test is totally isolated and performs its own cleanup/reset cd changedetectionio; ./run_all_tests.sh - # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ? # https://github.com/docker/buildx/issues/59 ? Needs to be one platform? diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 718731f8..88341426 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -464,7 +464,6 @@ def changedetection_app(config=None, datastore_o=None): else: flash('No notification URLs set, cannot send test.', 'error') - # Diff page [edit] link should go back to diff page if request.args.get("next") and request.args.get("next") == 'diff': return redirect(url_for('diff_history_page', uuid=uuid)) @@ -621,6 +620,7 @@ def changedetection_app(config=None, datastore_o=None): dates = list(watch['history'].keys()) # Convert to int, sort and back to str again + # @todo replace datastore getter that does this automatically dates = [int(i) for i in dates] dates.sort(reverse=True) dates = [str(i) for i in dates] @@ -631,13 +631,11 @@ def changedetection_app(config=None, datastore_o=None): # Save the current newest history as the most recently viewed datastore.set_last_viewed(uuid, dates[0]) - newest_file = watch['history'][dates[0]] with open(newest_file, 'r') as f: newest_version_file_contents = f.read() previous_version = request.args.get('previous_version') - try: previous_file = watch['history'][previous_version] except KeyError: @@ -843,8 +841,10 @@ def changedetection_app(config=None, datastore_o=None): threading.Thread(target=notification_runner).start() - # Check for new release version - threading.Thread(target=check_for_new_version).start() + # Check for new release version, but not when running in test/build + if not os.getenv("GITHUB_REF", False): + threading.Thread(target=check_for_new_version).start() + return app @@ -893,8 +893,6 @@ def notification_runner(): except Exception as e: print("Watch URL: {} Error {}".format(n_object['watch_url'], e)) - - # Thread runner to check every minute, look for new watches to feed into the Queue. def ticker_thread_check_time_launch_checks(): from changedetectionio import update_worker diff --git a/changedetectionio/diff.py b/changedetectionio/diff.py new file mode 100644 index 00000000..d2c2ab8a --- /dev/null +++ b/changedetectionio/diff.py @@ -0,0 +1,43 @@ +# used for the notifications, the front-end is using a JS library + +import difflib + +# like .compare but a little different output +def customSequenceMatcher(before, after, include_equal=False): + cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after) + + for tag, alo, ahi, blo, bhi in cruncher.get_opcodes(): + if include_equal and tag == 'equal': + g = before[alo:ahi] + yield g + elif tag == 'delete': + g = "(removed) {}".format(before[alo]) + yield g + elif tag == 'replace': + g = ["(changed) {}".format(before[alo]), "(-> into) {}".format(after[blo])] + yield g + elif tag == 'insert': + g = "(added) {}".format(after[blo]) + yield g + +# only_differences - only return info about the differences, no context +# line_feed_sep could be "
" or "
  • " or "\n" etc +def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"): + with open(newest_file, 'r') as f: + newest_version_file_contents = f.read() + newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()] + + if previous_file: + with open(previous_file, 'r') as f: + previous_version_file_contents = f.read() + previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()] + else: + previous_version_file_contents = "" + + rendered_diff = customSequenceMatcher(previous_version_file_contents, + newest_version_file_contents, + include_equal) + + # Recursively join lists + f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L]) + return f(rendered_diff) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 418bd5af..bc650165 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -6,7 +6,7 @@ from wtforms.fields import html5 from changedetectionio import content_fetcher import re -from changedetectionio.notification import default_notification_format, valid_notification_formats +from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title class StringListField(StringField): widget = widgets.TextArea() @@ -203,8 +203,8 @@ class quickWatchForm(Form): class commonSettingsForm(Form): notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()]) - notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {watch_url}', validators=[validators.Optional(), ValidateTokensList()]) - notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()]) + notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()]) + notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()]) notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format) trigger_check = BooleanField('Send test notification on save') fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()]) diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 96f18a90..5c5a1fb1 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -1,4 +1,3 @@ -import os import apprise from apprise import NotifyFormat @@ -8,6 +7,8 @@ valid_tokens = { 'watch_uuid': '', 'watch_title': '', 'watch_tag': '', + 'diff': '', + 'diff_full': '', 'diff_url': '', 'preview_url': '', 'current_snapshot': '' @@ -20,6 +21,8 @@ valid_notification_formats = { } default_notification_format = 'Text' +default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n' +default_notification_title = 'ChangeDetection.io Notification - {watch_url}' def process_notification(n_object, datastore): import logging @@ -33,8 +36,8 @@ def process_notification(n_object, datastore): apobj.add(url) # Get the notification body from datastore - n_body = n_object['notification_body'] - n_title = n_object['notification_title'] + n_body = n_object.get('notification_body', default_notification_body) + n_title = n_object.get('notification_title', default_notification_title) n_format = valid_notification_formats.get( n_object['notification_format'], valid_notification_formats[default_notification_format], @@ -88,15 +91,17 @@ def create_notification_parameters(n_object, datastore): # Valid_tokens also used as a field validator tokens.update( - { - 'base_url': base_url if base_url is not None else '', - 'watch_url': watch_url, - 'watch_uuid': uuid, - 'watch_title': watch_title if watch_title is not None else '', - 'watch_tag': watch_tag if watch_tag is not None else '', - 'diff_url': diff_url, - 'preview_url': preview_url, - 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '' - }) + { + 'base_url': base_url if base_url is not None else '', + 'watch_url': watch_url, + 'watch_uuid': uuid, + 'watch_title': watch_title if watch_title is not None else '', + 'watch_tag': watch_tag if watch_tag is not None else '', + 'diff_url': diff_url, + 'diff': n_object.get('diff', ''), # Null default in the case we use a test + 'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test + 'preview_url': preview_url, + 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '' + }) return tokens diff --git a/changedetectionio/run_all_tests.sh b/changedetectionio/run_all_tests.sh index b9f8e229..82b603f3 100755 --- a/changedetectionio/run_all_tests.sh +++ b/changedetectionio/run_all_tests.sh @@ -9,15 +9,16 @@ # exit when any command fails set -e - find tests/test_*py -type f|while read test_name do echo "TEST RUNNING $test_name" pytest $test_name done +echo "RUNNING WITH BASE_URL SET" # Now re-run some tests with BASE_URL enabled # Re #65 - Ability to include a link back to the installation, in the notification. export BASE_URL="https://really-unique-domain.io" pytest tests/test_notification.py + diff --git a/changedetectionio/store.py b/changedetectionio/store.py index b4ebbc2e..bb70672f 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -9,6 +9,8 @@ import time import threading import os +from changedetectionio.notification import default_notification_format, default_notification_body, default_notification_title + # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods? # Open a github issue if you know something :) # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change @@ -157,6 +159,7 @@ class ChangeDetectionStore: dates = list(self.__data['watching'][uuid]['history'].keys()) # Convert to int, sort and back to str again + # @todo replace datastore getter that does this automatically dates = [int(i) for i in dates] dates.sort(reverse=True) if len(dates): diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index 58b997fc..0719dc19 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -65,6 +65,14 @@ {preview_url} The URL of the preview page generated by changedetection.io. + + {diff} + The diff output - differences only + + + {diff_full} + The diff output - full difference output + {diff_url} The URL of the diff page generated by changedetection.io. diff --git a/changedetectionio/tests/conftest.py b/changedetectionio/tests/conftest.py index fd2806d8..f34ed5bb 100644 --- a/changedetectionio/tests/conftest.py +++ b/changedetectionio/tests/conftest.py @@ -22,7 +22,6 @@ def cleanup(datastore_path): for file in files: try: os.unlink("{}/{}".format(datastore_path, file)) - x = 1 except FileNotFoundError: pass diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 1cf4819c..3cfeecf9 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -55,6 +55,8 @@ def test_check_notification(client, live_server): "Preview: {preview_url}\n" "Diff URL: {diff_url}\n" "Snapshot: {current_snapshot}\n" + "Diff: {diff}\n" + "Diff Full: {diff_full}\n" ":-)", "notification_format": "Text", "url": test_url, @@ -114,6 +116,11 @@ def test_check_notification(client, live_server): assert test_url in notification_submission + # Diff was correctly executed + assert "Diff Full: (changed) Which is across multiple lines" in notification_submission + assert "(-> into) which has this one new line" in notification_submission + + if env_base_url: # Re #65 - did we see our BASE_URl ? logging.debug (">>> BASE_URL checking in notification: %s", env_base_url) diff --git a/changedetectionio/tests/unit/__init__.py b/changedetectionio/tests/unit/__init__.py new file mode 100644 index 00000000..368609ff --- /dev/null +++ b/changedetectionio/tests/unit/__init__.py @@ -0,0 +1 @@ +"""Unit tests for the app.""" diff --git a/changedetectionio/tests/unit/test-content/README.md b/changedetectionio/tests/unit/test-content/README.md new file mode 100644 index 00000000..a1e55fbc --- /dev/null +++ b/changedetectionio/tests/unit/test-content/README.md @@ -0,0 +1,5 @@ +# What is this? +This is test content for the python diff engine, we use the JS interface for the front end, because you can explore +differences in words etc, but we use (at the moment) the python difflib engine. + +This content `before.txt` and `after.txt` is for unit testing diff --git a/changedetectionio/tests/unit/test-content/after.txt b/changedetectionio/tests/unit/test-content/after.txt new file mode 100644 index 00000000..ffb0e3a5 --- /dev/null +++ b/changedetectionio/tests/unit/test-content/after.txt @@ -0,0 +1,6 @@ +After twenty years, as cursed as I may be +for having learned computerese, +I continue to examine bits, bytes and words +xok +and insure that I'm one of those computer nerds. +and something new \ No newline at end of file diff --git a/changedetectionio/tests/unit/test-content/before.txt b/changedetectionio/tests/unit/test-content/before.txt new file mode 100644 index 00000000..8ca997df --- /dev/null +++ b/changedetectionio/tests/unit/test-content/before.txt @@ -0,0 +1,5 @@ +After twenty years, as cursed as I may be +for having learned computerese, +I continue to examine bits, bytes and words +ok +and insure that I'm one of those computer nerds. diff --git a/changedetectionio/tests/unit/test_notification_diff.py b/changedetectionio/tests/unit/test_notification_diff.py new file mode 100755 index 00000000..a3ef3e72 --- /dev/null +++ b/changedetectionio/tests/unit/test_notification_diff.py @@ -0,0 +1,25 @@ +#!/usr/bin/python3 + +# run from dir above changedetectionio/ dir +# python3 -m unittest changedetectionio.tests.unit.test_notification_diff + +import unittest +import os + +from changedetectionio import diff + +# mostly +class TestDiffBuilder(unittest.TestCase): + + def test_expected_diff_output(self): + base_dir=os.path.dirname(__file__) + output = diff.render_diff(base_dir+"/test-content/before.txt", base_dir+"/test-content/after.txt") + output = output.split("\n") + self.assertIn("(changed) ok", output) + self.assertIn("(-> into) xok", output) + self.assertIn("(added) and something new", output) + + # @todo test blocks of changed, blocks of added, blocks of removed + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index ef31756e..4ab1d806 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -56,9 +56,8 @@ class update_worker(threading.Thread): try: self.datastore.update_watch(uuid=uuid, update_obj=update_obj) if changed_detected: - + n_object = {} # A change was detected - newest_version_file_contents = "" fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents) # Update history with the stripped text for future reference, this will also mean we save the first @@ -69,37 +68,55 @@ class update_worker(threading.Thread): print (">> Change detected in UUID {} - {}".format(uuid, watch['url'])) - # Get the newest snapshot data to be possibily used in a notification - newest_key = self.datastore.get_newest_history_key(uuid) - if newest_key: - with open(watch['history'][newest_key], 'r') as f: - newest_version_file_contents = f.read().strip() - - n_object = { - 'watch_url': watch['url'], - 'uuid': uuid, - 'current_snapshot': newest_version_file_contents - } - - # Did it have any notification alerts to hit? - if len(watch['notification_urls']): - print(">>> Notifications queued for UUID from watch {}".format(uuid)) - n_object['notification_urls'] = watch['notification_urls'] - n_object['notification_title'] = watch['notification_title'] - n_object['notification_body'] = watch['notification_body'] - n_object['notification_format'] = watch['notification_format'] - self.notification_q.put(n_object) - - # No? maybe theres a global setting, queue them all - elif len(self.datastore.data['settings']['application']['notification_urls']): - print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid)) - n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls'] - n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title'] - n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body'] - n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format'] - self.notification_q.put(n_object) - else: - print(">>> NO notifications queued, watch and global notification URLs were empty.") + # Notifications should only trigger on the second time (first time, we gather the initial snapshot) + if len(watch['history']) > 1: + + dates = list(watch['history'].keys()) + # Convert to int, sort and back to str again + # @todo replace datastore getter that does this automatically + dates = [int(i) for i in dates] + dates.sort(reverse=True) + dates = [str(i) for i in dates] + + prev_fname = watch['history'][dates[1]] + + + # Did it have any notification alerts to hit? + if len(watch['notification_urls']): + print(">>> Notifications queued for UUID from watch {}".format(uuid)) + n_object['notification_urls'] = watch['notification_urls'] + n_object['notification_title'] = watch['notification_title'] + n_object['notification_body'] = watch['notification_body'] + n_object['notification_format'] = watch['notification_format'] + + # No? maybe theres a global setting, queue them all + elif len(self.datastore.data['settings']['application']['notification_urls']): + print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid)) + n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls'] + n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title'] + n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body'] + n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format'] + else: + print(">>> NO notifications queued, watch and global notification URLs were empty.") + + # Only prepare to notify if the rules above matched + if 'notification_urls' in n_object: + # HTML needs linebreak, but MarkDown and Text can use a linefeed + if n_object['notification_format'] == 'HTML': + line_feed_sep = "
    " + else: + line_feed_sep = "\n" + + from changedetectionio import diff + n_object.update({ + 'watch_url': watch['url'], + 'uuid': uuid, + 'current_snapshot': str(contents), + 'diff_full': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep), + 'diff': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep) + }) + + self.notification_q.put(n_object) except Exception as e: print("!!!! Exception in update_worker !!!\n", e)