Send diff in notifications (#296)

.github/workflows/test-only.yml

@@ -14,6 +14,9 @@ jobs:
         with:
           python-version: 3.9
 
+      - name: Show env vars
+        run: set
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -27,12 +30,15 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
+      - name: Unit tests
+        run: |
+          python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
       - name: Test with pytest
         run: |
           # Each test is totally isolated and performs its own cleanup/reset
           cd changedetectionio; ./run_all_tests.sh
 
-
       # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
       # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
 

changedetectionio/__init__.py

@@ -464,7 +464,6 @@ def changedetection_app(config=None, datastore_o=None):
                 else:
                     flash('No notification URLs set, cannot send test.', 'error')
 
-
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff':
                 return redirect(url_for('diff_history_page', uuid=uuid))
@@ -621,6 +620,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         dates = list(watch['history'].keys())
         # Convert to int, sort and back to str again
+        # @todo replace datastore getter that does this automatically
         dates = [int(i) for i in dates]
         dates.sort(reverse=True)
         dates = [str(i) for i in dates]
@@ -631,13 +631,11 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Save the current newest history as the most recently viewed
         datastore.set_last_viewed(uuid, dates[0])
-
         newest_file = watch['history'][dates[0]]
         with open(newest_file, 'r') as f:
             newest_version_file_contents = f.read()
 
         previous_version = request.args.get('previous_version')
-
         try:
             previous_file = watch['history'][previous_version]
         except KeyError:
@@ -843,8 +841,10 @@ def changedetection_app(config=None, datastore_o=None):
 
     threading.Thread(target=notification_runner).start()
 
-    # Check for new release version
-    threading.Thread(target=check_for_new_version).start()
+    # Check for new release version, but not when running in test/build
+    if not os.getenv("GITHUB_REF", False):
+        threading.Thread(target=check_for_new_version).start()
+
     return app
 
 
@@ -893,8 +893,6 @@ def notification_runner():
             except Exception as e:
                 print("Watch URL: {}  Error {}".format(n_object['watch_url'], e))
 
-
-
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
     from changedetectionio import update_worker

changedetectionio/diff.py

@@ -0,0 +1,43 @@
+# used for the notifications, the front-end is using a JS library
+
+import difflib
+
+# like .compare but a little different output
+def customSequenceMatcher(before, after, include_equal=False):
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
+
+    for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
+        if include_equal and tag == 'equal':
+            g = before[alo:ahi]
+            yield g
+        elif tag == 'delete':
+            g = "(removed) {}".format(before[alo])
+            yield g
+        elif tag == 'replace':
+            g = ["(changed) {}".format(before[alo]), "(-> into) {}".format(after[blo])]
+            yield g
+        elif tag == 'insert':
+            g = "(added) {}".format(after[blo])
+            yield g
+
+# only_differences - only return info about the differences, no context
+# line_feed_sep could be "<br/>" or "<li>" or "\n" etc
+def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
+    with open(newest_file, 'r') as f:
+        newest_version_file_contents = f.read()
+        newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+
+    if previous_file:
+        with open(previous_file, 'r') as f:
+            previous_version_file_contents = f.read()
+            previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+    else:
+        previous_version_file_contents = ""
+
+    rendered_diff = customSequenceMatcher(previous_version_file_contents,
+                                          newest_version_file_contents,
+                                          include_equal)
+
+    # Recursively join lists
+    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
+    return f(rendered_diff)

changedetectionio/forms.py

@@ -6,7 +6,7 @@ from wtforms.fields import html5
 from changedetectionio import content_fetcher
 import re
 
-from changedetectionio.notification import default_notification_format, valid_notification_formats
+from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title
 
 class StringListField(StringField):
     widget = widgets.TextArea()
@@ -203,8 +203,8 @@ class quickWatchForm(Form):
 class commonSettingsForm(Form):
 
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
-    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {watch_url}', validators=[validators.Optional(), ValidateTokensList()])
-    notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()])
+    notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
+    notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
     notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format)
     trigger_check = BooleanField('Send test notification on save')
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])

changedetectionio/notification.py

@@ -1,4 +1,3 @@
-import os
 import apprise
 from apprise import NotifyFormat
 
@@ -8,6 +7,8 @@ valid_tokens = {
     'watch_uuid': '',
     'watch_title': '',
     'watch_tag': '',
+    'diff': '',
+    'diff_full': '',
     'diff_url': '',
     'preview_url': '',
     'current_snapshot': ''
@@ -20,6 +21,8 @@ valid_notification_formats = {
 }
 
 default_notification_format = 'Text'
+default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
 
 def process_notification(n_object, datastore):
     import logging
@@ -33,8 +36,8 @@ def process_notification(n_object, datastore):
         apobj.add(url)
 
     # Get the notification body from datastore
-    n_body = n_object['notification_body']
-    n_title = n_object['notification_title']
+    n_body = n_object.get('notification_body', default_notification_body)
+    n_title = n_object.get('notification_title', default_notification_title)
     n_format = valid_notification_formats.get(
         n_object['notification_format'],
         valid_notification_formats[default_notification_format],
@@ -88,15 +91,17 @@ def create_notification_parameters(n_object, datastore):
 
     # Valid_tokens also used as a field validator
     tokens.update(
-    {
-        'base_url': base_url if base_url is not None else '',
-        'watch_url': watch_url,
-        'watch_uuid': uuid,
-        'watch_title': watch_title if watch_title is not None else '',
-        'watch_tag': watch_tag if watch_tag is not None else '',
-        'diff_url': diff_url,
-        'preview_url': preview_url,
-        'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
-    })
+        {
+            'base_url': base_url if base_url is not None else '',
+            'watch_url': watch_url,
+            'watch_uuid': uuid,
+            'watch_title': watch_title if watch_title is not None else '',
+            'watch_tag': watch_tag if watch_tag is not None else '',
+            'diff_url': diff_url,
+            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
+            'preview_url': preview_url,
+            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
+        })
 
     return tokens

changedetectionio/run_all_tests.sh

@@ -9,15 +9,16 @@
 # exit when any command fails
 set -e
 
-
 find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"
   pytest $test_name
 done
 
+echo "RUNNING WITH BASE_URL SET"
 
 # Now re-run some tests with BASE_URL enabled
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://really-unique-domain.io"
 pytest tests/test_notification.py
+

changedetectionio/store.py

@@ -9,6 +9,8 @@ import time
 import threading
 import os
 
+from changedetectionio.notification import default_notification_format, default_notification_body, default_notification_title
+
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -157,6 +159,7 @@ class ChangeDetectionStore:
 
         dates = list(self.__data['watching'][uuid]['history'].keys())
         # Convert to int, sort and back to str again
+        # @todo replace datastore getter that does this automatically
         dates = [int(i) for i in dates]
         dates.sort(reverse=True)
         if len(dates):

changedetectionio/templates/_common_fields.jinja

@@ -65,6 +65,14 @@
                                         <td><code>{preview_url}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
+                                    <tr>
+                                        <td><code>{diff}</code></td>
+                                        <td>The diff output - differences only</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff_full}</code></td>
+                                        <td>The diff output - full difference output</td>
+                                    </tr>
                                     <tr>
                                         <td><code>{diff_url}</code></td>
                                         <td>The URL of the diff page generated by changedetection.io.</td>

changedetectionio/tests/conftest.py

@@ -22,7 +22,6 @@ def cleanup(datastore_path):
     for file in files:
         try:
             os.unlink("{}/{}".format(datastore_path, file))
-            x = 1
         except FileNotFoundError:
             pass
 

changedetectionio/tests/test_notification.py

@@ -55,6 +55,8 @@ def test_check_notification(client, live_server):
                                    "Preview: {preview_url}\n"
                                    "Diff URL: {diff_url}\n"
                                    "Snapshot: {current_snapshot}\n"
+                                   "Diff: {diff}\n"
+                                   "Diff Full: {diff_full}\n"
                                    ":-)",
               "notification_format": "Text",
               "url": test_url,
@@ -114,6 +116,11 @@ def test_check_notification(client, live_server):
 
     assert test_url in notification_submission
 
+    # Diff was correctly executed
+    assert "Diff Full: (changed) Which is across multiple lines" in notification_submission
+    assert "(-> into) which has this one new line" in notification_submission
+
+
     if env_base_url:
         # Re #65 - did we see our BASE_URl ?
         logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)

changedetectionio/tests/unit/__init__.py

@@ -0,0 +1 @@
+"""Unit tests for the app."""

changedetectionio/tests/unit/test-content/README.md

@@ -0,0 +1,5 @@
+# What is this?
+This is test content for the python diff engine, we use the JS interface for the front end, because you can explore 
+differences in words etc, but we use (at the moment) the python difflib engine.
+
+This content `before.txt` and `after.txt` is for unit testing

changedetectionio/tests/unit/test-content/after.txt

@@ -0,0 +1,6 @@
+After twenty years, as cursed as I may be
+for having learned computerese,
+I continue to examine bits, bytes and words
+xok
+and insure that I'm one of those computer nerds.
+and something new

changedetectionio/tests/unit/test-content/before.txt

@@ -0,0 +1,5 @@
+After twenty years, as cursed as I may be
+for having learned computerese,
+I continue to examine bits, bytes and words
+ok
+and insure that I'm one of those computer nerds.

changedetectionio/tests/unit/test_notification_diff.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
+import unittest
+import os
+
+from changedetectionio import diff
+
+# mostly
+class TestDiffBuilder(unittest.TestCase):
+
+    def test_expected_diff_output(self):
+        base_dir=os.path.dirname(__file__)
+        output = diff.render_diff(base_dir+"/test-content/before.txt", base_dir+"/test-content/after.txt")
+        output = output.split("\n")
+        self.assertIn("(changed) ok", output)
+        self.assertIn("(-> into) xok", output)
+        self.assertIn("(added) and something new", output)
+
+        # @todo test blocks of changed, blocks of added, blocks of removed
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/update_worker.py

@@ -56,9 +56,8 @@ class update_worker(threading.Thread):
                             try:
                                 self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
                                 if changed_detected:
-
+                                    n_object = {}
                                     # A change was detected
-                                    newest_version_file_contents = ""
                                     fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
 
                                     # Update history with the stripped text for future reference, this will also mean we save the first
@@ -69,37 +68,55 @@ class update_worker(threading.Thread):
 
                                     print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
 
-                                    # Get the newest snapshot data to be possibily used in a notification
-                                    newest_key = self.datastore.get_newest_history_key(uuid)
-                                    if newest_key:
-                                        with open(watch['history'][newest_key], 'r') as f:
-                                            newest_version_file_contents = f.read().strip()
-
-                                    n_object = {
-                                        'watch_url': watch['url'],
-                                        'uuid': uuid,
-                                        'current_snapshot': newest_version_file_contents
-                                    }
-
-                                    # Did it have any notification alerts to hit?
-                                    if len(watch['notification_urls']):
-                                        print(">>> Notifications queued for UUID from watch {}".format(uuid))
-                                        n_object['notification_urls'] = watch['notification_urls']
-                                        n_object['notification_title'] = watch['notification_title']
-                                        n_object['notification_body'] = watch['notification_body']
-                                        n_object['notification_format'] = watch['notification_format']
-                                        self.notification_q.put(n_object)
-
-                                    # No? maybe theres a global setting, queue them all
-                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
-                                        print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
-                                        n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
-                                        n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
-                                        n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
-                                        n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
-                                        self.notification_q.put(n_object)
-                                    else:
-                                        print(">>> NO notifications queued, watch and global notification URLs were empty.")
+                                    # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
+                                    if len(watch['history']) > 1:
+
+                                        dates = list(watch['history'].keys())
+                                        # Convert to int, sort and back to str again
+                                        # @todo replace datastore getter that does this automatically
+                                        dates = [int(i) for i in dates]
+                                        dates.sort(reverse=True)
+                                        dates = [str(i) for i in dates]
+
+                                        prev_fname = watch['history'][dates[1]]
+
+
+                                        # Did it have any notification alerts to hit?
+                                        if len(watch['notification_urls']):
+                                            print(">>> Notifications queued for UUID from watch {}".format(uuid))
+                                            n_object['notification_urls'] = watch['notification_urls']
+                                            n_object['notification_title'] = watch['notification_title']
+                                            n_object['notification_body'] = watch['notification_body']
+                                            n_object['notification_format'] = watch['notification_format']
+
+                                        # No? maybe theres a global setting, queue them all
+                                        elif len(self.datastore.data['settings']['application']['notification_urls']):
+                                            print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
+                                            n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+                                            n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
+                                            n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
+                                            n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
+                                        else:
+                                            print(">>> NO notifications queued, watch and global notification URLs were empty.")
+
+                                        # Only prepare to notify if the rules above matched
+                                        if 'notification_urls' in n_object:
+                                            # HTML needs linebreak, but MarkDown and Text can use a linefeed
+                                            if n_object['notification_format'] == 'HTML':
+                                                line_feed_sep = "</br>"
+                                            else:
+                                                line_feed_sep = "\n"
+
+                                            from changedetectionio import diff
+                                            n_object.update({
+                                                'watch_url': watch['url'],
+                                                'uuid': uuid,
+                                                'current_snapshot': str(contents),
+                                                'diff_full': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep),
+                                                'diff': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep)
+                                            })
+
+                                            self.notification_q.put(n_object)
 
                             except Exception as e:
                                 print("!!!! Exception in update_worker !!!\n", e)