Notification Improvements - New tokens `{{diff_added}}` and `{{diff_removed}}`, removed whitespace around `added` and `into` ( Issue #905 ) (#1454)

changedetectionio/diff.py

@@ -10,7 +10,7 @@ def same_slicer(l, a, b):
         return l[a:b]
 
 # like .compare but a little different output
-def customSequenceMatcher(before, after, include_equal=False):
+def customSequenceMatcher(before, after, include_equal=False, include_removed=True, include_added=True):
     cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
 
     # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
@@ -18,20 +18,20 @@ def customSequenceMatcher(before, after, include_equal=False):
         if include_equal and tag == 'equal':
             g = before[alo:ahi]
             yield g
-        elif tag == 'delete':
+        elif include_removed and tag == 'delete':
             g = ["(removed) " + i for i in same_slicer(before, alo, ahi)]
             yield g
         elif tag == 'replace':
             g = ["(changed) " + i for i in same_slicer(before, alo, ahi)]
-            g += ["(into   ) " + i for i in same_slicer(after, blo, bhi)]
+            g += ["(into) " + i for i in same_slicer(after, blo, bhi)]
             yield g
-        elif tag == 'insert':
+        elif include_added and tag == 'insert':
-            g = ["(added  ) " + i for i in same_slicer(after, blo, bhi)]
+            g = ["(added) " + i for i in same_slicer(after, blo, bhi)]
             yield g
 
 # only_differences - only return info about the differences, no context
 # line_feed_sep could be "<br/>" or "<li>" or "\n" etc
-def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
+def render_diff(previous_file, newest_file, include_equal=False, include_removed=True, include_added=True, line_feed_sep="\n"):
     with open(newest_file, 'r') as f:
         newest_version_file_contents = f.read()
         newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
@@ -45,7 +45,7 @@ def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="
 
     rendered_diff = customSequenceMatcher(previous_version_file_contents,
                                           newest_version_file_contents,
-                                          include_equal)
+                                          include_equal, include_removed, include_added)
 
     # Recursively join lists
     f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])

changedetectionio/notification.py

@@ -10,6 +10,8 @@ valid_tokens = {
     'watch_title': '',
     'watch_tag': '',
     'diff': '',
+    'diff_added': '',
+    'diff_removed': '',
     'diff_full': '',
     'diff_url': '',
     'preview_url': '',
@@ -215,6 +217,8 @@ def create_notification_parameters(n_object, datastore):
             'watch_tag': watch_tag if watch_tag is not None else '',
             'diff_url': diff_url,
             'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
+            'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
             'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
             'preview_url': preview_url,
             'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''

changedetectionio/templates/_common_fields.jinja

@@ -55,39 +55,51 @@
                                     </thead>
                                     <tbody>
                                     <tr>
-                                        <td><code>{{ '{{ base_url }}' }}</code></td>
+                                        <td><code>{{ '{{base_url}}' }}</code></td>
                                         <td>The URL of the changedetection.io instance you are running.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_url }}' }}</code></td>
+                                        <td><code>{{ '{{watch_url}}' }}</code></td>
                                         <td>The URL being watched.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_uuid }}' }}</code></td>
+                                        <td><code>{{ '{{watch_uuid}}' }}</code></td>
                                         <td>The UUID of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_title }}' }}</code></td>
+                                        <td><code>{{ '{{watch_title}}' }}</code></td>
                                         <td>The title of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_tag }}' }}</code></td>
+                                        <td><code>{{ '{{watch_tag}}' }}</code></td>
                                         <td>The watch label / tag</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ preview_url }}' }}</code></td>
+                                        <td><code>{{ '{{preview_url}}' }}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ diff_url }}' }}</code></td>
+                                        <td><code>{{ '{{diff_url}}' }}</code></td>
-                                        <td>The diff output - differences only</td>
+                                        <td>The URL of the diff output for the watch.</td>
                                     </tr>
 									<tr>
-                                        <td><code>{{ '{{ diff_full }}' }}</code></td>
+                                        <td><code>{{ '{{diff}}' }}</code></td>
+                                        <td>The diff output - only changes, additions, and removals</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_added}}' }}</code></td>
+                                        <td>The diff output - only changes and additions</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_removed}}' }}</code></td>
+                                        <td>The diff output - only changes and removals</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{{ '{{diff_full}}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ current_snapshot }}' }}</code></td>
+                                        <td><code>{{ '{{current_snapshot}}' }}</code></td>
                                         <td>The current snapshot value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
@@ -95,8 +107,10 @@
                                 </table>
                                 <div class="pure-form-message-inline">
                                     <br>
-                                    URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                    URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
                                     Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
+									<br>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removal%7D%7D-notification-tokens">More Here</a> </br>
                                 </div>
                             </div>
                         </div>

changedetectionio/tests/test_backend.py

@@ -82,7 +82,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'<rss' in res.data
 
     # re #16 should have the diff in here too
-    assert b'(into   ) which has this one new line' in res.data
+    assert b'(into) which has this one new line' in res.data
     assert b'CDATA' in res.data
 
     assert expected_url.encode('utf-8') in res.data

changedetectionio/tests/test_notification.py

@@ -100,6 +100,8 @@ def test_check_notification(client, live_server):
                                                    "Diff URL: {{diff_url}}\n"
                                                    "Snapshot: {{current_snapshot}}\n"
                                                    "Diff: {{diff}}\n"
+                                                   "Diff Added: {{diff_added}}\n"
+                                                   "Diff Removed: {{diff_removed}}\n"
                                                    "Diff Full: {{diff_full}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
@@ -147,7 +149,7 @@ def test_check_notification(client, live_server):
     assert ':-)' in notification_submission
     assert "Diff Full: Some initial text" in notification_submission
     assert "Diff: (changed) Which is across multiple lines" in notification_submission
-    assert "(into   ) which has this one new line" in notification_submission
+    assert "(into) which has this one new line" in notification_submission
     # Re #342 - check for accidental python byte encoding of non-utf8/string
     assert "b'" not in notification_submission
     assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)

changedetectionio/tests/unit/test_notification_diff.py

@@ -16,9 +16,9 @@ class TestDiffBuilder(unittest.TestCase):
         output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after.txt")
         output = output.split("\n")
         self.assertIn('(changed) ok', output)
-        self.assertIn('(into   ) xok', output)
+        self.assertIn('(into) xok', output)
-        self.assertIn('(into   ) next-x-ok', output)
+        self.assertIn('(into) next-x-ok', output)
-        self.assertIn('(added  ) and something new', output)
+        self.assertIn('(added) and something new', output)
 
 
         output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after-2.txt")
@@ -26,6 +26,21 @@ class TestDiffBuilder(unittest.TestCase):
         self.assertIn('(removed) for having learned computerese,', output)
         self.assertIn('(removed) I continue to examine bits, bytes and words', output)
         
+        #diff_removed
+        output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after.txt", include_equal=False, include_removed=True, include_added=False)
+        output = output.split("\n")
+        self.assertIn('(changed) ok', output)
+        self.assertIn('(into) xok', output)
+        self.assertIn('(into) next-x-ok', output)
+        self.assertNotIn('(added) and something new', output)
+        
+        #diff_removed
+        output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after-2.txt", include_equal=False, include_removed=True, include_added=False)
+        output = output.split("\n")
+        self.assertIn('(removed) for having learned computerese,', output)
+        self.assertIn('(removed) I continue to examine bits, bytes and words', output)
+        
+
         # @todo test blocks of changed, blocks of added, blocks of removed
 
 if __name__ == '__main__':

changedetectionio/update_worker.py

@@ -78,7 +78,9 @@ class update_worker(threading.Thread):
                 'screenshot': watch.get_screenshot_as_jpeg() if watch.get('notification_screenshot') else None,
                 'current_snapshot': snapshot_contents.decode('utf-8'),
                 'diff': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], line_feed_sep=line_feed_sep),
-                'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], True, line_feed_sep=line_feed_sep)
+                'diff_added': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], include_removed=False, line_feed_sep=line_feed_sep),
+                'diff_removed': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], include_added=False, line_feed_sep=line_feed_sep),
+                'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], include_equal=True, line_feed_sep=line_feed_sep)
             })
             logging.info (">> SENDING NOTIFICATION")
             self.notification_q.put(n_object)