Notification screenshot option should only be available to webdriver/playwright watches, screenshot sent as JPEG to save bandwidth, Simplify the logic around screenshot, (#1140)

Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     gcc \
     libc-dev \
     libffi-dev \
+    libjpeg-dev \
     libssl-dev \
     libxslt-dev \
     make \
@@ -36,13 +37,14 @@ ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 # Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl-dev \
-    libffi-dev \
+    g++ \
     gcc \
     libc-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libssl-dev \
     libxslt-dev \
-    zlib1g-dev \
-    g++
+    zlib1g-dev
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1

README.md

@@ -53,6 +53,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 - Override Request Headers, Specify `POST` or `GET` and other methods
 - Use the "Visual Selector" to help target specific elements
 - Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
+- Send a screenshot with the notification when a change is detected in the web page
 
 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
 

changedetectionio/__init__.py

@@ -644,12 +644,18 @@ def changedetection_app(config=None, datastore_o=None):
             except ModuleNotFoundError:
                 jq_support = False
 
+            watch = datastore.data['watching'].get(uuid)
+            system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+            is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
+                    watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
+
             output = render_template("edit.html",
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
+                                     is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                                      settings_application=datastore.data['settings']['application'],
@@ -657,7 +663,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      uuid=uuid,
                                      visualselector_data_is_ready=visualselector_data_is_ready,
                                      visualselector_enabled=visualselector_enabled,
-                                     watch=datastore.data['watching'][uuid],
+                                     watch=watch
                                      )
 
         return output

changedetectionio/model/Watch.py

@@ -248,7 +248,28 @@ class model(dict):
         if os.path.isfile(fname):
             return fname
 
-        return False
+        # False is not an option for AppRise, must be type None
+        return None
+
+    def get_screenshot_as_jpeg(self):
+        """Best used in notifications due to its smaller size"""
+        png_fname = os.path.join(self.watch_data_dir, "last-screenshot.png")
+        jpg_fname = os.path.join(self.watch_data_dir, "last-screenshot.jpg")
+
+        if os.path.isfile(jpg_fname):
+            return jpg_fname
+
+        if os.path.isfile(png_fname) and not os.path.isfile(jpg_fname):
+            # Doesnt exist, so create the JPEG from the PNG
+            from PIL import Image
+            im1 = Image.open(png_fname)
+            im1.convert('RGB').save(jpg_fname, quality=int(os.getenv("NOTIFICATION_SCREENSHOT_JPG_QUALITY", 75)))
+            return jpg_fname
+
+
+        # False is not an option for AppRise, must be type None
+        return None
+
 
     def __get_file_ctime(self, filename):
         fname = os.path.join(self.watch_data_dir, filename)

changedetectionio/notification.py

@@ -103,7 +103,7 @@ def process_notification(n_object, datastore):
                     body=n_body,
                     body_format=n_format,
                     # False is not an option for AppRise, must be type None
-                    attach=None if not n_object.get('screenshot') else n_object.get('screenshot')
+                    attach=n_object.get('screenshot', None)
                 )
 
                 apobj.clear()

changedetectionio/templates/edit.html

@@ -141,9 +141,14 @@ User-Agent: wonderbra 1.0") }}
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
-                    <div  class="pure-control-group inline-radio">
+                    {% if is_html_webdriver %}
+                    <span class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
+                        <span class="pure-form-message-inline">
+                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
+                        </span>
                     </div>
+                    {% endif %}
                     <div class="field-group" id="notification-field-group">
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">

changedetectionio/tests/test_notification.py

@@ -1,3 +1,4 @@
+import json
 import os
 import time
 import re
@@ -20,7 +21,6 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
-
     set_original_response()
 
     # Give the endpoint time to spin up
@@ -70,13 +70,14 @@ def test_check_notification(client, live_server):
     # Give the thread time to pick up the first version
     time.sleep(3)
 
-    testimage = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
+    # We write the PNG to disk, but a JPEG should appear in the notification
+    testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
     # Write the last screenshot png
 
     uuid = extract_UUID_from_client(client)
     datastore = 'test-datastore'
     with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
-        f.write(base64.b64decode(testimage))
+        f.write(base64.b64decode(testimage_png))
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -153,7 +154,19 @@ def test_check_notification(client, live_server):
     assert "preview/" in notification_submission
     assert ":-)" in notification_submission
     assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
-    assert testimage in notification_submission
+
+    # Check the attachment was added, and that it is a JPEG from the original PNG
+    notification_submission_object = json.loads(notification_submission)
+    assert notification_submission_object['attachments'][0]['filename'] == 'last-screenshot.jpg'
+    assert len(notification_submission_object['attachments'][0]['base64'])
+    assert notification_submission_object['attachments'][0]['mimetype'] == 'image/jpeg'
+    jpeg_in_attachment = base64.b64decode(notification_submission_object['attachments'][0]['base64'])
+    assert b'JFIF' in jpeg_in_attachment
+    assert testimage_png not in notification_submission
+    # Assert that the JPEG is readable (didn't get chewed up somewhere)
+    from PIL import Image
+    import io
+    assert Image.open(io.BytesIO(jpeg_in_attachment))
 
     if env_base_url:
         # Re #65 - did we see our BASE_URl ?

changedetectionio/update_worker.py

@@ -74,7 +74,7 @@ class update_worker(threading.Thread):
             n_object.update({
                 'watch_url': watch['url'],
                 'uuid': watch_uuid,
-                'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else False,
+                'screenshot': watch.get_screenshot_as_jpeg() if watch.get('notification_screenshot') else None,
                 'current_snapshot': snapshot_contents.decode('utf-8'),
                 'diff': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], line_feed_sep=line_feed_sep),
                 'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], True, line_feed_sep=line_feed_sep)
@@ -108,7 +108,7 @@ class update_worker(threading.Thread):
             n_object.update({
                 'watch_url': watch['url'],
                 'uuid': watch_uuid,
-                'screenshot': False
+                'screenshot': None
             })
             self.notification_q.put(n_object)
             print("Sent filter not found notification for {}".format(watch_uuid))

requirements.txt

@@ -54,5 +54,7 @@ jinja2-time
 # https://github.com/dgtlmoon/changedetection.io/pull/1009
 jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux"
 
+# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
+pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms