diff --git a/Dockerfile b/Dockerfile index a88ab5a6..2272ea01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ libc-dev \ libffi-dev \ + libjpeg-dev \ libssl-dev \ libxslt-dev \ make \ @@ -36,13 +37,14 @@ ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1 # Re #93, #73, excluding rustc (adds another 430Mb~) RUN apt-get update && apt-get install -y --no-install-recommends \ - libssl-dev \ - libffi-dev \ + g++ \ gcc \ libc-dev \ + libffi-dev \ + libjpeg-dev \ + libssl-dev \ libxslt-dev \ - zlib1g-dev \ - g++ + zlib1g-dev # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops ENV PYTHONUNBUFFERED=1 diff --git a/README.md b/README.md index 86d0cd52..790212fb 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W - Override Request Headers, Specify `POST` or `GET` and other methods - Use the "Visual Selector" to help target specific elements - Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration) +- Send a screenshot with the notification when a change is detected in the web page We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link. diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 97db72cf..0215f12c 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -644,12 +644,18 @@ def changedetection_app(config=None, datastore_o=None): except ModuleNotFoundError: jq_support = False + watch = datastore.data['watching'].get(uuid) + system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' + is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or ( + watch.get('fetch_backend', None) is None and system_uses_webdriver) else False + output = render_template("edit.html", current_base_url=datastore.data['settings']['application']['base_url'], emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), form=form, has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, has_empty_checktime=using_default_check_time, + is_html_webdriver=is_html_webdriver, jq_support=jq_support, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), settings_application=datastore.data['settings']['application'], @@ -657,7 +663,7 @@ def changedetection_app(config=None, datastore_o=None): uuid=uuid, visualselector_data_is_ready=visualselector_data_is_ready, visualselector_enabled=visualselector_enabled, - watch=datastore.data['watching'][uuid], + watch=watch ) return output diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 3689cf93..62eaf33b 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -248,7 +248,28 @@ class model(dict): if os.path.isfile(fname): return fname - return False + # False is not an option for AppRise, must be type None + return None + + def get_screenshot_as_jpeg(self): + """Best used in notifications due to its smaller size""" + png_fname = os.path.join(self.watch_data_dir, "last-screenshot.png") + jpg_fname = os.path.join(self.watch_data_dir, "last-screenshot.jpg") + + if os.path.isfile(jpg_fname): + return jpg_fname + + if os.path.isfile(png_fname) and not os.path.isfile(jpg_fname): + # Doesnt exist, so create the JPEG from the PNG + from PIL import Image + im1 = Image.open(png_fname) + im1.convert('RGB').save(jpg_fname, quality=int(os.getenv("NOTIFICATION_SCREENSHOT_JPG_QUALITY", 75))) + return jpg_fname + + + # False is not an option for AppRise, must be type None + return None + def __get_file_ctime(self, filename): fname = os.path.join(self.watch_data_dir, filename) diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 116c0cce..ec5e9d19 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -103,7 +103,7 @@ def process_notification(n_object, datastore): body=n_body, body_format=n_format, # False is not an option for AppRise, must be type None - attach=None if not n_object.get('screenshot') else n_object.get('screenshot') + attach=n_object.get('screenshot', None) ) apobj.clear() diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 0f8e2ce1..57113272 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -141,9 +141,14 @@ User-Agent: wonderbra 1.0") }}
{{ render_checkbox_field(form.notification_muted) }}
-
+ {% if is_html_webdriver %} + {{ render_checkbox_field(form.notification_screenshot) }} + + Use with caution! This will easily fill up your email storage quota or flood other storages. +
+ {% endif %}
{% if has_default_notification_urls %}
diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index dbd0d26e..b71e5470 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -1,3 +1,4 @@ +import json import os import time import re @@ -20,7 +21,6 @@ def test_setup(live_server): # Hard to just add more live server URLs when one test is already running (I think) # So we add our test here (was in a different file) def test_check_notification(client, live_server): - set_original_response() # Give the endpoint time to spin up @@ -70,13 +70,14 @@ def test_check_notification(client, live_server): # Give the thread time to pick up the first version time.sleep(3) - testimage = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=' + # We write the PNG to disk, but a JPEG should appear in the notification + testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=' # Write the last screenshot png uuid = extract_UUID_from_client(client) datastore = 'test-datastore' with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f: - f.write(base64.b64decode(testimage)) + f.write(base64.b64decode(testimage_png)) # Goto the edit page, add our ignore text # Add our URL to the import page @@ -153,7 +154,19 @@ def test_check_notification(client, live_server): assert "preview/" in notification_submission assert ":-)" in notification_submission assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission - assert testimage in notification_submission + + # Check the attachment was added, and that it is a JPEG from the original PNG + notification_submission_object = json.loads(notification_submission) + assert notification_submission_object['attachments'][0]['filename'] == 'last-screenshot.jpg' + assert len(notification_submission_object['attachments'][0]['base64']) + assert notification_submission_object['attachments'][0]['mimetype'] == 'image/jpeg' + jpeg_in_attachment = base64.b64decode(notification_submission_object['attachments'][0]['base64']) + assert b'JFIF' in jpeg_in_attachment + assert testimage_png not in notification_submission + # Assert that the JPEG is readable (didn't get chewed up somewhere) + from PIL import Image + import io + assert Image.open(io.BytesIO(jpeg_in_attachment)) if env_base_url: # Re #65 - did we see our BASE_URl ? diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 154187b1..4f2b144d 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -74,7 +74,7 @@ class update_worker(threading.Thread): n_object.update({ 'watch_url': watch['url'], 'uuid': watch_uuid, - 'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else False, + 'screenshot': watch.get_screenshot_as_jpeg() if watch.get('notification_screenshot') else None, 'current_snapshot': snapshot_contents.decode('utf-8'), 'diff': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], line_feed_sep=line_feed_sep), 'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], True, line_feed_sep=line_feed_sep) @@ -108,7 +108,7 @@ class update_worker(threading.Thread): n_object.update({ 'watch_url': watch['url'], 'uuid': watch_uuid, - 'screenshot': False + 'screenshot': None }) self.notification_q.put(n_object) print("Sent filter not found notification for {}".format(watch_uuid)) diff --git a/requirements.txt b/requirements.txt index bdf27618..86d03afa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,5 +54,7 @@ jinja2-time # https://github.com/dgtlmoon/changedetection.io/pull/1009 jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux" +# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future +pillow # playwright is installed at Dockerfile build time because it's not available on all platforms