From 1543edca242fff993a8c1c02f0c1878e3b0b6e3e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 24 Oct 2024 19:46:45 +0200 Subject: [PATCH 01/49] "Send test notification" in "Restock" mode was not working correclty when restock tokens "{{restock.price}}" were in the notification body (#2737) --- changedetectionio/flask_app.py | 24 ++++++++++++------- changedetectionio/static/js/notifications.js | 9 +++---- changedetectionio/tests/test_notification.py | 17 ++++--------- .../tests/test_restock_itemprop.py | 8 ++++++- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index a1bb862e..c50d0bc3 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -537,21 +537,27 @@ def changedetection_app(config=None, datastore_o=None): import random from .apprise_asset import asset apobj = apprise.Apprise(asset=asset) + # so that the custom endpoints are registered from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper is_global_settings_form = request.args.get('mode', '') == 'global-settings' is_group_settings_form = request.args.get('mode', '') == 'group-settings' + # Use an existing random one on the global/main settings form if not watch_uuid and (is_global_settings_form or is_group_settings_form) \ and datastore.data.get('watching'): - logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}") watch_uuid = random.choice(list(datastore.data['watching'].keys())) - watch = datastore.data['watching'].get(watch_uuid) - else: - watch = None - notification_urls = request.form['notification_urls'].strip().splitlines() + if not watch_uuid: + return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400) + + watch = datastore.data['watching'].get(watch_uuid) + + notification_urls = None + + if request.form.get('notification_urls'): + notification_urls = request.form['notification_urls'].strip().splitlines() if not notification_urls: logger.debug("Test notification - Trying by group/tag in the edit form if available") @@ -569,12 +575,12 @@ def changedetection_app(config=None, datastore_o=None): if not notification_urls: - return 'No Notification URLs set/found' + return 'Error: No Notification URLs set/found' for n_url in notification_urls: if len(n_url.strip()): if not apobj.add(n_url): - return f'Error - {n_url} is not a valid AppRise URL.' + return f'Error: {n_url} is not a valid AppRise URL.' try: # use the same as when it is triggered, but then override it with the form test values @@ -593,11 +599,13 @@ def changedetection_app(config=None, datastore_o=None): if 'notification_body' in request.form and request.form['notification_body'].strip(): n_object['notification_body'] = request.form.get('notification_body', '').strip() + n_object.update(watch.extra_notification_token_values()) + from . import update_worker new_worker = update_worker.update_worker(update_q, notification_q, app, datastore) new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch) except Exception as e: - return make_response({'error': str(e)}, 400) + return make_response(f"Error: str(e)", 400) return 'OK - Sent test notifications' diff --git a/changedetectionio/static/js/notifications.js b/changedetectionio/static/js/notifications.js index 95f3eacf..95a0763c 100644 --- a/changedetectionio/static/js/notifications.js +++ b/changedetectionio/static/js/notifications.js @@ -28,17 +28,14 @@ $(document).ready(function() { url: notification_base_url, data : data, statusCode: { - 400: function() { - // More than likely the CSRF token was lost when the server restarted - alert("There was a problem processing the request, please reload the page."); + 400: function(data) { + // More than likely the CSRF token was lost when the server restarted + alert(data.responseText); } } }).done(function(data){ console.log(data); alert(data); - }).fail(function(data){ - console.log(data); - alert('There was an error communicating with the server.'); }) }); }); diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 760c660e..0016acf4 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -429,24 +429,15 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage follow_redirects=True ) - #2727 - be sure a test notification when there are zero watches works ( should all be deleted now) - - os.unlink("test-datastore/notification.txt") - - - ######### Test global/system settings + ######### Test global/system settings - When everything is deleted it should give a helpful error + # See #2727 res = client.post( url_for("ajax_callback_send_notification_test")+"?mode=global-settings", data={"notification_urls": test_notification_url}, follow_redirects=True ) + assert res.status_code == 400 + assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data - assert res.status_code != 400 - assert res.status_code != 500 - # Give apprise time to fire - time.sleep(4) - with open("test-datastore/notification.txt", 'r') as f: - x = f.read() - assert 'change detection is cool 网站监测 内容更新了' in x diff --git a/changedetectionio/tests/test_restock_itemprop.py b/changedetectionio/tests/test_restock_itemprop.py index 7d0ad7cb..057099ad 100644 --- a/changedetectionio/tests/test_restock_itemprop.py +++ b/changedetectionio/tests/test_restock_itemprop.py @@ -3,7 +3,7 @@ import os import time from flask import url_for -from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output +from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, extract_UUID_from_client from ..notification import default_notification_format instock_props = [ @@ -367,6 +367,12 @@ def test_change_with_notification_values(client, live_server): assert "new price 1950.45" in notification assert "title new price 1950.45" in notification + ## Now test the "SEND TEST NOTIFICATION" is working + os.unlink("test-datastore/notification.txt") + uuid = extract_UUID_from_client(client) + res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True) + time.sleep(5) + assert os.path.isfile("test-datastore/notification.txt"), "Notification received" def test_data_sanity(client, live_server): From e84de7e8f4bb14fac4f8b313e3e7ffdf66981877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Silva?= Date: Thu, 24 Oct 2024 19:03:14 +0100 Subject: [PATCH 02/49] Restock detection - Add additional out-of-stock detection for PT language (#2738) --- changedetectionio/content_fetchers/res/stock-not-in-stock.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changedetectionio/content_fetchers/res/stock-not-in-stock.js b/changedetectionio/content_fetchers/res/stock-not-in-stock.js index 373c669e..ecfa74a7 100644 --- a/changedetectionio/content_fetchers/res/stock-not-in-stock.js +++ b/changedetectionio/content_fetchers/res/stock-not-in-stock.js @@ -30,6 +30,8 @@ function isItemInStock() { 'dieser artikel ist bald wieder verfügbar', 'dostępne wkrótce', 'en rupture de stock', + 'esgotado', + 'indisponível', 'isn\'t in stock right now', 'isnt in stock right now', 'isn’t in stock right now', @@ -57,6 +59,7 @@ function isItemInStock() { 'notify me when available', 'notify me', 'notify when available', + 'não disponível', 'não estamos a aceitar encomendas', 'out of stock', 'out-of-stock', From e8b82c47ca1ce73f55f22078caf724e7bce21c3a Mon Sep 17 00:00:00 2001 From: Christopher Charbonneau Wells <10456740+cdubz@users.noreply.github.com> Date: Mon, 28 Oct 2024 07:46:05 -0700 Subject: [PATCH 03/49] #2502 - Add jinja2 template handling to request body and headers (#2740) --- .gitignore | 1 + changedetectionio/forms.py | 35 ++++++++++++++++++++++-- changedetectionio/processors/__init__.py | 7 +++++ changedetectionio/templates/edit.html | 17 +++++++----- changedetectionio/tests/test_request.py | 11 +++++--- 5 files changed, 57 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 39fc0dd0..99839fec 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ dist venv test-datastore/* test-datastore +test-memory.log *.egg-info* .vscode/settings.json diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index f99496e5..81869988 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -515,6 +515,7 @@ class processor_text_json_diff_form(commonSettingsForm): if not super().validate(): return False + from changedetectionio.safe_jinja import render as jinja_render result = True # Fail form validation when a body is set for a GET @@ -524,18 +525,46 @@ class processor_text_json_diff_form(commonSettingsForm): # Attempt to validate jinja2 templates in the URL try: - from changedetectionio.safe_jinja import render as jinja_render jinja_render(template_str=self.url.data) except ModuleNotFoundError as e: # incase jinja2_time or others is missing logger.error(e) - self.url.errors.append(e) + self.url.errors.append(f'Invalid template syntax configuration: {e}') result = False except Exception as e: logger.error(e) - self.url.errors.append('Invalid template syntax') + self.url.errors.append(f'Invalid template syntax: {e}') result = False + # Attempt to validate jinja2 templates in the body + if self.body.data and self.body.data.strip(): + try: + jinja_render(template_str=self.body.data) + except ModuleNotFoundError as e: + # incase jinja2_time or others is missing + logger.error(e) + self.body.errors.append(f'Invalid template syntax configuration: {e}') + result = False + except Exception as e: + logger.error(e) + self.body.errors.append(f'Invalid template syntax: {e}') + result = False + + # Attempt to validate jinja2 templates in the headers + if len(self.headers.data) > 0: + try: + for header, value in self.headers.data.items(): + jinja_render(template_str=value) + except ModuleNotFoundError as e: + # incase jinja2_time or others is missing + logger.error(e) + self.headers.errors.append(f'Invalid template syntax configuration: {e}') + result = False + except Exception as e: + logger.error(e) + self.headers.errors.append(f'Invalid template syntax in "{header}" header: {e}') + result = False + return result class SingleExtraProxy(Form): diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index e0682a25..7586d4c7 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -102,6 +102,7 @@ class difference_detection_processor(): self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid')) # Tweak the base config with the per-watch ones + from changedetectionio.safe_jinja import render as jinja_render request_headers = CaseInsensitiveDict() ua = self.datastore.data['settings']['requests'].get('default_ua') @@ -118,9 +119,15 @@ class difference_detection_processor(): if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']: request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '') + for header_name in request_headers: + request_headers.update({header_name: jinja_render(template_str=request_headers.get(header_name))}) + timeout = self.datastore.data['settings']['requests'].get('timeout') request_body = self.watch.get('body') + if request_body: + request_body = jinja_render(template_str=self.watch.get('body')) + request_method = self.watch.get('method') ignore_status_codes = self.watch.get('ignore_status_codes', False) diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 5847962f..a0a8f988 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -65,8 +65,8 @@
{{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }} - Some sites use JavaScript to create the content, for this you should use the Chrome/WebDriver Fetcher
- You can use variables in the URL, perfect for inserting the current date and other logic, help and examples here
+
Some sites use JavaScript to create the content, for this you should use the Chrome/WebDriver Fetcher
+
Variables are supported in the URL (help and examples here).
{{ render_field(form.processor) }} @@ -149,21 +149,24 @@ {{ render_field(form.method) }}
- {{ render_field(form.body, rows=5, placeholder="Example + {{ render_field(form.body, rows=7, placeholder="Example { \"name\":\"John\", \"age\":30, - \"car\":null + \"car\":null, + \"year\":{% now 'Europe/Berlin', '%Y' %} }") }}
+
Variables are supported in the request body (help and examples here).