diff --git a/README.md b/README.md index c657b579..47f05922 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,20 @@ Using the **Browser Steps** configuration, add basic steps before performing cha After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in. Requires Playwright to be enabled. +### Awesome restock and price change notifications + +Enable the _"Re-stock & Price detection for single product pages"_ option to activate the best way to monitor product pricing. + +Easily organise and monitor prices for products from the dashboard, get alerts and notifications when the price of a product changes or comes back in stock again! + +[Easily keep an eye on product price changes directly from the UI](https://changedetection.io?src=github) + +Set price change notification parameters, upper and lower price, price change percentage and more. +Always know when a product for sale drops in price. + +[Set upper lower and percentage price change notification values](https://changedetection.io?src=github) + + ### Example use cases diff --git a/changedetectionio/blueprint/tags/__init__.py b/changedetectionio/blueprint/tags/__init__.py index 4b6d40ca..ca974666 100644 --- a/changedetectionio/blueprint/tags/__init__.py +++ b/changedetectionio/blueprint/tags/__init__.py @@ -106,12 +106,14 @@ def construct_blueprint(datastore: ChangeDetectionStore): form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None, data=default, + extra_notification_tokens=datastore.get_unique_notification_tokens_available() ) template_args = { 'data': default, 'form': form, - 'watch': default + 'watch': default, + 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(), } included_content = {} @@ -161,6 +163,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None, data=default, + extra_notification_tokens=datastore.get_unique_notification_tokens_available() ) # @todo subclass form so validation works #if not form.validate(): diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index 7e84fc1e..2ccc68a0 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -128,7 +128,7 @@ nav {% endif %} Use system defaults - {{ render_common_settings_form(form, emailprefix, settings_application) }} + {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }} diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index d5e36158..a66785b5 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -696,8 +696,9 @@ def changedetection_app(config=None, datastore_o=None): form_class = forms.processor_text_json_diff_form form = form_class(formdata=request.form if request.method == 'POST' else None, - data=default - ) + data=default, + extra_notification_tokens=default.extra_notification_token_values() + ) # For the form widget tag UUID back to "string name" for the field form.tags.datastore = datastore @@ -824,6 +825,7 @@ def changedetection_app(config=None, datastore_o=None): 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), 'extra_title': f" - Edit - {watch.label}", 'extra_processor_config': form.extra_tab_content(), + 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(), 'form': form, 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False, 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, @@ -878,7 +880,8 @@ def changedetection_app(config=None, datastore_o=None): # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None, - data=default + data=default, + extra_notification_tokens=datastore.get_unique_notification_tokens_available() ) # Remove the last option 'System default' @@ -930,6 +933,7 @@ def changedetection_app(config=None, datastore_o=None): output = render_template("settings.html", api_key=datastore.data['settings']['application'].get('api_access_token'), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(), form=form, hide_remove_pass=os.getenv("SALTED_PASS", False), min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)), diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 2cefae90..b0b19f99 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -231,9 +231,6 @@ class ValidateJinja2Template(object): """ Validates that a {token} is from a valid set """ - def __init__(self, message=None): - self.message = message - def __call__(self, form, field): from changedetectionio import notification @@ -248,6 +245,10 @@ class ValidateJinja2Template(object): try: jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader) jinja2_env.globals.update(notification.valid_tokens) + # Extra validation tokens provided on the form_class(... extra_tokens={}) setup + if hasattr(field, 'extra_notification_tokens'): + jinja2_env.globals.update(field.extra_notification_tokens) + jinja2_env.from_string(joined_data).render() except TemplateSyntaxError as e: raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e @@ -422,6 +423,12 @@ class quickWatchForm(Form): class commonSettingsForm(Form): from . import processors + def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs): + super().__init__(formdata, obj, prefix, data, meta, **kwargs) + self.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) + self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) + self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) + extract_title_as_title = BooleanField('Extract from document and use as watch title', default=False) fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()]) notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()]) @@ -429,8 +436,8 @@ class commonSettingsForm(Form): notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()]) notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()]) processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff") - webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, - message="Should contain one or more seconds")]) + webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")]) + class importForm(Form): from . import processors @@ -590,6 +597,11 @@ class globalSettingsForm(Form): # Define these as FormFields/"sub forms", this way it matches the JSON storage # datastore.data['settings']['application'].. # datastore.data['settings']['requests'].. + def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs): + super().__init__(formdata, obj, prefix, data, meta, **kwargs) + self.application.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) + self.application.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) + self.application.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) requests = FormField(globalSettingsRequestForm) application = FormField(globalSettingsApplicationForm) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index e4697f38..d3167bf9 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -432,6 +432,17 @@ class model(watch_base): def toggle_mute(self): self['notification_muted'] ^= True + def extra_notification_token_values(self): + # Used for providing extra tokens + # return {'widget': 555} + return {} + + def extra_notification_token_placeholder_info(self): + # Used for providing extra tokens + # return [('widget', "Get widget amounts")] + return [] + + def extract_regex_from_all_history(self, regex): import csv import re diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 41285ce4..ae01f001 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -272,19 +272,18 @@ def create_notification_parameters(n_object, datastore): tokens.update( { 'base_url': base_url, - 'current_snapshot': n_object.get('current_snapshot', ''), - 'diff': n_object.get('diff', ''), # Null default in the case we use a test - 'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test - 'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test - 'diff_patch': n_object.get('diff_patch', ''), # Null default in the case we use a test - 'diff_removed': n_object.get('diff_removed', ''), # Null default in the case we use a test 'diff_url': diff_url, 'preview_url': preview_url, - 'triggered_text': n_object.get('triggered_text', ''), 'watch_tag': watch_tag if watch_tag is not None else '', 'watch_title': watch_title if watch_title is not None else '', 'watch_url': watch_url, 'watch_uuid': uuid, }) + # n_object will contain diff, diff_added etc etc + tokens.update(n_object) + + if uuid: + tokens.update(datastore.data['watching'].get(uuid).extra_notification_token_values()) + return tokens diff --git a/changedetectionio/processors/restock_diff/__init__.py b/changedetectionio/processors/restock_diff/__init__.py index 3f0be03b..0cad3db1 100644 --- a/changedetectionio/processors/restock_diff/__init__.py +++ b/changedetectionio/processors/restock_diff/__init__.py @@ -68,3 +68,16 @@ class Watch(BaseWatch): super().clear_watch() self.update({'restock': Restock()}) + def extra_notification_token_values(self): + values = super().extra_notification_token_values() + values['restock'] = self.get('restock', {}) + return values + + def extra_notification_token_placeholder_info(self): + values = super().extra_notification_token_placeholder_info() + + values.append(('restock.price', "Price detected")) + values.append(('restock.original_price', "Original price at first check")) + + return values + diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 0d88f8b2..c3772557 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -631,6 +631,33 @@ class ChangeDetectionStore: return True return False + def get_unique_notification_tokens_available(self): + # Ask each type of watch if they have any extra notification token to add to the validation + extra_notification_tokens = {} + watch_processors_checked = set() + + for watch_uuid, watch in self.__data['watching'].items(): + processor = watch.get('processor') + if processor not in watch_processors_checked: + extra_notification_tokens.update(watch.extra_notification_token_values()) + watch_processors_checked.add(processor) + + return extra_notification_tokens + + def get_unique_notification_token_placeholders_available(self): + # The actual description of the tokens, could be combined with get_unique_notification_tokens_available instead of doing this twice + extra_notification_tokens = [] + watch_processors_checked = set() + + for watch_uuid, watch in self.__data['watching'].items(): + processor = watch.get('processor') + if processor not in watch_processors_checked: + extra_notification_tokens+=watch.extra_notification_token_placeholder_info() + watch_processors_checked.add(processor) + + return extra_notification_tokens + + def get_updates_available(self): import inspect updates_available = [] diff --git a/changedetectionio/templates/_common_fields.html b/changedetectionio/templates/_common_fields.html index 932f3fb7..14fa9147 100644 --- a/changedetectionio/templates/_common_fields.html +++ b/changedetectionio/templates/_common_fields.html @@ -1,7 +1,7 @@ {% from '_helpers.html' import render_field %} -{% macro render_common_settings_form(form, emailprefix, settings_application) %} +{% macro render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) %} <div class="pure-control-group"> {{ render_field(form.notification_urls, rows=5, placeholder="Examples: Gitter - gitter://token/room @@ -107,7 +107,15 @@ <tr> <td><code>{{ '{{triggered_text}}' }}</code></td> <td>Text that tripped the trigger from filters</td> - </tr> + + {% if extra_notification_token_placeholder_info %} + {% for token in extra_notification_token_placeholder_info %} + <tr> + <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td> + <td>{{ token[1] }}</td> + </tr> + {% endfor %} + {% endif %} </tbody> </table> <div class="pure-form-message-inline"> diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 0c3dfecb..4b9a7fc1 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -246,7 +246,7 @@ User-Agent: wonderbra 1.0") }} {% endif %} <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a> - {{ render_common_settings_form(form, emailprefix, settings_application) }} + {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }} </div> </fieldset> </div> diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 1212def0..e07abfa7 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -92,7 +92,7 @@ <div class="tab-pane-inner" id="notifications"> <fieldset> <div class="field-group"> - {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }} + {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }} </div> </fieldset> <div class="pure-control-group" id="notification-base-url"> diff --git a/changedetectionio/tests/test_restock_itemprop.py b/changedetectionio/tests/test_restock_itemprop.py index d1af83a0..649e9f23 100644 --- a/changedetectionio/tests/test_restock_itemprop.py +++ b/changedetectionio/tests/test_restock_itemprop.py @@ -1,8 +1,10 @@ #!/usr/bin/python3 +import os import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client +from ..notification import default_notification_format instock_props = [ # LD+JSON with non-standard list of 'type' https://github.com/dgtlmoon/changedetection.io/issues/1833 @@ -305,6 +307,70 @@ def test_itemprop_percent_threshold(client, live_server): res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data + + +def test_change_with_notification_values(client, live_server): + #live_server_setup(live_server) + + if os.path.isfile("test-datastore/notification.txt"): + os.unlink("test-datastore/notification.txt") + + test_url = url_for('test_endpoint', _external=True) + set_original_response(props_markup=instock_props[0], price='960.45') + + notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json') + + ###################### + # You must add a type of 'restock_diff' for its tokens to register as valid in the global settings + client.post( + url_for("form_quick_watch_add"), + data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'}, + follow_redirects=True + ) + + # A change in price, should trigger a change by default + wait_for_all_checks(client) + + # Should see new tokens register + res = client.get(url_for("settings_page")) + assert b'{{restock.original_price}}' in res.data + assert b'Original price at first check' in res.data + + ##################### + # Set this up for when we remove the notification from the watch, it should fallback with these details + res = client.post( + url_for("settings_page"), + data={"application-notification_urls": notification_url, + "application-notification_title": "title new price {{restock.price}}", + "application-notification_body": "new price {{restock.price}}", + "application-notification_format": default_notification_format, + "requests-time_between_check-minutes": 180, + 'application-fetch_backend': "html_requests"}, + follow_redirects=True + ) + + # check tag accepts without error + + # Check the watches in these modes add the tokens for validating + assert b"A variable or function is not defined" not in res.data + + assert b"Settings updated." in res.data + + + set_original_response(props_markup=instock_props[0], price='960.45') + # A change in price, should trigger a change by default + set_original_response(props_markup=instock_props[0], price='1950.45') + client.get(url_for("form_watch_checknow")) + wait_for_all_checks(client) + time.sleep(3) + assert os.path.isfile("test-datastore/notification.txt"), "Notification received" + with open("test-datastore/notification.txt", 'r') as f: + notification = f.read() + assert "new price 1950.45" in notification + assert "title new price 1950.45" in notification + + + def test_data_sanity(client, live_server): #live_server_setup(live_server) diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 104db24c..1b2207f8 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -81,6 +81,9 @@ class update_worker(threading.Thread): 'uuid': watch.get('uuid') if watch else None, 'watch_url': watch.get('url') if watch else None, }) + + n_object.update(watch.extra_notification_token_values()) + logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s") logger.debug("Queued notification for sending") notification_q.put(n_object) diff --git a/docs/restock-overview.png b/docs/restock-overview.png new file mode 100644 index 00000000..c4a4e78f Binary files /dev/null and b/docs/restock-overview.png differ diff --git a/docs/restock-settings.png b/docs/restock-settings.png new file mode 100644 index 00000000..1e468632 Binary files /dev/null and b/docs/restock-settings.png differ