From bde27c8a8fffb3abaaa9dd010e7d56c6f1a02f69 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 16 Jul 2024 17:23:39 +0200 Subject: [PATCH] Restock & Price detection - Ability to set up a tag/group that applies to all watches with price + restock limits --- changedetectionio/blueprint/tags/__init__.py | 56 ++++++++++++--- changedetectionio/blueprint/tags/form.py | 9 ++- .../blueprint/tags/templates/edit-tag.html | 9 +++ changedetectionio/flask_app.py | 10 ++- changedetectionio/model/Tag.py | 6 +- .../processors/restock_diff/__init__.py | 5 ++ .../processors/restock_diff/forms.py | 64 ++++++++++------- .../processors/restock_diff/processor.py | 37 ++++++---- changedetectionio/static/js/watch-settings.js | 1 - changedetectionio/store.py | 46 ++++++++++--- .../tests/test_restock_itemprop.py | 69 +++++++++++++++---- 11 files changed, 230 insertions(+), 82 deletions(-) diff --git a/changedetectionio/blueprint/tags/__init__.py b/changedetectionio/blueprint/tags/__init__.py index e826aea0..4b6d40ca 100644 --- a/changedetectionio/blueprint/tags/__init__.py +++ b/changedetectionio/blueprint/tags/__init__.py @@ -1,4 +1,6 @@ -from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect +from flask import Blueprint, request, render_template, flash, url_for, redirect + + from changedetectionio.store import ChangeDetectionStore from changedetectionio.flask_app import login_optionally_required @@ -96,22 +98,53 @@ def construct_blueprint(datastore: ChangeDetectionStore): @tags_blueprint.route("/edit/", methods=['GET']) @login_optionally_required def form_tag_edit(uuid): - from changedetectionio import forms - + from changedetectionio.blueprint.tags.form import group_restock_settings_form if uuid == 'first': uuid = list(datastore.data['settings']['application']['tags'].keys()).pop() default = datastore.data['settings']['application']['tags'].get(uuid) - form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None, - data=default, - ) - form.datastore=datastore # needed? + form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None, + data=default, + ) + + template_args = { + 'data': default, + 'form': form, + 'watch': default + } + + included_content = {} + if form.extra_form_content(): + # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/ + # And then render the code from the module + from jinja2 import Environment, FileSystemLoader + import importlib.resources + templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates')) + env = Environment(loader=FileSystemLoader(templates_dir)) + template_str = """{% from '_helpers.html' import render_field, render_checkbox_field, render_button %} + +
+
+
+ {{ render_checkbox_field(form.overrides_watch) }} + Used for watches in "Restock & Price detection" mode +
+
+ """ + template_str += form.extra_form_content() + template = env.from_string(template_str) + included_content = template.render(**template_args) output = render_template("edit-tag.html", - data=default, - form=form, settings_application=datastore.data['settings']['application'], + extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None, + extra_form_content=included_content, + **template_args ) return output @@ -120,13 +153,13 @@ def construct_blueprint(datastore: ChangeDetectionStore): @tags_blueprint.route("/edit/", methods=['POST']) @login_optionally_required def form_tag_edit_submit(uuid): - from changedetectionio import forms + from changedetectionio.blueprint.tags.form import group_restock_settings_form if uuid == 'first': uuid = list(datastore.data['settings']['application']['tags'].keys()).pop() default = datastore.data['settings']['application']['tags'].get(uuid) - form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None, + form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None, data=default, ) # @todo subclass form so validation works @@ -136,6 +169,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): # return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid)) datastore.data['settings']['application']['tags'][uuid].update(form.data) + datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff' datastore.needs_write_urgent = True flash("Updated") diff --git a/changedetectionio/blueprint/tags/form.py b/changedetectionio/blueprint/tags/form.py index 22e8b077..6ff3a503 100644 --- a/changedetectionio/blueprint/tags/form.py +++ b/changedetectionio/blueprint/tags/form.py @@ -1,16 +1,15 @@ from wtforms import ( - BooleanField, Form, - IntegerField, - RadioField, - SelectField, StringField, SubmitField, - TextAreaField, validators, ) +from wtforms.fields.simple import BooleanField +from changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form +class group_restock_settings_form(restock_settings_form): + overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False) class SingleTag(Form): diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index 9f316c55..7e84fc1e 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -26,6 +26,9 @@ @@ -97,6 +100,12 @@ nav + {# rendered sub Template #} + {% if extra_form_content %} +
+ {{ extra_form_content|safe }} +
+ {% endif %}
diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 59bfa67a..d5e36158 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -699,8 +699,12 @@ def changedetection_app(config=None, datastore_o=None): data=default ) - # For the form widget tag uuid lookup - form.tags.datastore = datastore # in _value + # For the form widget tag UUID back to "string name" for the field + form.tags.datastore = datastore + + # Used by some forms that need to dig deeper + form.datastore = datastore + form.watch = default for p in datastore.extra_browsers: form.fetch_backend.choices.append(p) @@ -1559,7 +1563,7 @@ def changedetection_app(config=None, datastore_o=None): datastore.data['watching'][uuid]['tags'].append(tag_uuid) - flash("{} watches assigned tag".format(len(uuids))) + flash(f"{len(uuids)} watches were tagged") return redirect(url_for('index')) diff --git a/changedetectionio/model/Tag.py b/changedetectionio/model/Tag.py index 60e27a48..6dca480c 100644 --- a/changedetectionio/model/Tag.py +++ b/changedetectionio/model/Tag.py @@ -1,14 +1,14 @@ from changedetectionio.model import watch_base + class model(watch_base): def __init__(self, *arg, **kw): - super(model, self).__init__(*arg, **kw) + self['overrides_watch'] = kw.get('default', {}).get('overrides_watch') + if kw.get('default'): self.update(kw['default']) del kw['default'] - - diff --git a/changedetectionio/processors/restock_diff/__init__.py b/changedetectionio/processors/restock_diff/__init__.py index 3c48eec9..3f0be03b 100644 --- a/changedetectionio/processors/restock_diff/__init__.py +++ b/changedetectionio/processors/restock_diff/__init__.py @@ -59,6 +59,11 @@ class Watch(BaseWatch): super().__init__(*arg, **kw) self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock() + self['restock_settings'] = kw['default']['restock_settings'] if kw.get('default',{}).get('restock_settings') else { + 'follow_price_changes': True, + 'in_stock_processing' : 'in_stock_only' + } #@todo update + def clear_watch(self): super().clear_watch() self.update({'restock': Restock()}) diff --git a/changedetectionio/processors/restock_diff/forms.py b/changedetectionio/processors/restock_diff/forms.py index 3b89d523..39334aa3 100644 --- a/changedetectionio/processors/restock_diff/forms.py +++ b/changedetectionio/processors/restock_diff/forms.py @@ -1,17 +1,25 @@ - from wtforms import ( BooleanField, validators, FloatField ) +from wtforms.fields.choices import RadioField +from wtforms.fields.form import FormField +from wtforms.form import Form from changedetectionio.forms import processor_text_json_diff_form -class processor_settings_form(processor_text_json_diff_form): - in_stock_only = BooleanField('Only trigger when product goes BACK to in-stock', default=True) - price_change_min = FloatField('Minimum amount to trigger notification', [validators.Optional()], + +class RestockSettingsForm(Form): + in_stock_processing = RadioField(label='Re-stock detection', choices=[ + ('in_stock_only', "In Stock only (Out Of Stock -> In Stock only)"), + ('all_changes', "Any availability changes"), + ('off', "Off, don't follow availability/restock"), + ], default="in_stock_only") + + price_change_min = FloatField('Below price to trigger notification', [validators.Optional()], render_kw={"placeholder": "No limit", "size": "10"}) - price_change_max = FloatField('Maximum amount to trigger notification', [validators.Optional()], + price_change_max = FloatField('Above price to trigger notification', [validators.Optional()], render_kw={"placeholder": "No limit", "size": "10"}) price_change_threshold_percent = FloatField('Threshold in % for price changes since the original price', validators=[ @@ -19,45 +27,55 @@ class processor_settings_form(processor_text_json_diff_form): validators.NumberRange(min=0, max=100, message="Should be between 0 and 100"), ], render_kw={"placeholder": "0%", "size": "5"}) - follow_price_changes = BooleanField('Follow price changes', default=False) + follow_price_changes = BooleanField('Follow price changes', default=True) + +class processor_settings_form(processor_text_json_diff_form): + restock_settings = FormField(RestockSettingsForm) def extra_tab_content(self): return 'Restock & Price Detection' def extra_form_content(self): - return """ + output = "" + + if getattr(self, 'watch', None) and getattr(self, 'datastore'): + for tag_uuid in self.watch.get('tags'): + tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {}) + if tag.get('overrides_watch'): + # @todo - Quick and dirty, cant access 'url_for' here because its out of scope somehow + output = f"""

Note! A Group tag overrides the restock and price detection here.

""" + + output += """ {% from '_helpers.html' import render_field, render_checkbox_field, render_button %} - -
+
-
- {{ render_checkbox_field(form.in_stock_only) }} - Only trigger re-stock notification when page changes from out of stock to back in stock +
+ {{ render_field(form.restock_settings.in_stock_processing) }}
- {{ render_checkbox_field(form.follow_price_changes) }} + {{ render_checkbox_field(form.restock_settings.follow_price_changes) }} Changes in price should trigger a notification -
- When OFF - Only care about restock detection
- {{ render_field(form.price_change_min, placeholder=watch['restock']['price']) }} - Minimum amount, only trigger a change when the price is less than this amount. + {{ render_field(form.restock_settings.price_change_min, placeholder=watch.get('restock', {}).get('price')) }} + Minimum amount, Trigger a change/notification when the price drops below this value.
- {{ render_field(form.price_change_max, placeholder=watch['restock']['price']) }} - Maximum amount, only trigger a change when the price is more than this amount. + {{ render_field(form.restock_settings.price_change_max, placeholder=watch.get('restock', {}).get('price')) }} + Maximum amount, Trigger a change/notification when the price rises above this value.
- {{ render_field(form.price_change_threshold_percent) }} - Price must change more than this % to trigger a change.
+ {{ render_field(form.restock_settings.price_change_threshold_percent) }} + Price must change more than this % to trigger a change since the first check.
For example, If the product is $1,000 USD originally, 2% would mean it has to change more than $20 since the first check.
-
""" \ No newline at end of file +
+ """ + return output \ No newline at end of file diff --git a/changedetectionio/processors/restock_diff/processor.py b/changedetectionio/processors/restock_diff/processor.py index 42dd9bd4..e3ee6247 100644 --- a/changedetectionio/processors/restock_diff/processor.py +++ b/changedetectionio/processors/restock_diff/processor.py @@ -132,6 +132,18 @@ class perform_site_check(difference_detection_processor): update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '') update_obj["last_check_status"] = self.fetcher.get_last_status_code() + # Which restock settings to compare against? + restock_settings = watch.get('restock_settings', {}) + + # See if any tags have 'activate for individual watches in this tag/group?' enabled and use the first we find + for tag_uuid in watch.get('tags'): + tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {}) + if tag.get('overrides_watch'): + restock_settings = tag.get('restock_settings', {}) + logger.info(f"Watch {watch.get('uuid')} - Tag '{tag.get('title')}' selected for restock settings override") + break + + itemprop_availability = {} try: itemprop_availability = get_itemprop_availability(html_content=self.fetcher.content) @@ -195,14 +207,14 @@ class perform_site_check(difference_detection_processor): # out of stock -> back in stock only? if watch.get('restock') and watch['restock'].get('in_stock') != update_obj['restock'].get('in_stock'): # Yes if we only care about it going to instock, AND we are in stock - if watch.get('in_stock_only') and update_obj['restock']['in_stock']: + if restock_settings.get('in_stock_processing') == 'in_stock_only' and update_obj['restock']['in_stock']: changed_detected = True - if not watch.get('in_stock_only'): + if restock_settings.get('in_stock_processing') == 'all_changes': # All cases changed_detected = True - if watch.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'): + if restock_settings.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'): price = float(update_obj['restock'].get('price')) # Default to current price if no previous price found if watch['restock'].get('original_price'): @@ -214,26 +226,25 @@ class perform_site_check(difference_detection_processor): # Minimum/maximum price limit if update_obj.get('restock') and update_obj['restock'].get('price'): logger.debug( - f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{watch.get('price_change_max', '')}' 'price_change_min' is '{watch.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.") + f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{restock_settings.get('price_change_max', '')}' 'price_change_min' is '{restock_settings.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.") if update_obj['restock'].get('price'): - min_limit = float(watch.get('price_change_min')) if watch.get('price_change_min') else None - max_limit = float(watch.get('price_change_max')) if watch.get('price_change_max') else None + min_limit = float(restock_settings.get('price_change_min')) if restock_settings.get('price_change_min') else None + max_limit = float(restock_settings.get('price_change_max')) if restock_settings.get('price_change_max') else None price = float(update_obj['restock'].get('price')) logger.debug(f"{watch.get('uuid')} after float conversion - Min limit: '{min_limit}' Max limit: '{max_limit}' Price: '{price}'") if min_limit or max_limit: if is_between(number=price, lower=min_limit, upper=max_limit): - logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}") - if changed_detected: - logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because price was inside threshold") - changed_detected = False + # Price was between min/max limit, so there was nothing todo in any case + logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, nothing to check, forcing changed_detected = False (was {changed_detected})") + changed_detected = False else: - logger.trace(f"{watch.get('uuid')} {price} is NOT between {min_limit} and {max_limit}") + logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, continuing normal comparison") # Price comparison by % - if watch['restock'].get('original_price') and changed_detected and watch.get('price_change_threshold_percent'): + if watch['restock'].get('original_price') and changed_detected and restock_settings.get('price_change_threshold_percent'): previous_price = float(watch['restock'].get('original_price')) - pc = float(watch.get('price_change_threshold_percent')) + pc = float(restock_settings.get('price_change_threshold_percent')) change = abs((price - previous_price) / previous_price * 100) if change and change <= pc: logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because % threshold ({pc}%) was {change:.3f}%") diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index 6d45dc76..a55d2813 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -27,6 +27,5 @@ $(document).ready(function () { toggleOpacity('#time_between_check_use_default', '#time_between_check', false); - }); diff --git a/changedetectionio/store.py b/changedetectionio/store.py index c9c7ad10..0d88f8b2 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -83,16 +83,14 @@ class ChangeDetectionStore: self.__data['settings']['application'].update(from_disk['settings']['application']) # Convert each existing watch back to the Watch.model object - for uuid, watch in self.__data['watching'].items(): - watch['uuid'] = uuid - watch_class = get_custom_watch_obj_for_processor(watch.get('processor')) - if watch.get('uuid') != 'text_json_diff': - logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}") - - self.__data['watching'][uuid] = watch_class(datastore_path=self.datastore_path, default=watch) + self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch) + logger.info(f"Watching: {uuid} {watch['url']}") - logger.info(f"Watching: {uuid} {self.__data['watching'][uuid]['url']}") + # And for Tags also, should be Restock type because it has extra settings + for uuid, tag in self.__data['settings']['application']['tags'].items(): + self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff') + logger.info(f"Tag: {uuid} {tag['title']}") # First time ran, Create the datastore. except (FileNotFoundError): @@ -147,6 +145,22 @@ class ChangeDetectionStore: # Finally start the thread that will manage periodic data saves to JSON save_data_thread = threading.Thread(target=self.save_datastore).start() + def rehydrate_entity(self, uuid, entity, processor_override=None): + """Set the dict back to the dict Watch object""" + entity['uuid'] = uuid + + if processor_override: + watch_class = get_custom_watch_obj_for_processor(processor_override) + entity['processor']=processor_override + else: + watch_class = get_custom_watch_obj_for_processor(entity.get('processor')) + + if entity.get('uuid') != 'text_json_diff': + logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}") + + entity = watch_class(datastore_path=self.datastore_path, default=entity) + return entity + def set_last_viewed(self, uuid, timestamp): logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}") self.data['watching'][uuid].update({'last_viewed': int(timestamp)}) @@ -185,6 +199,9 @@ class ChangeDetectionStore: @property def has_unviewed(self): + if not self.__data.get('watching'): + return None + for uuid, watch in self.__data['watching'].items(): if watch.history_n >= 2 and watch.viewed == False: return True @@ -850,4 +867,17 @@ class ChangeDetectionStore: watch['restock'] = Restock({'in_stock': watch.get('in_stock')}) del watch['in_stock'] + # Migrate old restock settings + def update_18(self): + for uuid, watch in self.data['watching'].items(): + if not watch.get('restock_settings'): + # So we enable price following by default + self.data['watching'][uuid]['restock_settings'] = {'follow_price_changes': True} + + # Migrate and cleanoff old value + self.data['watching'][uuid]['restock_settings']['in_stock_processing'] = 'in_stock_only' if watch.get( + 'in_stock_only') else 'all_changes' + + if self.data['watching'][uuid].get('in_stock_only'): + del (self.data['watching'][uuid]['in_stock_only']) diff --git a/changedetectionio/tests/test_restock_itemprop.py b/changedetectionio/tests/test_restock_itemprop.py index e33de270..d1af83a0 100644 --- a/changedetectionio/tests/test_restock_itemprop.py +++ b/changedetectionio/tests/test_restock_itemprop.py @@ -52,6 +52,8 @@ def test_restock_itemprop_basic(client, live_server): test_url = url_for('test_endpoint', _external=True) + # By default it should enable ('in_stock_processing') == 'all_changes' + for p in instock_props: set_original_response(props_markup=p) client.post( @@ -87,6 +89,7 @@ def test_restock_itemprop_basic(client, live_server): def test_itemprop_price_change(client, live_server): #live_server_setup(live_server) + # Out of the box 'Follow price changes' should be ON test_url = url_for('test_endpoint', _external=True) set_original_response(props_markup=instock_props[0], price="190.95") @@ -114,7 +117,7 @@ def test_itemprop_price_change(client, live_server): set_original_response(props_markup=instock_props[0], price='120.45') res = client.post( url_for("edit_page", uuid="first"), - data={"follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -128,8 +131,7 @@ def test_itemprop_price_change(client, live_server): res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data -def test_itemprop_price_minmax_limit(client, live_server): - #live_server_setup(live_server) +def _run_test_minmax_limit(client, extra_watch_edit_form): res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -146,17 +148,16 @@ def test_itemprop_price_minmax_limit(client, live_server): # A change in price, should trigger a change by default wait_for_all_checks(client) - + data = { + "tags": "", + "url": test_url, + "headers": "", + 'fetch_backend': "html_requests" + } + data.update(extra_watch_edit_form) res = client.post( url_for("edit_page", uuid="first"), - data={"follow_price_changes": "y", - "price_change_min": 900.0, - "price_change_max": 1100.10, - "url": test_url, - "tags": "", - "headers": "", - 'fetch_backend': "html_requests" - }, + data=data, follow_redirects=True ) assert b"Updated watch." in res.data @@ -164,7 +165,7 @@ def test_itemprop_price_minmax_limit(client, live_server): client.get(url_for("mark_all_viewed")) - # price changed to something greater than min (900), and less than max (1100).. should be no change + # price changed to something greater than min (900), BUT less than max (1100).. should be no change set_original_response(props_markup=instock_props[0], price='1000.45') client.get(url_for("form_watch_checknow")) wait_for_all_checks(client) @@ -201,6 +202,44 @@ def test_itemprop_price_minmax_limit(client, live_server): assert b'Deleted' in res.data +def test_restock_itemprop_minmax(client, live_server): +# live_server_setup(live_server) + extras = { + "restock_settings-follow_price_changes": "y", + "restock_settings-price_change_min": 900.0, + "restock_settings-price_change_max": 1100.10 + } + _run_test_minmax_limit(client, extra_watch_edit_form=extras) + +def test_restock_itemprop_with_tag(client, live_server): + #live_server_setup(live_server) + + res = client.post( + url_for("tags.form_tag_add"), + data={"name": "test-tag"}, + follow_redirects=True + ) + assert b"Tag added" in res.data + + res = client.post( + url_for("tags.form_tag_edit_submit", uuid="first"), + data={"name": "test-tag", + "restock_settings-follow_price_changes": "y", + "restock_settings-price_change_min": 900.0, + "restock_settings-price_change_max": 1100.10, + "overrides_watch": "y", #overrides_watch should be restock_overrides_watch + }, + follow_redirects=True + ) + + extras = { + "tags": "test-tag" + } + + _run_test_minmax_limit(client, extra_watch_edit_form=extras) + + + def test_itemprop_percent_threshold(client, live_server): #live_server_setup(live_server) @@ -221,8 +260,8 @@ def test_itemprop_percent_threshold(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"follow_price_changes": "y", - "price_change_threshold_percent": 5.0, + data={"restock_settings-follow_price_changes": "y", + "restock_settings-price_change_threshold_percent": 5.0, "url": test_url, "tags": "", "headers": "",