Restock & Price detection - Ability to set up a tag/group that applies to all watches with price + restock limits

drop-arm-v6-support
dgtlmoon 5 months ago committed by GitHub
parent 1405e962f0
commit bde27c8a8f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -1,4 +1,6 @@
from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
from flask import Blueprint, request, render_template, flash, url_for, redirect
from changedetectionio.store import ChangeDetectionStore
from changedetectionio.flask_app import login_optionally_required
@ -96,22 +98,53 @@ def construct_blueprint(datastore: ChangeDetectionStore):
@tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
@login_optionally_required
def form_tag_edit(uuid):
from changedetectionio import forms
from changedetectionio.blueprint.tags.form import group_restock_settings_form
if uuid == 'first':
uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
default = datastore.data['settings']['application']['tags'].get(uuid)
form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
data=default,
)
form.datastore=datastore # needed?
form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
data=default,
)
template_args = {
'data': default,
'form': form,
'watch': default
}
included_content = {}
if form.extra_form_content():
# So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
# And then render the code from the module
from jinja2 import Environment, FileSystemLoader
import importlib.resources
templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
env = Environment(loader=FileSystemLoader(templates_dir))
template_str = """{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
<script>
$(document).ready(function () {
toggleOpacity('#overrides_watch', '#restock-fieldset-price-group', true);
});
</script>
<fieldset>
<div class="pure-control-group">
<fieldset class="pure-group">
{{ render_checkbox_field(form.overrides_watch) }}
<span class="pure-form-message-inline">Used for watches in "Restock & Price detection" mode</span>
</fieldset>
</fieldset>
"""
template_str += form.extra_form_content()
template = env.from_string(template_str)
included_content = template.render(**template_args)
output = render_template("edit-tag.html",
data=default,
form=form,
settings_application=datastore.data['settings']['application'],
extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
extra_form_content=included_content,
**template_args
)
return output
@ -120,13 +153,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
@tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
@login_optionally_required
def form_tag_edit_submit(uuid):
from changedetectionio import forms
from changedetectionio.blueprint.tags.form import group_restock_settings_form
if uuid == 'first':
uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
default = datastore.data['settings']['application']['tags'].get(uuid)
form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
data=default,
)
# @todo subclass form so validation works
@ -136,6 +169,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
datastore.data['settings']['application']['tags'][uuid].update(form.data)
datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
datastore.needs_write_urgent = True
flash("Updated")

@ -1,16 +1,15 @@
from wtforms import (
BooleanField,
Form,
IntegerField,
RadioField,
SelectField,
StringField,
SubmitField,
TextAreaField,
validators,
)
from wtforms.fields.simple import BooleanField
from changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form
class group_restock_settings_form(restock_settings_form):
overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)
class SingleTag(Form):

@ -26,6 +26,9 @@
<ul>
<li class="tab" id=""><a href="#general">General</a></li>
<li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
{% if extra_tab_content %}
<li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
{% endif %}
<li class="tab"><a href="#notifications">Notifications</a></li>
</ul>
</div>
@ -97,6 +100,12 @@ nav
</div>
{# rendered sub Template #}
{% if extra_form_content %}
<div class="tab-pane-inner" id="extras_tab">
{{ extra_form_content|safe }}
</div>
{% endif %}
<div class="tab-pane-inner" id="notifications">
<fieldset>
<div class="pure-control-group inline-radio">

@ -699,8 +699,12 @@ def changedetection_app(config=None, datastore_o=None):
data=default
)
# For the form widget tag uuid lookup
form.tags.datastore = datastore # in _value
# For the form widget tag UUID back to "string name" for the field
form.tags.datastore = datastore
# Used by some forms that need to dig deeper
form.datastore = datastore
form.watch = default
for p in datastore.extra_browsers:
form.fetch_backend.choices.append(p)
@ -1559,7 +1563,7 @@ def changedetection_app(config=None, datastore_o=None):
datastore.data['watching'][uuid]['tags'].append(tag_uuid)
flash("{} watches assigned tag".format(len(uuids)))
flash(f"{len(uuids)} watches were tagged")
return redirect(url_for('index'))

@ -1,14 +1,14 @@
from changedetectionio.model import watch_base
class model(watch_base):
def __init__(self, *arg, **kw):
super(model, self).__init__(*arg, **kw)
self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')
if kw.get('default'):
self.update(kw['default'])
del kw['default']

@ -59,6 +59,11 @@ class Watch(BaseWatch):
super().__init__(*arg, **kw)
self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock()
self['restock_settings'] = kw['default']['restock_settings'] if kw.get('default',{}).get('restock_settings') else {
'follow_price_changes': True,
'in_stock_processing' : 'in_stock_only'
} #@todo update
def clear_watch(self):
super().clear_watch()
self.update({'restock': Restock()})

@ -1,17 +1,25 @@
from wtforms import (
BooleanField,
validators,
FloatField
)
from wtforms.fields.choices import RadioField
from wtforms.fields.form import FormField
from wtforms.form import Form
from changedetectionio.forms import processor_text_json_diff_form
class processor_settings_form(processor_text_json_diff_form):
in_stock_only = BooleanField('Only trigger when product goes BACK to in-stock', default=True)
price_change_min = FloatField('Minimum amount to trigger notification', [validators.Optional()],
class RestockSettingsForm(Form):
in_stock_processing = RadioField(label='Re-stock detection', choices=[
('in_stock_only', "In Stock only (Out Of Stock -> In Stock only)"),
('all_changes', "Any availability changes"),
('off', "Off, don't follow availability/restock"),
], default="in_stock_only")
price_change_min = FloatField('Below price to trigger notification', [validators.Optional()],
render_kw={"placeholder": "No limit", "size": "10"})
price_change_max = FloatField('Maximum amount to trigger notification', [validators.Optional()],
price_change_max = FloatField('Above price to trigger notification', [validators.Optional()],
render_kw={"placeholder": "No limit", "size": "10"})
price_change_threshold_percent = FloatField('Threshold in % for price changes since the original price', validators=[
@ -19,45 +27,55 @@ class processor_settings_form(processor_text_json_diff_form):
validators.NumberRange(min=0, max=100, message="Should be between 0 and 100"),
], render_kw={"placeholder": "0%", "size": "5"})
follow_price_changes = BooleanField('Follow price changes', default=False)
follow_price_changes = BooleanField('Follow price changes', default=True)
class processor_settings_form(processor_text_json_diff_form):
restock_settings = FormField(RestockSettingsForm)
def extra_tab_content(self):
return 'Restock & Price Detection'
def extra_form_content(self):
return """
output = ""
if getattr(self, 'watch', None) and getattr(self, 'datastore'):
for tag_uuid in self.watch.get('tags'):
tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
if tag.get('overrides_watch'):
# @todo - Quick and dirty, cant access 'url_for' here because its out of scope somehow
output = f"""<p><strong>Note! A Group tag overrides the restock and price detection here.</strong></p><style>#restock-fieldset-price-group {{ opacity: 0.6; }}</style>"""
output += """
{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
<script>
$(document).ready(function () {
toggleOpacity('#follow_price_changes', '.price-change-minmax', true);
toggleOpacity('#restock_settings-follow_price_changes', '.price-change-minmax', true);
});
</script>
<fieldset>
<fieldset id="restock-fieldset-price-group">
<div class="pure-control-group">
<fieldset class="pure-group">
{{ render_checkbox_field(form.in_stock_only) }}
<span class="pure-form-message-inline">Only trigger re-stock notification when page changes from <strong>out of stock</strong> to <strong>back in stock</strong></span>
<fieldset class="pure-group inline-radio">
{{ render_field(form.restock_settings.in_stock_processing) }}
</fieldset>
<fieldset class="pure-group">
{{ render_checkbox_field(form.follow_price_changes) }}
{{ render_checkbox_field(form.restock_settings.follow_price_changes) }}
<span class="pure-form-message-inline">Changes in price should trigger a notification</span>
<br>
<span class="pure-form-message-inline">When OFF - Only care about restock detection</span>
</fieldset>
<fieldset class="pure-group price-change-minmax">
{{ render_field(form.price_change_min, placeholder=watch['restock']['price']) }}
<span class="pure-form-message-inline">Minimum amount, only trigger a change when the price is less than this amount.</span>
{{ render_field(form.restock_settings.price_change_min, placeholder=watch.get('restock', {}).get('price')) }}
<span class="pure-form-message-inline">Minimum amount, Trigger a change/notification when the price drops <i>below</i> this value.</span>
</fieldset>
<fieldset class="pure-group price-change-minmax">
{{ render_field(form.price_change_max, placeholder=watch['restock']['price']) }}
<span class="pure-form-message-inline">Maximum amount, only trigger a change when the price is more than this amount.</span>
{{ render_field(form.restock_settings.price_change_max, placeholder=watch.get('restock', {}).get('price')) }}
<span class="pure-form-message-inline">Maximum amount, Trigger a change/notification when the price rises <i>above</i> this value.</span>
</fieldset>
<fieldset class="pure-group price-change-minmax">
{{ render_field(form.price_change_threshold_percent) }}
<span class="pure-form-message-inline">Price must change more than this % to trigger a change.</span><br>
{{ render_field(form.restock_settings.price_change_threshold_percent) }}
<span class="pure-form-message-inline">Price must change more than this % to trigger a change since the first check.</span><br>
<span class="pure-form-message-inline">For example, If the product is $1,000 USD originally, <strong>2%</strong> would mean it has to change more than $20 since the first check.</span><br>
</fieldset>
</div>
</fieldset>"""
</fieldset>
"""
return output

@ -132,6 +132,18 @@ class perform_site_check(difference_detection_processor):
update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
update_obj["last_check_status"] = self.fetcher.get_last_status_code()
# Which restock settings to compare against?
restock_settings = watch.get('restock_settings', {})
# See if any tags have 'activate for individual watches in this tag/group?' enabled and use the first we find
for tag_uuid in watch.get('tags'):
tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
if tag.get('overrides_watch'):
restock_settings = tag.get('restock_settings', {})
logger.info(f"Watch {watch.get('uuid')} - Tag '{tag.get('title')}' selected for restock settings override")
break
itemprop_availability = {}
try:
itemprop_availability = get_itemprop_availability(html_content=self.fetcher.content)
@ -195,14 +207,14 @@ class perform_site_check(difference_detection_processor):
# out of stock -> back in stock only?
if watch.get('restock') and watch['restock'].get('in_stock') != update_obj['restock'].get('in_stock'):
# Yes if we only care about it going to instock, AND we are in stock
if watch.get('in_stock_only') and update_obj['restock']['in_stock']:
if restock_settings.get('in_stock_processing') == 'in_stock_only' and update_obj['restock']['in_stock']:
changed_detected = True
if not watch.get('in_stock_only'):
if restock_settings.get('in_stock_processing') == 'all_changes':
# All cases
changed_detected = True
if watch.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
if restock_settings.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
price = float(update_obj['restock'].get('price'))
# Default to current price if no previous price found
if watch['restock'].get('original_price'):
@ -214,26 +226,25 @@ class perform_site_check(difference_detection_processor):
# Minimum/maximum price limit
if update_obj.get('restock') and update_obj['restock'].get('price'):
logger.debug(
f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{watch.get('price_change_max', '')}' 'price_change_min' is '{watch.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{restock_settings.get('price_change_max', '')}' 'price_change_min' is '{restock_settings.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
if update_obj['restock'].get('price'):
min_limit = float(watch.get('price_change_min')) if watch.get('price_change_min') else None
max_limit = float(watch.get('price_change_max')) if watch.get('price_change_max') else None
min_limit = float(restock_settings.get('price_change_min')) if restock_settings.get('price_change_min') else None
max_limit = float(restock_settings.get('price_change_max')) if restock_settings.get('price_change_max') else None
price = float(update_obj['restock'].get('price'))
logger.debug(f"{watch.get('uuid')} after float conversion - Min limit: '{min_limit}' Max limit: '{max_limit}' Price: '{price}'")
if min_limit or max_limit:
if is_between(number=price, lower=min_limit, upper=max_limit):
logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}")
if changed_detected:
logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because price was inside threshold")
changed_detected = False
# Price was between min/max limit, so there was nothing todo in any case
logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, nothing to check, forcing changed_detected = False (was {changed_detected})")
changed_detected = False
else:
logger.trace(f"{watch.get('uuid')} {price} is NOT between {min_limit} and {max_limit}")
logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, continuing normal comparison")
# Price comparison by %
if watch['restock'].get('original_price') and changed_detected and watch.get('price_change_threshold_percent'):
if watch['restock'].get('original_price') and changed_detected and restock_settings.get('price_change_threshold_percent'):
previous_price = float(watch['restock'].get('original_price'))
pc = float(watch.get('price_change_threshold_percent'))
pc = float(restock_settings.get('price_change_threshold_percent'))
change = abs((price - previous_price) / previous_price * 100)
if change and change <= pc:
logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because % threshold ({pc}%) was {change:.3f}%")

@ -27,6 +27,5 @@ $(document).ready(function () {
toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
});

@ -83,16 +83,14 @@ class ChangeDetectionStore:
self.__data['settings']['application'].update(from_disk['settings']['application'])
# Convert each existing watch back to the Watch.model object
for uuid, watch in self.__data['watching'].items():
watch['uuid'] = uuid
watch_class = get_custom_watch_obj_for_processor(watch.get('processor'))
if watch.get('uuid') != 'text_json_diff':
logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
self.__data['watching'][uuid] = watch_class(datastore_path=self.datastore_path, default=watch)
self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
logger.info(f"Watching: {uuid} {watch['url']}")
logger.info(f"Watching: {uuid} {self.__data['watching'][uuid]['url']}")
# And for Tags also, should be Restock type because it has extra settings
for uuid, tag in self.__data['settings']['application']['tags'].items():
self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff')
logger.info(f"Tag: {uuid} {tag['title']}")
# First time ran, Create the datastore.
except (FileNotFoundError):
@ -147,6 +145,22 @@ class ChangeDetectionStore:
# Finally start the thread that will manage periodic data saves to JSON
save_data_thread = threading.Thread(target=self.save_datastore).start()
def rehydrate_entity(self, uuid, entity, processor_override=None):
"""Set the dict back to the dict Watch object"""
entity['uuid'] = uuid
if processor_override:
watch_class = get_custom_watch_obj_for_processor(processor_override)
entity['processor']=processor_override
else:
watch_class = get_custom_watch_obj_for_processor(entity.get('processor'))
if entity.get('uuid') != 'text_json_diff':
logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
entity = watch_class(datastore_path=self.datastore_path, default=entity)
return entity
def set_last_viewed(self, uuid, timestamp):
logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
@ -185,6 +199,9 @@ class ChangeDetectionStore:
@property
def has_unviewed(self):
if not self.__data.get('watching'):
return None
for uuid, watch in self.__data['watching'].items():
if watch.history_n >= 2 and watch.viewed == False:
return True
@ -850,4 +867,17 @@ class ChangeDetectionStore:
watch['restock'] = Restock({'in_stock': watch.get('in_stock')})
del watch['in_stock']
# Migrate old restock settings
def update_18(self):
for uuid, watch in self.data['watching'].items():
if not watch.get('restock_settings'):
# So we enable price following by default
self.data['watching'][uuid]['restock_settings'] = {'follow_price_changes': True}
# Migrate and cleanoff old value
self.data['watching'][uuid]['restock_settings']['in_stock_processing'] = 'in_stock_only' if watch.get(
'in_stock_only') else 'all_changes'
if self.data['watching'][uuid].get('in_stock_only'):
del (self.data['watching'][uuid]['in_stock_only'])

@ -52,6 +52,8 @@ def test_restock_itemprop_basic(client, live_server):
test_url = url_for('test_endpoint', _external=True)
# By default it should enable ('in_stock_processing') == 'all_changes'
for p in instock_props:
set_original_response(props_markup=p)
client.post(
@ -87,6 +89,7 @@ def test_restock_itemprop_basic(client, live_server):
def test_itemprop_price_change(client, live_server):
#live_server_setup(live_server)
# Out of the box 'Follow price changes' should be ON
test_url = url_for('test_endpoint', _external=True)
set_original_response(props_markup=instock_props[0], price="190.95")
@ -114,7 +117,7 @@ def test_itemprop_price_change(client, live_server):
set_original_response(props_markup=instock_props[0], price='120.45')
res = client.post(
url_for("edit_page", uuid="first"),
data={"follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@ -128,8 +131,7 @@ def test_itemprop_price_change(client, live_server):
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
def test_itemprop_price_minmax_limit(client, live_server):
#live_server_setup(live_server)
def _run_test_minmax_limit(client, extra_watch_edit_form):
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
@ -146,17 +148,16 @@ def test_itemprop_price_minmax_limit(client, live_server):
# A change in price, should trigger a change by default
wait_for_all_checks(client)
data = {
"tags": "",
"url": test_url,
"headers": "",
'fetch_backend': "html_requests"
}
data.update(extra_watch_edit_form)
res = client.post(
url_for("edit_page", uuid="first"),
data={"follow_price_changes": "y",
"price_change_min": 900.0,
"price_change_max": 1100.10,
"url": test_url,
"tags": "",
"headers": "",
'fetch_backend': "html_requests"
},
data=data,
follow_redirects=True
)
assert b"Updated watch." in res.data
@ -164,7 +165,7 @@ def test_itemprop_price_minmax_limit(client, live_server):
client.get(url_for("mark_all_viewed"))
# price changed to something greater than min (900), and less than max (1100).. should be no change
# price changed to something greater than min (900), BUT less than max (1100).. should be no change
set_original_response(props_markup=instock_props[0], price='1000.45')
client.get(url_for("form_watch_checknow"))
wait_for_all_checks(client)
@ -201,6 +202,44 @@ def test_itemprop_price_minmax_limit(client, live_server):
assert b'Deleted' in res.data
def test_restock_itemprop_minmax(client, live_server):
# live_server_setup(live_server)
extras = {
"restock_settings-follow_price_changes": "y",
"restock_settings-price_change_min": 900.0,
"restock_settings-price_change_max": 1100.10
}
_run_test_minmax_limit(client, extra_watch_edit_form=extras)
def test_restock_itemprop_with_tag(client, live_server):
#live_server_setup(live_server)
res = client.post(
url_for("tags.form_tag_add"),
data={"name": "test-tag"},
follow_redirects=True
)
assert b"Tag added" in res.data
res = client.post(
url_for("tags.form_tag_edit_submit", uuid="first"),
data={"name": "test-tag",
"restock_settings-follow_price_changes": "y",
"restock_settings-price_change_min": 900.0,
"restock_settings-price_change_max": 1100.10,
"overrides_watch": "y", #overrides_watch should be restock_overrides_watch
},
follow_redirects=True
)
extras = {
"tags": "test-tag"
}
_run_test_minmax_limit(client, extra_watch_edit_form=extras)
def test_itemprop_percent_threshold(client, live_server):
#live_server_setup(live_server)
@ -221,8 +260,8 @@ def test_itemprop_percent_threshold(client, live_server):
res = client.post(
url_for("edit_page", uuid="first"),
data={"follow_price_changes": "y",
"price_change_threshold_percent": 5.0,
data={"restock_settings-follow_price_changes": "y",
"restock_settings-price_change_threshold_percent": 5.0,
"url": test_url,
"tags": "",
"headers": "",

Loading…
Cancel
Save