UI/Functionality - Ability to manage/apply filters and notifications across tags/groups

pull/1641/head
dgtlmoon 11 months ago committed by GitHub
parent 72311fb845
commit 52f2c00308
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -317,25 +317,21 @@ def changedetection_app(config=None, datastore_o=None):
return "Access denied, bad token", 403
from . import diff
limit_tag = request.args.get('tag')
limit_tag = request.args.get('tag', '').lower().strip()
# Be sure limit_tag is a uuid
for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
if limit_tag == tag.get('title', '').lower().strip():
limit_tag = uuid
# Sort by last_changed and add the uuid which is usually the key..
sorted_watches = []
# @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
for uuid, watch in datastore.data['watching'].items():
if limit_tag != None:
# Support for comma separated list of tags.
for tag_in_watch in watch['tag'].split(','):
tag_in_watch = tag_in_watch.strip()
if tag_in_watch == limit_tag:
watch['uuid'] = uuid
sorted_watches.append(watch)
else:
watch['uuid'] = uuid
sorted_watches.append(watch)
if limit_tag and not limit_tag in watch['tags']:
continue
watch['uuid'] = uuid
sorted_watches.append(watch)
sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
@ -392,9 +388,17 @@ def changedetection_app(config=None, datastore_o=None):
@app.route("/", methods=['GET'])
@login_optionally_required
def index():
global datastore
from changedetectionio import forms
limit_tag = request.args.get('tag')
limit_tag = request.args.get('tag', '').lower().strip()
# Be sure limit_tag is a uuid
for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
if limit_tag == tag.get('title', '').lower().strip():
limit_tag = uuid
# Redirect for the old rss path which used the /?rss=true
if request.args.get('rss'):
return redirect(url_for('rss', tag=limit_tag))
@ -414,30 +418,15 @@ def changedetection_app(config=None, datastore_o=None):
sorted_watches = []
search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
for uuid, watch in datastore.data['watching'].items():
if limit_tag:
# Support for comma separated list of tags.
if not watch.get('tag'):
if limit_tag and not limit_tag in watch['tags']:
continue
for tag_in_watch in watch.get('tag', '').split(','):
tag_in_watch = tag_in_watch.strip()
if tag_in_watch == limit_tag:
watch['uuid'] = uuid
if search_q:
if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
sorted_watches.append(watch)
else:
sorted_watches.append(watch)
else:
#watch['uuid'] = uuid
if search_q:
if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
sorted_watches.append(watch)
else:
if search_q:
if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
sorted_watches.append(watch)
else:
sorted_watches.append(watch)
existing_tags = datastore.get_all_tags()
form = forms.quickWatchForm(request.form)
page = request.args.get(get_page_parameter(), type=int, default=1)
total_count = len(sorted_watches)
@ -452,6 +441,7 @@ def changedetection_app(config=None, datastore_o=None):
# Don't link to hosting when we're on the hosting environment
active_tag=limit_tag,
app_rss_token=datastore.data['settings']['application']['rss_access_token'],
datastore=datastore,
form=form,
guid=datastore.data['app_guid'],
has_proxies=datastore.proxy_list,
@ -463,7 +453,7 @@ def changedetection_app(config=None, datastore_o=None):
sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
tags=existing_tags,
tags=datastore.data['settings']['application'].get('tags'),
watches=sorted_watches
)
@ -606,9 +596,13 @@ def changedetection_app(config=None, datastore_o=None):
# proxy_override set to the json/text list of the items
form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
data=default,
data=default
)
# For the form widget tag uuid lookup
form.tags.datastore = datastore # in _value
form.fetch_backend.choices.append(("system", 'System settings default'))
# form.browser_steps[0] can be assumed that we 'goto url' first
@ -659,6 +653,16 @@ def changedetection_app(config=None, datastore_o=None):
extra_update_obj['filter_text_replaced'] = True
extra_update_obj['filter_text_removed'] = True
# Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
tag_uuids = []
if form.data.get('tags'):
# Sometimes in testing this can be list, dont know why
if type(form.data.get('tags')) == list:
extra_update_obj['tags'] = form.data.get('tags')
else:
for t in form.data.get('tags').split(','):
tag_uuids.append(datastore.add_tag(name=t))
extra_update_obj['tags'] = tag_uuids
datastore.data['watching'][uuid].update(form.data)
datastore.data['watching'][uuid].update(extra_update_obj)
@ -713,7 +717,7 @@ def changedetection_app(config=None, datastore_o=None):
form=form,
has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
has_empty_checktime=using_default_check_time,
has_extra_headers_file=watch.has_extra_headers_file or datastore.has_extra_headers_file,
has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
is_html_webdriver=is_html_webdriver,
jq_support=jq_support,
playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
@ -1110,8 +1114,8 @@ def changedetection_app(config=None, datastore_o=None):
os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
) as f:
for uuid in datastore.data["watching"]:
url = datastore.data["watching"][uuid]["url"]
tag = datastore.data["watching"][uuid]["tag"]
url = datastore.data["watching"][uuid].get('url')
tag = datastore.data["watching"][uuid].get('tags', {})
f.write("{} {}\r\n".format(url, tag))
# Add it to the Zip
@ -1199,7 +1203,7 @@ def changedetection_app(config=None, datastore_o=None):
add_paused = request.form.get('edit_and_watch_submit_button') != None
processor = request.form.get('processor', 'text_json_diff')
new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused, 'processor': processor})
new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
if new_uuid:
if add_paused:
@ -1267,9 +1271,11 @@ def changedetection_app(config=None, datastore_o=None):
elif tag != None:
# Items that have this current tag
for watch_uuid, watch in datastore.data['watching'].items():
if (tag != None and tag in watch['tag']):
if (tag != None and tag in watch.get('tags', {})):
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
update_q.put(
queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
)
i += 1
else:
@ -1357,6 +1363,17 @@ def changedetection_app(config=None, datastore_o=None):
datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
flash("{} watches set to use default notification settings".format(len(uuids)))
elif (op == 'assign-tag'):
op_extradata = request.form.get('op_extradata')
tag_uuid = datastore.add_tag(name=op_extradata)
if op_extradata and tag_uuid:
for uuid in uuids:
uuid = uuid.strip()
if datastore.data['watching'].get(uuid):
datastore.data['watching'][uuid]['tags'].append(tag_uuid)
flash("{} watches assigned tag".format(len(uuids)))
return redirect(url_for('index'))
@app.route("/api/share-url", methods=['GET'])
@ -1366,7 +1383,6 @@ def changedetection_app(config=None, datastore_o=None):
the share-link can be imported/added"""
import requests
import json
tag = request.args.get('tag')
uuid = request.args.get('uuid')
# more for testing
@ -1419,6 +1435,8 @@ def changedetection_app(config=None, datastore_o=None):
import changedetectionio.blueprint.price_data_follower as price_data_follower
app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
import changedetectionio.blueprint.tags as tags
app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
# @todo handle ctrl break
ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()

@ -218,6 +218,11 @@ class CreateWatch(Resource):
return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
extras = copy.deepcopy(json_data)
# Because we renamed 'tag' to 'tags' but dont want to change the API (can do this in v2 of the API)
if extras.get('tag'):
extras['tags'] = extras.get('tag')
del extras['url']
new_uuid = self.datastore.add_watch(url=url, extras=extras)
@ -259,13 +264,16 @@ class CreateWatch(Resource):
"""
list = {}
tag_limit = request.args.get('tag', None)
for k, watch in self.datastore.data['watching'].items():
if tag_limit:
if not tag_limit.lower() in watch.all_tags:
continue
tag_limit = request.args.get('tag', '').lower()
for uuid, watch in self.datastore.data['watching'].items():
# Watch tags by name (replace the other calls?)
tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
continue
list[k] = {'url': watch['url'],
list[uuid] = {'url': watch['url'],
'title': watch['title'],
'last_checked': watch['last_checked'],
'last_changed': watch.last_changed,

@ -0,0 +1,9 @@
# Groups tags
## How it works
Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
So a tag is like an abstract of a watch

@ -0,0 +1,131 @@
from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
from changedetectionio.store import ChangeDetectionStore
from changedetectionio import login_optionally_required
def construct_blueprint(datastore: ChangeDetectionStore):
tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
@tags_blueprint.route("/list", methods=['GET'])
@login_optionally_required
def tags_overview_page():
from .form import SingleTag
add_form = SingleTag(request.form)
output = render_template("groups-overview.html",
form=add_form,
available_tags=datastore.data['settings']['application'].get('tags', {}),
)
return output
@tags_blueprint.route("/add", methods=['POST'])
@login_optionally_required
def form_tag_add():
from .form import SingleTag
add_form = SingleTag(request.form)
if not add_form.validate():
for widget, l in add_form.errors.items():
flash(','.join(l), 'error')
return redirect(url_for('tags.tags_overview_page'))
title = request.form.get('name').strip()
if datastore.tag_exists_by_name(title):
flash(f'The tag "{title}" already exists', "error")
return redirect(url_for('tags.tags_overview_page'))
datastore.add_tag(title)
flash("Tag added")
return redirect(url_for('tags.tags_overview_page'))
@tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
@login_optionally_required
def mute(uuid):
if datastore.data['settings']['application']['tags'].get(uuid):
datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
return redirect(url_for('tags.tags_overview_page'))
@tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
@login_optionally_required
def delete(uuid):
removed = 0
# Delete the tag, and any tag reference
if datastore.data['settings']['application']['tags'].get(uuid):
del datastore.data['settings']['application']['tags'][uuid]
for watch_uuid, watch in datastore.data['watching'].items():
if watch.get('tags') and uuid in watch['tags']:
removed += 1
watch['tags'].remove(uuid)
flash(f"Tag deleted and removed from {removed} watches")
return redirect(url_for('tags.tags_overview_page'))
@tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
@login_optionally_required
def unlink(uuid):
unlinked = 0
for watch_uuid, watch in datastore.data['watching'].items():
if watch.get('tags') and uuid in watch['tags']:
unlinked += 1
watch['tags'].remove(uuid)
flash(f"Tag unlinked removed from {unlinked} watches")
return redirect(url_for('tags.tags_overview_page'))
@tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
@login_optionally_required
def form_tag_edit(uuid):
from changedetectionio import forms
if uuid == 'first':
uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
default = datastore.data['settings']['application']['tags'].get(uuid)
form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
data=default,
)
form.datastore=datastore # needed?
output = render_template("edit-tag.html",
data=default,
form=form,
settings_application=datastore.data['settings']['application'],
)
return output
@tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
@login_optionally_required
def form_tag_edit_submit(uuid):
from changedetectionio import forms
if uuid == 'first':
uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
default = datastore.data['settings']['application']['tags'].get(uuid)
form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
data=default,
)
# @todo subclass form so validation works
#if not form.validate():
# for widget, l in form.errors.items():
# flash(','.join(l), 'error')
# return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
datastore.data['settings']['application']['tags'][uuid].update(form.data)
datastore.needs_write_urgent = True
flash("Updated")
return redirect(url_for('tags.tags_overview_page'))
@tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
def form_tag_delete(uuid):
return redirect(url_for('tags.tags_overview_page'))
return tags_blueprint

@ -0,0 +1,22 @@
from wtforms import (
BooleanField,
Form,
IntegerField,
RadioField,
SelectField,
StringField,
SubmitField,
TextAreaField,
validators,
)
class SingleTag(Form):
name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})

@ -0,0 +1,131 @@
{% extends 'base.html' %}
{% block content %}
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.jinja' import render_common_settings_form %}
<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
<script>
/*{% if emailprefix %}*/
/*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
/*{% endif %}*/
</script>
<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
<div class="edit-form monospaced-textarea">
<div class="tabs collapsable">
<ul>
<li class="tab" id=""><a href="#general">General</a></li>
<li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
<li class="tab"><a href="#notifications">Notifications</a></li>
</ul>
</div>
<div class="box-wrap inner">
<form class="pure-form pure-form-stacked"
action="{{ url_for('tags.form_tag_edit', uuid=data.uuid) }}" method="POST">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<div class="tab-pane-inner" id="general">
<fieldset>
<div class="pure-control-group">
{{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
</div>
</fieldset>
</div>
<div class="tab-pane-inner" id="filters-and-triggers">
<div class="pure-control-group">
{% set field = render_field(form.include_filters,
rows=5,
placeholder="#example
xpath://body/div/span[contains(@class, 'example-class')]",
class="m-d")
%}
{{ field }}
{% if '/text()' in field %}
<span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
{% endif %}
<span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
<ul>
<li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
<li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
<ul>
<li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required, <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
{% if jq_support %}
<li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
{% else %}
<li>jq support not installed</li>
{% endif %}
</ul>
</li>
<li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
<ul>
<li>Example: <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
href="http://xpather.com/" target="new">test your XPath here</a></li>
<li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
</ul>
</li>
</ul>
Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
</span>
</div>
<fieldset class="pure-control-group">
{{ render_field(form.subtractive_selectors, rows=5, placeholder="header
footer
nav
.stockticker") }}
<span class="pure-form-message-inline">
<ul>
<li> Remove HTML element(s) by CSS selector before text conversion. </li>
<li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
</ul>
</span>
</fieldset>
</div>
<div class="tab-pane-inner" id="notifications">
<fieldset>
<div class="pure-control-group inline-radio">
{{ render_checkbox_field(form.notification_muted) }}
</div>
{% if is_html_webdriver %}
<div class="pure-control-group inline-radio">
{{ render_checkbox_field(form.notification_screenshot) }}
<span class="pure-form-message-inline">
<strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
</span>
</div>
{% endif %}
<div class="field-group" id="notification-field-group">
{% if has_default_notification_urls %}
<div class="inline-warning">
<img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
</div>
{% endif %}
<a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
{{ render_common_settings_form(form, emailprefix, settings_application) }}
</div>
</fieldset>
</div>
<div id="actions">
<div class="pure-control-group">
{{ render_button(form.save_button) }}
</div>
</div>
</form>
</div>
</div>
{% endblock %}

@ -0,0 +1,60 @@
{% extends 'base.html' %}
{% block content %}
{% from '_helpers.jinja' import render_simple_field, render_field %}
<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
<div class="box">
<form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
<fieldset>
<legend>Add a new organisational tag</legend>
<div id="watch-add-wrapper-zone">
<div>
{{ render_simple_field(form.name, placeholder="watch label / tag") }}
</div>
<div>
{{ render_simple_field(form.save_button, title="Save" ) }}
</div>
</div>
<br>
<div style="color: #fff;">Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.</div>
</fieldset>
</form>
<!-- @todo maybe some overview matrix, 'tick' with which has notification, filter rules etc -->
<div id="watch-table-wrapper">
<table class="pure-table pure-table-striped watch-table group-overview-table">
<thead>
<tr>
<th></th>
<th>Tag / Label name</th>
<th></th>
</tr>
</thead>
<tbody>
<!--
@Todo - connect Last checked, Last Changed, Number of Watches etc
--->
{% if not available_tags|length %}
<tr>
<td colspan="3">No website organisational tags/groups configured</td>
</tr>
{% endif %}
{% for uuid, tag in available_tags.items() %}
<tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
<td class="watch-controls">
<a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
</td>
<td class="title-col inline">{{tag.title}}</td>
<td>
<a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
<a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
<a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endblock %}

@ -28,6 +28,8 @@ from changedetectionio.notification import (
from wtforms.fields import FormField
dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
valid_method = {
'GET',
'POST',
@ -90,6 +92,29 @@ class SaltyPasswordField(StringField):
else:
self.data = False
class StringTagUUID(StringField):
# process_formdata(self, valuelist) handled manually in POST handler
# Is what is shown when field <input> is rendered
def _value(self):
# Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)
if self.data and type(self.data) is list:
tag_titles = []
for i in self.data:
tag = self.datastore.data['settings']['application']['tags'].get(i)
if tag:
tag_title = tag.get('title')
if tag_title:
tag_titles.append(tag_title)
return ', '.join(tag_titles)
if not self.data:
return ''
return 'error'
class TimeBetweenCheckForm(Form):
weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@ -347,7 +372,7 @@ class quickWatchForm(Form):
from . import processors
url = fields.URLField('URL', validators=[validateURL()])
tag = StringField('Group tag', [validators.Optional()])
tags = StringTagUUID('Group tag', [validators.Optional()])
watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
@ -355,6 +380,7 @@ class quickWatchForm(Form):
# Common to a single watch and the global settings
class commonSettingsForm(Form):
notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@ -382,7 +408,7 @@ class SingleBrowserStep(Form):
class watchForm(commonSettingsForm):
url = fields.URLField('URL', validators=[validateURL()])
tag = StringField('Group tag', [validators.Optional()], default='')
tags = StringTagUUID('Group tag', [validators.Optional()], default='')
time_between_check = FormField(TimeBetweenCheckForm)

@ -120,9 +120,9 @@ class import_distill_io_json(Importer):
except IndexError:
pass
# Does this need to be here anymore?
if d.get('tags', False):
extras['tag'] = ", ".join(d['tags'])
extras['tags'] = ", ".join(d['tags'])
new_uuid = datastore.add_watch(url=d['uri'].strip(),
extras=extras,

@ -43,6 +43,7 @@ class model(dict):
'schema_version' : 0,
'shared_diff_access': False,
'webdriver_delay': None , # Extra delay in seconds before extracting text
'tags': {} #@todo use Tag.model initialisers
}
}
}

@ -0,0 +1,19 @@
from .Watch import base_config
import uuid
class model(dict):
def __init__(self, *arg, **kw):
self.update(base_config)
self['uuid'] = str(uuid.uuid4())
if kw.get('default'):
self.update(kw['default'])
del kw['default']
# Goes at the end so we update the default object with the initialiser
super(model, self).__init__(*arg, **kw)

@ -52,7 +52,8 @@ base_config = {
'previous_md5_before_filters': False, # Used for skipping changedetection entirely
'proxy': None, # Preferred proxy connection
'subtractive_selectors': [],
'tag': None,
'tag': '', # Old system of text name for a tag, to be removed
'tags': [], # list of UUIDs to App.Tags
'text_should_not_be_present': [], # Text that should not present
# Re #110, so then if this is set to None, we know to use the default value instead
# Requires setting to None on submit if it's the same as the default
@ -455,10 +456,6 @@ class model(dict):
return csv_output_filename
@property
# Return list of tags, stripped and lowercase, used for searching
def all_tags(self):
return [s.strip().lower() for s in self.get('tag','').split(',')]
def has_special_diff_filter_options_set(self):
@ -473,40 +470,6 @@ class model(dict):
# None is set
return False
@property
def has_extra_headers_file(self):
if os.path.isfile(os.path.join(self.watch_data_dir, 'headers.txt')):
return True
for f in self.all_tags:
fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
filepath = os.path.join(self.__datastore_path, fname)
if os.path.isfile(filepath):
return True
return False
def get_all_headers(self):
from .App import parse_headers_from_text_file
headers = self.get('headers', {}).copy()
# Available headers on the disk could 'headers.txt' in the watch data dir
filepath = os.path.join(self.watch_data_dir, 'headers.txt')
try:
if os.path.isfile(filepath):
headers.update(parse_headers_from_text_file(filepath))
except Exception as e:
print(f"ERROR reading headers.txt at {filepath}", str(e))
# Or each by tag, as tagname.txt in the main datadir
for f in self.all_tags:
fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
filepath = os.path.join(self.__datastore_path, fname)
try:
if os.path.isfile(filepath):
headers.update(parse_headers_from_text_file(filepath))
except Exception as e:
print(f"ERROR reading headers.txt at {filepath}", str(e))
return headers
def get_last_fetched_before_filters(self):
import brotli

@ -186,8 +186,13 @@ def create_notification_parameters(n_object, datastore):
uuid = n_object['uuid'] if 'uuid' in n_object else ''
if uuid != '':
watch_title = datastore.data['watching'][uuid]['title']
watch_tag = datastore.data['watching'][uuid]['tag']
watch_title = datastore.data['watching'][uuid].get('title', '')
tag_list = []
tags = datastore.get_all_tags_for_watch(uuid)
if tags:
for tag_uuid, tag in tags.items():
tag_list.append(tag.get('title'))
watch_tag = ', '.join(tag_list)
else:
watch_title = 'Change Detection'
watch_tag = ''

@ -42,11 +42,10 @@ class perform_site_check(difference_detection_processor):
# Unset any existing notification error
update_obj = {'last_notification_error': False, 'last_error': False}
extra_headers = watch.get('headers', [])
# Tweak the base config with the per-watch ones
request_headers = deepcopy(self.datastore.data['settings']['headers'])
request_headers.update(extra_headers)
request_headers = watch.get('headers', [])
request_headers.update(self.datastore.get_all_base_headers())
request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
# https://github.com/psf/requests/issues/4525
# Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot

@ -57,7 +57,6 @@ class perform_site_check(difference_detection_processor):
# DeepCopy so we can be sure we don't accidently change anything by reference
watch = deepcopy(self.datastore.data['watching'].get(uuid))
if not watch:
raise Exception("Watch no longer exists.")
@ -71,9 +70,9 @@ class perform_site_check(difference_detection_processor):
update_obj = {'last_notification_error': False, 'last_error': False}
# Tweak the base config with the per-watch ones
extra_headers = watch.get_all_headers()
request_headers = self.datastore.get_all_headers()
request_headers.update(extra_headers)
request_headers = watch.get('headers', [])
request_headers.update(self.datastore.get_all_base_headers())
request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
# https://github.com/psf/requests/issues/4525
# Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
@ -191,21 +190,23 @@ class perform_site_check(difference_detection_processor):
fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
# Better would be if Watch.model could access the global data also
# and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
# https://realpython.com/inherit-python-dict/ instead of doing it procedurely
include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags]
include_filters_rule = deepcopy(watch.get('include_filters', []))
# include_filters_rule = watch['include_filters']
subtractive_selectors = watch.get(
"subtractive_selectors", []
) + self.datastore.data["settings"]["application"].get(
"global_subtractive_selectors", []
)
subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
*watch.get("subtractive_selectors", []),
*self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
]
# Inject a virtual LD+JSON price tracker rule
if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
if is_json and not has_filter_rule:
include_filters_rule.append("json:$")

@ -4,6 +4,9 @@ $(function () {
$(this).closest('.unviewed').removeClass('unviewed');
});
$("#checkbox-assign-tag").click(function (e) {
$('#op_extradata').val(prompt("Enter a tag name"));
});
$('.with-share-link > *').click(function () {
$("#copied-clipboard").remove();

@ -16,6 +16,8 @@ import threading
import time
import uuid as uuid_builder
dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
# Open a github issue if you know something :)
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@ -178,20 +180,6 @@ class ChangeDetectionStore:
return self.__data
def get_all_tags(self):
tags = []
for uuid, watch in self.data['watching'].items():
if watch['tag'] is None:
continue
# Support for comma separated list of tags.
for tag in watch['tag'].split(','):
tag = tag.strip()
if tag not in tags:
tags.append(tag)
tags.sort()
return tags
# Delete a single watch by UUID
def delete(self, uuid):
import pathlib
@ -218,9 +206,9 @@ class ChangeDetectionStore:
# Clone a watch by UUID
def clone(self, uuid):
url = self.data['watching'][uuid]['url']
tag = self.data['watching'][uuid]['tag']
tag = self.data['watching'][uuid].get('tags',[])
extras = self.data['watching'][uuid]
new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
new_uuid = self.add_watch(url=url, tag_uuids=tag, extras=extras)
return new_uuid
def url_exists(self, url):
@ -255,10 +243,11 @@ class ChangeDetectionStore:
self.needs_write_urgent = True
def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
if extras is None:
extras = {}
# should always be str
if tag is None or not tag:
tag = ''
@ -291,6 +280,7 @@ class ChangeDetectionStore:
'processor',
'subtractive_selectors',
'tag',
'tags',
'text_should_not_be_present',
'title',
'trigger_text',
@ -313,25 +303,34 @@ class ChangeDetectionStore:
flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
return None
with self.lock:
# #Re 569
new_watch = Watch.model(datastore_path=self.datastore_path, default={
'url': url,
'tag': tag,
'date_created': int(time.time())
})
new_uuid = new_watch['uuid']
logging.debug("Added URL {} - {}".format(url, new_uuid))
# #Re 569
# Could be in 'tags', var or extras, smash them together and strip
apply_extras['tags'] = []
if tag or extras.get('tags'):
tags = list(filter(None, list(set().union(tag.split(','), extras.get('tags', '').split(',')))))
for t in list(map(str.strip, tags)):
# for each stripped tag, add tag as UUID
apply_extras['tags'].append(self.add_tag(t))
# Or if UUIDs given directly
if tag_uuids:
apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
new_watch = Watch.model(datastore_path=self.datastore_path, url=url)
for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
if k in apply_extras:
del apply_extras[k]
new_uuid = new_watch.get('uuid')
new_watch.update(apply_extras)
self.__data['watching'][new_uuid] = new_watch
logging.debug("Added URL {} - {}".format(url, new_uuid))
for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
if k in apply_extras:
del apply_extras[k]
new_watch.update(apply_extras)
new_watch.ensure_data_dir_exists()
self.__data['watching'][new_uuid] = new_watch
self.__data['watching'][new_uuid].ensure_data_dir_exists()
if write_to_disk_now:
self.sync_to_json()
@ -511,10 +510,19 @@ class ChangeDetectionStore:
filepath = os.path.join(self.datastore_path, 'headers.txt')
return os.path.isfile(filepath)
def get_all_headers(self):
def get_all_base_headers(self):
from .model.App import parse_headers_from_text_file
headers = {}
# Global app settings
headers.update(self.data['settings'].get('headers', {}))
return headers
def get_all_headers_in_textfile_for_watch(self, uuid):
from .model.App import parse_headers_from_text_file
headers = copy(self.data['settings'].get('headers', {}))
headers = {}
# Global in /datastore/headers.txt
filepath = os.path.join(self.datastore_path, 'headers.txt')
try:
if os.path.isfile(filepath):
@ -522,8 +530,76 @@ class ChangeDetectionStore:
except Exception as e:
print(f"ERROR reading headers.txt at {filepath}", str(e))
watch = self.data['watching'].get(uuid)
if watch:
# In /datastore/xyz-xyz/headers.txt
filepath = os.path.join(watch.watch_data_dir, 'headers.txt')
try:
if os.path.isfile(filepath):
headers.update(parse_headers_from_text_file(filepath))
except Exception as e:
print(f"ERROR reading headers.txt at {filepath}", str(e))
# In /datastore/tag-name.txt
tags = self.get_all_tags_for_watch(uuid=uuid)
for tag_uuid, tag in tags.items():
fname = "headers-"+re.sub(r'[\W_]', '', tag.get('title')).lower().strip() + ".txt"
filepath = os.path.join(self.datastore_path, fname)
try:
if os.path.isfile(filepath):
headers.update(parse_headers_from_text_file(filepath))
except Exception as e:
print(f"ERROR reading headers.txt at {filepath}", str(e))
return headers
def get_tag_overrides_for_watch(self, uuid, attr):
tags = self.get_all_tags_for_watch(uuid=uuid)
ret = []
if tags:
for tag_uuid, tag in tags.items():
if attr in tag and tag[attr]:
ret=[*ret, *tag[attr]]
return ret
def add_tag(self, name):
print (">>> Adding new tag -", name)
# If name exists, return that
n = name.strip().lower()
for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
if n == tag.get('title', '').lower().strip():
print (f">>> Tag {name} already exists")
return uuid
# Eventually almost everything todo with a watch will apply as a Tag
# So we use the same model as a Watch
with self.lock:
new_tag = Watch.model(datastore_path=self.datastore_path, default={
'title': name.strip(),
'date_created': int(time.time())
})
new_uuid = new_tag.get('uuid')
self.__data['settings']['application']['tags'][new_uuid] = new_tag
return new_uuid
def get_all_tags_for_watch(self, uuid):
"""This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet"""
watch = self.data['watching'].get(uuid)
# Should return a dict of full tag info linked by UUID
if watch:
return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
return {}
def tag_exists_by_name(self, tag_name):
return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
# Run all updates
# IMPORTANT - Each update could be run even when they have a new install and the schema is correct
@ -710,3 +786,16 @@ class ChangeDetectionStore:
i+=1
return
# Create tag objects and their references from existing tag text
def update_12(self):
i = 0
for uuid, watch in self.data['watching'].items():
# Split out and convert old tag string
tag = watch.get('tag')
if tag:
tag_uuids = []
for t in tag.split(','):
tag_uuids.append(self.add_tag(name=t))
self.data['watching'][uuid]['tags'] = tag_uuids

@ -58,6 +58,9 @@
{% if current_user.is_authenticated or not has_password %}
{% if not
current_diff_url %}
<li class="pure-menu-item">
<a href="{{ url_for('tags.tags_overview_page')}}" class="pure-menu-link">GROUPS</a>
</li>
<li class="pure-menu-item">
<a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
</li>
@ -86,7 +89,7 @@
<!-- We use GET here so it offers people a chance to set bookmarks etc -->
<form name="searchForm" action="" method="GET">
<input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
<input name="tag" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
<input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
<button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
{% include "svgs/search-icon.svg" %}
</button>

@ -75,7 +75,7 @@
{{ render_field(form.title, class="m-d") }}
</div>
<div class="pure-control-group">
{{ render_field(form.tag) }}
{{ render_field(form.tags) }}
<span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
</div>
<div class="pure-control-group">

@ -13,7 +13,7 @@
<div id="watch-add-wrapper-zone">
<div>
{{ render_simple_field(form.url, placeholder="https://...", required=true) }}
{{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
{{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
</div>
<div>
{{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@ -30,12 +30,14 @@
<form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
<input type="hidden" id="op_extradata" name="op_extradata" value="" >
<div id="checkbox-operations">
<button class="pure-button button-secondary button-xsmall" name="op" value="pause">Pause</button>
<button class="pure-button button-secondary button-xsmall" name="op" value="unpause">UnPause</button>
<button class="pure-button button-secondary button-xsmall" name="op" value="mute">Mute</button>
<button class="pure-button button-secondary button-xsmall" name="op" value="unmute">UnMute</button>
<button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
<button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
<button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
<button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
<button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
@ -47,9 +49,9 @@
{% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
<div>
<a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
{% for tag in tags %}
{% for uuid, tag in tags.items() %}
{% if tag != "" %}
<a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
<a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
{% endif %}
{% endfor %}
</div>
@ -143,9 +145,11 @@
</span>
{% endif %}
{% if not active_tag %}
<span class="watch-tag-list">{{ watch.tag}}</span>
{% endif %}
{% for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() %}
<span class="watch-tag-list">{{ watch_tag.title }}</span>
{% endfor %}
</td>
<td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
<td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
@ -178,7 +182,7 @@
{% endif %}
<li>
<a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
</li>
<li>
<a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>

@ -28,7 +28,7 @@ def test_preferred_proxy(client, live_server):
"fetch_backend": "html_requests",
"headers": "",
"proxy": "proxy-two",
"tag": "",
"tags": "",
"url": url,
},
follow_redirects=True

@ -77,7 +77,7 @@ def test_restock_detection(client, live_server):
client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": '', 'processor': 'restock_diff'},
data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
follow_redirects=True
)

@ -45,6 +45,15 @@ def test_check_access_control(app, client, live_server):
res = client.get(url_for("diff_history_page", uuid="first"))
assert b'Random content' in res.data
# Check wrong password does not let us in
res = c.post(
url_for("login"),
data={"password": "WRONG PASSWORD"},
follow_redirects=True
)
assert b"LOG OUT" not in res.data
assert b"Incorrect password" in res.data
# Menu should not be available yet

@ -2,7 +2,7 @@
import time
from flask import url_for
from .util import live_server_setup
from .util import live_server_setup, wait_for_all_checks
from changedetectionio import html_tools
@ -39,7 +39,6 @@ def test_setup(client, live_server):
live_server_setup(live_server)
def test_check_removed_line_contains_trigger(client, live_server):
sleep_time_for_fetch_thread = 3
# Give the endpoint time to spin up
time.sleep(1)
@ -54,7 +53,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
assert b"1 Imported" in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# Goto the edit page, add our ignore text
# Add our URL to the import page
@ -67,20 +66,20 @@ def test_check_removed_line_contains_trigger(client, live_server):
follow_redirects=True
)
assert b"Updated watch." in res.data
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
set_original(excluding='Something irrelevant')
# A line thats not the trigger should not trigger anything
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
assert b'1 watches queued for rechecking.' in res.data
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' not in res.data
# The trigger line is REMOVED, this should trigger
set_original(excluding='The golden line')
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' in res.data
@ -89,14 +88,14 @@ def test_check_removed_line_contains_trigger(client, live_server):
client.get(url_for("mark_all_viewed"), follow_redirects=True)
set_original(excluding=None)
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' not in res.data
# Remove it again, and we should get a trigger
set_original(excluding='The golden line')
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' in res.data
@ -105,8 +104,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
def test_check_add_line_contains_trigger(client, live_server):
sleep_time_for_fetch_thread = 3
#live_server_setup(live_server)
# Give the endpoint time to spin up
time.sleep(1)
@ -136,8 +134,7 @@ def test_check_add_line_contains_trigger(client, live_server):
assert b"1 Imported" in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# Goto the edit page, add our ignore text
# Add our URL to the import page
res = client.post(
@ -150,23 +147,25 @@ def test_check_add_line_contains_trigger(client, live_server):
follow_redirects=True
)
assert b"Updated watch." in res.data
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
set_original(excluding='Something irrelevant')
# A line thats not the trigger should not trigger anything
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
assert b'1 watches queued for rechecking.' in res.data
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' not in res.data
# The trigger line is ADDED, this should trigger
set_original(add_line='<p>Oh yes please</p>')
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' in res.data
# Takes a moment for apprise to fire
time.sleep(3)
with open("test-datastore/notification.txt", 'r') as f:
response= f.read()
assert '-Oh yes please-' in response

@ -2,7 +2,7 @@
import time
from flask import url_for
from .util import live_server_setup, extract_api_key_from_UI
from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks
import json
import uuid
@ -57,6 +57,7 @@ def test_setup(client, live_server):
live_server_setup(live_server)
def test_api_simple(client, live_server):
#live_server_setup(live_server)
api_key = extract_api_key_from_UI(client)
@ -86,7 +87,7 @@ def test_api_simple(client, live_server):
watch_uuid = res.json.get('uuid')
assert res.status_code == 201
time.sleep(3)
wait_for_all_checks(client)
# Verify its in the list and that recheck worked
res = client.get(
@ -107,7 +108,7 @@ def test_api_simple(client, live_server):
)
assert len(res.json) == 0
time.sleep(2)
wait_for_all_checks(client)
set_modified_response()
# Trigger recheck of all ?recheck_all=1
@ -115,7 +116,7 @@ def test_api_simple(client, live_server):
url_for("createwatch", recheck_all='1'),
headers={'x-api-key': api_key},
)
time.sleep(3)
wait_for_all_checks(client)
# Did the recheck fire?
res = client.get(
@ -297,6 +298,8 @@ def test_api_watch_PUT_update(client, live_server):
url_for("edit_page", uuid=watch_uuid),
)
assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
assert b"One" in res.data, "Tag 'One' was found"
assert b"Two" in res.data, "Tag 'Two' was found"
# HTTP PUT ( UPDATE an existing watch )
res = client.put(
@ -319,7 +322,8 @@ def test_api_watch_PUT_update(client, live_server):
)
assert b"new title" in res.data, "new title found in edit page"
assert b"552" in res.data, "552 minutes found in edit page"
assert b"One, Two" in res.data, "Tag 'One, Two' was found"
assert b"One" in res.data, "Tag 'One' was found"
assert b"Two" in res.data, "Tag 'Two' was found"
assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section"
######################################################

@ -24,7 +24,7 @@ def test_basic_auth(client, live_server):
# Check form validation
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data

@ -2,7 +2,7 @@
import time
from flask import url_for
from . util import live_server_setup
from .util import live_server_setup, wait_for_all_checks
from changedetectionio import html_tools
def set_original_ignore_response():
@ -61,7 +61,7 @@ def set_modified_response_minus_block_text():
def test_check_block_changedetection_text_NOT_present(client, live_server):
sleep_time_for_fetch_thread = 3
live_server_setup(live_server)
# Use a mix of case in ZzZ to prove it works case-insensitive.
ignore_text = "out of stoCk\r\nfoobar"
@ -81,7 +81,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
assert b"1 Imported" in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# Goto the edit page, add our ignore text
# Add our URL to the import page
@ -96,7 +96,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
assert b"Updated watch." in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# Check it saved
res = client.get(
url_for("edit_page", uuid="first"),
@ -107,7 +107,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# It should report nothing found (no new 'unviewed' class)
res = client.get(url_for("index"))
@ -120,7 +120,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# It should report nothing found (no new 'unviewed' class)
res = client.get(url_for("index"))
@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
# Now we set a change where the text is gone, it should now trigger
set_modified_response_minus_block_text()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' in res.data

@ -96,7 +96,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@ -157,7 +157,7 @@ def test_check_multiple_filters(client, live_server):
url_for("edit_page", uuid="first"),
data={"include_filters": include_filters,
"url": test_url,
"tag": "",
"tags": "",
"headers": "",
'fetch_backend': "html_requests"},
follow_redirects=True

@ -129,7 +129,7 @@ def test_element_removal_full(client, live_server):
data={
"subtractive_selectors": subtractive_selectors_data,
"url": test_url,
"tag": "",
"tags": "",
"headers": "",
"fetch_backend": "html_requests",
},

@ -91,7 +91,7 @@ def test_check_filter_multiline(client, live_server):
data={"include_filters": '',
'extract_text': '/something.+?6 billion.+?lines/si',
"url": test_url,
"tag": "",
"tags": "",
"headers": "",
'fetch_backend': "html_requests"
},
@ -146,7 +146,7 @@ def test_check_filter_and_regex_extract(client, live_server):
data={"include_filters": include_filters,
'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
"url": test_url,
"tag": "",
"tags": "",
"headers": "",
'fetch_backend': "html_requests"
},

@ -56,7 +56,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": 'cinema'},
data={"url": test_url, "tags": 'cinema'},
follow_redirects=True
)
assert b"Watch added" in res.data
@ -89,7 +89,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
notification_form_data.update({
"url": test_url,
"tag": "my tag",
"tags": "my tag",
"title": "my title",
"headers": "",
"include_filters": '.ticket-available',

@ -1,7 +1,7 @@
import os
import time
from flask import url_for
from .util import set_original_response, live_server_setup, extract_UUID_from_client
from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks
from changedetectionio.model import App
@ -37,14 +37,14 @@ def run_filter_test(client, content_filter):
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": ''},
data={"url": test_url, "tags": ''},
follow_redirects=True
)
assert b"Watch added" in res.data
# Give the thread time to pick up the first version
time.sleep(3)
wait_for_all_checks(client)
# Goto the edit page, add our ignore text
# Add our URL to the import page
@ -71,8 +71,8 @@ def run_filter_test(client, content_filter):
notification_form_data.update({
"url": test_url,
"tag": "my tag",
"title": "my title",
"tags": "my tag",
"title": "my title 123",
"headers": "",
"filter_failure_notification_send": 'y',
"include_filters": content_filter,
@ -85,43 +85,55 @@ def run_filter_test(client, content_filter):
)
assert b"Updated watch." in res.data
time.sleep(3)
wait_for_all_checks(client)
# Now the notification should not exist, because we didnt reach the threshold
assert not os.path.isfile("test-datastore/notification.txt")
for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
# -2 because we would have checked twice above (on adding and on edit)
for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(3)
wait_for_all_checks(client)
assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
# We should see something in the frontend
assert b'Warning, no filters were found' in res.data
# One more check should trigger it (see -2 above)
client.get(url_for("form_watch_checknow"), follow_redirects=True)
wait_for_all_checks(client)
client.get(url_for("form_watch_checknow"), follow_redirects=True)
wait_for_all_checks(client)
# Now it should exist and contain our "filter not found" alert
assert os.path.isfile("test-datastore/notification.txt")
notification = False
with open("test-datastore/notification.txt", 'r') as f:
notification = f.read()
assert 'CSS/xPath filter was not present in the page' in notification
assert content_filter.replace('"', '\\"') in notification
# Remove it and prove that it doesnt trigger when not expected
# Remove it and prove that it doesn't trigger when not expected
# It should register a change, but no 'filter not found'
os.unlink("test-datastore/notification.txt")
set_response_with_filter()
# Try several times, it should NOT have 'filter not found'
for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(3)
wait_for_all_checks(client)
# It should have sent a notification, but..
assert os.path.isfile("test-datastore/notification.txt")
# but it should not contain the info about the failed filter
# but it should not contain the info about a failed filter (because there was none in this case)
with open("test-datastore/notification.txt", 'r') as f:
notification = f.read()
assert not 'CSS/xPath filter was not present in the page' in notification
# Re #1247 - All tokens got replaced
# Re #1247 - All tokens got replaced correctly in the notification
res = client.get(url_for("index"))
uuid = extract_UUID_from_client(client)
# UUID is correct, but notification contains tag uuid as UUIID wtf
assert uuid in notification
# cleanup for the next
@ -137,7 +149,7 @@ def test_setup(live_server):
def test_check_include_filters_failure_notification(client, live_server):
set_original_response()
time.sleep(1)
wait_for_all_checks(client)
run_filter_test(client, '#nope-doesnt-exist')
def test_check_xpath_filter_failure_notification(client, live_server):

@ -0,0 +1,262 @@
#!/usr/bin/python3
import time
from flask import url_for
from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name
import os
def test_setup(client, live_server):
live_server_setup(live_server)
def set_original_response():
test_return_data = """<html>
<body>
Some initial text<br>
<p id="only-this">Should be only this</p>
<br>
<p id="not-this">And never this</p>
</body>
</html>
"""
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
def set_modified_response():
test_return_data = """<html>
<body>
Some initial text<br>
<p id="only-this">Should be REALLY only this</p>
<br>
<p id="not-this">And never this</p>
</body>
</html>
"""
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
def test_setup_group_tag(client, live_server):
#live_server_setup(live_server)
set_original_response()
# Add a tag with some config, import a tag and it should roughly work
res = client.post(
url_for("tags.form_tag_add"),
data={"name": "test-tag"},
follow_redirects=True
)
assert b"Tag added" in res.data
assert b"test-tag" in res.data
res = client.post(
url_for("tags.form_tag_edit_submit", uuid="first"),
data={"name": "test-tag",
"include_filters": '#only-this',
"subtractive_selectors": '#not-this'},
follow_redirects=True
)
assert b"Updated" in res.data
tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
res = client.get(
url_for("tags.form_tag_edit", uuid="first")
)
assert b"#only-this" in res.data
assert b"#not-this" in res.data
# Tag should be setup and ready, now add a watch
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("import_page"),
data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"},
follow_redirects=True
)
assert b"1 Imported" in res.data
res = client.get(url_for("index"))
assert b'import-tag' in res.data
assert b'extra-import-tag' in res.data
res = client.get(
url_for("tags.tags_overview_page"),
follow_redirects=True
)
assert b'import-tag' in res.data
assert b'extra-import-tag' in res.data
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'Warning, no filters were found' not in res.data
res = client.get(
url_for("preview_page", uuid="first"),
follow_redirects=True
)
assert b'Should be only this' in res.data
assert b'And never this' not in res.data
# RSS Group tag filter
# An extra one that should be excluded
res = client.post(
url_for("import_page"),
data={"urls": test_url + "?should-be-excluded=1 some-tag"},
follow_redirects=True
)
assert b"1 Imported" in res.data
wait_for_all_checks(client)
set_modified_response()
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
wait_for_all_checks(client)
rss_token = extract_rss_token_from_UI(client)
res = client.get(
url_for("rss", token=rss_token, tag="extra-import-tag", _external=True),
follow_redirects=True
)
assert b"should-be-excluded" not in res.data
assert res.status_code == 200
assert b"first-imported=1" in res.data
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
def test_tag_import_singular(client, live_server):
#live_server_setup(live_server)
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("import_page"),
data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"},
follow_redirects=True
)
assert b"2 Imported" in res.data
res = client.get(
url_for("tags.tags_overview_page"),
follow_redirects=True
)
# Should be only 1 tag because they both had the same
assert res.data.count(b'test-tag') == 1
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
def test_tag_add_in_ui(client, live_server):
#live_server_setup(live_server)
#
res = client.post(
url_for("tags.form_tag_add"),
data={"name": "new-test-tag"},
follow_redirects=True
)
assert b"Tag added" in res.data
assert b"new-test-tag" in res.data
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
def test_group_tag_notification(client, live_server):
#live_server_setup(live_server)
set_original_response()
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tags": 'test-tag, other-tag'},
follow_redirects=True
)
assert b"Watch added" in res.data
notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
notification_form_data = {"notification_urls": notification_url,
"notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
"notification_body": "BASE URL: {{base_url}}\n"
"Watch URL: {{watch_url}}\n"
"Watch UUID: {{watch_uuid}}\n"
"Watch title: {{watch_title}}\n"
"Watch tag: {{watch_tag}}\n"
"Preview: {{preview_url}}\n"
"Diff URL: {{diff_url}}\n"
"Snapshot: {{current_snapshot}}\n"
"Diff: {{diff}}\n"
"Diff Added: {{diff_added}}\n"
"Diff Removed: {{diff_removed}}\n"
"Diff Full: {{diff_full}}\n"
":-)",
"notification_screenshot": True,
"notification_format": "Text",
"title": "test-tag"}
res = client.post(
url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
data=notification_form_data,
follow_redirects=True
)
assert b"Updated" in res.data
wait_for_all_checks(client)
set_modified_response()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(3)
assert os.path.isfile("test-datastore/notification.txt")
# Verify what was sent as a notification, this file should exist
with open("test-datastore/notification.txt", "r") as f:
notification_submission = f.read()
os.unlink("test-datastore/notification.txt")
# Did we see the URL that had a change, in the notification?
# Diff was correctly executed
assert test_url in notification_submission
assert ':-)' in notification_submission
assert "Diff Full: Some initial text" in notification_submission
assert "New GROUP TAG ChangeDetection.io" in notification_submission
assert "test-tag" in notification_submission
assert "other-tag" in notification_submission
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
#@todo Test that multiple notifications fired
#@todo Test that each of multiple notifications with different settings
def test_limit_tag_ui(client, live_server):
#live_server_setup(live_server)
test_url = url_for('test_endpoint', _external=True)
urls=[]
for i in range(20):
urls.append(test_url+"?x="+str(i)+" test-tag")
for i in range(20):
urls.append(test_url+"?non-grouped="+str(i))
res = client.post(
url_for("import_page"),
data={"urls": "\r\n".join(urls)},
follow_redirects=True
)
assert b"40 Imported" in res.data
res = client.get(url_for("index"))
assert b'test-tag' in res.data
# All should be here
assert res.data.count(b'processor-text_json_diff') == 40
tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
res = client.get(url_for("index", tag=tag_uuid))
# Just a subset should be here
assert b'test-tag' in res.data
assert res.data.count(b'processor-text_json_diff') == 20
assert b"object at" not in res.data

@ -2,7 +2,7 @@
import time
from flask import url_for
from . util import live_server_setup
from .util import live_server_setup, wait_for_all_checks
from changedetectionio import html_tools
def test_setup(live_server):
@ -84,7 +84,6 @@ def set_modified_ignore_response():
def test_check_ignore_text_functionality(client, live_server):
sleep_time_for_fetch_thread = 3
# Use a mix of case in ZzZ to prove it works case-insensitive.
ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
@ -103,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
assert b"1 Imported" in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# Goto the edit page, add our ignore text
# Add our URL to the import page
@ -124,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# It should report nothing found (no new 'unviewed' class)
res = client.get(url_for("index"))
@ -137,7 +136,7 @@ def test_check_ignore_text_functionality(client, live_server):
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# It should report nothing found (no new 'unviewed' class)
res = client.get(url_for("index"))
@ -151,7 +150,7 @@ def test_check_ignore_text_functionality(client, live_server):
# Just to be sure.. set a regular modified change..
set_modified_original_ignore_response()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' in res.data
@ -167,7 +166,6 @@ def test_check_ignore_text_functionality(client, live_server):
assert b'Deleted' in res.data
def test_check_global_ignore_text_functionality(client, live_server):
sleep_time_for_fetch_thread = 3
# Give the endpoint time to spin up
time.sleep(1)
@ -198,7 +196,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
assert b"1 Imported" in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# Goto the edit page of the item, add our ignore text
@ -220,7 +218,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# so that we are sure everything is viewed and in a known 'nothing changed' state
res = client.get(url_for("diff_history_page", uuid="first"))
@ -237,7 +235,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# It should report nothing found (no new 'unviewed' class)
res = client.get(url_for("index"))
@ -247,7 +245,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
# Just to be sure.. set a regular modified change that will trigger it
set_modified_original_ignore_response()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' in res.data

@ -2,7 +2,7 @@
import time
from flask import url_for
from . util import live_server_setup
from .util import live_server_setup, wait_for_all_checks
def test_setup(live_server):
@ -40,7 +40,7 @@ def set_some_changed_response():
def test_normal_page_check_works_with_ignore_status_code(client, live_server):
sleep_time_for_fetch_thread = 3
# Give the endpoint time to spin up
time.sleep(1)
@ -68,15 +68,15 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
set_some_changed_response()
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# It should report nothing found (no new 'unviewed' class)
res = client.get(url_for("index"))
@ -109,13 +109,13 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# Make a change
set_some_changed_response()
@ -123,7 +123,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
# It should have 'unviewed' still
# Because it should be looking at only that 'sametext' id

@ -20,7 +20,7 @@ def test_jinja2_in_url_query(client, live_server):
"date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
res = client.post(
url_for("form_quick_watch_add"),
data={"url": full_url, "tag": "test"},
data={"url": full_url, "tags": "test"},
follow_redirects=True
)
assert b"Watch added" in res.data

@ -208,7 +208,7 @@ def test_check_json_without_filter(client, live_server):
)
# Give the thread time to pick it up
time.sleep(3)
wait_for_all_checks(client)
res = client.get(
url_for("preview_page", uuid="first"),
@ -238,7 +238,7 @@ def check_json_filter(json_filter, client, live_server):
assert b"1 Imported" in res.data
# Give the thread time to pick it up
time.sleep(3)
wait_for_all_checks(client)
# Goto the edit page, add our ignore text
# Add our URL to the import page
@ -246,7 +246,7 @@ def check_json_filter(json_filter, client, live_server):
url_for("edit_page", uuid="first"),
data={"include_filters": json_filter,
"url": test_url,
"tag": "",
"tags": "",
"headers": "",
"fetch_backend": "html_requests"
},
@ -261,14 +261,14 @@ def check_json_filter(json_filter, client, live_server):
assert bytes(escape(json_filter).encode('utf-8')) in res.data
# Give the thread time to pick it up
time.sleep(3)
wait_for_all_checks(client)
# Make a change
set_modified_response()
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(4)
wait_for_all_checks(client)
# It should have 'unviewed' still
res = client.get(url_for("index"))
@ -306,14 +306,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(3)
wait_for_all_checks(client)
# Goto the edit page, add our ignore text
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": json_filter,
"url": test_url,
"tag": "",
"tags": "",
"headers": "",
"fetch_backend": "html_requests"
},
@ -322,14 +322,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
assert b"Updated watch." in res.data
# Give the thread time to pick it up
time.sleep(3)
wait_for_all_checks(client)
# Make a change
set_modified_response()
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(3)
wait_for_all_checks(client)
res = client.get(url_for("diff_history_page", uuid="first"))
# But the change should be there, tho its hard to test the change was detected because it will show old and new versions
@ -366,7 +366,7 @@ def check_json_ext_filter(json_filter, client, live_server):
assert b"1 Imported" in res.data
# Give the thread time to pick it up
time.sleep(3)
wait_for_all_checks(client)
# Goto the edit page, add our ignore text
# Add our URL to the import page
@ -374,7 +374,7 @@ def check_json_ext_filter(json_filter, client, live_server):
url_for("edit_page", uuid="first"),
data={"include_filters": json_filter,
"url": test_url,
"tag": "",
"tags": "",
"headers": "",
"fetch_backend": "html_requests"
},
@ -389,14 +389,14 @@ def check_json_ext_filter(json_filter, client, live_server):
assert bytes(escape(json_filter).encode('utf-8')) in res.data
# Give the thread time to pick it up
time.sleep(3)
wait_for_all_checks(client)
# Make a change
set_modified_ext_response()
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(4)
wait_for_all_checks(client)
# It should have 'unviewed'
res = client.get(url_for("index"))
@ -428,14 +428,14 @@ def test_ignore_json_order(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(2)
wait_for_all_checks(client)
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write('{"world" : 123, "hello": 123}')
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(2)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' not in res.data
@ -446,7 +446,7 @@ def test_ignore_json_order(client, live_server):
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(2)
wait_for_all_checks(client)
res = client.get(url_for("index"))
assert b'unviewed' in res.data

@ -3,7 +3,7 @@ import os
import time
import re
from flask import url_for
from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
from . util import extract_UUID_from_client
import logging
import base64
@ -21,10 +21,11 @@ def test_setup(live_server):
# Hard to just add more live server URLs when one test is already running (I think)
# So we add our test here (was in a different file)
def test_check_notification(client, live_server):
#live_server_setup(live_server)
set_original_response()
# Give the endpoint time to spin up
time.sleep(3)
time.sleep(1)
# Re 360 - new install should have defaults set
res = client.get(url_for("settings_page"))
@ -62,13 +63,13 @@ def test_check_notification(client, live_server):
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": ''},
data={"url": test_url, "tags": ''},
follow_redirects=True
)
assert b"Watch added" in res.data
# Give the thread time to pick up the first version
time.sleep(3)
wait_for_all_checks(client)
# We write the PNG to disk, but a JPEG should appear in the notification
# Write the last screenshot png
@ -105,7 +106,7 @@ def test_check_notification(client, live_server):
notification_form_data.update({
"url": test_url,
"tag": "my tag",
"tags": "my tag, my second tag",
"title": "my title",
"headers": "",
"fetch_backend": "html_requests"})
@ -128,7 +129,7 @@ def test_check_notification(client, live_server):
## Now recheck, and it should have sent the notification
time.sleep(3)
wait_for_all_checks(client)
set_modified_response()
# Trigger a check
@ -150,7 +151,7 @@ def test_check_notification(client, live_server):
assert "b'" not in notification_submission
assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
assert "Watch title: my title" in notification_submission
assert "Watch tag: my tag" in notification_submission
assert "Watch tag: my tag, my second tag" in notification_submission
assert "diff/" in notification_submission
assert "preview/" in notification_submission
assert ":-)" in notification_submission
@ -193,11 +194,11 @@ def test_check_notification(client, live_server):
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(1)
wait_for_all_checks(client)
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(1)
wait_for_all_checks(client)
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(1)
wait_for_all_checks(client)
assert os.path.exists("test-datastore/notification.txt") == False
res = client.get(url_for("notification_logs"))
@ -209,7 +210,7 @@ def test_check_notification(client, live_server):
url_for("edit_page", uuid="first"),
data={
"url": test_url,
"tag": "my tag",
"tags": "my tag",
"title": "my title",
"notification_urls": '',
"notification_title": '',
@ -243,7 +244,7 @@ def test_notification_validation(client, live_server):
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": 'nice one'},
data={"url": test_url, "tags": 'nice one'},
follow_redirects=True
)
@ -303,13 +304,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": 'nice one'},
data={"url": test_url, "tags": 'nice one'},
follow_redirects=True
)
assert b"Watch added" in res.data
time.sleep(2)
wait_for_all_checks(client)
set_modified_response()
client.get(url_for("form_watch_checknow"), follow_redirects=True)

@ -17,7 +17,7 @@ def test_check_notification_error_handling(client, live_server):
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": ''},
data={"url": test_url, "tags": ''},
follow_redirects=True
)
assert b"Watch added" in res.data
@ -32,7 +32,7 @@ def test_check_notification_error_handling(client, live_server):
"notification_body": "xxxxx",
"notification_format": "Text",
"url": test_url,
"tag": "",
"tags": "",
"title": "",
"headers": "",
"time_between_check-minutes": "180",

@ -25,7 +25,7 @@ def test_headers_in_request(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(1)
wait_for_all_checks(client)
res = client.post(
url_for("import_page"),
@ -43,7 +43,7 @@ def test_headers_in_request(client, live_server):
url_for("edit_page", uuid="first"),
data={
"url": test_url,
"tag": "",
"tags": "",
"fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
"headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
follow_redirects=True
@ -95,14 +95,14 @@ def test_body_in_request(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(3)
wait_for_all_checks(client)
# add the first 'version'
res = client.post(
url_for("edit_page", uuid="first"),
data={
"url": test_url,
"tag": "",
"tags": "",
"method": "POST",
"fetch_backend": "html_requests",
"body": "something something"},
@ -110,7 +110,7 @@ def test_body_in_request(client, live_server):
)
assert b"Updated watch." in res.data
time.sleep(3)
wait_for_all_checks(client)
# Now the change which should trigger a change
body_value = 'Test Body Value'
@ -118,7 +118,7 @@ def test_body_in_request(client, live_server):
url_for("edit_page", uuid="first"),
data={
"url": test_url,
"tag": "",
"tags": "",
"method": "POST",
"fetch_backend": "html_requests",
"body": body_value},
@ -126,7 +126,7 @@ def test_body_in_request(client, live_server):
)
assert b"Updated watch." in res.data
time.sleep(3)
wait_for_all_checks(client)
# The service should echo back the body
res = client.get(
@ -163,7 +163,7 @@ def test_body_in_request(client, live_server):
url_for("edit_page", uuid="first"),
data={
"url": test_url,
"tag": "",
"tags": "",
"method": "GET",
"fetch_backend": "html_requests",
"body": "invalid"},
@ -187,7 +187,7 @@ def test_method_in_request(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(2)
wait_for_all_checks(client)
res = client.post(
url_for("import_page"),
data={"urls": test_url},
@ -195,14 +195,14 @@ def test_method_in_request(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(2)
wait_for_all_checks(client)
# Attempt to add a method which is not valid
res = client.post(
url_for("edit_page", uuid="first"),
data={
"url": test_url,
"tag": "",
"tags": "",
"fetch_backend": "html_requests",
"method": "invalid"},
follow_redirects=True
@ -214,7 +214,7 @@ def test_method_in_request(client, live_server):
url_for("edit_page", uuid="first"),
data={
"url": test_url,
"tag": "",
"tags": "",
"fetch_backend": "html_requests",
"method": "PATCH"},
follow_redirects=True
@ -222,7 +222,7 @@ def test_method_in_request(client, live_server):
assert b"Updated watch." in res.data
# Give the thread time to pick up the first version
time.sleep(2)
wait_for_all_checks(client)
# The service should echo back the request verb
res = client.get(
@ -233,7 +233,7 @@ def test_method_in_request(client, live_server):
# The test call service will return the verb as the body
assert b"PATCH" in res.data
time.sleep(2)
wait_for_all_checks(client)
watches_with_method = 0
with open('test-datastore/url-watches.json') as f:
@ -265,7 +265,7 @@ def test_headers_textfile_in_request(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(1)
wait_for_all_checks(client)
# Add some headers to a request
@ -273,7 +273,7 @@ def test_headers_textfile_in_request(client, live_server):
url_for("edit_page", uuid="first"),
data={
"url": test_url,
"tag": "testtag",
"tags": "testtag",
"fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
"headers": "xxx:ooo\ncool:yeah\r\n"},
follow_redirects=True

@ -28,7 +28,7 @@ def test_basic_search(client, live_server):
res = client.post(
url_for("edit_page", uuid="first"),
data={"title": "xxx-title", "url": urls[0], "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@ -62,7 +62,7 @@ def test_search_in_tag_limit(client, live_server):
# By Title
res = client.post(
url_for("edit_page", uuid="first"),
data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tag": urls[0].split(' ')[1], "headers": "",
data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tags": urls[0].split(' ')[1], "headers": "",
'fetch_backend': "html_requests"},
follow_redirects=True
)

@ -18,7 +18,7 @@ def test_bad_access(client, live_server):
url_for("edit_page", uuid="first"),
data={
"url": 'javascript:alert(document.domain)',
"tag": "",
"tags": "",
"method": "GET",
"fetch_backend": "html_requests",
"body": ""},
@ -29,7 +29,7 @@ def test_bad_access(client, live_server):
res = client.post(
url_for("form_quick_watch_add"),
data={"url": ' javascript:alert(123)', "tag": ''},
data={"url": ' javascript:alert(123)', "tags": ''},
follow_redirects=True
)
@ -37,7 +37,7 @@ def test_bad_access(client, live_server):
res = client.post(
url_for("form_quick_watch_add"),
data={"url": '%20%20%20javascript:alert(123)%20%20', "tag": ''},
data={"url": '%20%20%20javascript:alert(123)%20%20', "tags": ''},
follow_redirects=True
)
@ -46,7 +46,7 @@ def test_bad_access(client, live_server):
res = client.post(
url_for("form_quick_watch_add"),
data={"url": ' source:javascript:alert(document.domain)', "tag": ''},
data={"url": ' source:javascript:alert(document.domain)', "tags": ''},
follow_redirects=True
)
@ -56,7 +56,7 @@ def test_bad_access(client, live_server):
client.post(
url_for("form_quick_watch_add"),
data={"url": 'file:///tasty/disk/drive', "tag": ''},
data={"url": 'file:///tasty/disk/drive', "tags": ''},
follow_redirects=True
)
time.sleep(1)

@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data

@ -3,7 +3,7 @@
import time
from flask import url_for
from urllib.request import urlopen
from .util import set_original_response, set_modified_response, live_server_setup
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
sleep_time_for_fetch_thread = 3
@ -42,7 +42,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
assert b'1 watches queued for rechecking.' in res.data
time.sleep(5)
wait_for_all_checks(client)
# Now something should be ready, indicated by having a 'unviewed' class
res = client.get(url_for("index"))
@ -60,7 +60,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
# `subtractive_selectors` should still work in `source:` type requests
def test_check_ignore_elements(client, live_server):
set_original_response()
time.sleep(2)
time.sleep(1)
test_url = 'source:'+url_for('test_endpoint', _external=True)
# Add our URL to the import page
res = client.post(
@ -71,14 +71,14 @@ def test_check_ignore_elements(client, live_server):
assert b"1 Imported" in res.data
time.sleep(sleep_time_for_fetch_thread)
wait_for_all_checks(client)
#####################
# We want <span> and <p> ONLY, but ignore span with .foobar-detection
client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
follow_redirects=True
)

@ -26,7 +26,7 @@ def test_check_watch_field_storage(client, live_server):
"title" : "My title",
"ignore_text" : "ignore this",
"url": test_url,
"tag": "woohoo",
"tags": "woohoo",
"headers": "curl:foo",
'fetch_backend': "html_requests"
},

@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
time.sleep(1)
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
time.sleep(1)
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@ -189,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
@ -231,7 +231,7 @@ def test_xpath_validation(client, live_server):
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"is not a valid XPath expression" in res.data
@ -261,7 +261,7 @@ def test_check_with_prefix_include_filters(client, live_server):
res = client.post(
url_for("edit_page", uuid="first"),
data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)

@ -70,6 +70,16 @@ def extract_api_key_from_UI(client):
api_key = m.group(1)
return api_key.strip()
# kinda funky, but works for now
def get_UUID_for_tag_name(client, name):
app_config = client.application.config.get('DATASTORE').data
for uuid, tag in app_config['settings']['application'].get('tags', {}).items():
if name == tag.get('title', '').lower().strip():
return uuid
return None
# kinda funky, but works for now
def extract_rss_token_from_UI(client):
import re

@ -19,7 +19,7 @@ def test_visual_selector_content_ready(client, live_server):
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
follow_redirects=True
)
assert b"Watch added in Paused state, saving will unpause" in res.data
@ -28,7 +28,7 @@ def test_visual_selector_content_ready(client, live_server):
url_for("edit_page", uuid="first", unpause_on_save=1),
data={
"url": test_url,
"tag": "",
"tags": "",
"headers": "",
'fetch_backend': "html_webdriver",
'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'

@ -26,10 +26,48 @@ class update_worker(threading.Thread):
self.datastore = datastore
super().__init__(*args, **kwargs)
def send_content_changed_notification(self, t, watch_uuid):
def queue_notification_for_watch(self, n_object, watch):
from changedetectionio import diff
watch_history = watch.history
dates = list(watch_history.keys())
# HTML needs linebreak, but MarkDown and Text can use a linefeed
if n_object['notification_format'] == 'HTML':
line_feed_sep = "<br>"
else:
line_feed_sep = "\n"
# Add text that was triggered
snapshot_contents = watch.get_history_snapshot(dates[-1])
trigger_text = watch.get('trigger_text', [])
triggered_text = ''
if len(trigger_text):
from . import html_tools
triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
if triggered_text:
triggered_text = line_feed_sep.join(triggered_text)
n_object.update({
'current_snapshot': snapshot_contents,
'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
'triggered_text': triggered_text,
'uuid': watch.get('uuid'),
'watch_url': watch.get('url'),
})
logging.info (">> SENDING NOTIFICATION")
self.notification_q.put(n_object)
def send_content_changed_notification(self, watch_uuid):
from changedetectionio.notification import (
default_notification_format_for_watch
)
@ -48,8 +86,9 @@ class update_worker(threading.Thread):
"History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?"
)
n_object['notification_urls'] = watch['notification_urls'] if len(watch['notification_urls']) else \
self.datastore.data['settings']['application']['notification_urls']
# Should be a better parent getter in the model object
# Prefer - Individual watch settings > Tag settings > Global settings (in that order)
n_object['notification_urls'] = watch.get('notification_urls')
n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
self.datastore.data['settings']['application']['notification_title']
@ -60,47 +99,51 @@ class update_worker(threading.Thread):
n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
self.datastore.data['settings']['application']['notification_format']
# Only prepare to notify if the rules above matched
# (Individual watch) Only prepare to notify if the rules above matched
sent = False
if 'notification_urls' in n_object and n_object['notification_urls']:
# HTML needs linebreak, but MarkDown and Text can use a linefeed
if n_object['notification_format'] == 'HTML':
line_feed_sep = "<br>"
else:
line_feed_sep = "\n"
sent = True
self.queue_notification_for_watch(n_object, watch)
# Add text that was triggered
snapshot_contents = watch.get_history_snapshot(dates[-1])
trigger_text = watch.get('trigger_text', [])
triggered_text = ''
# (Group tags) try by group tag
if not sent:
# Else, Try by tag, and use system default vars for format, body etc as fallback
tags = self.datastore.get_all_tags_for_watch(uuid=watch_uuid)
for tag_uuid, tag in tags.items():
n_object = {}
n_object['notification_urls'] = tag.get('notification_urls')
if len(trigger_text):
from . import html_tools
triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
if triggered_text:
triggered_text = line_feed_sep.join(triggered_text)
n_object['notification_title'] = tag.get('notification_title') if tag.get('notification_title') else \
self.datastore.data['settings']['application']['notification_title']
n_object['notification_body'] = tag.get('notification_body') if tag.get('notification_body') else \
self.datastore.data['settings']['application']['notification_body']
n_object['notification_format'] = tag.get('notification_format') if tag.get('notification_format') != default_notification_format_for_watch else \
self.datastore.data['settings']['application']['notification_format']
if 'notification_urls' in n_object and n_object.get('notification_urls') and not tag.get('notification_muted'):
sent = True
self.queue_notification_for_watch(n_object, watch)
# (Group tags) try by global
if not sent:
# leave this as is, but repeat in a loop for each tag also
n_object['notification_urls'] = self.datastore.data['settings']['application'].get('notification_urls')
n_object['notification_title'] = self.datastore.data['settings']['application'].get('notification_title')
n_object['notification_body'] = self.datastore.data['settings']['application'].get('notification_body')
n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
if n_object.get('notification_urls') and n_object.get('notification_body') and n_object.get('notification_title'):
sent = True
self.queue_notification_for_watch(n_object, watch)
return sent
n_object.update({
'current_snapshot': snapshot_contents,
'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
'triggered_text': triggered_text,
'uuid': watch_uuid,
'watch_url': watch['url'],
})
logging.info (">> SENDING NOTIFICATION")
self.notification_q.put(n_object)
else:
logging.info (">> NO Notification sent, notification_url was empty in both watch and system")
def send_filter_failure_notification(self, watch_uuid):
threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
watch = self.datastore.data['watching'].get(watch_uuid, False)
watch = self.datastore.data['watching'].get(watch_uuid)
if not watch:
return
@ -177,7 +220,7 @@ class update_worker(threading.Thread):
uuid = queued_item_data.item.get('uuid')
self.current_uuid = uuid
if uuid in list(self.datastore.data['watching'].keys()):
if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
changed_detected = False
contents = b''
process_changedetection_results = True
@ -360,7 +403,7 @@ class update_worker(threading.Thread):
# Notifications should only trigger on the second time (first time, we gather the initial snapshot)
if watch.history_n >= 2:
if not self.datastore.data['watching'][uuid].get('notification_muted'):
self.send_content_changed_notification(self, watch_uuid=uuid)
self.send_content_changed_notification(watch_uuid=uuid)
except Exception as e:

Loading…
Cancel
Save