diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index ea2acf50..b17f5cfb 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -317,25 +317,21 @@ def changedetection_app(config=None, datastore_o=None): return "Access denied, bad token", 403 from . import diff - limit_tag = request.args.get('tag') + limit_tag = request.args.get('tag', '').lower().strip() + # Be sure limit_tag is a uuid + for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): + if limit_tag == tag.get('title', '').lower().strip(): + limit_tag = uuid # Sort by last_changed and add the uuid which is usually the key.. sorted_watches = [] # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away for uuid, watch in datastore.data['watching'].items(): - - if limit_tag != None: - # Support for comma separated list of tags. - for tag_in_watch in watch['tag'].split(','): - tag_in_watch = tag_in_watch.strip() - if tag_in_watch == limit_tag: - watch['uuid'] = uuid - sorted_watches.append(watch) - - else: - watch['uuid'] = uuid - sorted_watches.append(watch) + if limit_tag and not limit_tag in watch['tags']: + continue + watch['uuid'] = uuid + sorted_watches.append(watch) sorted_watches.sort(key=lambda x: x.last_changed, reverse=False) @@ -392,9 +388,17 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/", methods=['GET']) @login_optionally_required def index(): + global datastore from changedetectionio import forms - limit_tag = request.args.get('tag') + limit_tag = request.args.get('tag', '').lower().strip() + + # Be sure limit_tag is a uuid + for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): + if limit_tag == tag.get('title', '').lower().strip(): + limit_tag = uuid + + # Redirect for the old rss path which used the /?rss=true if request.args.get('rss'): return redirect(url_for('rss', tag=limit_tag)) @@ -414,30 +418,15 @@ def changedetection_app(config=None, datastore_o=None): sorted_watches = [] search_q = request.args.get('q').strip().lower() if request.args.get('q') else False for uuid, watch in datastore.data['watching'].items(): - - if limit_tag: - # Support for comma separated list of tags. - if not watch.get('tag'): + if limit_tag and not limit_tag in watch['tags']: continue - for tag_in_watch in watch.get('tag', '').split(','): - tag_in_watch = tag_in_watch.strip() - if tag_in_watch == limit_tag: - watch['uuid'] = uuid - if search_q: - if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower(): - sorted_watches.append(watch) - else: - sorted_watches.append(watch) - else: - #watch['uuid'] = uuid - if search_q: - if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower(): - sorted_watches.append(watch) - else: + if search_q: + if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower(): sorted_watches.append(watch) + else: + sorted_watches.append(watch) - existing_tags = datastore.get_all_tags() form = forms.quickWatchForm(request.form) page = request.args.get(get_page_parameter(), type=int, default=1) total_count = len(sorted_watches) @@ -452,6 +441,7 @@ def changedetection_app(config=None, datastore_o=None): # Don't link to hosting when we're on the hosting environment active_tag=limit_tag, app_rss_token=datastore.data['settings']['application']['rss_access_token'], + datastore=datastore, form=form, guid=datastore.data['app_guid'], has_proxies=datastore.proxy_list, @@ -463,7 +453,7 @@ def changedetection_app(config=None, datastore_o=None): sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'), sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'), system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'), - tags=existing_tags, + tags=datastore.data['settings']['application'].get('tags'), watches=sorted_watches ) @@ -606,9 +596,13 @@ def changedetection_app(config=None, datastore_o=None): # proxy_override set to the json/text list of the items form = forms.watchForm(formdata=request.form if request.method == 'POST' else None, - data=default, + data=default ) + # For the form widget tag uuid lookup + form.tags.datastore = datastore # in _value + + form.fetch_backend.choices.append(("system", 'System settings default')) # form.browser_steps[0] can be assumed that we 'goto url' first @@ -659,6 +653,16 @@ def changedetection_app(config=None, datastore_o=None): extra_update_obj['filter_text_replaced'] = True extra_update_obj['filter_text_removed'] = True + # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs + tag_uuids = [] + if form.data.get('tags'): + # Sometimes in testing this can be list, dont know why + if type(form.data.get('tags')) == list: + extra_update_obj['tags'] = form.data.get('tags') + else: + for t in form.data.get('tags').split(','): + tag_uuids.append(datastore.add_tag(name=t)) + extra_update_obj['tags'] = tag_uuids datastore.data['watching'][uuid].update(form.data) datastore.data['watching'][uuid].update(extra_update_obj) @@ -713,7 +717,7 @@ def changedetection_app(config=None, datastore_o=None): form=form, has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, has_empty_checktime=using_default_check_time, - has_extra_headers_file=watch.has_extra_headers_file or datastore.has_extra_headers_file, + has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, is_html_webdriver=is_html_webdriver, jq_support=jq_support, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), @@ -1110,8 +1114,8 @@ def changedetection_app(config=None, datastore_o=None): os.path.join(datastore_o.datastore_path, list_with_tags_file), "w" ) as f: for uuid in datastore.data["watching"]: - url = datastore.data["watching"][uuid]["url"] - tag = datastore.data["watching"][uuid]["tag"] + url = datastore.data["watching"][uuid].get('url') + tag = datastore.data["watching"][uuid].get('tags', {}) f.write("{} {}\r\n".format(url, tag)) # Add it to the Zip @@ -1199,7 +1203,7 @@ def changedetection_app(config=None, datastore_o=None): add_paused = request.form.get('edit_and_watch_submit_button') != None processor = request.form.get('processor', 'text_json_diff') - new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused, 'processor': processor}) + new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor}) if new_uuid: if add_paused: @@ -1267,9 +1271,11 @@ def changedetection_app(config=None, datastore_o=None): elif tag != None: # Items that have this current tag for watch_uuid, watch in datastore.data['watching'].items(): - if (tag != None and tag in watch['tag']): + if (tag != None and tag in watch.get('tags', {})): if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})) + update_q.put( + queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}) + ) i += 1 else: @@ -1357,6 +1363,17 @@ def changedetection_app(config=None, datastore_o=None): datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch flash("{} watches set to use default notification settings".format(len(uuids))) + elif (op == 'assign-tag'): + op_extradata = request.form.get('op_extradata') + tag_uuid = datastore.add_tag(name=op_extradata) + if op_extradata and tag_uuid: + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid]['tags'].append(tag_uuid) + + flash("{} watches assigned tag".format(len(uuids))) + return redirect(url_for('index')) @app.route("/api/share-url", methods=['GET']) @@ -1366,7 +1383,6 @@ def changedetection_app(config=None, datastore_o=None): the share-link can be imported/added""" import requests import json - tag = request.args.get('tag') uuid = request.args.get('uuid') # more for testing @@ -1419,6 +1435,8 @@ def changedetection_app(config=None, datastore_o=None): import changedetectionio.blueprint.price_data_follower as price_data_follower app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower') + import changedetectionio.blueprint.tags as tags + app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags') # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py index b30c0d63..07e4022e 100644 --- a/changedetectionio/api/api_v1.py +++ b/changedetectionio/api/api_v1.py @@ -218,6 +218,11 @@ class CreateWatch(Resource): return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400 extras = copy.deepcopy(json_data) + + # Because we renamed 'tag' to 'tags' but dont want to change the API (can do this in v2 of the API) + if extras.get('tag'): + extras['tags'] = extras.get('tag') + del extras['url'] new_uuid = self.datastore.add_watch(url=url, extras=extras) @@ -259,13 +264,16 @@ class CreateWatch(Resource): """ list = {} - tag_limit = request.args.get('tag', None) - for k, watch in self.datastore.data['watching'].items(): - if tag_limit: - if not tag_limit.lower() in watch.all_tags: - continue + tag_limit = request.args.get('tag', '').lower() + + + for uuid, watch in self.datastore.data['watching'].items(): + # Watch tags by name (replace the other calls?) + tags = self.datastore.get_all_tags_for_watch(uuid=uuid) + if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()): + continue - list[k] = {'url': watch['url'], + list[uuid] = {'url': watch['url'], 'title': watch['title'], 'last_checked': watch['last_checked'], 'last_changed': watch.last_changed, diff --git a/changedetectionio/blueprint/tags/README.md b/changedetectionio/blueprint/tags/README.md new file mode 100644 index 00000000..c61159c2 --- /dev/null +++ b/changedetectionio/blueprint/tags/README.md @@ -0,0 +1,9 @@ +# Groups tags + +## How it works + +Watch has a list() of tag UUID's, which relate to a config under application.settings.tags + +The 'tag' is actually a watch, because they basically will eventually share 90% of the same config. + +So a tag is like an abstract of a watch diff --git a/changedetectionio/blueprint/tags/__init__.py b/changedetectionio/blueprint/tags/__init__.py new file mode 100644 index 00000000..3e1c732a --- /dev/null +++ b/changedetectionio/blueprint/tags/__init__.py @@ -0,0 +1,131 @@ +from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect +from changedetectionio.store import ChangeDetectionStore +from changedetectionio import login_optionally_required + + +def construct_blueprint(datastore: ChangeDetectionStore): + tags_blueprint = Blueprint('tags', __name__, template_folder="templates") + + @tags_blueprint.route("/list", methods=['GET']) + @login_optionally_required + def tags_overview_page(): + from .form import SingleTag + add_form = SingleTag(request.form) + output = render_template("groups-overview.html", + form=add_form, + available_tags=datastore.data['settings']['application'].get('tags', {}), + ) + + return output + + @tags_blueprint.route("/add", methods=['POST']) + @login_optionally_required + def form_tag_add(): + from .form import SingleTag + add_form = SingleTag(request.form) + + if not add_form.validate(): + for widget, l in add_form.errors.items(): + flash(','.join(l), 'error') + return redirect(url_for('tags.tags_overview_page')) + + title = request.form.get('name').strip() + + if datastore.tag_exists_by_name(title): + flash(f'The tag "{title}" already exists', "error") + return redirect(url_for('tags.tags_overview_page')) + + datastore.add_tag(title) + flash("Tag added") + + + return redirect(url_for('tags.tags_overview_page')) + + @tags_blueprint.route("/mute/", methods=['GET']) + @login_optionally_required + def mute(uuid): + if datastore.data['settings']['application']['tags'].get(uuid): + datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted'] + return redirect(url_for('tags.tags_overview_page')) + + @tags_blueprint.route("/delete/", methods=['GET']) + @login_optionally_required + def delete(uuid): + removed = 0 + # Delete the tag, and any tag reference + if datastore.data['settings']['application']['tags'].get(uuid): + del datastore.data['settings']['application']['tags'][uuid] + + for watch_uuid, watch in datastore.data['watching'].items(): + if watch.get('tags') and uuid in watch['tags']: + removed += 1 + watch['tags'].remove(uuid) + + flash(f"Tag deleted and removed from {removed} watches") + return redirect(url_for('tags.tags_overview_page')) + + @tags_blueprint.route("/unlink/", methods=['GET']) + @login_optionally_required + def unlink(uuid): + unlinked = 0 + for watch_uuid, watch in datastore.data['watching'].items(): + if watch.get('tags') and uuid in watch['tags']: + unlinked += 1 + watch['tags'].remove(uuid) + + flash(f"Tag unlinked removed from {unlinked} watches") + return redirect(url_for('tags.tags_overview_page')) + + @tags_blueprint.route("/edit/", methods=['GET']) + @login_optionally_required + def form_tag_edit(uuid): + from changedetectionio import forms + + if uuid == 'first': + uuid = list(datastore.data['settings']['application']['tags'].keys()).pop() + + default = datastore.data['settings']['application']['tags'].get(uuid) + + form = forms.watchForm(formdata=request.form if request.method == 'POST' else None, + data=default, + ) + form.datastore=datastore # needed? + + output = render_template("edit-tag.html", + data=default, + form=form, + settings_application=datastore.data['settings']['application'], + ) + + return output + + + @tags_blueprint.route("/edit/", methods=['POST']) + @login_optionally_required + def form_tag_edit_submit(uuid): + from changedetectionio import forms + if uuid == 'first': + uuid = list(datastore.data['settings']['application']['tags'].keys()).pop() + + default = datastore.data['settings']['application']['tags'].get(uuid) + + form = forms.watchForm(formdata=request.form if request.method == 'POST' else None, + data=default, + ) + # @todo subclass form so validation works + #if not form.validate(): +# for widget, l in form.errors.items(): +# flash(','.join(l), 'error') +# return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid)) + + datastore.data['settings']['application']['tags'][uuid].update(form.data) + datastore.needs_write_urgent = True + flash("Updated") + + return redirect(url_for('tags.tags_overview_page')) + + + @tags_blueprint.route("/delete/", methods=['GET']) + def form_tag_delete(uuid): + return redirect(url_for('tags.tags_overview_page')) + return tags_blueprint diff --git a/changedetectionio/blueprint/tags/form.py b/changedetectionio/blueprint/tags/form.py new file mode 100644 index 00000000..22e8b077 --- /dev/null +++ b/changedetectionio/blueprint/tags/form.py @@ -0,0 +1,22 @@ +from wtforms import ( + BooleanField, + Form, + IntegerField, + RadioField, + SelectField, + StringField, + SubmitField, + TextAreaField, + validators, +) + + + +class SingleTag(Form): + + name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"}) + save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"}) + + + + diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html new file mode 100644 index 00000000..618b81fa --- /dev/null +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -0,0 +1,131 @@ +{% extends 'base.html' %} +{% block content %} +{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} +{% from '_common_fields.jinja' import render_common_settings_form %} + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+
+ {{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }} +
+
+
+ +
+
+ {% set field = render_field(form.include_filters, + rows=5, + placeholder="#example +xpath://body/div/span[contains(@class, 'example-class')]", + class="m-d") + %} + {{ field }} + {% if '/text()' in field %} + Note!: //text() function does not work where the <element> contains <![CDATA[]]>
+ {% endif %} + One rule per line, any rules that matches will be used.
+ +
    +
  • CSS - Limit text to this CSS rule, only text matching this CSS rule is included.
    • +
  • JSON - Limit text to this JSON rule, using either JSONPath or jq (if installed). +
      +
    • JSONPath: Prefix with json:, use json:$ to force re-formatting if required, test your JSONPath here.
      • + {% if jq_support %} +
    • jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here.
      • + {% else %} +
    • jq support not installed
      • + {% endif %} +
    +
    • +
  • XPath - Limit text to this XPath rule, simply start with a forward-slash, +
      +
    • Example: //*[contains(@class, 'sametext')] or xpath://*[contains(@class, 'sametext')], test your XPath here
      • +
    • Example: Get all titles from an RSS feed //title/text()
      • +
    +
    • +
+ Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! here for more CSS selector help.
+ +
+
+ {{ render_field(form.subtractive_selectors, rows=5, placeholder="header +footer +nav +.stockticker") }} + +
    +
  • Remove HTML element(s) by CSS selector before text conversion.
    • +
  • Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML.
    • +
+ +
+ +
+ +
+
+
+ {{ render_checkbox_field(form.notification_muted) }} +
+ {% if is_html_webdriver %} +
+ {{ render_checkbox_field(form.notification_screenshot) }} + + Use with caution! This will easily fill up your email storage quota or flood other storages. + +
+ {% endif %} +
+ {% if has_default_notification_urls %} +
+ Look out! + There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications. +
+ {% endif %} + Use system defaults + + {{ render_common_settings_form(form, emailprefix, settings_application) }} +
+
+
+ +
+
+ {{ render_button(form.save_button) }} +
+
+
+
+
+ +{% endblock %} diff --git a/changedetectionio/blueprint/tags/templates/groups-overview.html b/changedetectionio/blueprint/tags/templates/groups-overview.html new file mode 100644 index 00000000..cab8d5e6 --- /dev/null +++ b/changedetectionio/blueprint/tags/templates/groups-overview.html @@ -0,0 +1,60 @@ +{% extends 'base.html' %} +{% block content %} +{% from '_helpers.jinja' import render_simple_field, render_field %} + + +
+
+ +
+ Add a new organisational tag +
+
+ {{ render_simple_field(form.name, placeholder="watch label / tag") }} +
+
+ {{ render_simple_field(form.save_button, title="Save" ) }} +
+
+
+
Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.
+
+
+ +
+ + + + + + + + + + + + {% if not available_tags|length %} + + + + {% endif %} + {% for uuid, tag in available_tags.items() %} + + + + + + {% endfor %} + +
Tag / Label name
No website organisational tags/groups configured
+ Mute notifications + {{tag.title}} + Edit  + Delete + Unlink +
+
+
+{% endblock %} diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 1cca60cf..7199b445 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -28,6 +28,8 @@ from changedetectionio.notification import ( from wtforms.fields import FormField +dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ]) + valid_method = { 'GET', 'POST', @@ -90,6 +92,29 @@ class SaltyPasswordField(StringField): else: self.data = False +class StringTagUUID(StringField): + + # process_formdata(self, valuelist) handled manually in POST handler + + # Is what is shown when field is rendered + def _value(self): + # Tag UUID to name, on submit it will convert it back (in the submit handler of init.py) + if self.data and type(self.data) is list: + tag_titles = [] + for i in self.data: + tag = self.datastore.data['settings']['application']['tags'].get(i) + if tag: + tag_title = tag.get('title') + if tag_title: + tag_titles.append(tag_title) + + return ', '.join(tag_titles) + + if not self.data: + return '' + + return 'error' + class TimeBetweenCheckForm(Form): weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")]) days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")]) @@ -347,7 +372,7 @@ class quickWatchForm(Form): from . import processors url = fields.URLField('URL', validators=[validateURL()]) - tag = StringField('Group tag', [validators.Optional()]) + tags = StringTagUUID('Group tag', [validators.Optional()]) watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"}) processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff") edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"}) @@ -355,6 +380,7 @@ class quickWatchForm(Form): # Common to a single watch and the global settings class commonSettingsForm(Form): + notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()]) notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()]) notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()]) @@ -382,7 +408,7 @@ class SingleBrowserStep(Form): class watchForm(commonSettingsForm): url = fields.URLField('URL', validators=[validateURL()]) - tag = StringField('Group tag', [validators.Optional()], default='') + tags = StringTagUUID('Group tag', [validators.Optional()], default='') time_between_check = FormField(TimeBetweenCheckForm) diff --git a/changedetectionio/importer.py b/changedetectionio/importer.py index c39b7a68..a420ea1e 100644 --- a/changedetectionio/importer.py +++ b/changedetectionio/importer.py @@ -120,9 +120,9 @@ class import_distill_io_json(Importer): except IndexError: pass - +# Does this need to be here anymore? if d.get('tags', False): - extras['tag'] = ", ".join(d['tags']) + extras['tags'] = ", ".join(d['tags']) new_uuid = datastore.add_watch(url=d['uri'].strip(), extras=extras, diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 29d3e49b..697d0d00 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -43,6 +43,7 @@ class model(dict): 'schema_version' : 0, 'shared_diff_access': False, 'webdriver_delay': None , # Extra delay in seconds before extracting text + 'tags': {} #@todo use Tag.model initialisers } } } diff --git a/changedetectionio/model/Tag.py b/changedetectionio/model/Tag.py new file mode 100644 index 00000000..1592cf08 --- /dev/null +++ b/changedetectionio/model/Tag.py @@ -0,0 +1,19 @@ +from .Watch import base_config +import uuid + +class model(dict): + + def __init__(self, *arg, **kw): + + self.update(base_config) + + self['uuid'] = str(uuid.uuid4()) + + if kw.get('default'): + self.update(kw['default']) + del kw['default'] + + + # Goes at the end so we update the default object with the initialiser + super(model, self).__init__(*arg, **kw) + diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 77c07497..f07f3dfe 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -52,7 +52,8 @@ base_config = { 'previous_md5_before_filters': False, # Used for skipping changedetection entirely 'proxy': None, # Preferred proxy connection 'subtractive_selectors': [], - 'tag': None, + 'tag': '', # Old system of text name for a tag, to be removed + 'tags': [], # list of UUIDs to App.Tags 'text_should_not_be_present': [], # Text that should not present # Re #110, so then if this is set to None, we know to use the default value instead # Requires setting to None on submit if it's the same as the default @@ -455,10 +456,6 @@ class model(dict): return csv_output_filename - @property - # Return list of tags, stripped and lowercase, used for searching - def all_tags(self): - return [s.strip().lower() for s in self.get('tag','').split(',')] def has_special_diff_filter_options_set(self): @@ -473,40 +470,6 @@ class model(dict): # None is set return False - @property - def has_extra_headers_file(self): - if os.path.isfile(os.path.join(self.watch_data_dir, 'headers.txt')): - return True - - for f in self.all_tags: - fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt" - filepath = os.path.join(self.__datastore_path, fname) - if os.path.isfile(filepath): - return True - - return False - - def get_all_headers(self): - from .App import parse_headers_from_text_file - headers = self.get('headers', {}).copy() - # Available headers on the disk could 'headers.txt' in the watch data dir - filepath = os.path.join(self.watch_data_dir, 'headers.txt') - try: - if os.path.isfile(filepath): - headers.update(parse_headers_from_text_file(filepath)) - except Exception as e: - print(f"ERROR reading headers.txt at {filepath}", str(e)) - - # Or each by tag, as tagname.txt in the main datadir - for f in self.all_tags: - fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt" - filepath = os.path.join(self.__datastore_path, fname) - try: - if os.path.isfile(filepath): - headers.update(parse_headers_from_text_file(filepath)) - except Exception as e: - print(f"ERROR reading headers.txt at {filepath}", str(e)) - return headers def get_last_fetched_before_filters(self): import brotli diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 58e028cb..a9b31702 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -186,8 +186,13 @@ def create_notification_parameters(n_object, datastore): uuid = n_object['uuid'] if 'uuid' in n_object else '' if uuid != '': - watch_title = datastore.data['watching'][uuid]['title'] - watch_tag = datastore.data['watching'][uuid]['tag'] + watch_title = datastore.data['watching'][uuid].get('title', '') + tag_list = [] + tags = datastore.get_all_tags_for_watch(uuid) + if tags: + for tag_uuid, tag in tags.items(): + tag_list.append(tag.get('title')) + watch_tag = ', '.join(tag_list) else: watch_title = 'Change Detection' watch_tag = '' diff --git a/changedetectionio/processors/restock_diff.py b/changedetectionio/processors/restock_diff.py index 6caf82d7..09cae6e7 100644 --- a/changedetectionio/processors/restock_diff.py +++ b/changedetectionio/processors/restock_diff.py @@ -42,11 +42,10 @@ class perform_site_check(difference_detection_processor): # Unset any existing notification error update_obj = {'last_notification_error': False, 'last_error': False} - extra_headers = watch.get('headers', []) - # Tweak the base config with the per-watch ones - request_headers = deepcopy(self.datastore.data['settings']['headers']) - request_headers.update(extra_headers) + request_headers = watch.get('headers', []) + request_headers.update(self.datastore.get_all_base_headers()) + request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) # https://github.com/psf/requests/issues/4525 # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index e0fc0cd2..63d6e64b 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -57,7 +57,6 @@ class perform_site_check(difference_detection_processor): # DeepCopy so we can be sure we don't accidently change anything by reference watch = deepcopy(self.datastore.data['watching'].get(uuid)) - if not watch: raise Exception("Watch no longer exists.") @@ -71,9 +70,9 @@ class perform_site_check(difference_detection_processor): update_obj = {'last_notification_error': False, 'last_error': False} # Tweak the base config with the per-watch ones - extra_headers = watch.get_all_headers() - request_headers = self.datastore.get_all_headers() - request_headers.update(extra_headers) + request_headers = watch.get('headers', []) + request_headers.update(self.datastore.get_all_base_headers()) + request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) # https://github.com/psf/requests/issues/4525 # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot @@ -191,21 +190,23 @@ class perform_site_check(difference_detection_processor): fetcher.content = fetcher.content.replace('', metadata + '') + # Better would be if Watch.model could access the global data also + # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__ + # https://realpython.com/inherit-python-dict/ instead of doing it procedurely + include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters') + include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags] - include_filters_rule = deepcopy(watch.get('include_filters', [])) - # include_filters_rule = watch['include_filters'] - subtractive_selectors = watch.get( - "subtractive_selectors", [] - ) + self.datastore.data["settings"]["application"].get( - "global_subtractive_selectors", [] - ) + subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'), + *watch.get("subtractive_selectors", []), + *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", []) + ] # Inject a virtual LD+JSON price tracker rule if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT: include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR) - has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip()) - has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip()) + has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip()) + has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip()) if is_json and not has_filter_rule: include_filters_rule.append("json:$") diff --git a/changedetectionio/static/js/watch-overview.js b/changedetectionio/static/js/watch-overview.js index 4b747c14..e5fb69c9 100644 --- a/changedetectionio/static/js/watch-overview.js +++ b/changedetectionio/static/js/watch-overview.js @@ -4,6 +4,9 @@ $(function () { $(this).closest('.unviewed').removeClass('unviewed'); }); + $("#checkbox-assign-tag").click(function (e) { + $('#op_extradata').val(prompt("Enter a tag name")); + }); $('.with-share-link > *').click(function () { $("#copied-clipboard").remove(); diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 0b8002c9..8711413e 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -16,6 +16,8 @@ import threading import time import uuid as uuid_builder +dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ]) + # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods? # Open a github issue if you know something :) # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change @@ -178,20 +180,6 @@ class ChangeDetectionStore: return self.__data - def get_all_tags(self): - tags = [] - for uuid, watch in self.data['watching'].items(): - if watch['tag'] is None: - continue - # Support for comma separated list of tags. - for tag in watch['tag'].split(','): - tag = tag.strip() - if tag not in tags: - tags.append(tag) - - tags.sort() - return tags - # Delete a single watch by UUID def delete(self, uuid): import pathlib @@ -218,9 +206,9 @@ class ChangeDetectionStore: # Clone a watch by UUID def clone(self, uuid): url = self.data['watching'][uuid]['url'] - tag = self.data['watching'][uuid]['tag'] + tag = self.data['watching'][uuid].get('tags',[]) extras = self.data['watching'][uuid] - new_uuid = self.add_watch(url=url, tag=tag, extras=extras) + new_uuid = self.add_watch(url=url, tag_uuids=tag, extras=extras) return new_uuid def url_exists(self, url): @@ -255,10 +243,11 @@ class ChangeDetectionStore: self.needs_write_urgent = True - def add_watch(self, url, tag="", extras=None, write_to_disk_now=True): + def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True): if extras is None: extras = {} + # should always be str if tag is None or not tag: tag = '' @@ -291,6 +280,7 @@ class ChangeDetectionStore: 'processor', 'subtractive_selectors', 'tag', + 'tags', 'text_should_not_be_present', 'title', 'trigger_text', @@ -313,25 +303,34 @@ class ChangeDetectionStore: flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error') return None - with self.lock: - # #Re 569 - new_watch = Watch.model(datastore_path=self.datastore_path, default={ - 'url': url, - 'tag': tag, - 'date_created': int(time.time()) - }) - new_uuid = new_watch['uuid'] - logging.debug("Added URL {} - {}".format(url, new_uuid)) + # #Re 569 + # Could be in 'tags', var or extras, smash them together and strip + apply_extras['tags'] = [] + if tag or extras.get('tags'): + tags = list(filter(None, list(set().union(tag.split(','), extras.get('tags', '').split(','))))) + for t in list(map(str.strip, tags)): + # for each stripped tag, add tag as UUID + apply_extras['tags'].append(self.add_tag(t)) + + # Or if UUIDs given directly + if tag_uuids: + apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids)) + + new_watch = Watch.model(datastore_path=self.datastore_path, url=url) - for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']: - if k in apply_extras: - del apply_extras[k] + new_uuid = new_watch.get('uuid') - new_watch.update(apply_extras) - self.__data['watching'][new_uuid] = new_watch + logging.debug("Added URL {} - {}".format(url, new_uuid)) + + for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']: + if k in apply_extras: + del apply_extras[k] + + new_watch.update(apply_extras) + new_watch.ensure_data_dir_exists() + self.__data['watching'][new_uuid] = new_watch - self.__data['watching'][new_uuid].ensure_data_dir_exists() if write_to_disk_now: self.sync_to_json() @@ -511,10 +510,19 @@ class ChangeDetectionStore: filepath = os.path.join(self.datastore_path, 'headers.txt') return os.path.isfile(filepath) - def get_all_headers(self): + def get_all_base_headers(self): + from .model.App import parse_headers_from_text_file + headers = {} + # Global app settings + headers.update(self.data['settings'].get('headers', {})) + + return headers + + def get_all_headers_in_textfile_for_watch(self, uuid): from .model.App import parse_headers_from_text_file - headers = copy(self.data['settings'].get('headers', {})) + headers = {} + # Global in /datastore/headers.txt filepath = os.path.join(self.datastore_path, 'headers.txt') try: if os.path.isfile(filepath): @@ -522,8 +530,76 @@ class ChangeDetectionStore: except Exception as e: print(f"ERROR reading headers.txt at {filepath}", str(e)) + watch = self.data['watching'].get(uuid) + if watch: + + # In /datastore/xyz-xyz/headers.txt + filepath = os.path.join(watch.watch_data_dir, 'headers.txt') + try: + if os.path.isfile(filepath): + headers.update(parse_headers_from_text_file(filepath)) + except Exception as e: + print(f"ERROR reading headers.txt at {filepath}", str(e)) + + # In /datastore/tag-name.txt + tags = self.get_all_tags_for_watch(uuid=uuid) + for tag_uuid, tag in tags.items(): + fname = "headers-"+re.sub(r'[\W_]', '', tag.get('title')).lower().strip() + ".txt" + filepath = os.path.join(self.datastore_path, fname) + try: + if os.path.isfile(filepath): + headers.update(parse_headers_from_text_file(filepath)) + except Exception as e: + print(f"ERROR reading headers.txt at {filepath}", str(e)) + return headers + def get_tag_overrides_for_watch(self, uuid, attr): + tags = self.get_all_tags_for_watch(uuid=uuid) + ret = [] + + if tags: + for tag_uuid, tag in tags.items(): + if attr in tag and tag[attr]: + ret=[*ret, *tag[attr]] + + return ret + + def add_tag(self, name): + print (">>> Adding new tag -", name) + # If name exists, return that + n = name.strip().lower() + for uuid, tag in self.__data['settings']['application'].get('tags', {}).items(): + if n == tag.get('title', '').lower().strip(): + print (f">>> Tag {name} already exists") + return uuid + + # Eventually almost everything todo with a watch will apply as a Tag + # So we use the same model as a Watch + with self.lock: + new_tag = Watch.model(datastore_path=self.datastore_path, default={ + 'title': name.strip(), + 'date_created': int(time.time()) + }) + + new_uuid = new_tag.get('uuid') + + self.__data['settings']['application']['tags'][new_uuid] = new_tag + + return new_uuid + + def get_all_tags_for_watch(self, uuid): + """This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet""" + watch = self.data['watching'].get(uuid) + + # Should return a dict of full tag info linked by UUID + if watch: + return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', [])) + + return {} + + def tag_exists_by_name(self, tag_name): + return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items()) # Run all updates # IMPORTANT - Each update could be run even when they have a new install and the schema is correct @@ -710,3 +786,16 @@ class ChangeDetectionStore: i+=1 return + # Create tag objects and their references from existing tag text + def update_12(self): + i = 0 + for uuid, watch in self.data['watching'].items(): + # Split out and convert old tag string + tag = watch.get('tag') + if tag: + tag_uuids = [] + for t in tag.split(','): + tag_uuids.append(self.add_tag(name=t)) + + self.data['watching'][uuid]['tags'] = tag_uuids + diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index f0044362..2147efe3 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -58,6 +58,9 @@ {% if current_user.is_authenticated or not has_password %} {% if not current_diff_url %} +
  • + GROUPS +
  • SETTINGS
    • @@ -86,7 +89,7 @@
    - + diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index cfcc827b..81e102ce 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -75,7 +75,7 @@ {{ render_field(form.title, class="m-d") }}
    - {{ render_field(form.tag) }} + {{ render_field(form.tags) }} Organisational tag/group name used in the main listing page
    diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index fe61f626..968bfc80 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -13,7 +13,7 @@
    {{ render_simple_field(form.url, placeholder="https://...", required=true) }} - {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }} + {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
    {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }} @@ -30,12 +30,14 @@ +
    + @@ -47,9 +49,9 @@ {% if search_q %}
    Searching "{{search_q}}"
    {% endif %}
    All - {% for tag in tags %} + {% for uuid, tag in tags.items() %} {% if tag != "" %} - {{ tag }} + {{ tag.title }} {% endif %} {% endfor %}
    @@ -143,9 +145,11 @@ {% endif %} - {% if not active_tag %} - {{ watch.tag}} - {% endif %} + + {% for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() %} + {{ watch_tag.title }} + {% endfor %} + {{watch|format_last_checked_time|safe}} {% if watch.history_n >=2 and watch.last_changed >0 %} @@ -178,7 +182,7 @@ {% endif %}
  • Recheck - all {% if active_tag%}in "{{active_tag}}"{%endif%} + all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}
  • RSS Feed diff --git a/changedetectionio/tests/proxy_list/test_multiple_proxy.py b/changedetectionio/tests/proxy_list/test_multiple_proxy.py index b329836e..859f1563 100644 --- a/changedetectionio/tests/proxy_list/test_multiple_proxy.py +++ b/changedetectionio/tests/proxy_list/test_multiple_proxy.py @@ -28,7 +28,7 @@ def test_preferred_proxy(client, live_server): "fetch_backend": "html_requests", "headers": "", "proxy": "proxy-two", - "tag": "", + "tags": "", "url": url, }, follow_redirects=True diff --git a/changedetectionio/tests/restock/test_restock.py b/changedetectionio/tests/restock/test_restock.py index 7711f247..d00fed10 100644 --- a/changedetectionio/tests/restock/test_restock.py +++ b/changedetectionio/tests/restock/test_restock.py @@ -77,7 +77,7 @@ def test_restock_detection(client, live_server): client.post( url_for("form_quick_watch_add"), - data={"url": test_url, "tag": '', 'processor': 'restock_diff'}, + data={"url": test_url, "tags": '', 'processor': 'restock_diff'}, follow_redirects=True ) diff --git a/changedetectionio/tests/test_access_control.py b/changedetectionio/tests/test_access_control.py index f931f66e..b8063683 100644 --- a/changedetectionio/tests/test_access_control.py +++ b/changedetectionio/tests/test_access_control.py @@ -45,6 +45,15 @@ def test_check_access_control(app, client, live_server): res = client.get(url_for("diff_history_page", uuid="first")) assert b'Random content' in res.data + # Check wrong password does not let us in + res = c.post( + url_for("login"), + data={"password": "WRONG PASSWORD"}, + follow_redirects=True + ) + + assert b"LOG OUT" not in res.data + assert b"Incorrect password" in res.data # Menu should not be available yet diff --git a/changedetectionio/tests/test_add_replace_remove_filter.py b/changedetectionio/tests/test_add_replace_remove_filter.py index fa74fe96..4ad9ecf8 100644 --- a/changedetectionio/tests/test_add_replace_remove_filter.py +++ b/changedetectionio/tests/test_add_replace_remove_filter.py @@ -2,7 +2,7 @@ import time from flask import url_for -from .util import live_server_setup +from .util import live_server_setup, wait_for_all_checks from changedetectionio import html_tools @@ -39,7 +39,6 @@ def test_setup(client, live_server): live_server_setup(live_server) def test_check_removed_line_contains_trigger(client, live_server): - sleep_time_for_fetch_thread = 3 # Give the endpoint time to spin up time.sleep(1) @@ -54,7 +53,7 @@ def test_check_removed_line_contains_trigger(client, live_server): assert b"1 Imported" in res.data # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # Goto the edit page, add our ignore text # Add our URL to the import page @@ -67,20 +66,20 @@ def test_check_removed_line_contains_trigger(client, live_server): follow_redirects=True ) assert b"Updated watch." in res.data - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) set_original(excluding='Something irrelevant') # A line thats not the trigger should not trigger anything res = client.get(url_for("form_watch_checknow"), follow_redirects=True) assert b'1 watches queued for rechecking.' in res.data - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data # The trigger line is REMOVED, this should trigger set_original(excluding='The golden line') client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data @@ -89,14 +88,14 @@ def test_check_removed_line_contains_trigger(client, live_server): client.get(url_for("mark_all_viewed"), follow_redirects=True) set_original(excluding=None) client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data # Remove it again, and we should get a trigger set_original(excluding='The golden line') client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data @@ -105,8 +104,7 @@ def test_check_removed_line_contains_trigger(client, live_server): def test_check_add_line_contains_trigger(client, live_server): - - sleep_time_for_fetch_thread = 3 + #live_server_setup(live_server) # Give the endpoint time to spin up time.sleep(1) @@ -136,8 +134,7 @@ def test_check_add_line_contains_trigger(client, live_server): assert b"1 Imported" in res.data # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) - + wait_for_all_checks(client) # Goto the edit page, add our ignore text # Add our URL to the import page res = client.post( @@ -150,23 +147,25 @@ def test_check_add_line_contains_trigger(client, live_server): follow_redirects=True ) assert b"Updated watch." in res.data - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) set_original(excluding='Something irrelevant') # A line thats not the trigger should not trigger anything res = client.get(url_for("form_watch_checknow"), follow_redirects=True) assert b'1 watches queued for rechecking.' in res.data - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data # The trigger line is ADDED, this should trigger set_original(add_line='

    Oh yes please

    ') client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data + # Takes a moment for apprise to fire + time.sleep(3) with open("test-datastore/notification.txt", 'r') as f: response= f.read() assert '-Oh yes please-' in response diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index 4fa6fa85..c19812ab 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -2,7 +2,7 @@ import time from flask import url_for -from .util import live_server_setup, extract_api_key_from_UI +from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks import json import uuid @@ -57,6 +57,7 @@ def test_setup(client, live_server): live_server_setup(live_server) def test_api_simple(client, live_server): + #live_server_setup(live_server) api_key = extract_api_key_from_UI(client) @@ -86,7 +87,7 @@ def test_api_simple(client, live_server): watch_uuid = res.json.get('uuid') assert res.status_code == 201 - time.sleep(3) + wait_for_all_checks(client) # Verify its in the list and that recheck worked res = client.get( @@ -107,7 +108,7 @@ def test_api_simple(client, live_server): ) assert len(res.json) == 0 - time.sleep(2) + wait_for_all_checks(client) set_modified_response() # Trigger recheck of all ?recheck_all=1 @@ -115,7 +116,7 @@ def test_api_simple(client, live_server): url_for("createwatch", recheck_all='1'), headers={'x-api-key': api_key}, ) - time.sleep(3) + wait_for_all_checks(client) # Did the recheck fire? res = client.get( @@ -297,6 +298,8 @@ def test_api_watch_PUT_update(client, live_server): url_for("edit_page", uuid=watch_uuid), ) assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section" + assert b"One" in res.data, "Tag 'One' was found" + assert b"Two" in res.data, "Tag 'Two' was found" # HTTP PUT ( UPDATE an existing watch ) res = client.put( @@ -319,7 +322,8 @@ def test_api_watch_PUT_update(client, live_server): ) assert b"new title" in res.data, "new title found in edit page" assert b"552" in res.data, "552 minutes found in edit page" - assert b"One, Two" in res.data, "Tag 'One, Two' was found" + assert b"One" in res.data, "Tag 'One' was found" + assert b"Two" in res.data, "Tag 'Two' was found" assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section" ###################################################### diff --git a/changedetectionio/tests/test_auth.py b/changedetectionio/tests/test_auth.py index 7ae16dfd..b84f8cf7 100644 --- a/changedetectionio/tests/test_auth.py +++ b/changedetectionio/tests/test_auth.py @@ -24,7 +24,7 @@ def test_basic_auth(client, live_server): # Check form validation res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_block_while_text_present.py b/changedetectionio/tests/test_block_while_text_present.py index 0538060b..2669b52a 100644 --- a/changedetectionio/tests/test_block_while_text_present.py +++ b/changedetectionio/tests/test_block_while_text_present.py @@ -2,7 +2,7 @@ import time from flask import url_for -from . util import live_server_setup +from .util import live_server_setup, wait_for_all_checks from changedetectionio import html_tools def set_original_ignore_response(): @@ -61,7 +61,7 @@ def set_modified_response_minus_block_text(): def test_check_block_changedetection_text_NOT_present(client, live_server): - sleep_time_for_fetch_thread = 3 + live_server_setup(live_server) # Use a mix of case in ZzZ to prove it works case-insensitive. ignore_text = "out of stoCk\r\nfoobar" @@ -81,7 +81,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server): assert b"1 Imported" in res.data # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # Goto the edit page, add our ignore text # Add our URL to the import page @@ -96,7 +96,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server): assert b"Updated watch." in res.data # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # Check it saved res = client.get( url_for("edit_page", uuid="first"), @@ -107,7 +107,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server): client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) @@ -120,7 +120,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) @@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server): # Now we set a change where the text is gone, it should now trigger set_modified_response_minus_block_text() client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data diff --git a/changedetectionio/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py index d8d09578..0dfe2af7 100644 --- a/changedetectionio/tests/test_css_selector.py +++ b/changedetectionio/tests/test_css_selector.py @@ -96,7 +96,7 @@ def test_check_markup_include_filters_restriction(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -157,7 +157,7 @@ def test_check_multiple_filters(client, live_server): url_for("edit_page", uuid="first"), data={"include_filters": include_filters, "url": test_url, - "tag": "", + "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True diff --git a/changedetectionio/tests/test_element_removal.py b/changedetectionio/tests/test_element_removal.py index 093764f7..3c280d22 100644 --- a/changedetectionio/tests/test_element_removal.py +++ b/changedetectionio/tests/test_element_removal.py @@ -129,7 +129,7 @@ def test_element_removal_full(client, live_server): data={ "subtractive_selectors": subtractive_selectors_data, "url": test_url, - "tag": "", + "tags": "", "headers": "", "fetch_backend": "html_requests", }, diff --git a/changedetectionio/tests/test_extract_regex.py b/changedetectionio/tests/test_extract_regex.py index 9fe54c0b..fec939f1 100644 --- a/changedetectionio/tests/test_extract_regex.py +++ b/changedetectionio/tests/test_extract_regex.py @@ -91,7 +91,7 @@ def test_check_filter_multiline(client, live_server): data={"include_filters": '', 'extract_text': '/something.+?6 billion.+?lines/si', "url": test_url, - "tag": "", + "tags": "", "headers": "", 'fetch_backend': "html_requests" }, @@ -146,7 +146,7 @@ def test_check_filter_and_regex_extract(client, live_server): data={"include_filters": include_filters, 'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i', "url": test_url, - "tag": "", + "tags": "", "headers": "", 'fetch_backend': "html_requests" }, diff --git a/changedetectionio/tests/test_filter_exist_changes.py b/changedetectionio/tests/test_filter_exist_changes.py index 451a0510..24f84455 100644 --- a/changedetectionio/tests/test_filter_exist_changes.py +++ b/changedetectionio/tests/test_filter_exist_changes.py @@ -56,7 +56,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se test_url = url_for('test_endpoint', _external=True) res = client.post( url_for("form_quick_watch_add"), - data={"url": test_url, "tag": 'cinema'}, + data={"url": test_url, "tags": 'cinema'}, follow_redirects=True ) assert b"Watch added" in res.data @@ -89,7 +89,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se notification_form_data.update({ "url": test_url, - "tag": "my tag", + "tags": "my tag", "title": "my title", "headers": "", "include_filters": '.ticket-available', diff --git a/changedetectionio/tests/test_filter_failure_notification.py b/changedetectionio/tests/test_filter_failure_notification.py index acbfed64..3ec5bfb7 100644 --- a/changedetectionio/tests/test_filter_failure_notification.py +++ b/changedetectionio/tests/test_filter_failure_notification.py @@ -1,7 +1,7 @@ import os import time from flask import url_for -from .util import set_original_response, live_server_setup, extract_UUID_from_client +from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks from changedetectionio.model import App @@ -37,14 +37,14 @@ def run_filter_test(client, content_filter): test_url = url_for('test_endpoint', _external=True) res = client.post( url_for("form_quick_watch_add"), - data={"url": test_url, "tag": ''}, + data={"url": test_url, "tags": ''}, follow_redirects=True ) assert b"Watch added" in res.data # Give the thread time to pick up the first version - time.sleep(3) + wait_for_all_checks(client) # Goto the edit page, add our ignore text # Add our URL to the import page @@ -71,8 +71,8 @@ def run_filter_test(client, content_filter): notification_form_data.update({ "url": test_url, - "tag": "my tag", - "title": "my title", + "tags": "my tag", + "title": "my title 123", "headers": "", "filter_failure_notification_send": 'y', "include_filters": content_filter, @@ -85,43 +85,55 @@ def run_filter_test(client, content_filter): ) assert b"Updated watch." in res.data - time.sleep(3) + wait_for_all_checks(client) # Now the notification should not exist, because we didnt reach the threshold assert not os.path.isfile("test-datastore/notification.txt") - for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT): + # -2 because we would have checked twice above (on adding and on edit) + for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2): res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(3) + wait_for_all_checks(client) + assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}" # We should see something in the frontend assert b'Warning, no filters were found' in res.data + # One more check should trigger it (see -2 above) + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) # Now it should exist and contain our "filter not found" alert assert os.path.isfile("test-datastore/notification.txt") - notification = False + with open("test-datastore/notification.txt", 'r') as f: notification = f.read() + assert 'CSS/xPath filter was not present in the page' in notification assert content_filter.replace('"', '\\"') in notification - # Remove it and prove that it doesnt trigger when not expected + # Remove it and prove that it doesn't trigger when not expected + # It should register a change, but no 'filter not found' os.unlink("test-datastore/notification.txt") set_response_with_filter() + # Try several times, it should NOT have 'filter not found' for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT): client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(3) + wait_for_all_checks(client) # It should have sent a notification, but.. assert os.path.isfile("test-datastore/notification.txt") - # but it should not contain the info about the failed filter + # but it should not contain the info about a failed filter (because there was none in this case) with open("test-datastore/notification.txt", 'r') as f: notification = f.read() assert not 'CSS/xPath filter was not present in the page' in notification - # Re #1247 - All tokens got replaced + # Re #1247 - All tokens got replaced correctly in the notification + res = client.get(url_for("index")) uuid = extract_UUID_from_client(client) + # UUID is correct, but notification contains tag uuid as UUIID wtf assert uuid in notification # cleanup for the next @@ -137,7 +149,7 @@ def test_setup(live_server): def test_check_include_filters_failure_notification(client, live_server): set_original_response() - time.sleep(1) + wait_for_all_checks(client) run_filter_test(client, '#nope-doesnt-exist') def test_check_xpath_filter_failure_notification(client, live_server): diff --git a/changedetectionio/tests/test_group.py b/changedetectionio/tests/test_group.py new file mode 100644 index 00000000..1518356c --- /dev/null +++ b/changedetectionio/tests/test_group.py @@ -0,0 +1,262 @@ +#!/usr/bin/python3 + +import time +from flask import url_for +from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name +import os + + +def test_setup(client, live_server): + live_server_setup(live_server) + +def set_original_response(): + test_return_data = """ + + Some initial text
    +

    Should be only this

    +
    +

    And never this

    + + + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + return None + +def set_modified_response(): + test_return_data = """ + + Some initial text
    +

    Should be REALLY only this

    +
    +

    And never this

    + + + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + return None + +def test_setup_group_tag(client, live_server): + #live_server_setup(live_server) + set_original_response() + + # Add a tag with some config, import a tag and it should roughly work + res = client.post( + url_for("tags.form_tag_add"), + data={"name": "test-tag"}, + follow_redirects=True + ) + assert b"Tag added" in res.data + assert b"test-tag" in res.data + + res = client.post( + url_for("tags.form_tag_edit_submit", uuid="first"), + data={"name": "test-tag", + "include_filters": '#only-this', + "subtractive_selectors": '#not-this'}, + follow_redirects=True + ) + assert b"Updated" in res.data + tag_uuid = get_UUID_for_tag_name(client, name="test-tag") + res = client.get( + url_for("tags.form_tag_edit", uuid="first") + ) + assert b"#only-this" in res.data + assert b"#not-this" in res.data + + # Tag should be setup and ready, now add a watch + + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + res = client.get(url_for("index")) + assert b'import-tag' in res.data + assert b'extra-import-tag' in res.data + + res = client.get( + url_for("tags.tags_overview_page"), + follow_redirects=True + ) + assert b'import-tag' in res.data + assert b'extra-import-tag' in res.data + + wait_for_all_checks(client) + + res = client.get(url_for("index")) + assert b'Warning, no filters were found' not in res.data + + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + assert b'Should be only this' in res.data + assert b'And never this' not in res.data + + + # RSS Group tag filter + # An extra one that should be excluded + res = client.post( + url_for("import_page"), + data={"urls": test_url + "?should-be-excluded=1 some-tag"}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + set_modified_response() + res = client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + rss_token = extract_rss_token_from_UI(client) + res = client.get( + url_for("rss", token=rss_token, tag="extra-import-tag", _external=True), + follow_redirects=True + ) + assert b"should-be-excluded" not in res.data + assert res.status_code == 200 + assert b"first-imported=1" in res.data + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + +def test_tag_import_singular(client, live_server): + #live_server_setup(live_server) + + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"}, + follow_redirects=True + ) + assert b"2 Imported" in res.data + + res = client.get( + url_for("tags.tags_overview_page"), + follow_redirects=True + ) + # Should be only 1 tag because they both had the same + assert res.data.count(b'test-tag') == 1 + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + +def test_tag_add_in_ui(client, live_server): + #live_server_setup(live_server) +# + res = client.post( + url_for("tags.form_tag_add"), + data={"name": "new-test-tag"}, + follow_redirects=True + ) + assert b"Tag added" in res.data + assert b"new-test-tag" in res.data + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + +def test_group_tag_notification(client, live_server): + #live_server_setup(live_server) + set_original_response() + + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("form_quick_watch_add"), + data={"url": test_url, "tags": 'test-tag, other-tag'}, + follow_redirects=True + ) + + assert b"Watch added" in res.data + + notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json') + notification_form_data = {"notification_urls": notification_url, + "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}", + "notification_body": "BASE URL: {{base_url}}\n" + "Watch URL: {{watch_url}}\n" + "Watch UUID: {{watch_uuid}}\n" + "Watch title: {{watch_title}}\n" + "Watch tag: {{watch_tag}}\n" + "Preview: {{preview_url}}\n" + "Diff URL: {{diff_url}}\n" + "Snapshot: {{current_snapshot}}\n" + "Diff: {{diff}}\n" + "Diff Added: {{diff_added}}\n" + "Diff Removed: {{diff_removed}}\n" + "Diff Full: {{diff_full}}\n" + ":-)", + "notification_screenshot": True, + "notification_format": "Text", + "title": "test-tag"} + + res = client.post( + url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")), + data=notification_form_data, + follow_redirects=True + ) + assert b"Updated" in res.data + + wait_for_all_checks(client) + + set_modified_response() + client.get(url_for("form_watch_checknow"), follow_redirects=True) + time.sleep(3) + + assert os.path.isfile("test-datastore/notification.txt") + + # Verify what was sent as a notification, this file should exist + with open("test-datastore/notification.txt", "r") as f: + notification_submission = f.read() + os.unlink("test-datastore/notification.txt") + + # Did we see the URL that had a change, in the notification? + # Diff was correctly executed + assert test_url in notification_submission + assert ':-)' in notification_submission + assert "Diff Full: Some initial text" in notification_submission + assert "New GROUP TAG ChangeDetection.io" in notification_submission + assert "test-tag" in notification_submission + assert "other-tag" in notification_submission + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + + #@todo Test that multiple notifications fired + #@todo Test that each of multiple notifications with different settings + + +def test_limit_tag_ui(client, live_server): + #live_server_setup(live_server) + + test_url = url_for('test_endpoint', _external=True) + urls=[] + + for i in range(20): + urls.append(test_url+"?x="+str(i)+" test-tag") + + for i in range(20): + urls.append(test_url+"?non-grouped="+str(i)) + + res = client.post( + url_for("import_page"), + data={"urls": "\r\n".join(urls)}, + follow_redirects=True + ) + + assert b"40 Imported" in res.data + + res = client.get(url_for("index")) + assert b'test-tag' in res.data + + # All should be here + assert res.data.count(b'processor-text_json_diff') == 40 + + tag_uuid = get_UUID_for_tag_name(client, name="test-tag") + + res = client.get(url_for("index", tag=tag_uuid)) + + # Just a subset should be here + assert b'test-tag' in res.data + assert res.data.count(b'processor-text_json_diff') == 20 + assert b"object at" not in res.data diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index 2d64f369..f3918663 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -2,7 +2,7 @@ import time from flask import url_for -from . util import live_server_setup +from .util import live_server_setup, wait_for_all_checks from changedetectionio import html_tools def test_setup(live_server): @@ -84,7 +84,6 @@ def set_modified_ignore_response(): def test_check_ignore_text_functionality(client, live_server): - sleep_time_for_fetch_thread = 3 # Use a mix of case in ZzZ to prove it works case-insensitive. ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff" @@ -103,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server): assert b"1 Imported" in res.data # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # Goto the edit page, add our ignore text # Add our URL to the import page @@ -124,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server): client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) @@ -137,7 +136,7 @@ def test_check_ignore_text_functionality(client, live_server): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) @@ -151,7 +150,7 @@ def test_check_ignore_text_functionality(client, live_server): # Just to be sure.. set a regular modified change.. set_modified_original_ignore_response() client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data @@ -167,7 +166,6 @@ def test_check_ignore_text_functionality(client, live_server): assert b'Deleted' in res.data def test_check_global_ignore_text_functionality(client, live_server): - sleep_time_for_fetch_thread = 3 # Give the endpoint time to spin up time.sleep(1) @@ -198,7 +196,7 @@ def test_check_global_ignore_text_functionality(client, live_server): assert b"1 Imported" in res.data # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # Goto the edit page of the item, add our ignore text @@ -220,7 +218,7 @@ def test_check_global_ignore_text_functionality(client, live_server): client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # so that we are sure everything is viewed and in a known 'nothing changed' state res = client.get(url_for("diff_history_page", uuid="first")) @@ -237,7 +235,7 @@ def test_check_global_ignore_text_functionality(client, live_server): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) @@ -247,7 +245,7 @@ def test_check_global_ignore_text_functionality(client, live_server): # Just to be sure.. set a regular modified change that will trigger it set_modified_original_ignore_response() client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index 01bc2fa9..74999b24 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -2,7 +2,7 @@ import time from flask import url_for -from . util import live_server_setup +from .util import live_server_setup, wait_for_all_checks def test_setup(live_server): @@ -40,7 +40,7 @@ def set_some_changed_response(): def test_normal_page_check_works_with_ignore_status_code(client, live_server): - sleep_time_for_fetch_thread = 3 + # Give the endpoint time to spin up time.sleep(1) @@ -68,15 +68,15 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server): ) assert b"1 Imported" in res.data - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) set_some_changed_response() - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) @@ -109,13 +109,13 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # Make a change set_some_changed_response() @@ -123,7 +123,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) # It should have 'unviewed' still # Because it should be looking at only that 'sametext' id diff --git a/changedetectionio/tests/test_jinja2.py b/changedetectionio/tests/test_jinja2.py index 9c6baa9f..771dc5ff 100644 --- a/changedetectionio/tests/test_jinja2.py +++ b/changedetectionio/tests/test_jinja2.py @@ -20,7 +20,7 @@ def test_jinja2_in_url_query(client, live_server): "date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", ) res = client.post( url_for("form_quick_watch_add"), - data={"url": full_url, "tag": "test"}, + data={"url": full_url, "tags": "test"}, follow_redirects=True ) assert b"Watch added" in res.data diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index f18cafe5..7dc4d68f 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -208,7 +208,7 @@ def test_check_json_without_filter(client, live_server): ) # Give the thread time to pick it up - time.sleep(3) + wait_for_all_checks(client) res = client.get( url_for("preview_page", uuid="first"), @@ -238,7 +238,7 @@ def check_json_filter(json_filter, client, live_server): assert b"1 Imported" in res.data # Give the thread time to pick it up - time.sleep(3) + wait_for_all_checks(client) # Goto the edit page, add our ignore text # Add our URL to the import page @@ -246,7 +246,7 @@ def check_json_filter(json_filter, client, live_server): url_for("edit_page", uuid="first"), data={"include_filters": json_filter, "url": test_url, - "tag": "", + "tags": "", "headers": "", "fetch_backend": "html_requests" }, @@ -261,14 +261,14 @@ def check_json_filter(json_filter, client, live_server): assert bytes(escape(json_filter).encode('utf-8')) in res.data # Give the thread time to pick it up - time.sleep(3) + wait_for_all_checks(client) # Make a change set_modified_response() # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(4) + wait_for_all_checks(client) # It should have 'unviewed' still res = client.get(url_for("index")) @@ -306,14 +306,14 @@ def check_json_filter_bool_val(json_filter, client, live_server): ) assert b"1 Imported" in res.data - time.sleep(3) + wait_for_all_checks(client) # Goto the edit page, add our ignore text # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), data={"include_filters": json_filter, "url": test_url, - "tag": "", + "tags": "", "headers": "", "fetch_backend": "html_requests" }, @@ -322,14 +322,14 @@ def check_json_filter_bool_val(json_filter, client, live_server): assert b"Updated watch." in res.data # Give the thread time to pick it up - time.sleep(3) + wait_for_all_checks(client) # Make a change set_modified_response() # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(3) + wait_for_all_checks(client) res = client.get(url_for("diff_history_page", uuid="first")) # But the change should be there, tho its hard to test the change was detected because it will show old and new versions @@ -366,7 +366,7 @@ def check_json_ext_filter(json_filter, client, live_server): assert b"1 Imported" in res.data # Give the thread time to pick it up - time.sleep(3) + wait_for_all_checks(client) # Goto the edit page, add our ignore text # Add our URL to the import page @@ -374,7 +374,7 @@ def check_json_ext_filter(json_filter, client, live_server): url_for("edit_page", uuid="first"), data={"include_filters": json_filter, "url": test_url, - "tag": "", + "tags": "", "headers": "", "fetch_backend": "html_requests" }, @@ -389,14 +389,14 @@ def check_json_ext_filter(json_filter, client, live_server): assert bytes(escape(json_filter).encode('utf-8')) in res.data # Give the thread time to pick it up - time.sleep(3) + wait_for_all_checks(client) # Make a change set_modified_ext_response() # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up - time.sleep(4) + wait_for_all_checks(client) # It should have 'unviewed' res = client.get(url_for("index")) @@ -428,14 +428,14 @@ def test_ignore_json_order(client, live_server): ) assert b"1 Imported" in res.data - time.sleep(2) + wait_for_all_checks(client) with open("test-datastore/endpoint-content.txt", "w") as f: f.write('{"world" : 123, "hello": 123}') # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(2) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data @@ -446,7 +446,7 @@ def test_ignore_json_order(client, live_server): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(2) + wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 036bf730..a94909ef 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -3,7 +3,7 @@ import os import time import re from flask import url_for -from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup +from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks from . util import extract_UUID_from_client import logging import base64 @@ -21,10 +21,11 @@ def test_setup(live_server): # Hard to just add more live server URLs when one test is already running (I think) # So we add our test here (was in a different file) def test_check_notification(client, live_server): + #live_server_setup(live_server) set_original_response() # Give the endpoint time to spin up - time.sleep(3) + time.sleep(1) # Re 360 - new install should have defaults set res = client.get(url_for("settings_page")) @@ -62,13 +63,13 @@ def test_check_notification(client, live_server): test_url = url_for('test_endpoint', _external=True) res = client.post( url_for("form_quick_watch_add"), - data={"url": test_url, "tag": ''}, + data={"url": test_url, "tags": ''}, follow_redirects=True ) assert b"Watch added" in res.data # Give the thread time to pick up the first version - time.sleep(3) + wait_for_all_checks(client) # We write the PNG to disk, but a JPEG should appear in the notification # Write the last screenshot png @@ -105,7 +106,7 @@ def test_check_notification(client, live_server): notification_form_data.update({ "url": test_url, - "tag": "my tag", + "tags": "my tag, my second tag", "title": "my title", "headers": "", "fetch_backend": "html_requests"}) @@ -128,7 +129,7 @@ def test_check_notification(client, live_server): ## Now recheck, and it should have sent the notification - time.sleep(3) + wait_for_all_checks(client) set_modified_response() # Trigger a check @@ -150,7 +151,7 @@ def test_check_notification(client, live_server): assert "b'" not in notification_submission assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE) assert "Watch title: my title" in notification_submission - assert "Watch tag: my tag" in notification_submission + assert "Watch tag: my tag, my second tag" in notification_submission assert "diff/" in notification_submission assert "preview/" in notification_submission assert ":-)" in notification_submission @@ -193,11 +194,11 @@ def test_check_notification(client, live_server): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(1) + wait_for_all_checks(client) client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(1) + wait_for_all_checks(client) client.get(url_for("form_watch_checknow"), follow_redirects=True) - time.sleep(1) + wait_for_all_checks(client) assert os.path.exists("test-datastore/notification.txt") == False res = client.get(url_for("notification_logs")) @@ -209,7 +210,7 @@ def test_check_notification(client, live_server): url_for("edit_page", uuid="first"), data={ "url": test_url, - "tag": "my tag", + "tags": "my tag", "title": "my title", "notification_urls": '', "notification_title": '', @@ -243,7 +244,7 @@ def test_notification_validation(client, live_server): test_url = url_for('test_endpoint', _external=True) res = client.post( url_for("form_quick_watch_add"), - data={"url": test_url, "tag": 'nice one'}, + data={"url": test_url, "tags": 'nice one'}, follow_redirects=True ) @@ -303,13 +304,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server): test_url = url_for('test_endpoint', _external=True) res = client.post( url_for("form_quick_watch_add"), - data={"url": test_url, "tag": 'nice one'}, + data={"url": test_url, "tags": 'nice one'}, follow_redirects=True ) assert b"Watch added" in res.data - time.sleep(2) + wait_for_all_checks(client) set_modified_response() client.get(url_for("form_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_notification_errors.py b/changedetectionio/tests/test_notification_errors.py index c7daee58..da6d851a 100644 --- a/changedetectionio/tests/test_notification_errors.py +++ b/changedetectionio/tests/test_notification_errors.py @@ -17,7 +17,7 @@ def test_check_notification_error_handling(client, live_server): test_url = url_for('test_endpoint', _external=True) res = client.post( url_for("form_quick_watch_add"), - data={"url": test_url, "tag": ''}, + data={"url": test_url, "tags": ''}, follow_redirects=True ) assert b"Watch added" in res.data @@ -32,7 +32,7 @@ def test_check_notification_error_handling(client, live_server): "notification_body": "xxxxx", "notification_format": "Text", "url": test_url, - "tag": "", + "tags": "", "title": "", "headers": "", "time_between_check-minutes": "180", diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 51cc3970..55c2e342 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -25,7 +25,7 @@ def test_headers_in_request(client, live_server): ) assert b"1 Imported" in res.data - time.sleep(1) + wait_for_all_checks(client) res = client.post( url_for("import_page"), @@ -43,7 +43,7 @@ def test_headers_in_request(client, live_server): url_for("edit_page", uuid="first"), data={ "url": test_url, - "tag": "", + "tags": "", "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests', "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header}, follow_redirects=True @@ -95,14 +95,14 @@ def test_body_in_request(client, live_server): ) assert b"1 Imported" in res.data - time.sleep(3) + wait_for_all_checks(client) # add the first 'version' res = client.post( url_for("edit_page", uuid="first"), data={ "url": test_url, - "tag": "", + "tags": "", "method": "POST", "fetch_backend": "html_requests", "body": "something something"}, @@ -110,7 +110,7 @@ def test_body_in_request(client, live_server): ) assert b"Updated watch." in res.data - time.sleep(3) + wait_for_all_checks(client) # Now the change which should trigger a change body_value = 'Test Body Value' @@ -118,7 +118,7 @@ def test_body_in_request(client, live_server): url_for("edit_page", uuid="first"), data={ "url": test_url, - "tag": "", + "tags": "", "method": "POST", "fetch_backend": "html_requests", "body": body_value}, @@ -126,7 +126,7 @@ def test_body_in_request(client, live_server): ) assert b"Updated watch." in res.data - time.sleep(3) + wait_for_all_checks(client) # The service should echo back the body res = client.get( @@ -163,7 +163,7 @@ def test_body_in_request(client, live_server): url_for("edit_page", uuid="first"), data={ "url": test_url, - "tag": "", + "tags": "", "method": "GET", "fetch_backend": "html_requests", "body": "invalid"}, @@ -187,7 +187,7 @@ def test_method_in_request(client, live_server): ) assert b"1 Imported" in res.data - time.sleep(2) + wait_for_all_checks(client) res = client.post( url_for("import_page"), data={"urls": test_url}, @@ -195,14 +195,14 @@ def test_method_in_request(client, live_server): ) assert b"1 Imported" in res.data - time.sleep(2) + wait_for_all_checks(client) # Attempt to add a method which is not valid res = client.post( url_for("edit_page", uuid="first"), data={ "url": test_url, - "tag": "", + "tags": "", "fetch_backend": "html_requests", "method": "invalid"}, follow_redirects=True @@ -214,7 +214,7 @@ def test_method_in_request(client, live_server): url_for("edit_page", uuid="first"), data={ "url": test_url, - "tag": "", + "tags": "", "fetch_backend": "html_requests", "method": "PATCH"}, follow_redirects=True @@ -222,7 +222,7 @@ def test_method_in_request(client, live_server): assert b"Updated watch." in res.data # Give the thread time to pick up the first version - time.sleep(2) + wait_for_all_checks(client) # The service should echo back the request verb res = client.get( @@ -233,7 +233,7 @@ def test_method_in_request(client, live_server): # The test call service will return the verb as the body assert b"PATCH" in res.data - time.sleep(2) + wait_for_all_checks(client) watches_with_method = 0 with open('test-datastore/url-watches.json') as f: @@ -265,7 +265,7 @@ def test_headers_textfile_in_request(client, live_server): ) assert b"1 Imported" in res.data - time.sleep(1) + wait_for_all_checks(client) # Add some headers to a request @@ -273,7 +273,7 @@ def test_headers_textfile_in_request(client, live_server): url_for("edit_page", uuid="first"), data={ "url": test_url, - "tag": "testtag", + "tags": "testtag", "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests', "headers": "xxx:ooo\ncool:yeah\r\n"}, follow_redirects=True diff --git a/changedetectionio/tests/test_search.py b/changedetectionio/tests/test_search.py index 0cd63f1d..453a0b8a 100644 --- a/changedetectionio/tests/test_search.py +++ b/changedetectionio/tests/test_search.py @@ -28,7 +28,7 @@ def test_basic_search(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"title": "xxx-title", "url": urls[0], "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -62,7 +62,7 @@ def test_search_in_tag_limit(client, live_server): # By Title res = client.post( url_for("edit_page", uuid="first"), - data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tag": urls[0].split(' ')[1], "headers": "", + data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tags": urls[0].split(' ')[1], "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) diff --git a/changedetectionio/tests/test_security.py b/changedetectionio/tests/test_security.py index 7d0d3667..08a69eeb 100644 --- a/changedetectionio/tests/test_security.py +++ b/changedetectionio/tests/test_security.py @@ -18,7 +18,7 @@ def test_bad_access(client, live_server): url_for("edit_page", uuid="first"), data={ "url": 'javascript:alert(document.domain)', - "tag": "", + "tags": "", "method": "GET", "fetch_backend": "html_requests", "body": ""}, @@ -29,7 +29,7 @@ def test_bad_access(client, live_server): res = client.post( url_for("form_quick_watch_add"), - data={"url": ' javascript:alert(123)', "tag": ''}, + data={"url": ' javascript:alert(123)', "tags": ''}, follow_redirects=True ) @@ -37,7 +37,7 @@ def test_bad_access(client, live_server): res = client.post( url_for("form_quick_watch_add"), - data={"url": '%20%20%20javascript:alert(123)%20%20', "tag": ''}, + data={"url": '%20%20%20javascript:alert(123)%20%20', "tags": ''}, follow_redirects=True ) @@ -46,7 +46,7 @@ def test_bad_access(client, live_server): res = client.post( url_for("form_quick_watch_add"), - data={"url": ' source:javascript:alert(document.domain)', "tag": ''}, + data={"url": ' source:javascript:alert(document.domain)', "tags": ''}, follow_redirects=True ) @@ -56,7 +56,7 @@ def test_bad_access(client, live_server): client.post( url_for("form_quick_watch_add"), - data={"url": 'file:///tasty/disk/drive', "tag": ''}, + data={"url": 'file:///tasty/disk/drive', "tags": ''}, follow_redirects=True ) time.sleep(1) diff --git a/changedetectionio/tests/test_share_watch.py b/changedetectionio/tests/test_share_watch.py index e328bf81..bf76fabc 100644 --- a/changedetectionio/tests/test_share_watch.py +++ b/changedetectionio/tests/test_share_watch.py @@ -29,7 +29,7 @@ def test_share_watch(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_source.py b/changedetectionio/tests/test_source.py index 695418c9..f46e8ad8 100644 --- a/changedetectionio/tests/test_source.py +++ b/changedetectionio/tests/test_source.py @@ -3,7 +3,7 @@ import time from flask import url_for from urllib.request import urlopen -from .util import set_original_response, set_modified_response, live_server_setup +from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks sleep_time_for_fetch_thread = 3 @@ -42,7 +42,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server): res = client.get(url_for("form_watch_checknow"), follow_redirects=True) assert b'1 watches queued for rechecking.' in res.data - time.sleep(5) + wait_for_all_checks(client) # Now something should be ready, indicated by having a 'unviewed' class res = client.get(url_for("index")) @@ -60,7 +60,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server): # `subtractive_selectors` should still work in `source:` type requests def test_check_ignore_elements(client, live_server): set_original_response() - time.sleep(2) + time.sleep(1) test_url = 'source:'+url_for('test_endpoint', _external=True) # Add our URL to the import page res = client.post( @@ -71,14 +71,14 @@ def test_check_ignore_elements(client, live_server): assert b"1 Imported" in res.data - time.sleep(sleep_time_for_fetch_thread) + wait_for_all_checks(client) ##################### # We want and

    ONLY, but ignore span with .foobar-detection client.post( url_for("edit_page", uuid="first"), - data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"}, + data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"}, follow_redirects=True ) diff --git a/changedetectionio/tests/test_watch_fields_storage.py b/changedetectionio/tests/test_watch_fields_storage.py index 5db29a72..5044598c 100644 --- a/changedetectionio/tests/test_watch_fields_storage.py +++ b/changedetectionio/tests/test_watch_fields_storage.py @@ -26,7 +26,7 @@ def test_check_watch_field_storage(client, live_server): "title" : "My title", "ignore_text" : "ignore this", "url": test_url, - "tag": "woohoo", + "tags": "woohoo", "headers": "curl:foo", 'fetch_backend': "html_requests" }, diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index a0172a77..fa8f4e8d 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server): time.sleep(1) res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server): time.sleep(1) res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -189,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -231,7 +231,7 @@ def test_xpath_validation(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"is not a valid XPath expression" in res.data @@ -261,7 +261,7 @@ def test_check_with_prefix_include_filters(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index 65a3e513..13e3fff9 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -70,6 +70,16 @@ def extract_api_key_from_UI(client): api_key = m.group(1) return api_key.strip() + +# kinda funky, but works for now +def get_UUID_for_tag_name(client, name): + app_config = client.application.config.get('DATASTORE').data + for uuid, tag in app_config['settings']['application'].get('tags', {}).items(): + if name == tag.get('title', '').lower().strip(): + return uuid + return None + + # kinda funky, but works for now def extract_rss_token_from_UI(client): import re diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py index 63c85438..08180bb0 100644 --- a/changedetectionio/tests/visualselector/test_fetch_data.py +++ b/changedetectionio/tests/visualselector/test_fetch_data.py @@ -19,7 +19,7 @@ def test_visual_selector_content_ready(client, live_server): res = client.post( url_for("form_quick_watch_add"), - data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'}, + data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'}, follow_redirects=True ) assert b"Watch added in Paused state, saving will unpause" in res.data @@ -28,7 +28,7 @@ def test_visual_selector_content_ready(client, live_server): url_for("edit_page", uuid="first", unpause_on_save=1), data={ "url": test_url, - "tag": "", + "tags": "", "headers": "", 'fetch_backend': "html_webdriver", 'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();' diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 18fe9cfb..5287e68b 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -26,10 +26,48 @@ class update_worker(threading.Thread): self.datastore = datastore super().__init__(*args, **kwargs) - def send_content_changed_notification(self, t, watch_uuid): + def queue_notification_for_watch(self, n_object, watch): from changedetectionio import diff + watch_history = watch.history + dates = list(watch_history.keys()) + + # HTML needs linebreak, but MarkDown and Text can use a linefeed + if n_object['notification_format'] == 'HTML': + line_feed_sep = "
    " + else: + line_feed_sep = "\n" + + # Add text that was triggered + snapshot_contents = watch.get_history_snapshot(dates[-1]) + trigger_text = watch.get('trigger_text', []) + triggered_text = '' + + if len(trigger_text): + from . import html_tools + triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text) + if triggered_text: + triggered_text = line_feed_sep.join(triggered_text) + + + n_object.update({ + 'current_snapshot': snapshot_contents, + 'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep), + 'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep), + 'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep), + 'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep), + 'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None, + 'triggered_text': triggered_text, + 'uuid': watch.get('uuid'), + 'watch_url': watch.get('url'), + }) + logging.info (">> SENDING NOTIFICATION") + self.notification_q.put(n_object) + + + def send_content_changed_notification(self, watch_uuid): + from changedetectionio.notification import ( default_notification_format_for_watch ) @@ -48,8 +86,9 @@ class update_worker(threading.Thread): "History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?" ) - n_object['notification_urls'] = watch['notification_urls'] if len(watch['notification_urls']) else \ - self.datastore.data['settings']['application']['notification_urls'] + # Should be a better parent getter in the model object + # Prefer - Individual watch settings > Tag settings > Global settings (in that order) + n_object['notification_urls'] = watch.get('notification_urls') n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \ self.datastore.data['settings']['application']['notification_title'] @@ -60,47 +99,51 @@ class update_worker(threading.Thread): n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \ self.datastore.data['settings']['application']['notification_format'] - - # Only prepare to notify if the rules above matched + # (Individual watch) Only prepare to notify if the rules above matched + sent = False if 'notification_urls' in n_object and n_object['notification_urls']: - # HTML needs linebreak, but MarkDown and Text can use a linefeed - if n_object['notification_format'] == 'HTML': - line_feed_sep = "
    " - else: - line_feed_sep = "\n" + sent = True + self.queue_notification_for_watch(n_object, watch) - # Add text that was triggered - snapshot_contents = watch.get_history_snapshot(dates[-1]) - trigger_text = watch.get('trigger_text', []) - triggered_text = '' + # (Group tags) try by group tag + if not sent: + # Else, Try by tag, and use system default vars for format, body etc as fallback + tags = self.datastore.get_all_tags_for_watch(uuid=watch_uuid) + for tag_uuid, tag in tags.items(): + n_object = {} + n_object['notification_urls'] = tag.get('notification_urls') - if len(trigger_text): - from . import html_tools - triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text) - if triggered_text: - triggered_text = line_feed_sep.join(triggered_text) + n_object['notification_title'] = tag.get('notification_title') if tag.get('notification_title') else \ + self.datastore.data['settings']['application']['notification_title'] + n_object['notification_body'] = tag.get('notification_body') if tag.get('notification_body') else \ + self.datastore.data['settings']['application']['notification_body'] + + n_object['notification_format'] = tag.get('notification_format') if tag.get('notification_format') != default_notification_format_for_watch else \ + self.datastore.data['settings']['application']['notification_format'] + + if 'notification_urls' in n_object and n_object.get('notification_urls') and not tag.get('notification_muted'): + sent = True + self.queue_notification_for_watch(n_object, watch) + + # (Group tags) try by global + if not sent: + # leave this as is, but repeat in a loop for each tag also + n_object['notification_urls'] = self.datastore.data['settings']['application'].get('notification_urls') + n_object['notification_title'] = self.datastore.data['settings']['application'].get('notification_title') + n_object['notification_body'] = self.datastore.data['settings']['application'].get('notification_body') + n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format') + if n_object.get('notification_urls') and n_object.get('notification_body') and n_object.get('notification_title'): + sent = True + self.queue_notification_for_watch(n_object, watch) + + return sent - n_object.update({ - 'current_snapshot': snapshot_contents, - 'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep), - 'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep), - 'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep), - 'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep), - 'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None, - 'triggered_text': triggered_text, - 'uuid': watch_uuid, - 'watch_url': watch['url'], - }) - logging.info (">> SENDING NOTIFICATION") - self.notification_q.put(n_object) - else: - logging.info (">> NO Notification sent, notification_url was empty in both watch and system") def send_filter_failure_notification(self, watch_uuid): threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts') - watch = self.datastore.data['watching'].get(watch_uuid, False) + watch = self.datastore.data['watching'].get(watch_uuid) if not watch: return @@ -177,7 +220,7 @@ class update_worker(threading.Thread): uuid = queued_item_data.item.get('uuid') self.current_uuid = uuid - if uuid in list(self.datastore.data['watching'].keys()): + if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'): changed_detected = False contents = b'' process_changedetection_results = True @@ -360,7 +403,7 @@ class update_worker(threading.Thread): # Notifications should only trigger on the second time (first time, we gather the initial snapshot) if watch.history_n >= 2: if not self.datastore.data['watching'][uuid].get('notification_muted'): - self.send_content_changed_notification(self, watch_uuid=uuid) + self.send_content_changed_notification(watch_uuid=uuid) except Exception as e: