Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512)

pull/563/head
dgtlmoon 3 years ago committed by GitHub
parent 1e8aa6158b
commit 9e708810d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -519,7 +519,7 @@ def changedetection_app(config=None, datastore_o=None):
def edit_page(uuid): def edit_page(uuid):
from changedetectionio import forms from changedetectionio import forms
using_default_check_time = True
# More for testing, possible to return the first/only # More for testing, possible to return the first/only
if not datastore.data['watching'].keys(): if not datastore.data['watching'].keys():
flash("No watches to edit", "error") flash("No watches to edit", "error")
@ -532,27 +532,39 @@ def changedetection_app(config=None, datastore_o=None):
flash("No watch with the UUID %s found." % (uuid), "error") flash("No watch with the UUID %s found." % (uuid), "error")
return redirect(url_for('index')) return redirect(url_for('index'))
# be sure we update with a copy instead of accidently editing the live object by reference
default = deepcopy(datastore.data['watching'][uuid])
# Show system wide default if nothing configured
if datastore.data['watching'][uuid]['fetch_backend'] is None:
default['fetch_backend'] = datastore.data['settings']['application']['fetch_backend']
# Show system wide default if nothing configured
if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])
form = forms.watchForm(formdata=request.form if request.method == 'POST' else None, form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
data=datastore.data['watching'][uuid] data=default
) )
if request.method == 'GET':
# Set some defaults that refer to the main config when None, we do the same in POST,
# probably there should be a nice little handler for this.
if datastore.data['watching'][uuid]['fetch_backend'] is None:
form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
if datastore.data['watching'][uuid]['minutes_between_check'] is None:
form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
if request.method == 'POST' and form.validate(): if request.method == 'POST' and form.validate():
extra_update_obj = {}
# Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']: # Assume we use the default value, unless something relevant is different, then use the form value
form.minutes_between_check.data = None # values could be None, 0 etc.
# Set to None unless the next for: says that something is different
extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data)
for k, v in form.time_between_check.data.items():
if v and v != datastore.data['settings']['requests']['time_between_check'][k]:
extra_update_obj['time_between_check'] = form.time_between_check.data
using_default_check_time = False
break
# Use the default if its the same as system wide
if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']: if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
form.fetch_backend.data = None extra_update_obj['fetch_backend'] = None
extra_update_obj = {}
# Notification URLs # Notification URLs
datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
@ -578,7 +590,7 @@ def changedetection_app(config=None, datastore_o=None):
# Re #286 - We wait for syncing new data to disk in another thread every 60 seconds # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
# But in the case something is added we should save straight away # But in the case something is added we should save straight away
datastore.sync_to_json() datastore.needs_write_urgent = True
# Queue the watch for immediate recheck # Queue the watch for immediate recheck
update_q.put(uuid) update_q.put(uuid)
@ -597,12 +609,12 @@ def changedetection_app(config=None, datastore_o=None):
if request.method == 'POST' and not form.validate(): if request.method == 'POST' and not form.validate():
flash("An error occurred, please see below.", "error") flash("An error occurred, please see below.", "error")
has_empty_checktime = datastore.data['watching'][uuid].has_empty_checktime
output = render_template("edit.html", output = render_template("edit.html",
uuid=uuid, uuid=uuid,
watch=datastore.data['watching'][uuid], watch=datastore.data['watching'][uuid],
form=form, form=form,
has_empty_checktime=has_empty_checktime, has_empty_checktime=using_default_check_time,
current_base_url=datastore.data['settings']['application']['base_url'], current_base_url=datastore.data['settings']['application']['base_url'],
emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False) emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
) )
@ -632,15 +644,15 @@ def changedetection_app(config=None, datastore_o=None):
if form.validate(): if form.validate():
datastore.data['settings']['application'].update(form.data['application']) datastore.data['settings']['application'].update(form.data['application'])
datastore.data['settings']['requests'].update(form.data['requests']) datastore.data['settings']['requests'].update(form.data['requests'])
datastore.needs_write = True
if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password): if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
datastore.needs_write = True datastore.needs_write_urgent = True
flash("Password protection enabled.", 'notice') flash("Password protection enabled.", 'notice')
flask_login.logout_user() flask_login.logout_user()
return redirect(url_for('index')) return redirect(url_for('index'))
datastore.needs_write_urgent = True
flash("Settings updated.") flash("Settings updated.")
else: else:
@ -1177,7 +1189,7 @@ def ticker_thread_check_time_launch_checks():
now = time.time() now = time.time()
recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60)) recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
recheck_time_system_seconds = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60 recheck_time_system_seconds = datastore.threshold_seconds
for uuid, watch in copied_datastore.data['watching'].items(): for uuid, watch in copied_datastore.data['watching'].items():
@ -1187,8 +1199,9 @@ def ticker_thread_check_time_launch_checks():
# If they supplied an individual entry minutes to threshold. # If they supplied an individual entry minutes to threshold.
threshold = now threshold = now
if watch.threshold_seconds: watch_threshold_seconds = watch.threshold_seconds()
threshold -= watch.threshold_seconds if watch_threshold_seconds:
threshold -= watch_threshold_seconds
else: else:
threshold -= recheck_time_system_seconds threshold -= recheck_time_system_seconds

@ -164,8 +164,8 @@ class perform_site_check():
else: else:
fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest() fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
# On the first run of a site, watch['previous_md5'] will be an empty string, set it the current one. # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
if not len(watch['previous_md5']): if not watch.get('previous_md5'):
watch['previous_md5'] = fetched_md5 watch['previous_md5'] = fetched_md5
update_obj["previous_md5"] = fetched_md5 update_obj["previous_md5"] = fetched_md5

@ -89,6 +89,13 @@ class SaltyPasswordField(StringField):
else: else:
self.data = False self.data = False
class TimeBetweenCheckForm(Form):
weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
hours = IntegerField('Hours', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
minutes = IntegerField('Minutes', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
seconds = IntegerField('Seconds', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
# @todo add total seconds minimum validatior = minimum_seconds_recheck_time
# Separated by key:value # Separated by key:value
class StringDictKeyValue(StringField): class StringDictKeyValue(StringField):
@ -317,8 +324,7 @@ class watchForm(commonSettingsForm):
url = fields.URLField('URL', validators=[validateURL()]) url = fields.URLField('URL', validators=[validateURL()])
tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)], default='') tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)], default='')
minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck', time_between_check = FormField(TimeBetweenCheckForm)
[validators.Optional(), validators.NumberRange(min=1)])
css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='') css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='')
@ -351,8 +357,9 @@ class watchForm(commonSettingsForm):
# datastore.data['settings']['requests'].. # datastore.data['settings']['requests']..
class globalSettingsRequestForm(Form): class globalSettingsRequestForm(Form):
minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck', time_between_check = FormField(TimeBetweenCheckForm)
[validators.NumberRange(min=1)])
# datastore.data['settings']['application'].. # datastore.data['settings']['application']..
class globalSettingsApplicationForm(commonSettingsForm): class globalSettingsApplicationForm(commonSettingsForm):

@ -10,9 +10,7 @@ from changedetectionio.notification import (
) )
class model(dict): class model(dict):
def __init__(self, *arg, **kw): base_config = {
super(model, self).__init__(*arg, **kw)
self.update({
'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!", 'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
'watching': {}, 'watching': {},
'settings': { 'settings': {
@ -24,8 +22,7 @@ class model(dict):
}, },
'requests': { 'requests': {
'timeout': 15, # Default 15 seconds 'timeout': 15, # Default 15 seconds
# Default 3 hours 'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
'minutes_between_check': 3 * 60, # Default 3 hours
'workers': 10 # Number of threads, lower is better for slow connections 'workers': 10 # Number of threads, lower is better for slow connections
}, },
'application': { 'application': {
@ -46,4 +43,8 @@ class model(dict):
'schema_version' : 0 'schema_version' : 0
} }
} }
}) }
def __init__(self, *arg, **kw):
super(model, self).__init__(*arg, **kw)
self.update(self.base_config)

@ -12,17 +12,16 @@ from changedetectionio.notification import (
class model(dict): class model(dict):
def __init__(self, *arg, **kw): base_config = {
self.update({
'url': None, 'url': None,
'tag': None, 'tag': None,
'last_checked': 0, 'last_checked': 0,
'last_changed': 0, 'last_changed': 0,
'paused': False, 'paused': False,
'last_viewed': 0, # history key value of the last viewed via the [diff] link 'last_viewed': 0, # history key value of the last viewed via the [diff] link
'newest_history_key': "", 'newest_history_key': 0,
'title': None, 'title': None,
'previous_md5': "", 'previous_md5': False,
'uuid': str(uuid_builder.uuid4()), 'uuid': str(uuid_builder.uuid4()),
'headers': {}, # Extra headers to send 'headers': {}, # Extra headers to send
'body': None, 'body': None,
@ -42,21 +41,27 @@ class model(dict):
# Re #110, so then if this is set to None, we know to use the default value instead # Re #110, so then if this is set to None, we know to use the default value instead
# Requires setting to None on submit if it's the same as the default # Requires setting to None on submit if it's the same as the default
# Should be all None by default, so we use the system default in this case. # Should be all None by default, so we use the system default in this case.
'minutes_between_check': None 'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
}) }
def __init__(self, *arg, **kw):
self.update(self.base_config)
# goes at the end so we update the default object with the initialiser # goes at the end so we update the default object with the initialiser
super(model, self).__init__(*arg, **kw) super(model, self).__init__(*arg, **kw)
@property @property
def has_empty_checktime(self): def has_empty_checktime(self):
if self.get('minutes_between_check', None): # using all() + dictionary comprehension
return False # Check if all values are 0 in dictionary
return True res = all(x == None or x == False or x==0 for x in self.get('time_between_check', {}).values())
return res
@property
def threshold_seconds(self): def threshold_seconds(self):
sec = self.get('minutes_between_check', None) seconds = 0
if sec: mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
sec = sec * 60 for m, n in mtable.items():
return sec x = self.get('time_between_check', {}).get(m, None)
if x:
seconds += x * n
return seconds

@ -440,3 +440,8 @@ ul {
padding-left: 1em; padding-left: 1em;
padding-top: 0px; padding-top: 0px;
margin-top: 4px; } margin-top: 4px; }
.time-check-widget tr {
display: inline; }
.time-check-widget tr input[type="number"] {
width: 4em; }

@ -624,4 +624,13 @@ ul {
padding-left: 1em; padding-left: 1em;
padding-top: 0px; padding-top: 0px;
margin-top: 4px; margin-top: 4px;
}
.time-check-widget {
tr {
display: inline;
input[type="number"] {
width: 4em;
}
}
} }

@ -7,6 +7,7 @@ import uuid as uuid_builder
from copy import deepcopy from copy import deepcopy
from os import mkdir, path, unlink from os import mkdir, path, unlink
from threading import Lock from threading import Lock
import re
from changedetectionio.model import Watch, App from changedetectionio.model import Watch, App
@ -16,6 +17,11 @@ from changedetectionio.model import Watch, App
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
class ChangeDetectionStore: class ChangeDetectionStore:
lock = Lock() lock = Lock()
# For general updates/writes that can wait a few seconds
needs_write = False
# For when we edit, we should write to disk
needs_write_urgent = False
def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"): def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
# Should only be active for docker # Should only be active for docker
@ -100,6 +106,9 @@ class ChangeDetectionStore:
secret = secrets.token_hex(16) secret = secrets.token_hex(16)
self.__data['settings']['application']['rss_access_token'] = secret self.__data['settings']['application']['rss_access_token'] = secret
# Bump the update version by running updates
self.run_updates()
self.needs_write = True self.needs_write = True
# Finally start the thread that will manage periodic data saves to JSON # Finally start the thread that will manage periodic data saves to JSON
@ -145,6 +154,17 @@ class ChangeDetectionStore:
self.needs_write = True self.needs_write = True
@property
def threshold_seconds(self):
seconds = 0
mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 5))
for m, n in mtable.items():
x = self.__data['settings']['requests']['time_between_check'].get(m)
if x:
seconds += x * n
return max(seconds, minimum_seconds_recheck_time)
@property @property
def data(self): def data(self):
has_unviewed = False has_unviewed = False
@ -339,7 +359,7 @@ class ChangeDetectionStore:
def sync_to_json(self): def sync_to_json(self):
logging.info("Saving JSON..") logging.info("Saving JSON..")
print("Saving JSON..")
try: try:
data = deepcopy(self.__data) data = deepcopy(self.__data)
except RuntimeError as e: except RuntimeError as e:
@ -361,6 +381,7 @@ class ChangeDetectionStore:
logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e)) logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
self.needs_write = False self.needs_write = False
self.needs_write_urgent = False
# Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
# by just running periodically in one thread, according to python, dict updates are threadsafe. # by just running periodically in one thread, according to python, dict updates are threadsafe.
@ -371,14 +392,14 @@ class ChangeDetectionStore:
print("Shutting down datastore thread") print("Shutting down datastore thread")
return return
if self.needs_write: if self.needs_write or self.needs_write_urgent:
self.sync_to_json() self.sync_to_json()
# Once per minute is enough, more and it can cause high CPU usage # Once per minute is enough, more and it can cause high CPU usage
# better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here
for i in range(30): for i in range(120):
time.sleep(2) time.sleep(0.5)
if self.stop_thread: if self.stop_thread or self.needs_write_urgent:
break break
# Go through the datastore path and remove any snapshots that are not mentioned in the index # Go through the datastore path and remove any snapshots that are not mentioned in the index
@ -398,3 +419,49 @@ class ChangeDetectionStore:
if not str(item) in index: if not str(item) in index:
print ("Removing",item) print ("Removing",item)
unlink(item) unlink(item)
# Run all updates
# IMPORTANT - Each update could be run even when they have a new install and the schema is correct
# So therefor - each `update_n` should be very careful about checking if it needs to actually run
# Probably we should bump the current update schema version with each tag release version?
def run_updates(self):
import inspect
import shutil
updates_available = []
for i, o in inspect.getmembers(self, predicate=inspect.ismethod):
m = re.search(r'update_(\d+)$', i)
if m:
updates_available.append(int(m.group(1)))
updates_available.sort()
for update_n in updates_available:
if update_n > self.__data['settings']['application']['schema_version']:
print ("Applying update_{}".format((update_n)))
# Wont exist on fresh installs
if os.path.exists(self.json_store_path):
shutil.copyfile(self.json_store_path, self.datastore_path+"/url-watches-before-{}.json".format(update_n))
try:
update_method = getattr(self, "update_{}".format(update_n))()
except Exception as e:
print("Error while trying update_{}".format((update_n)))
print(e)
# Don't run any more updates
return
else:
# Bump the version, important
self.__data['settings']['application']['schema_version'] = update_n
# Convert minutes to seconds on settings and each watch
def update_1(self):
if self.data['settings']['requests'].get('minutes_between_check'):
self.data['settings']['requests']['time_between_check']['minutes'] = self.data['settings']['requests']['minutes_between_check']
# Remove the default 'hours' that is set from the model
self.data['settings']['requests']['time_between_check']['hours'] = None
for uuid, watch in self.data['watching'].items():
if 'minutes_between_check' in watch:
# Only upgrade individual watch time if it was set
if watch.get('minutes_between_check', False):
self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']

@ -41,7 +41,7 @@
<span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span> <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
</div> </div>
<div class="pure-control-group"> <div class="pure-control-group">
{{ render_field(form.minutes_between_check) }} {{ render_field(form.time_between_check, class="time-check-widget") }}
{% if has_empty_checktime %} {% if has_empty_checktime %}
<span class="pure-form-message-inline">Currently using the <a <span class="pure-form-message-inline">Currently using the <a
href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span> href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>

@ -28,7 +28,7 @@
<div class="tab-pane-inner" id="general"> <div class="tab-pane-inner" id="general">
<fieldset> <fieldset>
<div class="pure-control-group"> <div class="pure-control-group">
{{ render_field(form.requests.form.minutes_between_check) }} {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
<span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span> <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
</div> </div>
<div class="pure-control-group"> <div class="pure-control-group">

@ -13,7 +13,7 @@ def test_check_access_control(app, client):
res = c.post( res = c.post(
url_for("settings_page"), url_for("settings_page"),
data={"application-password": "foobar", data={"application-password": "foobar",
"requests-minutes_between_check": 180, "requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_requests"}, 'application-fetch_backend': "html_requests"},
follow_redirects=True follow_redirects=True
) )
@ -46,7 +46,7 @@ def test_check_access_control(app, client):
assert b"BACKUP" in res.data assert b"BACKUP" in res.data
assert b"IMPORT" in res.data assert b"IMPORT" in res.data
assert b"LOG OUT" in res.data assert b"LOG OUT" in res.data
assert b"minutes_between_check" in res.data assert b"time_between_check-minutes" in res.data
assert b"fetch_backend" in res.data assert b"fetch_backend" in res.data
################################################## ##################################################
@ -55,7 +55,7 @@ def test_check_access_control(app, client):
res = c.post( res = c.post(
url_for("settings_page"), url_for("settings_page"),
data={ data={
"requests-minutes_between_check": 180, "requests-time_between_check-minutes": 180,
"application-fetch_backend": "html_webdriver", "application-fetch_backend": "html_webdriver",
"application-removepassword_button": "Remove password" "application-removepassword_button": "Remove password"
}, },
@ -70,7 +70,7 @@ def test_check_access_control(app, client):
res = c.post( res = c.post(
url_for("settings_page"), url_for("settings_page"),
data={"application-password": "", data={"application-password": "",
"requests-minutes_between_check": 180, "requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_requests"}, 'application-fetch_backend': "html_requests"},
follow_redirects=True follow_redirects=True
) )

@ -109,7 +109,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
# Enable auto pickup of <title> in settings # Enable auto pickup of <title> in settings
res = client.post( res = client.post(
url_for("settings_page"), url_for("settings_page"),
data={"application-extract_title_as_title": "1", "requests-minutes_between_check": 180, 'application-fetch_backend': "html_requests"}, data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"},
follow_redirects=True follow_redirects=True
) )

@ -171,11 +171,24 @@ def test_check_ignore_text_functionality(client, live_server):
def test_check_global_ignore_text_functionality(client, live_server): def test_check_global_ignore_text_functionality(client, live_server):
sleep_time_for_fetch_thread = 3 sleep_time_for_fetch_thread = 3
# Give the endpoint time to spin up
time.sleep(1)
ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ" ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
set_original_ignore_response() set_original_ignore_response()
# Give the endpoint time to spin up # Goto the settings page, add our ignore text
time.sleep(1) res = client.post(
url_for("settings_page"),
data={
"requests-time_between_check-minutes": 180,
"application-global_ignore_text": ignore_text,
'application-fetch_backend': "html_requests"
},
follow_redirects=True
)
assert b"Settings updated." in res.data
# Add our URL to the import page # Add our URL to the import page
test_url = url_for('test_endpoint', _external=True) test_url = url_for('test_endpoint', _external=True)
@ -192,17 +205,6 @@ def test_check_global_ignore_text_functionality(client, live_server):
# Give the thread time to pick it up # Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)
# Goto the settings page, add our ignore text
res = client.post(
url_for("settings_page"),
data={
"requests-minutes_between_check": 180,
"application-global_ignore_text": ignore_text,
'application-fetch_backend': "html_requests"
},
follow_redirects=True
)
assert b"Settings updated." in res.data
# Goto the edit page of the item, add our ignore text # Goto the edit page of the item, add our ignore text
# Add our URL to the import page # Add our URL to the import page
@ -225,12 +227,16 @@ def test_check_global_ignore_text_functionality(client, live_server):
# Give the thread time to pick it up # Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)
# so that we are sure everything is viewed and in a known 'nothing changed' state
res = client.get(url_for("diff_history_page", uuid="first"))
# It should report nothing found (no new 'unviewed' class) # It should report nothing found (no new 'unviewed' class)
res = client.get(url_for("index")) res = client.get(url_for("index"))
assert b'unviewed' not in res.data assert b'unviewed' not in res.data
assert b'/test-endpoint' in res.data assert b'/test-endpoint' in res.data
# Make a change
# Make a change which includes the ignore text
set_modified_ignore_response() set_modified_ignore_response()
# Trigger a check # Trigger a check
@ -243,7 +249,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
assert b'unviewed' not in res.data assert b'unviewed' not in res.data
assert b'/test-endpoint' in res.data assert b'/test-endpoint' in res.data
# Just to be sure.. set a regular modified change.. # Just to be sure.. set a regular modified change that will trigger it
set_modified_original_ignore_response() set_modified_original_ignore_response()
client.get(url_for("api_watch_checknow"), follow_redirects=True) client.get(url_for("api_watch_checknow"), follow_redirects=True)
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)

@ -51,12 +51,11 @@ def test_render_anchor_tag_content_true(client, live_server):
# set original html text # set original html text
set_original_ignore_response() set_original_ignore_response()
# Goto the settings page, choose not to ignore links # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content")
res = client.post( res = client.post(
url_for("settings_page"), url_for("settings_page"),
data={ data={
"requests-minutes_between_check": 180, "requests-time_between_check-minutes": 180,
"application-render_anchor_tag_content": "true",
"application-fetch_backend": "html_requests", "application-fetch_backend": "html_requests",
}, },
follow_redirects=True, follow_redirects=True,
@ -85,135 +84,42 @@ def test_render_anchor_tag_content_true(client, live_server):
# Give the thread time to pick it up # Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)
# check that the anchor tag content is rendered # We should not see the rendered anchor tag
res = client.get(url_for("preview_page", uuid="first"))
assert '(/modified_link)' in res.data.decode()
# since the link has changed, and we chose to render anchor tag content,
# we should detect a change (new 'unviewed' class)
res = client.get(url_for("index"))
assert b"unviewed" in res.data
assert b"/test-endpoint" in res.data
# Cleanup everything
res = client.get(url_for("api_delete", uuid="all"),
follow_redirects=True)
assert b'Deleted' in res.data
def test_render_anchor_tag_content_false(client, live_server):
"""Testing that anchor tag content changes are ignored when
render_anchor_tag_content setting is set to false"""
sleep_time_for_fetch_thread = 3
# Give the endpoint time to spin up
time.sleep(1)
# set the original html text
set_original_ignore_response()
# Goto the settings page, choose to ignore hyperlinks
res = client.post(
url_for("settings_page"),
data={
"requests-minutes_between_check": 180,
"application-render_anchor_tag_content": "false",
"application-fetch_backend": "html_requests",
},
follow_redirects=True,
)
assert b"Settings updated." in res.data
# Add our URL to the import page
test_url = url_for("test_endpoint", _external=True)
res = client.post(
url_for("import_page"), data={"urls": test_url}, follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(sleep_time_for_fetch_thread)
# Trigger a check
client.get(url_for("api_watch_checknow"), follow_redirects=True)
# set a new html text, with a modified link
set_modified_ignore_response()
time.sleep(sleep_time_for_fetch_thread)
# Trigger a check
client.get(url_for("api_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
# check that the anchor tag content is not rendered
res = client.get(url_for("preview_page", uuid="first")) res = client.get(url_for("preview_page", uuid="first"))
assert '(/modified_link)' not in res.data.decode() assert '(/modified_link)' not in res.data.decode()
# even though the link has changed, we shouldn't detect a change since # Goto the settings page, ENABLE render anchor tag
# we selected to not render anchor tag content (no new 'unviewed' class)
res = client.get(url_for("index"))
assert b"unviewed" not in res.data
assert b"/test-endpoint" in res.data
# Cleanup everything
res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
def test_render_anchor_tag_content_default(client, live_server):
"""Testing that anchor tag content changes are ignored when the
render_anchor_tag_content setting is not explicitly selected"""
sleep_time_for_fetch_thread = 3
# Give the endpoint time to spin up
time.sleep(1)
# set the original html text
set_original_ignore_response()
# Goto the settings page, not passing the render_anchor_tag_content setting
res = client.post( res = client.post(
url_for("settings_page"), url_for("settings_page"),
data={ data={
"requests-minutes_between_check": 180, "requests-time_between_check-minutes": 180,
"application-render_anchor_tag_content": "true",
"application-fetch_backend": "html_requests", "application-fetch_backend": "html_requests",
}, },
follow_redirects=True, follow_redirects=True,
) )
assert b"Settings updated." in res.data assert b"Settings updated." in res.data
# Add our URL to the import page
test_url = url_for("test_endpoint", _external=True)
res = client.post(
url_for("import_page"), data={"urls": test_url}, follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(sleep_time_for_fetch_thread)
# Trigger a check # Trigger a check
client.get(url_for("api_watch_checknow"), follow_redirects=True) client.get(url_for("api_watch_checknow"), follow_redirects=True)
# set a new html text, with a modified link # Give the thread time to pick it up
set_modified_ignore_response()
time.sleep(sleep_time_for_fetch_thread) time.sleep(sleep_time_for_fetch_thread)
# Trigger a check
client.get(url_for("api_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
# check that the anchor tag content is not rendered # check that the anchor tag content is rendered
res = client.get(url_for("preview_page", uuid="first")) res = client.get(url_for("preview_page", uuid="first"))
assert '(/modified_link)' not in res.data.decode() assert '(/modified_link)' in res.data.decode()
# even though the link has changed, we shouldn't detect a change since # since the link has changed, and we chose to render anchor tag content,
# we did not select the setting and the default behaviour is to not # we should detect a change (new 'unviewed' class)
# render anchor tag content (no new 'unviewed' class)
res = client.get(url_for("index")) res = client.get(url_for("index"))
assert b"unviewed" not in res.data assert b"unviewed" in res.data
assert b"/test-endpoint" in res.data assert b"/test-endpoint" in res.data
# Cleanup everything # Cleanup everything
res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True) res = client.get(url_for("api_delete", uuid="all"),
follow_redirects=True)
assert b'Deleted' in res.data assert b'Deleted' in res.data

@ -51,7 +51,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
res = client.post( res = client.post(
url_for("settings_page"), url_for("settings_page"),
data={ data={
"requests-minutes_between_check": 180, "requests-time_between_check-minutes": 180,
"application-ignore_status_codes": "y", "application-ignore_status_codes": "y",
'application-fetch_backend': "html_requests" 'application-fetch_backend': "html_requests"
}, },

@ -61,7 +61,7 @@ def test_check_ignore_whitespace(client, live_server):
res = client.post( res = client.post(
url_for("settings_page"), url_for("settings_page"),
data={ data={
"requests-minutes_between_check": 180, "requests-time_between_check-minutes": 180,
"application-ignore_whitespace": "y", "application-ignore_whitespace": "y",
"application-fetch_backend": "html_requests" "application-fetch_backend": "html_requests"
}, },

@ -270,6 +270,7 @@ def test_check_json_filter_bool_val(client, live_server):
) )
assert b"1 Imported" in res.data assert b"1 Imported" in res.data
time.sleep(3)
# Goto the edit page, add our ignore text # Goto the edit page, add our ignore text
# Add our URL to the import page # Add our URL to the import page
res = client.post( res = client.post(
@ -284,6 +285,7 @@ def test_check_json_filter_bool_val(client, live_server):
) )
assert b"Updated watch." in res.data assert b"Updated watch." in res.data
time.sleep(3)
# Trigger a check # Trigger a check
client.get(url_for("api_watch_checknow"), follow_redirects=True) client.get(url_for("api_watch_checknow"), follow_redirects=True)

@ -199,7 +199,7 @@ def test_notification_validation(client, live_server):
"application-notification_body": "Rubbish: {rubbish}\n", "application-notification_body": "Rubbish: {rubbish}\n",
"application-notification_format": "Text", "application-notification_format": "Text",
"application-notification_urls": "json://localhost/foobar", "application-notification_urls": "json://localhost/foobar",
"requests-minutes_between_check": 180, "requests-time_between_check-minutes": 180,
"fetch_backend": "html_requests" "fetch_backend": "html_requests"
}, },
follow_redirects=True follow_redirects=True

@ -35,7 +35,7 @@ def test_check_notification_error_handling(client, live_server):
"tag": "", "tag": "",
"title": "", "title": "",
"headers": "", "headers": "",
"minutes_between_check": "180", "time_between_check-minutes": "180",
"fetch_backend": "html_requests", "fetch_backend": "html_requests",
"trigger_check": "y"}, "trigger_check": "y"},
follow_redirects=True follow_redirects=True

@ -21,7 +21,7 @@ def test_check_watch_field_storage(client, live_server):
res = client.post( res = client.post(
url_for("edit_page", uuid="first"), url_for("edit_page", uuid="first"),
data={ "notification_urls": "json://127.0.0.1:30000\r\njson://128.0.0.1\r\n", data={ "notification_urls": "json://127.0.0.1:30000\r\njson://128.0.0.1\r\n",
"minutes_between_check": 126, "time_between_check-minutes": 126,
"css_filter" : ".fooclass", "css_filter" : ".fooclass",
"title" : "My title", "title" : "My title",
"ignore_text" : "ignore this", "ignore_text" : "ignore this",
@ -62,7 +62,7 @@ def test_check_recheck_global_setting(client, live_server):
res = client.post( res = client.post(
url_for("settings_page"), url_for("settings_page"),
data={ data={
"requests-minutes_between_check": 1566, "requests-time_between_check-minutes": 1566,
'application-fetch_backend': "html_requests" 'application-fetch_backend': "html_requests"
}, },
follow_redirects=True follow_redirects=True
@ -94,7 +94,7 @@ def test_check_recheck_global_setting(client, live_server):
res = client.post( res = client.post(
url_for("settings_page"), url_for("settings_page"),
data={ data={
"requests-minutes_between_check": 222, "requests-time_between_check-minutes": 222,
'application-fetch_backend': "html_requests" 'application-fetch_backend': "html_requests"
}, },
follow_redirects=True follow_redirects=True
@ -114,7 +114,7 @@ def test_check_recheck_global_setting(client, live_server):
res = client.post( res = client.post(
url_for("edit_page", uuid="first"), url_for("edit_page", uuid="first"),
data={"url": test_url, data={"url": test_url,
"minutes_between_check": 55, "time_between_check-minutes": 55,
'fetch_backend': "html_requests" 'fetch_backend': "html_requests"
}, },
follow_redirects=True follow_redirects=True
@ -130,8 +130,8 @@ def test_check_recheck_global_setting(client, live_server):
res = client.post( res = client.post(
url_for("settings_page"), url_for("settings_page"),
data={ data={
"requests-minutes_between_check": 666, "requests-time_between_check-minutes": 666,
'application-fetch_backend': "html_requests" "application-fetch_backend": "html_requests"
}, },
follow_redirects=True follow_redirects=True
) )
@ -140,7 +140,7 @@ def test_check_recheck_global_setting(client, live_server):
res = client.post( res = client.post(
url_for("edit_page", uuid="first"), url_for("edit_page", uuid="first"),
data={"url": test_url, data={"url": test_url,
"minutes_between_check": "", "time_between_check-minutes": "",
'fetch_backend': "html_requests" 'fetch_backend': "html_requests"
}, },
follow_redirects=True follow_redirects=True
@ -153,4 +153,3 @@ def test_check_recheck_global_setting(client, live_server):
follow_redirects=True follow_redirects=True
) )
assert b"666" in res.data assert b"666" in res.data

Loading…
Cancel
Save