diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index aa9c810b..4baa923e 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -519,7 +519,7 @@ def changedetection_app(config=None, datastore_o=None): def edit_page(uuid): from changedetectionio import forms - + using_default_check_time = True # More for testing, possible to return the first/only if not datastore.data['watching'].keys(): flash("No watches to edit", "error") @@ -532,27 +532,39 @@ def changedetection_app(config=None, datastore_o=None): flash("No watch with the UUID %s found." % (uuid), "error") return redirect(url_for('index')) + # be sure we update with a copy instead of accidently editing the live object by reference + default = deepcopy(datastore.data['watching'][uuid]) + + # Show system wide default if nothing configured + if datastore.data['watching'][uuid]['fetch_backend'] is None: + default['fetch_backend'] = datastore.data['settings']['application']['fetch_backend'] + + # Show system wide default if nothing configured + if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()): + default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check']) + form = forms.watchForm(formdata=request.form if request.method == 'POST' else None, - data=datastore.data['watching'][uuid] + data=default ) - if request.method == 'GET': - # Set some defaults that refer to the main config when None, we do the same in POST, - # probably there should be a nice little handler for this. - if datastore.data['watching'][uuid]['fetch_backend'] is None: - form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend'] - if datastore.data['watching'][uuid]['minutes_between_check'] is None: - form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check'] if request.method == 'POST' and form.validate(): + extra_update_obj = {} # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default - if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']: - form.minutes_between_check.data = None + # Assume we use the default value, unless something relevant is different, then use the form value + # values could be None, 0 etc. + # Set to None unless the next for: says that something is different + extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data) + for k, v in form.time_between_check.data.items(): + if v and v != datastore.data['settings']['requests']['time_between_check'][k]: + extra_update_obj['time_between_check'] = form.time_between_check.data + using_default_check_time = False + break + + # Use the default if its the same as system wide if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']: - form.fetch_backend.data = None - - extra_update_obj = {} + extra_update_obj['fetch_backend'] = None # Notification URLs datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data @@ -578,7 +590,7 @@ def changedetection_app(config=None, datastore_o=None): # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds # But in the case something is added we should save straight away - datastore.sync_to_json() + datastore.needs_write_urgent = True # Queue the watch for immediate recheck update_q.put(uuid) @@ -597,12 +609,12 @@ def changedetection_app(config=None, datastore_o=None): if request.method == 'POST' and not form.validate(): flash("An error occurred, please see below.", "error") - has_empty_checktime = datastore.data['watching'][uuid].has_empty_checktime + output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], form=form, - has_empty_checktime=has_empty_checktime, + has_empty_checktime=using_default_check_time, current_base_url=datastore.data['settings']['application']['base_url'], emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False) ) @@ -632,15 +644,15 @@ def changedetection_app(config=None, datastore_o=None): if form.validate(): datastore.data['settings']['application'].update(form.data['application']) datastore.data['settings']['requests'].update(form.data['requests']) - datastore.needs_write = True if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password): datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password - datastore.needs_write = True + datastore.needs_write_urgent = True flash("Password protection enabled.", 'notice') flask_login.logout_user() return redirect(url_for('index')) + datastore.needs_write_urgent = True flash("Settings updated.") else: @@ -1177,7 +1189,7 @@ def ticker_thread_check_time_launch_checks(): now = time.time() recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60)) - recheck_time_system_seconds = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60 + recheck_time_system_seconds = datastore.threshold_seconds for uuid, watch in copied_datastore.data['watching'].items(): @@ -1187,8 +1199,9 @@ def ticker_thread_check_time_launch_checks(): # If they supplied an individual entry minutes to threshold. threshold = now - if watch.threshold_seconds: - threshold -= watch.threshold_seconds + watch_threshold_seconds = watch.threshold_seconds() + if watch_threshold_seconds: + threshold -= watch_threshold_seconds else: threshold -= recheck_time_system_seconds diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 7fd86611..55bf49dc 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -164,8 +164,8 @@ class perform_site_check(): else: fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest() - # On the first run of a site, watch['previous_md5'] will be an empty string, set it the current one. - if not len(watch['previous_md5']): + # On the first run of a site, watch['previous_md5'] will be None, set it the current one. + if not watch.get('previous_md5'): watch['previous_md5'] = fetched_md5 update_obj["previous_md5"] = fetched_md5 diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 45c1457a..72dae639 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -89,6 +89,13 @@ class SaltyPasswordField(StringField): else: self.data = False +class TimeBetweenCheckForm(Form): + weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")]) + days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")]) + hours = IntegerField('Hours', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")]) + minutes = IntegerField('Minutes', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")]) + seconds = IntegerField('Seconds', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")]) + # @todo add total seconds minimum validatior = minimum_seconds_recheck_time # Separated by key:value class StringDictKeyValue(StringField): @@ -317,8 +324,7 @@ class watchForm(commonSettingsForm): url = fields.URLField('URL', validators=[validateURL()]) tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)], default='') - minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck', - [validators.Optional(), validators.NumberRange(min=1)]) + time_between_check = FormField(TimeBetweenCheckForm) css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='') @@ -351,8 +357,9 @@ class watchForm(commonSettingsForm): # datastore.data['settings']['requests'].. class globalSettingsRequestForm(Form): - minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck', - [validators.NumberRange(min=1)]) + time_between_check = FormField(TimeBetweenCheckForm) + + # datastore.data['settings']['application'].. class globalSettingsApplicationForm(commonSettingsForm): diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index c54df507..ebd5731a 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -10,9 +10,7 @@ from changedetectionio.notification import ( ) class model(dict): - def __init__(self, *arg, **kw): - super(model, self).__init__(*arg, **kw) - self.update({ + base_config = { 'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!", 'watching': {}, 'settings': { @@ -24,8 +22,7 @@ class model(dict): }, 'requests': { 'timeout': 15, # Default 15 seconds - # Default 3 hours - 'minutes_between_check': 3 * 60, # Default 3 hours + 'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None}, 'workers': 10 # Number of threads, lower is better for slow connections }, 'application': { @@ -46,4 +43,8 @@ class model(dict): 'schema_version' : 0 } } - }) + } + + def __init__(self, *arg, **kw): + super(model, self).__init__(*arg, **kw) + self.update(self.base_config) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index b86b930e..8283002c 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -12,17 +12,16 @@ from changedetectionio.notification import ( class model(dict): - def __init__(self, *arg, **kw): - self.update({ + base_config = { 'url': None, 'tag': None, 'last_checked': 0, 'last_changed': 0, 'paused': False, 'last_viewed': 0, # history key value of the last viewed via the [diff] link - 'newest_history_key': "", + 'newest_history_key': 0, 'title': None, - 'previous_md5': "", + 'previous_md5': False, 'uuid': str(uuid_builder.uuid4()), 'headers': {}, # Extra headers to send 'body': None, @@ -42,21 +41,27 @@ class model(dict): # Re #110, so then if this is set to None, we know to use the default value instead # Requires setting to None on submit if it's the same as the default # Should be all None by default, so we use the system default in this case. - 'minutes_between_check': None - }) + 'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None} + } + + def __init__(self, *arg, **kw): + self.update(self.base_config) # goes at the end so we update the default object with the initialiser super(model, self).__init__(*arg, **kw) @property def has_empty_checktime(self): - if self.get('minutes_between_check', None): - return False - return True + # using all() + dictionary comprehension + # Check if all values are 0 in dictionary + res = all(x == None or x == False or x==0 for x in self.get('time_between_check', {}).values()) + return res - @property def threshold_seconds(self): - sec = self.get('minutes_between_check', None) - if sec: - sec = sec * 60 - return sec + seconds = 0 + mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7} + for m, n in mtable.items(): + x = self.get('time_between_check', {}).get(m, None) + if x: + seconds += x * n + return seconds diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 2ff07ee1..174c9aea 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -440,3 +440,8 @@ ul { padding-left: 1em; padding-top: 0px; margin-top: 4px; } + +.time-check-widget tr { + display: inline; } + .time-check-widget tr input[type="number"] { + width: 4em; } diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss index 79e12909..3b305b45 100644 --- a/changedetectionio/static/styles/styles.scss +++ b/changedetectionio/static/styles/styles.scss @@ -624,4 +624,13 @@ ul { padding-left: 1em; padding-top: 0px; margin-top: 4px; +} + +.time-check-widget { + tr { + display: inline; + input[type="number"] { + width: 4em; + } + } } \ No newline at end of file diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 9aeebeeb..853f63fe 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -7,6 +7,7 @@ import uuid as uuid_builder from copy import deepcopy from os import mkdir, path, unlink from threading import Lock +import re from changedetectionio.model import Watch, App @@ -16,6 +17,11 @@ from changedetectionio.model import Watch, App # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change class ChangeDetectionStore: lock = Lock() + # For general updates/writes that can wait a few seconds + needs_write = False + + # For when we edit, we should write to disk + needs_write_urgent = False def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"): # Should only be active for docker @@ -100,6 +106,9 @@ class ChangeDetectionStore: secret = secrets.token_hex(16) self.__data['settings']['application']['rss_access_token'] = secret + # Bump the update version by running updates + self.run_updates() + self.needs_write = True # Finally start the thread that will manage periodic data saves to JSON @@ -145,6 +154,17 @@ class ChangeDetectionStore: self.needs_write = True + @property + def threshold_seconds(self): + seconds = 0 + mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7} + minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 5)) + for m, n in mtable.items(): + x = self.__data['settings']['requests']['time_between_check'].get(m) + if x: + seconds += x * n + return max(seconds, minimum_seconds_recheck_time) + @property def data(self): has_unviewed = False @@ -339,7 +359,7 @@ class ChangeDetectionStore: def sync_to_json(self): logging.info("Saving JSON..") - + print("Saving JSON..") try: data = deepcopy(self.__data) except RuntimeError as e: @@ -361,6 +381,7 @@ class ChangeDetectionStore: logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e)) self.needs_write = False + self.needs_write_urgent = False # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON # by just running periodically in one thread, according to python, dict updates are threadsafe. @@ -371,14 +392,14 @@ class ChangeDetectionStore: print("Shutting down datastore thread") return - if self.needs_write: + if self.needs_write or self.needs_write_urgent: self.sync_to_json() # Once per minute is enough, more and it can cause high CPU usage # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here - for i in range(30): - time.sleep(2) - if self.stop_thread: + for i in range(120): + time.sleep(0.5) + if self.stop_thread or self.needs_write_urgent: break # Go through the datastore path and remove any snapshots that are not mentioned in the index @@ -398,3 +419,49 @@ class ChangeDetectionStore: if not str(item) in index: print ("Removing",item) unlink(item) + + # Run all updates + # IMPORTANT - Each update could be run even when they have a new install and the schema is correct + # So therefor - each `update_n` should be very careful about checking if it needs to actually run + # Probably we should bump the current update schema version with each tag release version? + def run_updates(self): + import inspect + import shutil + + updates_available = [] + for i, o in inspect.getmembers(self, predicate=inspect.ismethod): + m = re.search(r'update_(\d+)$', i) + if m: + updates_available.append(int(m.group(1))) + updates_available.sort() + + for update_n in updates_available: + if update_n > self.__data['settings']['application']['schema_version']: + print ("Applying update_{}".format((update_n))) + # Wont exist on fresh installs + if os.path.exists(self.json_store_path): + shutil.copyfile(self.json_store_path, self.datastore_path+"/url-watches-before-{}.json".format(update_n)) + + try: + update_method = getattr(self, "update_{}".format(update_n))() + except Exception as e: + print("Error while trying update_{}".format((update_n))) + print(e) + # Don't run any more updates + return + else: + # Bump the version, important + self.__data['settings']['application']['schema_version'] = update_n + + # Convert minutes to seconds on settings and each watch + def update_1(self): + if self.data['settings']['requests'].get('minutes_between_check'): + self.data['settings']['requests']['time_between_check']['minutes'] = self.data['settings']['requests']['minutes_between_check'] + # Remove the default 'hours' that is set from the model + self.data['settings']['requests']['time_between_check']['hours'] = None + + for uuid, watch in self.data['watching'].items(): + if 'minutes_between_check' in watch: + # Only upgrade individual watch time if it was set + if watch.get('minutes_between_check', False): + self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check'] diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index d06c2208..07af9c4d 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -41,7 +41,7 @@ Organisational tag/group name used in the main listing page
- {{ render_field(form.minutes_between_check) }} + {{ render_field(form.time_between_check, class="time-check-widget") }} {% if has_empty_checktime %} Currently using the default global settings, change to another value if you want to be specific. diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 64cdbc92..fcac7097 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -28,7 +28,7 @@
- {{ render_field(form.requests.form.minutes_between_check) }} + {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }} Default time for all watches, when the watch does not have a specific time setting.
diff --git a/changedetectionio/tests/test_access_control.py b/changedetectionio/tests/test_access_control.py index b1fa8492..8b36923e 100644 --- a/changedetectionio/tests/test_access_control.py +++ b/changedetectionio/tests/test_access_control.py @@ -13,7 +13,7 @@ def test_check_access_control(app, client): res = c.post( url_for("settings_page"), data={"application-password": "foobar", - "requests-minutes_between_check": 180, + "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, follow_redirects=True ) @@ -46,7 +46,7 @@ def test_check_access_control(app, client): assert b"BACKUP" in res.data assert b"IMPORT" in res.data assert b"LOG OUT" in res.data - assert b"minutes_between_check" in res.data + assert b"time_between_check-minutes" in res.data assert b"fetch_backend" in res.data ################################################## @@ -55,7 +55,7 @@ def test_check_access_control(app, client): res = c.post( url_for("settings_page"), data={ - "requests-minutes_between_check": 180, + "requests-time_between_check-minutes": 180, "application-fetch_backend": "html_webdriver", "application-removepassword_button": "Remove password" }, @@ -70,7 +70,7 @@ def test_check_access_control(app, client): res = c.post( url_for("settings_page"), data={"application-password": "", - "requests-minutes_between_check": 180, + "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, follow_redirects=True ) diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index 4d718d5e..85bf4007 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -109,7 +109,7 @@ def test_check_basic_change_detection_functionality(client, live_server): # Enable auto pickup of in settings res = client.post( url_for("settings_page"), - data={"application-extract_title_as_title": "1", "requests-minutes_between_check": 180, 'application-fetch_backend': "html_requests"}, + data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, follow_redirects=True ) diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index da6ceb23..4adbfedd 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -171,11 +171,24 @@ def test_check_ignore_text_functionality(client, live_server): def test_check_global_ignore_text_functionality(client, live_server): sleep_time_for_fetch_thread = 3 + # Give the endpoint time to spin up + time.sleep(1) + ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ" set_original_ignore_response() - # Give the endpoint time to spin up - time.sleep(1) + # Goto the settings page, add our ignore text + res = client.post( + url_for("settings_page"), + data={ + "requests-time_between_check-minutes": 180, + "application-global_ignore_text": ignore_text, + 'application-fetch_backend': "html_requests" + }, + follow_redirects=True + ) + assert b"Settings updated." in res.data + # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -192,17 +205,6 @@ def test_check_global_ignore_text_functionality(client, live_server): # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) - # Goto the settings page, add our ignore text - res = client.post( - url_for("settings_page"), - data={ - "requests-minutes_between_check": 180, - "application-global_ignore_text": ignore_text, - 'application-fetch_backend': "html_requests" - }, - follow_redirects=True - ) - assert b"Settings updated." in res.data # Goto the edit page of the item, add our ignore text # Add our URL to the import page @@ -225,12 +227,16 @@ def test_check_global_ignore_text_functionality(client, live_server): # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) + # so that we are sure everything is viewed and in a known 'nothing changed' state + res = client.get(url_for("diff_history_page", uuid="first")) + # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) assert b'unviewed' not in res.data assert b'/test-endpoint' in res.data - # Make a change + + # Make a change which includes the ignore text set_modified_ignore_response() # Trigger a check @@ -243,7 +249,7 @@ def test_check_global_ignore_text_functionality(client, live_server): assert b'unviewed' not in res.data assert b'/test-endpoint' in res.data - # Just to be sure.. set a regular modified change.. + # Just to be sure.. set a regular modified change that will trigger it set_modified_original_ignore_response() client.get(url_for("api_watch_checknow"), follow_redirects=True) time.sleep(sleep_time_for_fetch_thread) diff --git a/changedetectionio/tests/test_ignorehyperlinks.py b/changedetectionio/tests/test_ignorehyperlinks.py index 920d0864..54aaf1d1 100644 --- a/changedetectionio/tests/test_ignorehyperlinks.py +++ b/changedetectionio/tests/test_ignorehyperlinks.py @@ -51,12 +51,11 @@ def test_render_anchor_tag_content_true(client, live_server): # set original html text set_original_ignore_response() - # Goto the settings page, choose not to ignore links + # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content") res = client.post( url_for("settings_page"), data={ - "requests-minutes_between_check": 180, - "application-render_anchor_tag_content": "true", + "requests-time_between_check-minutes": 180, "application-fetch_backend": "html_requests", }, follow_redirects=True, @@ -85,135 +84,42 @@ def test_render_anchor_tag_content_true(client, live_server): # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) - # check that the anchor tag content is rendered - res = client.get(url_for("preview_page", uuid="first")) - assert '(/modified_link)' in res.data.decode() - - # since the link has changed, and we chose to render anchor tag content, - # we should detect a change (new 'unviewed' class) - res = client.get(url_for("index")) - assert b"unviewed" in res.data - assert b"/test-endpoint" in res.data - - # Cleanup everything - res = client.get(url_for("api_delete", uuid="all"), - follow_redirects=True) - assert b'Deleted' in res.data - - -def test_render_anchor_tag_content_false(client, live_server): - """Testing that anchor tag content changes are ignored when - render_anchor_tag_content setting is set to false""" - sleep_time_for_fetch_thread = 3 - - # Give the endpoint time to spin up - time.sleep(1) - - # set the original html text - set_original_ignore_response() - - # Goto the settings page, choose to ignore hyperlinks - res = client.post( - url_for("settings_page"), - data={ - "requests-minutes_between_check": 180, - "application-render_anchor_tag_content": "false", - "application-fetch_backend": "html_requests", - }, - follow_redirects=True, - ) - assert b"Settings updated." in res.data - - # Add our URL to the import page - test_url = url_for("test_endpoint", _external=True) - res = client.post( - url_for("import_page"), data={"urls": test_url}, follow_redirects=True - ) - assert b"1 Imported" in res.data - - time.sleep(sleep_time_for_fetch_thread) - # Trigger a check - client.get(url_for("api_watch_checknow"), follow_redirects=True) - - # set a new html text, with a modified link - set_modified_ignore_response() - time.sleep(sleep_time_for_fetch_thread) - - # Trigger a check - client.get(url_for("api_watch_checknow"), follow_redirects=True) - - # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) - - # check that the anchor tag content is not rendered + # We should not see the rendered anchor tag res = client.get(url_for("preview_page", uuid="first")) assert '(/modified_link)' not in res.data.decode() - # even though the link has changed, we shouldn't detect a change since - # we selected to not render anchor tag content (no new 'unviewed' class) - res = client.get(url_for("index")) - assert b"unviewed" not in res.data - assert b"/test-endpoint" in res.data - - # Cleanup everything - res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True) - assert b'Deleted' in res.data - - -def test_render_anchor_tag_content_default(client, live_server): - """Testing that anchor tag content changes are ignored when the - render_anchor_tag_content setting is not explicitly selected""" - sleep_time_for_fetch_thread = 3 - - # Give the endpoint time to spin up - time.sleep(1) - - # set the original html text - set_original_ignore_response() - - # Goto the settings page, not passing the render_anchor_tag_content setting + # Goto the settings page, ENABLE render anchor tag res = client.post( url_for("settings_page"), data={ - "requests-minutes_between_check": 180, + "requests-time_between_check-minutes": 180, + "application-render_anchor_tag_content": "true", "application-fetch_backend": "html_requests", }, follow_redirects=True, ) assert b"Settings updated." in res.data - # Add our URL to the import page - test_url = url_for("test_endpoint", _external=True) - res = client.post( - url_for("import_page"), data={"urls": test_url}, follow_redirects=True - ) - assert b"1 Imported" in res.data - - time.sleep(sleep_time_for_fetch_thread) # Trigger a check client.get(url_for("api_watch_checknow"), follow_redirects=True) - # set a new html text, with a modified link - set_modified_ignore_response() + # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) - # Trigger a check - client.get(url_for("api_watch_checknow"), follow_redirects=True) - # Give the thread time to pick it up - time.sleep(sleep_time_for_fetch_thread) - # check that the anchor tag content is not rendered + # check that the anchor tag content is rendered res = client.get(url_for("preview_page", uuid="first")) - assert '(/modified_link)' not in res.data.decode() + assert '(/modified_link)' in res.data.decode() - # even though the link has changed, we shouldn't detect a change since - # we did not select the setting and the default behaviour is to not - # render anchor tag content (no new 'unviewed' class) + # since the link has changed, and we chose to render anchor tag content, + # we should detect a change (new 'unviewed' class) res = client.get(url_for("index")) - assert b"unviewed" not in res.data + assert b"unviewed" in res.data assert b"/test-endpoint" in res.data # Cleanup everything - res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("api_delete", uuid="all"), + follow_redirects=True) assert b'Deleted' in res.data + diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index 03df6cf3..09070c7b 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -51,7 +51,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server): res = client.post( url_for("settings_page"), data={ - "requests-minutes_between_check": 180, + "requests-time_between_check-minutes": 180, "application-ignore_status_codes": "y", 'application-fetch_backend': "html_requests" }, diff --git a/changedetectionio/tests/test_ignorewhitespace.py b/changedetectionio/tests/test_ignorewhitespace.py index fbc98495..ef8b002c 100644 --- a/changedetectionio/tests/test_ignorewhitespace.py +++ b/changedetectionio/tests/test_ignorewhitespace.py @@ -61,7 +61,7 @@ def test_check_ignore_whitespace(client, live_server): res = client.post( url_for("settings_page"), data={ - "requests-minutes_between_check": 180, + "requests-time_between_check-minutes": 180, "application-ignore_whitespace": "y", "application-fetch_backend": "html_requests" }, diff --git a/changedetectionio/tests/test_jsonpath_selector.py b/changedetectionio/tests/test_jsonpath_selector.py index dbe4725d..42fcfce8 100644 --- a/changedetectionio/tests/test_jsonpath_selector.py +++ b/changedetectionio/tests/test_jsonpath_selector.py @@ -270,6 +270,7 @@ def test_check_json_filter_bool_val(client, live_server): ) assert b"1 Imported" in res.data + time.sleep(3) # Goto the edit page, add our ignore text # Add our URL to the import page res = client.post( @@ -284,6 +285,7 @@ def test_check_json_filter_bool_val(client, live_server): ) assert b"Updated watch." in res.data + time.sleep(3) # Trigger a check client.get(url_for("api_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 02f59cfa..5cd2d0c7 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -199,7 +199,7 @@ def test_notification_validation(client, live_server): "application-notification_body": "Rubbish: {rubbish}\n", "application-notification_format": "Text", "application-notification_urls": "json://localhost/foobar", - "requests-minutes_between_check": 180, + "requests-time_between_check-minutes": 180, "fetch_backend": "html_requests" }, follow_redirects=True diff --git a/changedetectionio/tests/test_notification_errors.py b/changedetectionio/tests/test_notification_errors.py index 6ed860af..c90d0bbd 100644 --- a/changedetectionio/tests/test_notification_errors.py +++ b/changedetectionio/tests/test_notification_errors.py @@ -35,7 +35,7 @@ def test_check_notification_error_handling(client, live_server): "tag": "", "title": "", "headers": "", - "minutes_between_check": "180", + "time_between_check-minutes": "180", "fetch_backend": "html_requests", "trigger_check": "y"}, follow_redirects=True diff --git a/changedetectionio/tests/test_watch_fields_storage.py b/changedetectionio/tests/test_watch_fields_storage.py index 513c474e..bd9c9e65 100644 --- a/changedetectionio/tests/test_watch_fields_storage.py +++ b/changedetectionio/tests/test_watch_fields_storage.py @@ -21,7 +21,7 @@ def test_check_watch_field_storage(client, live_server): res = client.post( url_for("edit_page", uuid="first"), data={ "notification_urls": "json://127.0.0.1:30000\r\njson://128.0.0.1\r\n", - "minutes_between_check": 126, + "time_between_check-minutes": 126, "css_filter" : ".fooclass", "title" : "My title", "ignore_text" : "ignore this", @@ -62,7 +62,7 @@ def test_check_recheck_global_setting(client, live_server): res = client.post( url_for("settings_page"), data={ - "requests-minutes_between_check": 1566, + "requests-time_between_check-minutes": 1566, 'application-fetch_backend': "html_requests" }, follow_redirects=True @@ -94,7 +94,7 @@ def test_check_recheck_global_setting(client, live_server): res = client.post( url_for("settings_page"), data={ - "requests-minutes_between_check": 222, + "requests-time_between_check-minutes": 222, 'application-fetch_backend': "html_requests" }, follow_redirects=True @@ -114,7 +114,7 @@ def test_check_recheck_global_setting(client, live_server): res = client.post( url_for("edit_page", uuid="first"), data={"url": test_url, - "minutes_between_check": 55, + "time_between_check-minutes": 55, 'fetch_backend': "html_requests" }, follow_redirects=True @@ -130,8 +130,8 @@ def test_check_recheck_global_setting(client, live_server): res = client.post( url_for("settings_page"), data={ - "requests-minutes_between_check": 666, - 'application-fetch_backend': "html_requests" + "requests-time_between_check-minutes": 666, + "application-fetch_backend": "html_requests" }, follow_redirects=True ) @@ -140,7 +140,7 @@ def test_check_recheck_global_setting(client, live_server): res = client.post( url_for("edit_page", uuid="first"), data={"url": test_url, - "minutes_between_check": "", + "time_between_check-minutes": "", 'fetch_backend': "html_requests" }, follow_redirects=True @@ -153,4 +153,3 @@ def test_check_recheck_global_setting(client, live_server): follow_redirects=True ) assert b"666" in res.data -