Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512)

changedetectionio/__init__.py

@@ -519,7 +519,7 @@ def changedetection_app(config=None, datastore_o=None):
     def edit_page(uuid):
         from changedetectionio import forms
 
-
+        using_default_check_time = True
         # More for testing, possible to return the first/only
         if not datastore.data['watching'].keys():
             flash("No watches to edit", "error")
@@ -532,27 +532,39 @@ def changedetection_app(config=None, datastore_o=None):
             flash("No watch with the UUID %s found." % (uuid), "error")
             return redirect(url_for('index'))
 
+        # be sure we update with a copy instead of accidently editing the live object by reference
+        default = deepcopy(datastore.data['watching'][uuid])
+
+        # Show system wide default if nothing configured
+        if datastore.data['watching'][uuid]['fetch_backend'] is None:
+            default['fetch_backend'] = datastore.data['settings']['application']['fetch_backend']
+
+        # Show system wide default if nothing configured
+        if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
+            default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])
+
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=datastore.data['watching'][uuid]
+                                        data=default
                                         )
 
-        if request.method == 'GET':
-            # Set some defaults that refer to the main config when None, we do the same in POST,
-            # probably there should be a nice little handler for this.
-            if datastore.data['watching'][uuid]['fetch_backend'] is None:
-                form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
-            if datastore.data['watching'][uuid]['minutes_between_check'] is None:
-                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
 
         if request.method == 'POST' and form.validate():
+            extra_update_obj = {}
 
             # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
-            if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
-                form.minutes_between_check.data = None
+            # Assume we use the default value, unless something relevant is different, then use the form value
+            # values could be None, 0 etc.
+            # Set to None unless the next for: says that something is different
+            extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data)
+            for k, v in form.time_between_check.data.items():
+                if v and v != datastore.data['settings']['requests']['time_between_check'][k]:
+                    extra_update_obj['time_between_check'] = form.time_between_check.data
+                    using_default_check_time = False
+                    break
+
+            # Use the default if its the same as system wide
             if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
-                form.fetch_backend.data = None
-
-            extra_update_obj = {}
+                extra_update_obj['fetch_backend'] = None
 
             # Notification URLs
             datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
@@ -578,7 +590,7 @@ def changedetection_app(config=None, datastore_o=None):
 
             # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
             # But in the case something is added we should save straight away
-            datastore.sync_to_json()
+            datastore.needs_write_urgent = True
 
             # Queue the watch for immediate recheck
             update_q.put(uuid)
@@ -597,12 +609,12 @@ def changedetection_app(config=None, datastore_o=None):
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
-            has_empty_checktime = datastore.data['watching'][uuid].has_empty_checktime
+
             output = render_template("edit.html",
                                      uuid=uuid,
                                      watch=datastore.data['watching'][uuid],
                                      form=form,
-                                     has_empty_checktime=has_empty_checktime,
+                                     has_empty_checktime=using_default_check_time,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
@@ -632,15 +644,15 @@ def changedetection_app(config=None, datastore_o=None):
             if form.validate():
                 datastore.data['settings']['application'].update(form.data['application'])
                 datastore.data['settings']['requests'].update(form.data['requests'])
-                datastore.needs_write = True
 
                 if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
                     datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
-                    datastore.needs_write = True
+                    datastore.needs_write_urgent = True
                     flash("Password protection enabled.", 'notice')
                     flask_login.logout_user()
                     return redirect(url_for('index'))
 
+                datastore.needs_write_urgent = True
                 flash("Settings updated.")
 
             else:
@@ -1177,7 +1189,7 @@ def ticker_thread_check_time_launch_checks():
         now = time.time()
 
         recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
-        recheck_time_system_seconds = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60
+        recheck_time_system_seconds = datastore.threshold_seconds
 
         for uuid, watch in copied_datastore.data['watching'].items():
 
@@ -1187,8 +1199,9 @@ def ticker_thread_check_time_launch_checks():
 
             # If they supplied an individual entry minutes to threshold.
             threshold = now
-            if watch.threshold_seconds:
-                threshold -= watch.threshold_seconds
+            watch_threshold_seconds = watch.threshold_seconds()
+            if watch_threshold_seconds:
+                threshold -= watch_threshold_seconds
             else:
                 threshold -= recheck_time_system_seconds
 

changedetectionio/fetch_site_status.py

@@ -164,8 +164,8 @@ class perform_site_check():
         else:
             fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
 
-        # On the first run of a site, watch['previous_md5'] will be an empty string, set it the current one.
-        if not len(watch['previous_md5']):
+        # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
+        if not watch.get('previous_md5'):
             watch['previous_md5'] = fetched_md5
             update_obj["previous_md5"] = fetched_md5
 

changedetectionio/forms.py

@@ -89,6 +89,13 @@ class SaltyPasswordField(StringField):
         else:
             self.data = False
 
+class TimeBetweenCheckForm(Form):
+    weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    hours = IntegerField('Hours', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    minutes = IntegerField('Minutes', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    seconds = IntegerField('Seconds', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    # @todo add total seconds minimum validatior = minimum_seconds_recheck_time
 
 # Separated by  key:value
 class StringDictKeyValue(StringField):
@@ -317,8 +324,7 @@ class watchForm(commonSettingsForm):
     url = fields.URLField('URL', validators=[validateURL()])
     tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)], default='')
 
-    minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck',
-                                               [validators.Optional(), validators.NumberRange(min=1)])
+    time_between_check = FormField(TimeBetweenCheckForm)
 
     css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='')
 
@@ -351,8 +357,9 @@ class watchForm(commonSettingsForm):
 
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
-    minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck',
-                                               [validators.NumberRange(min=1)])
+    time_between_check = FormField(TimeBetweenCheckForm)
+
+
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
 

changedetectionio/model/App.py

@@ -10,9 +10,7 @@ from changedetectionio.notification import (
 )
 
 class model(dict):
-    def __init__(self, *arg, **kw):
-        super(model, self).__init__(*arg, **kw)
-        self.update({
+    base_config = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
             'settings': {
@@ -24,8 +22,7 @@ class model(dict):
                 },
                 'requests': {
                     'timeout': 15,  # Default 15 seconds
-                    # Default 3 hours
-                    'minutes_between_check': 3 * 60,  # Default 3 hours
+                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'workers': 10  # Number of threads, lower is better for slow connections
                 },
                 'application': {
@@ -46,4 +43,8 @@ class model(dict):
                     'schema_version' : 0
                 }
             }
-        })
+        }
+
+    def __init__(self, *arg, **kw):
+        super(model, self).__init__(*arg, **kw)
+        self.update(self.base_config)

changedetectionio/model/Watch.py

@@ -12,17 +12,16 @@ from changedetectionio.notification import (
 
 
 class model(dict):
-    def __init__(self, *arg, **kw):
-        self.update({
+    base_config = {
             'url': None,
             'tag': None,
             'last_checked': 0,
             'last_changed': 0,
             'paused': False,
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            'newest_history_key': "",
+            'newest_history_key': 0,
             'title': None,
-            'previous_md5': "",
+            'previous_md5': False,
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
             'body': None,
@@ -42,21 +41,27 @@ class model(dict):
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.
-            'minutes_between_check': None
-        })
+            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
+        }
+
+    def __init__(self, *arg, **kw):
+        self.update(self.base_config)
         # goes at the end so we update the default object with the initialiser
         super(model, self).__init__(*arg, **kw)
 
 
     @property
     def has_empty_checktime(self):
-        if self.get('minutes_between_check', None):
-            return False
-        return True
+        # using all() + dictionary comprehension
+        # Check if all values are 0 in dictionary
+        res = all(x == None or x == False or x==0 for x in self.get('time_between_check', {}).values())
+        return res
 
-    @property
     def threshold_seconds(self):
-        sec = self.get('minutes_between_check', None)
-        if sec:
-            sec = sec * 60
-        return sec
+        seconds = 0
+        mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
+        for m, n in mtable.items():
+            x = self.get('time_between_check', {}).get(m, None)
+            if x:
+                seconds += x * n
+        return seconds

changedetectionio/static/styles/styles.css

@@ -440,3 +440,8 @@ ul {
   padding-left: 1em;
   padding-top: 0px;
   margin-top: 4px; }
+
+.time-check-widget tr {
+  display: inline; }
+  .time-check-widget tr input[type="number"] {
+    width: 4em; }

changedetectionio/static/styles/styles.scss

@@ -624,4 +624,13 @@ ul {
     padding-left: 1em;
     padding-top: 0px;
     margin-top: 4px;
+}
+
+.time-check-widget {
+    tr {
+        display: inline;
+        input[type="number"] {
+            width: 4em;
+        }
+    }
 }

changedetectionio/store.py

@@ -7,6 +7,7 @@ import uuid as uuid_builder
 from copy import deepcopy
 from os import mkdir, path, unlink
 from threading import Lock
+import re
 
 from changedetectionio.model import Watch, App
 
@@ -16,6 +17,11 @@ from changedetectionio.model import Watch, App
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
 class ChangeDetectionStore:
     lock = Lock()
+    # For general updates/writes that can wait a few seconds
+    needs_write = False
+
+    # For when we edit, we should write to disk
+    needs_write_urgent = False
 
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
@@ -100,6 +106,9 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
+        # Bump the update version by running updates
+        self.run_updates()
+
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
@@ -145,6 +154,17 @@ class ChangeDetectionStore:
 
         self.needs_write = True
 
+    @property
+    def threshold_seconds(self):
+        seconds = 0
+        mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
+        minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 5))
+        for m, n in mtable.items():
+            x = self.__data['settings']['requests']['time_between_check'].get(m)
+            if x:
+                seconds += x * n
+        return max(seconds, minimum_seconds_recheck_time)
+
     @property
     def data(self):
         has_unviewed = False
@@ -339,7 +359,7 @@ class ChangeDetectionStore:
 
     def sync_to_json(self):
         logging.info("Saving JSON..")
-
+        print("Saving JSON..")
         try:
             data = deepcopy(self.__data)
         except RuntimeError as e:
@@ -361,6 +381,7 @@ class ChangeDetectionStore:
                 logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
 
             self.needs_write = False
+            self.needs_write_urgent = False
 
     # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
     # by just running periodically in one thread, according to python, dict updates are threadsafe.
@@ -371,14 +392,14 @@ class ChangeDetectionStore:
                 print("Shutting down datastore thread")
                 return
 
-            if self.needs_write:
+            if self.needs_write or self.needs_write_urgent:
                 self.sync_to_json()
 
             # Once per minute is enough, more and it can cause high CPU usage
             # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here
-            for i in range(30):
-                time.sleep(2)
-                if self.stop_thread:
+            for i in range(120):
+                time.sleep(0.5)
+                if self.stop_thread or self.needs_write_urgent:
                     break
 
     # Go through the datastore path and remove any snapshots that are not mentioned in the index
@@ -398,3 +419,49 @@ class ChangeDetectionStore:
             if not str(item) in index:
                 print ("Removing",item)
                 unlink(item)
+
+    # Run all updates
+    # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
+    #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
+    #             Probably we should bump the current update schema version with each tag release version?
+    def run_updates(self):
+        import inspect
+        import shutil
+
+        updates_available = []
+        for i, o in inspect.getmembers(self, predicate=inspect.ismethod):
+            m = re.search(r'update_(\d+)$', i)
+            if m:
+                updates_available.append(int(m.group(1)))
+        updates_available.sort()
+
+        for update_n in updates_available:
+            if update_n > self.__data['settings']['application']['schema_version']:
+                print ("Applying update_{}".format((update_n)))
+                # Wont exist on fresh installs
+                if os.path.exists(self.json_store_path):
+                    shutil.copyfile(self.json_store_path, self.datastore_path+"/url-watches-before-{}.json".format(update_n))
+
+                try:
+                    update_method = getattr(self, "update_{}".format(update_n))()
+                except Exception as e:
+                    print("Error while trying update_{}".format((update_n)))
+                    print(e)
+                    # Don't run any more updates
+                    return
+                else:
+                    # Bump the version, important
+                    self.__data['settings']['application']['schema_version'] = update_n
+
+    # Convert minutes to seconds on settings and each watch
+    def update_1(self):
+        if self.data['settings']['requests'].get('minutes_between_check'):
+            self.data['settings']['requests']['time_between_check']['minutes'] = self.data['settings']['requests']['minutes_between_check']
+            # Remove the default 'hours' that is set from the model
+            self.data['settings']['requests']['time_between_check']['hours'] = None
+
+        for uuid, watch in self.data['watching'].items():
+            if 'minutes_between_check' in watch:
+                # Only upgrade individual watch time if it was set
+                if watch.get('minutes_between_check', False):
+                    self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']

changedetectionio/templates/edit.html

@@ -41,7 +41,7 @@
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.minutes_between_check) }}
+                        {{ render_field(form.time_between_check, class="time-check-widget") }}
                         {% if has_empty_checktime %}
                         <span class="pure-form-message-inline">Currently using the <a
                                 href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>

changedetectionio/templates/settings.html

@@ -28,7 +28,7 @@
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
-                        {{ render_field(form.requests.form.minutes_between_check) }}
+                        {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
                         <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
                     <div class="pure-control-group">

changedetectionio/tests/test_access_control.py

@@ -13,7 +13,7 @@ def test_check_access_control(app, client):
         res = c.post(
             url_for("settings_page"),
             data={"application-password": "foobar",
-                  "requests-minutes_between_check": 180,
+                  "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
             follow_redirects=True
         )
@@ -46,7 +46,7 @@ def test_check_access_control(app, client):
         assert b"BACKUP" in res.data
         assert b"IMPORT" in res.data
         assert b"LOG OUT" in res.data
-        assert b"minutes_between_check" in res.data
+        assert b"time_between_check-minutes" in res.data
         assert b"fetch_backend" in res.data
 
         ##################################################
@@ -55,7 +55,7 @@ def test_check_access_control(app, client):
         res = c.post(
             url_for("settings_page"),
             data={
-                "requests-minutes_between_check": 180,
+                "requests-time_between_check-minutes": 180,
                 "application-fetch_backend": "html_webdriver",
                 "application-removepassword_button": "Remove password"
             },
@@ -70,7 +70,7 @@ def test_check_access_control(app, client):
         res = c.post(
             url_for("settings_page"),
             data={"application-password": "",
-                  "requests-minutes_between_check": 180,
+                  "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
             follow_redirects=True
         )

changedetectionio/tests/test_backend.py

@@ -109,7 +109,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # Enable auto pickup of <title> in settings
     res = client.post(
         url_for("settings_page"),
-        data={"application-extract_title_as_title": "1", "requests-minutes_between_check": 180, 'application-fetch_backend': "html_requests"},
+        data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_ignore_text.py

@@ -171,11 +171,24 @@ def test_check_ignore_text_functionality(client, live_server):
 def test_check_global_ignore_text_functionality(client, live_server):
     sleep_time_for_fetch_thread = 3
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
     set_original_ignore_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-global_ignore_text": ignore_text,
+            'application-fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -192,17 +205,6 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # Goto the settings page, add our ignore text
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-minutes_between_check": 180,
-            "application-global_ignore_text": ignore_text,
-            'application-fetch_backend': "html_requests"
-        },
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
 
     # Goto the edit page of the item, add our ignore text
     # Add our URL to the import page
@@ -225,12 +227,16 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
+    # so that we are sure everything is viewed and in a known 'nothing changed' state
+    res = client.get(url_for("diff_history_page", uuid="first"))
+
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
-    #  Make a change
+
+    #  Make a change which includes the ignore text
     set_modified_ignore_response()
 
     # Trigger a check
@@ -243,7 +249,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
-    # Just to be sure.. set a regular modified change..
+    # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_ignorehyperlinks.py

@@ -51,12 +51,11 @@ def test_render_anchor_tag_content_true(client, live_server):
     # set original html text
     set_original_ignore_response()
 
-    # Goto the settings page, choose not to ignore links
+    # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content")
     res = client.post(
         url_for("settings_page"),
         data={
-            "requests-minutes_between_check": 180,
-            "application-render_anchor_tag_content": "true",
+            "requests-time_between_check-minutes": 180,
             "application-fetch_backend": "html_requests",
         },
         follow_redirects=True,
@@ -85,135 +84,42 @@ def test_render_anchor_tag_content_true(client, live_server):
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # check that the anchor tag content is rendered
-    res = client.get(url_for("preview_page", uuid="first"))
-    assert '(/modified_link)' in res.data.decode()
-
-    # since the link has changed, and we chose to render anchor tag content,
-    # we should detect a change (new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b"unviewed" in res.data
-    assert b"/test-endpoint" in res.data
-
-    # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"),
-                     follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_render_anchor_tag_content_false(client, live_server):
-    """Testing that anchor tag content changes are ignored when
-    render_anchor_tag_content setting is set to false"""
-    sleep_time_for_fetch_thread = 3
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # set the original html text
-    set_original_ignore_response()
-
-    # Goto the settings page, choose to ignore hyperlinks
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-minutes_between_check": 180,
-            "application-render_anchor_tag_content": "false",
-            "application-fetch_backend": "html_requests",
-        },
-        follow_redirects=True,
-    )
-    assert b"Settings updated." in res.data
-
-    # Add our URL to the import page
-    test_url = url_for("test_endpoint", _external=True)
-    res = client.post(
-        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    time.sleep(sleep_time_for_fetch_thread)
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # set a new html text, with a modified link
-    set_modified_ignore_response()
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # check that the anchor tag content is not rendered
+    # We should not see the rendered anchor tag
     res = client.get(url_for("preview_page", uuid="first"))
     assert '(/modified_link)' not in res.data.decode()
 
-    # even though the link has changed, we shouldn't detect a change since
-    # we selected to not render anchor tag content (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b"unviewed" not in res.data
-    assert b"/test-endpoint" in res.data
-
-    # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_render_anchor_tag_content_default(client, live_server):
-    """Testing that anchor tag content changes are ignored when the
-    render_anchor_tag_content setting is not explicitly selected"""
-    sleep_time_for_fetch_thread = 3
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # set the original html text
-    set_original_ignore_response()
-
-    # Goto the settings page, not passing the render_anchor_tag_content setting
+    # Goto the settings page, ENABLE render anchor tag
     res = client.post(
         url_for("settings_page"),
         data={
-            "requests-minutes_between_check": 180,
+            "requests-time_between_check-minutes": 180,
+            "application-render_anchor_tag_content": "true",
             "application-fetch_backend": "html_requests",
         },
         follow_redirects=True,
     )
     assert b"Settings updated." in res.data
 
-    # Add our URL to the import page
-    test_url = url_for("test_endpoint", _external=True)
-    res = client.post(
-        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
-    # set a new html text, with a modified link
-    set_modified_ignore_response()
+    # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
 
-    # check that the anchor tag content is not rendered
+    # check that the anchor tag content is rendered
     res = client.get(url_for("preview_page", uuid="first"))
-    assert '(/modified_link)' not in res.data.decode()
+    assert '(/modified_link)' in res.data.decode()
 
-    # even though the link has changed, we shouldn't detect a change since
-    # we did not select the setting and the default behaviour is to not
-    # render anchor tag content (no new 'unviewed' class)
+    # since the link has changed, and we chose to render anchor tag content,
+    # we should detect a change (new 'unviewed' class)
     res = client.get(url_for("index"))
-    assert b"unviewed" not in res.data
+    assert b"unviewed" in res.data
     assert b"/test-endpoint" in res.data
 
     # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"),
+                     follow_redirects=True)
     assert b'Deleted' in res.data
+

changedetectionio/tests/test_ignorestatuscode.py

@@ -51,7 +51,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "requests-minutes_between_check": 180,
+            "requests-time_between_check-minutes": 180,
             "application-ignore_status_codes": "y",
             'application-fetch_backend': "html_requests"
         },

changedetectionio/tests/test_ignorewhitespace.py

@@ -61,7 +61,7 @@ def test_check_ignore_whitespace(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "requests-minutes_between_check": 180,
+            "requests-time_between_check-minutes": 180,
             "application-ignore_whitespace": "y",
             "application-fetch_backend": "html_requests"
         },

changedetectionio/tests/test_jsonpath_selector.py

@@ -270,6 +270,7 @@ def test_check_json_filter_bool_val(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(3)
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
@@ -284,6 +285,7 @@ def test_check_json_filter_bool_val(client, live_server):
     )
     assert b"Updated watch." in res.data
 
+    time.sleep(3)
 
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_notification.py

@@ -199,7 +199,7 @@ def test_notification_validation(client, live_server):
               "application-notification_body": "Rubbish: {rubbish}\n",
               "application-notification_format": "Text",
               "application-notification_urls": "json://localhost/foobar",
-              "requests-minutes_between_check": 180,
+              "requests-time_between_check-minutes": 180,
               "fetch_backend": "html_requests"
               },
         follow_redirects=True

changedetectionio/tests/test_notification_errors.py

@@ -35,7 +35,7 @@ def test_check_notification_error_handling(client, live_server):
               "tag": "",
               "title": "",
               "headers": "",
-              "minutes_between_check": "180",
+              "time_between_check-minutes": "180",
               "fetch_backend": "html_requests",
               "trigger_check": "y"},
         follow_redirects=True

changedetectionio/tests/test_watch_fields_storage.py

@@ -21,7 +21,7 @@ def test_check_watch_field_storage(client, live_server):
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={ "notification_urls": "json://127.0.0.1:30000\r\njson://128.0.0.1\r\n",
-               "minutes_between_check": 126,
+               "time_between_check-minutes": 126,
                "css_filter" : ".fooclass",
                "title" : "My title",
                "ignore_text" : "ignore this",
@@ -62,7 +62,7 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "requests-minutes_between_check": 1566,
+               "requests-time_between_check-minutes": 1566,
                'application-fetch_backend': "html_requests"
                },
         follow_redirects=True
@@ -94,7 +94,7 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "requests-minutes_between_check": 222,
+               "requests-time_between_check-minutes": 222,
                 'application-fetch_backend': "html_requests"
                },
         follow_redirects=True
@@ -114,7 +114,7 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"url": test_url,
-              "minutes_between_check": 55,
+              "time_between_check-minutes": 55,
               'fetch_backend': "html_requests"
               },
         follow_redirects=True
@@ -130,8 +130,8 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "requests-minutes_between_check": 666,
-                'application-fetch_backend': "html_requests"
+               "requests-time_between_check-minutes": 666,
+                "application-fetch_backend": "html_requests"
                },
         follow_redirects=True
     )
@@ -140,7 +140,7 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"url": test_url,
-              "minutes_between_check": "",
+              "time_between_check-minutes": "",
               'fetch_backend': "html_requests"
               },
         follow_redirects=True
@@ -153,4 +153,3 @@ def test_check_recheck_global_setting(client, live_server):
         follow_redirects=True
     )
     assert b"666" in res.data
-