Re #580 - New functionality - Random "jitter" delay to requests (#681)

changedetectionio/__init__.py

@@ -1287,8 +1287,11 @@ def notification_runner():
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
+    import random
     from changedetectionio import update_worker
-    import logging
+
+    recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20))
+    print("System env MINIMUM_SECONDS_RECHECK_TIME", recheck_time_minimum_seconds)
 
     # Spin up Workers that do the fetching
     # Can be overriden by ENV or use the default settings
@@ -1321,14 +1324,12 @@ def ticker_thread_check_time_launch_checks():
         while update_q.qsize() >= 2000:
             time.sleep(1)
 
-        # Check for watches outside of the time threshold to put in the thread queue.
-        now = time.time()
 
-        recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
+        recheck_time_system_seconds = int(datastore.threshold_seconds)
-        recheck_time_system_seconds = datastore.threshold_seconds
 
+        # Check for watches outside of the time threshold to put in the thread queue.
         for uuid in watch_uuid_list:
-
+            now = time.time()
             watch = datastore.data['watching'].get(uuid)
             if not watch:
                 logging.error("Watch: {} no longer present.".format(uuid))
@@ -1339,20 +1340,33 @@ def ticker_thread_check_time_launch_checks():
                 continue
 
             # If they supplied an individual entry minutes to threshold.
-            threshold = now
+
             watch_threshold_seconds = watch.threshold_seconds()
-            if watch_threshold_seconds:
+            threshold = watch_threshold_seconds if watch_threshold_seconds > 0 else recheck_time_system_seconds
-                threshold -= watch_threshold_seconds
+
-            else:
+            # #580 - Jitter plus/minus amount of time to make the check seem more random to the server
-                threshold -= recheck_time_system_seconds
+            jitter = datastore.data['settings']['requests'].get('jitter_seconds', 0)
+            if jitter > 0:
+                if watch.jitter_seconds == 0:
+                    watch.jitter_seconds = random.uniform(-abs(jitter), jitter)
 
-            # Yeah, put it in the queue, it's more than time
+
-            if watch['last_checked'] <= max(threshold, recheck_time_minimum_seconds):
+            seconds_since_last_recheck = now - watch['last_checked']
+            if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
                 if not uuid in running_uuids and uuid not in update_q.queue:
+                    print("Queued watch UUID {} last checked at {} queued at {:0.2f} jitter {:0.2f}s, {:0.2f}s since last checked".format(uuid,
+                                                                                                         watch['last_checked'],
+                                                                                                         now,
+                                                                                                         watch.jitter_seconds,
+                                                                                                         now - watch['last_checked']))
+                    # Into the queue with you
                     update_q.put(uuid)
 
-        # Wait a few seconds before checking the list again
+                    # Reset for next time
-        time.sleep(3)
+                    watch.jitter_seconds = 0
+
+        # Wait before checking the list again - saves CPU
+        time.sleep(1)
 
         # Should be low so we can break this out in testing
         app.config.exit.wait(1)

changedetectionio/forms.py

@@ -363,7 +363,9 @@ class watchForm(commonSettingsForm):
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
     proxy = RadioField('Proxy')
-
+    jitter_seconds = IntegerField('Random jitter seconds ± check',
+                                  render_kw={"style": "width: 5em;"},
+                                  validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):

changedetectionio/model/App.py

@@ -23,6 +23,7 @@ class model(dict):
                 'requests': {
                     'timeout': 15,  # Default 15 seconds
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'jitter_seconds': 0,
                     'workers': 10,  # Number of threads, lower is better for slow connections
                     'proxy': None # Preferred proxy connection
                 },

changedetectionio/model/Watch.py

@@ -13,7 +13,6 @@ from changedetectionio.notification import (
 class model(dict):
     __newest_history_key = None
     __history_n=0
-
     __base_config = {
             'url': None,
             'tag': None,
@@ -48,7 +47,8 @@ class model(dict):
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
             'webdriver_delay': None
         }
-
+    jitter_seconds = 0
+    mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
     def __init__(self, *arg, **kw):
         import uuid
         self.update(self.__base_config)
@@ -157,8 +157,7 @@ class model(dict):
 
     def threshold_seconds(self):
         seconds = 0
-        mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
+        for m, n in self.mtable.items():
-        for m, n in mtable.items():
             x = self.get('time_between_check', {}).get(m, None)
             if x:
                 seconds += x * n

changedetectionio/run_all_tests.sh

@@ -9,6 +9,8 @@
 # exit when any command fails
 set -e
 
+export MINIMUM_SECONDS_RECHECK_TIME=0
+
 find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"

changedetectionio/store.py

@@ -159,12 +159,11 @@ class ChangeDetectionStore:
     def threshold_seconds(self):
         seconds = 0
         mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
-        minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
         for m, n in mtable.items():
             x = self.__data['settings']['requests']['time_between_check'].get(m)
             if x:
                 seconds += x * n
-        return max(seconds, minimum_seconds_recheck_time)
+        return seconds
 
     @property
     def has_unviewed(self):

changedetectionio/templates/settings.html

@@ -32,6 +32,11 @@
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
                         <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
+                        <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
+                    </div>
+
                     <div class="pure-control-group">
                         {% if not hide_remove_pass %}
                             {% if current_user.is_authenticated %}