Refactor the wait-for-check

changedetectionio/flask_app.py

@@ -45,7 +45,6 @@ running_update_threads = []
 ticker_thread = None
 
 extra_stylesheets = []
-
 update_q = queue.PriorityQueue()
 notification_q = queue.Queue()
 
@@ -1538,6 +1537,11 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
+    @app.route("/queue_size", methods=['GET'])
+    @login_optionally_required
+    def get_queue_size():
+        return update_q.qsize()
+
     @app.route("/highlight_submit_ignore_url", methods=['POST'])
     @login_optionally_required
     def highlight_submit_ignore_url():

changedetectionio/tests/test_filter_failure_notification.py

@@ -22,7 +22,6 @@ def set_response_with_filter():
     return None
 
 def run_filter_test(client, content_filter):
-    time.sleep(1)
 
     # cleanup for the next
     client.get(
@@ -104,6 +103,10 @@ def run_filter_test(client, content_filter):
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
+
+    # Give apprise time to fire
+    time.sleep(3)
+
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
 
@@ -123,6 +126,9 @@ def run_filter_test(client, content_filter):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
+    # Give apprise time to fire
+    time.sleep(3)
+
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
     # but it should not contain the info about a failed filter (because there was none in this case)
@@ -148,11 +154,13 @@ def test_setup(live_server):
     live_server_setup(live_server)
 
 def test_check_include_filters_failure_notification(client, live_server):
+    #live_server_setup(live_server)
     set_original_response()
     wait_for_all_checks(client)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):
+    # live_server_setup(live_server)
     set_original_response()
     time.sleep(1)
     run_filter_test(client, '//*[@id="nope-doesnt-exist"]')

changedetectionio/tests/util.py

@@ -122,16 +122,16 @@ def extract_UUID_from_client(client):
 
 
 def wait_for_all_checks(client):
-    # Loop waiting until done..
-    attempt = 0
-    while attempt < 60:
-        time.sleep(1)
-        # should be greater than update_worker.py:            self.app.config.exit.wait(0.5)
-        res = client.get(url_for("index"))
-        if not b'Checking now' in res.data:
+    now = time.time()
+    while time.time() - now <= 30:
+        time.sleep(0.1)
+        p = client.application.view_functions['get_queue_size']()
+        if not p:
             break
-        logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))
-        attempt += 1
+        logging.getLogger().info(f"Waiting for queue to be empty, queue size {p}  - {time.time() - now}")
+
+    # Empty queue still means that one could be processing, give time for the processing to complete
+    time.sleep(0.2)
 
 def live_server_setup(live_server):
 

changedetectionio/update_worker.py

@@ -160,8 +160,8 @@ class update_worker(threading.Thread):
 
 
     def send_filter_failure_notification(self, watch_uuid):
-
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+        logger.trace(f"Watch UUID: {watch_uuid} - Sending filter failure notification - threshold attempts {threshold}")
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -335,6 +335,7 @@ class update_worker(threading.Thread):
                         process_changedetection_results = False
 
                     except FilterNotFoundInResponse as e:
+                        logger.debug(f"Watch UUID: {uuid} - Got FilterNotFoundInResponse exception, Consecutive failures - {self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)} - handling..")
                         if not self.datastore.data['watching'].get(uuid):
                             continue