Testing - Tidyup (#2693)

changedetectionio/content_fetchers/requests.py

@@ -75,6 +75,7 @@ class fetcher(Fetcher):
         self.headers = r.headers
 
         if not r.content or not len(r.content):
+            logger.debug(f"Requests returned empty content for '{url}'")
             if not empty_pages_are_a_change:
                 raise EmptyReply(url=url, status_code=r.status_code)
             else:

changedetectionio/flask_app.py

@@ -788,7 +788,6 @@ def changedetection_app(config=None, datastore_o=None):
             # Recast it if need be to right data Watch handler
             watch_class = get_custom_watch_obj_for_processor(form.data.get('processor'))
             datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid])
-
             flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")
 
             # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
@@ -1487,7 +1486,6 @@ def changedetection_app(config=None, datastore_o=None):
                         continue
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                     i += 1
-
         flash(f"{i} watches queued for rechecking.")
         return redirect(url_for('index', tag=tag))
 

changedetectionio/processors/text_json_diff/processor.py

@@ -221,6 +221,7 @@ class perform_site_check(difference_detection_processor):
         # @todo whitespace coming from missing rtrim()?
         # stripped_text_from_html could be based on their preferences, replace the processed text with only that which they want to know about.
         # Rewrite's the processing text based on only what diff result they want to see
+
         if watch.has_special_diff_filter_options_set() and len(watch.history.keys()):
             # Now the content comes from the diff-parser and not the returned HTTP traffic, so could be some differences
             from changedetectionio import diff

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -77,6 +77,8 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
 
     # The trigger line is REMOVED,  this should trigger
     set_original(excluding='The golden line')
+
+    # Check in the processor here what's going on, its triggering empty-reply and no change.
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
@@ -151,7 +153,6 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     assert b'1 watches queued for rechecking.' in res.data
 
     wait_for_all_checks(client)
@@ -173,6 +174,5 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
         assert b'-Oh yes please-' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 
-
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_element_removal.py

@@ -5,7 +5,7 @@ import time
 from flask import url_for
 
 from ..html_tools import *
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
 def test_setup(live_server):
@@ -119,12 +119,10 @@ across multiple lines
 
 
 def test_element_removal_full(client, live_server, measure_memory_usage):
-    sleep_time_for_fetch_thread = 3
+    #live_server_setup(live_server)
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for("test_endpoint", _external=True)
@@ -132,7 +130,8 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
         url_for("import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
+
     # Goto the edit page, add the filter data
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"
@@ -148,6 +147,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
         follow_redirects=True,
     )
     assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
 
     # Check it saved
     res = client.get(
@@ -156,10 +156,10 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
 
-    # Give the thread time to pick it up
+    wait_for_all_checks(client)
-    time.sleep(sleep_time_for_fetch_thread)
 
     # so that we set the state to 'unviewed' after all the edits
     client.get(url_for("diff_history_page", uuid="first"))
@@ -168,10 +168,11 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # There should not be an unviewed change, as changes should be removed
     res = client.get(url_for("index"))