From b629276ab9a92ec2e0c219709aaa9f142c5b6402 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 17 Aug 2022 12:56:02 +0200 Subject: [PATCH] Be sure visual-selector data is set when filter is not found --- changedetectionio/fetch_site_status.py | 7 +++- changedetectionio/run_all_tests.sh | 3 +- .../tests/fetchers/test_content.py | 12 ++----- changedetectionio/tests/util.py | 28 +++++++++++++++ .../tests/visualselector/__init__.py | 2 ++ .../tests/visualselector/conftest.py | 3 ++ .../tests/visualselector/test_fetch_data.py | 35 +++++++++++++++++++ changedetectionio/update_worker.py | 11 +++--- 8 files changed, 85 insertions(+), 16 deletions(-) create mode 100644 changedetectionio/tests/visualselector/__init__.py create mode 100644 changedetectionio/tests/visualselector/conftest.py create mode 100644 changedetectionio/tests/visualselector/test_fetch_data.py diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index ea94181e..bee0d50c 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -13,6 +13,8 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # Some common stuff here that can be moved to a base class # (set_proxy_from_list) class perform_site_check(): + screenshot = None + xpath_data = None def __init__(self, *args, datastore, **kwargs): super().__init__(*args, **kwargs) @@ -127,6 +129,9 @@ class perform_site_check(): fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch['css_filter']) fetcher.quit() + self.screenshot = fetcher.screenshot + self.xpath_data = fetcher.xpath_data + # Fetching complete, now filters # @todo move to class / maybe inside of fetcher abstract base? @@ -312,4 +317,4 @@ class perform_site_check(): if not watch.get('previous_md5'): watch['previous_md5'] = fetched_md5 - return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot, fetcher.xpath_data + return changed_detected, update_obj, text_content_before_ignored_filter diff --git a/changedetectionio/run_all_tests.sh b/changedetectionio/run_all_tests.sh index d111105f..ce428f12 100755 --- a/changedetectionio/run_all_tests.sh +++ b/changedetectionio/run_all_tests.sh @@ -38,13 +38,14 @@ docker kill $$-test_selenium echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..." # Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt -pip3 install playwright~=1.22 +pip3 install playwright~=1.24 docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable # takes a while to spin up sleep 5 export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/fetchers/test_content.py pytest tests/test_errorhandling.py +pytest tests/visualselector/test_fetch_data.py unset PLAYWRIGHT_DRIVER_URL docker kill $$-test_browserless \ No newline at end of file diff --git a/changedetectionio/tests/fetchers/test_content.py b/changedetectionio/tests/fetchers/test_content.py index 02c2c026..d50d8210 100644 --- a/changedetectionio/tests/fetchers/test_content.py +++ b/changedetectionio/tests/fetchers/test_content.py @@ -2,7 +2,7 @@ import time from flask import url_for -from ..util import live_server_setup +from ..util import live_server_setup, wait_for_all_checks import logging @@ -29,14 +29,8 @@ def test_fetch_webdriver_content(client, live_server): assert b"1 Imported" in res.data time.sleep(3) - attempt = 0 - while attempt < 20: - res = client.get(url_for("index")) - if not b'Checking now' in res.data: - break - logging.getLogger().info("Waiting for check to not say 'Checking now'..") - time.sleep(3) - attempt += 1 + + wait_for_all_checks(client) res = client.get( diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index af32d8ae..e93d9a40 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -2,6 +2,8 @@ from flask import make_response, request from flask import url_for +import logging +import time def set_original_response(): test_return_data = """ @@ -68,6 +70,31 @@ def extract_api_key_from_UI(client): api_key = m.group(1) return api_key.strip() + +# kinda funky, but works for now +def extract_UUID_from_client(client): + import re + res = client.get( + url_for("index"), + ) + # {{api_key}} + + m = re.search('edit/(.+?)"', str(res.data)) + uuid = m.group(1) + return uuid.strip() + +def wait_for_all_checks(client): + # Loop waiting until done.. + attempt=0 + while attempt < 60: + time.sleep(1) + res = client.get(url_for("index")) + if not b'Checking now' in res.data: + break + logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt)) + + attempt += 1 + def live_server_setup(live_server): @live_server.app.route('/test-endpoint') @@ -133,3 +160,4 @@ def live_server_setup(live_server): return ret live_server.start() + diff --git a/changedetectionio/tests/visualselector/__init__.py b/changedetectionio/tests/visualselector/__init__.py new file mode 100644 index 00000000..085b3d78 --- /dev/null +++ b/changedetectionio/tests/visualselector/__init__.py @@ -0,0 +1,2 @@ +"""Tests for the app.""" + diff --git a/changedetectionio/tests/visualselector/conftest.py b/changedetectionio/tests/visualselector/conftest.py new file mode 100644 index 00000000..430513d4 --- /dev/null +++ b/changedetectionio/tests/visualselector/conftest.py @@ -0,0 +1,3 @@ +#!/usr/bin/python3 + +from .. import conftest diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py new file mode 100644 index 00000000..dc27d1ba --- /dev/null +++ b/changedetectionio/tests/visualselector/test_fetch_data.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import time +from flask import url_for +from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client + +# Add a site in paused mode, add an invalid filter, we should still have visual selector data ready +def test_visual_selector_content_ready(client, live_server): + import os + + assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test" + live_server_setup(live_server) + time.sleep(1) + + # Add our URL to the import page, maybe better to use something we control? + # We use an external URL because the docker container is too difficult to setup to connect back to the pytest socket + test_url = 'https://news.ycombinator.com' + res = client.post( + url_for("form_quick_watch_add"), + data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'}, + follow_redirects=True + ) + assert b"Watch added in Paused state, saving will unpause" in res.data + + res = client.post( + url_for("edit_page", uuid="first", unpause_on_save=1), + data={"css_filter": ".does-not-exist", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_webdriver"}, + follow_redirects=True + ) + assert b"unpaused" in res.data + time.sleep(1) + wait_for_all_checks(client) + uuid = extract_UUID_from_client(client) + assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist" + assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist" diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index d6c990ec..0f9758d2 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -142,7 +142,7 @@ class update_worker(threading.Thread): now = time.time() try: - changed_detected, update_obj, contents, screenshot, xpath_data = update_handler.run(uuid) + changed_detected, update_obj, contents = update_handler.run(uuid) # Re #342 # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes. # We then convert/.decode('utf-8') for the notification etc @@ -222,6 +222,7 @@ class update_worker(threading.Thread): self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text, 'last_check_status': e.status_code}) except content_fetcher.PageUnloadable as e: + # @todo connection-refused ? err_text = "Page request from server didnt respond correctly" if e.screenshot: self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True) @@ -280,10 +281,10 @@ class update_worker(threading.Thread): 'last_checked': round(time.time())}) # Always save the screenshot if it's available - if screenshot: - self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot) - if xpath_data: - self.datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data) + if update_handler.screenshot: + self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot) + if update_handler.xpath_data: + self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data) self.current_uuid = None # Done