Self-Hosted
/
changedetection.io
mirror of https://github.com/dgtlmoon/changedetection.io.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Be sure visual-selector data is set when xPath/CSS filter is not yet found (#851)

Browse Source
pull/852/head^2
dgtlmoon 2 years ago committed by GitHub
parent 06ab34b6af
commit d67c654f37
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 85 additions and 16 deletions
Show Stats Download Patch File Download Diff File

7
changedetectionio/fetch_site_status.py
Unescape View File

@ -13,6 +13,8 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# Some common stuff here that can be moved to a base class # Some common stuff here that can be moved to a base class
# (set_proxy_from_list) # (set_proxy_from_list)
class perform_site_check(): class perform_site_check():
screenshot = None
xpath_data = None
def __init__(self, *args, datastore, **kwargs): def __init__(self, *args, datastore, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@ -127,6 +129,9 @@ class perform_site_check():
fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch['css_filter']) fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch['css_filter'])
fetcher.quit() fetcher.quit()
self.screenshot = fetcher.screenshot
self.xpath_data = fetcher.xpath_data
# Fetching complete, now filters # Fetching complete, now filters
# @todo move to class / maybe inside of fetcher abstract base? # @todo move to class / maybe inside of fetcher abstract base?
@ -312,4 +317,4 @@ class perform_site_check():
if not watch.get('previous_md5'): if not watch.get('previous_md5'):
watch['previous_md5'] = fetched_md5 watch['previous_md5'] = fetched_md5
return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot, fetcher.xpath_data return changed_detected, update_obj, text_content_before_ignored_filter

3
changedetectionio/run_all_tests.sh
Unescape View File

@ -38,13 +38,14 @@ docker kill $$-test_selenium
echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..." echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..."
# Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt # Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt
pip3 install playwright~=1.22 pip3 install playwright~=1.24
docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable
# takes a while to spin up # takes a while to spin up
sleep 5 sleep 5
export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000
pytest tests/fetchers/test_content.py pytest tests/fetchers/test_content.py
pytest tests/test_errorhandling.py pytest tests/test_errorhandling.py
pytest tests/visualselector/test_fetch_data.py
unset PLAYWRIGHT_DRIVER_URL unset PLAYWRIGHT_DRIVER_URL
docker kill $$-test_browserless docker kill $$-test_browserless

12
changedetectionio/tests/fetchers/test_content.py
Unescape View File

@ -2,7 +2,7 @@
import time import time
from flask import url_for from flask import url_for
from ..util import live_server_setup from ..util import live_server_setup, wait_for_all_checks
import logging import logging
@ -29,14 +29,8 @@ def test_fetch_webdriver_content(client, live_server):
assert b"1 Imported" in res.data assert b"1 Imported" in res.data
time.sleep(3) time.sleep(3)
attempt = 0
while attempt < 20: wait_for_all_checks(client)
res = client.get(url_for("index"))
if not b'Checking now' in res.data:
break
logging.getLogger().info("Waiting for check to not say 'Checking now'..")
time.sleep(3)
attempt += 1
res = client.get( res = client.get(

28
changedetectionio/tests/util.py
Unescape View File

@ -2,6 +2,8 @@
from flask import make_response, request from flask import make_response, request
from flask import url_for from flask import url_for
import logging
import time
def set_original_response(): def set_original_response():
test_return_data = """<html> test_return_data = """<html>
@ -68,6 +70,31 @@ def extract_api_key_from_UI(client):
api_key = m.group(1) api_key = m.group(1)
return api_key.strip() return api_key.strip()
# kinda funky, but works for now
def extract_UUID_from_client(client):
import re
res = client.get(
url_for("index"),
)
# <span id="api-key">{{api_key}}</span>
m = re.search('edit/(.+?)"', str(res.data))
uuid = m.group(1)
return uuid.strip()
def wait_for_all_checks(client):
# Loop waiting until done..
attempt=0
while attempt < 60:
time.sleep(1)
res = client.get(url_for("index"))
if not b'Checking now' in res.data:
break
logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))
attempt += 1
def live_server_setup(live_server): def live_server_setup(live_server):
@live_server.app.route('/test-endpoint') @live_server.app.route('/test-endpoint')
@ -133,3 +160,4 @@ def live_server_setup(live_server):
return ret return ret
live_server.start() live_server.start()

2
changedetectionio/tests/visualselector/__init__.py
Unescape View File

@ -0,0 +1,2 @@
"""Tests for the app."""

3
changedetectionio/tests/visualselector/conftest.py
Unescape View File

@ -0,0 +1,3 @@
#!/usr/bin/python3
from .. import conftest

35
changedetectionio/tests/visualselector/test_fetch_data.py
Unescape View File

@ -0,0 +1,35 @@
#!/usr/bin/python3
import time
from flask import url_for
from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
# Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
def test_visual_selector_content_ready(client, live_server):
import os
assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
live_server_setup(live_server)
time.sleep(1)
# Add our URL to the import page, maybe better to use something we control?
# We use an external URL because the docker container is too difficult to setup to connect back to the pytest socket
test_url = 'https://news.ycombinator.com'
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
follow_redirects=True
)
assert b"Watch added in Paused state, saving will unpause" in res.data
res = client.post(
url_for("edit_page", uuid="first", unpause_on_save=1),
data={"css_filter": ".does-not-exist", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_webdriver"},
follow_redirects=True
)
assert b"unpaused" in res.data
time.sleep(1)
wait_for_all_checks(client)
uuid = extract_UUID_from_client(client)
assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"

11
changedetectionio/update_worker.py
Unescape View File

@ -142,7 +142,7 @@ class update_worker(threading.Thread):
now = time.time() now = time.time()
try: try:
changed_detected, update_obj, contents, screenshot, xpath_data = update_handler.run(uuid) changed_detected, update_obj, contents = update_handler.run(uuid)
# Re #342 # Re #342
# In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes. # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
# We then convert/.decode('utf-8') for the notification etc # We then convert/.decode('utf-8') for the notification etc
@ -222,6 +222,7 @@ class update_worker(threading.Thread):
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text, self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code}) 'last_check_status': e.status_code})
except content_fetcher.PageUnloadable as e: except content_fetcher.PageUnloadable as e:
# @todo connection-refused ?
err_text = "Page request from server didnt respond correctly" err_text = "Page request from server didnt respond correctly"
if e.screenshot: if e.screenshot:
self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True) self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
@ -280,10 +281,10 @@ class update_worker(threading.Thread):
'last_checked': round(time.time())}) 'last_checked': round(time.time())})
# Always save the screenshot if it's available # Always save the screenshot if it's available
if screenshot: if update_handler.screenshot:
self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot) self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
if xpath_data: if update_handler.xpath_data:
self.datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data) self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
self.current_uuid = None # Done self.current_uuid = None # Done

Write Preview
Loading…
Cancel
Save