oh yeah 456' in res.data
+
+ # Should be in base.html
+ assert b'csrftoken' in res.data
+
diff --git a/changedetectionio/tests/test_ignorehyperlinks.py b/changedetectionio/tests/test_ignorehyperlinks.py
index f917125f..d739eb58 100644
--- a/changedetectionio/tests/test_ignorehyperlinks.py
+++ b/changedetectionio/tests/test_ignorehyperlinks.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
"""Test suite for the render/not render anchor tag content functionality"""
import time
@@ -40,7 +40,7 @@ def set_modified_ignore_response():
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
-def test_render_anchor_tag_content_true(client, live_server):
+def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage):
"""Testing that the link changes are detected when
render_anchor_tag_content setting is set to true"""
sleep_time_for_fetch_thread = 3
diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py
index 74999b24..9ec8086a 100644
--- a/changedetectionio/tests/test_ignorestatuscode.py
+++ b/changedetectionio/tests/test_ignorestatuscode.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -39,7 +39,7 @@ def set_some_changed_response():
f.write(test_return_data)
-def test_normal_page_check_works_with_ignore_status_code(client, live_server):
+def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
# Give the endpoint time to spin up
@@ -85,7 +85,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
# Tests the whole stack works with staus codes ignored
-def test_403_page_check_works_with_ignore_status_code(client, live_server):
+def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
sleep_time_for_fetch_thread = 3
set_original_response()
diff --git a/changedetectionio/tests/test_ignorewhitespace.py b/changedetectionio/tests/test_ignorewhitespace.py
index a0db9ead..25d16244 100644
--- a/changedetectionio/tests/test_ignorewhitespace.py
+++ b/changedetectionio/tests/test_ignorewhitespace.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -49,7 +49,7 @@ def set_original_ignore_response():
# If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_check_ignore_whitespace(client, live_server):
+def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
sleep_time_for_fetch_thread = 3
# Give the endpoint time to spin up
diff --git a/changedetectionio/tests/test_import.py b/changedetectionio/tests/test_import.py
index ed080e0e..4b25d654 100644
--- a/changedetectionio/tests/test_import.py
+++ b/changedetectionio/tests/test_import.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import io
import os
import time
@@ -8,10 +8,10 @@ from flask import url_for
from .util import live_server_setup, wait_for_all_checks
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
live_server_setup(live_server)
-def test_import(client, live_server):
+def test_import(client, live_server, measure_memory_usage):
# Give the endpoint time to spin up
wait_for_all_checks(client)
@@ -34,7 +34,7 @@ https://example.com tag1, other tag"""
res = client.get( url_for("index"))
res = client.get( url_for("index"))
-def xtest_import_skip_url(client, live_server):
+def xtest_import_skip_url(client, live_server, measure_memory_usage):
# Give the endpoint time to spin up
@@ -57,7 +57,7 @@ def xtest_import_skip_url(client, live_server):
# Clear flask alerts
res = client.get( url_for("index"))
-def test_import_distillio(client, live_server):
+def test_import_distillio(client, live_server, measure_memory_usage):
distill_data='''
{
@@ -123,7 +123,7 @@ def test_import_distillio(client, live_server):
# Clear flask alerts
res = client.get(url_for("index"))
-def test_import_custom_xlsx(client, live_server):
+def test_import_custom_xlsx(client, live_server, measure_memory_usage):
"""Test can upload a excel spreadsheet and the watches are created correctly"""
#live_server_setup(live_server)
@@ -172,7 +172,7 @@ def test_import_custom_xlsx(client, live_server):
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
-def test_import_watchete_xlsx(client, live_server):
+def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
"""Test can upload a excel spreadsheet and the watches are created correctly"""
#live_server_setup(live_server)
diff --git a/changedetectionio/tests/test_jinja2.py b/changedetectionio/tests/test_jinja2.py
index 1e08691b..fba9f227 100644
--- a/changedetectionio/tests/test_jinja2.py
+++ b/changedetectionio/tests/test_jinja2.py
@@ -1,15 +1,15 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
from .util import live_server_setup, wait_for_all_checks
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
live_server_setup(live_server)
# If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_jinja2_in_url_query(client, live_server):
+def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
# Add our URL to the import page
@@ -34,7 +34,7 @@ def test_jinja2_in_url_query(client, live_server):
assert b'date=2' in res.data
# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
-def test_jinja2_security_url_query(client, live_server):
+def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
# Add our URL to the import page
diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py
index 1202849f..d139e1cf 100644
--- a/changedetectionio/tests/test_jsonpath_jq_selector.py
+++ b/changedetectionio/tests/test_jsonpath_jq_selector.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# coding=utf-8
import time
@@ -41,19 +41,26 @@ and it can also be repeated
from .. import html_tools
# See that we can find the second ',
+ # LD JSON
+ '',
+ ''
+ # Microdata
+ '
Example Product
This is a sample product description.
'
+]
+
+out_of_stock_props = [
+ # out of stock AND contains multiples
+ ''
+]
+
+def set_original_response(props_markup='', price="121.95"):
+
+ props_markup=props_markup.replace('$$PRICE$$', price)
+ test_return_data = f"""
+
+ Some initial text
+
Which is across multiple lines
+
+ So let's see what happens.
+
price: ${price}
+ {props_markup}
+
+
+ """
+
+ with open("test-datastore/endpoint-content.txt", "w") as f:
+ f.write(test_return_data)
+ time.sleep(1)
+ return None
+
+
+
+
+def test_setup(client, live_server):
+
+ live_server_setup(live_server)
+
+def test_restock_itemprop_basic(client, live_server):
+
+ #live_server_setup(live_server)
+
+ test_url = url_for('test_endpoint', _external=True)
+
+ # By default it should enable ('in_stock_processing') == 'all_changes'
+
+ for p in instock_props:
+ set_original_response(props_markup=p)
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'more than one price detected' not in res.data
+ assert b'has-restock-info' in res.data
+ assert b' in-stock' in res.data
+ assert b' not-in-stock' not in res.data
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+
+ for p in out_of_stock_props:
+ set_original_response(props_markup=p)
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+
+ assert b'has-restock-info not-in-stock' in res.data
+
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+def test_itemprop_price_change(client, live_server):
+ #live_server_setup(live_server)
+
+ # Out of the box 'Follow price changes' should be ON
+ test_url = url_for('test_endpoint', _external=True)
+
+ set_original_response(props_markup=instock_props[0], price="190.95")
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+
+ # A change in price, should trigger a change by default
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'190.95' in res.data
+
+ # basic price change, look for notification
+ set_original_response(props_markup=instock_props[0], price='180.45')
+ client.get(url_for("form_watch_checknow"), follow_redirects=True)
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'180.45' in res.data
+ assert b'unviewed' in res.data
+ client.get(url_for("mark_all_viewed"), follow_redirects=True)
+
+ # turning off price change trigger, but it should show the new price, with no change notification
+ set_original_response(props_markup=instock_props[0], price='120.45')
+ res = client.post(
+ url_for("edit_page", uuid="first"),
+ data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+ follow_redirects=True
+ )
+ assert b"Updated watch." in res.data
+ client.get(url_for("form_watch_checknow"), follow_redirects=True)
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'120.45' in res.data
+ assert b'unviewed' not in res.data
+
+
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+def _run_test_minmax_limit(client, extra_watch_edit_form):
+
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+ test_url = url_for('test_endpoint', _external=True)
+
+ set_original_response(props_markup=instock_props[0], price="950.95")
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+
+ # A change in price, should trigger a change by default
+ wait_for_all_checks(client)
+
+ data = {
+ "tags": "",
+ "url": test_url,
+ "headers": "",
+ 'fetch_backend': "html_requests"
+ }
+ data.update(extra_watch_edit_form)
+ res = client.post(
+ url_for("edit_page", uuid="first"),
+ data=data,
+ follow_redirects=True
+ )
+ assert b"Updated watch." in res.data
+ wait_for_all_checks(client)
+
+ client.get(url_for("mark_all_viewed"))
+
+ # price changed to something greater than min (900), BUT less than max (1100).. should be no change
+ set_original_response(props_markup=instock_props[0], price='1000.45')
+ client.get(url_for("form_watch_checknow"))
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+
+ assert b'more than one price detected' not in res.data
+ # BUT the new price should show, even tho its within limits
+ assert b'1,000.45' or b'1000.45' in res.data #depending on locale
+ assert b'unviewed' not in res.data
+
+
+ # price changed to something LESS than min (900), SHOULD be a change
+ set_original_response(props_markup=instock_props[0], price='890.45')
+ # let previous runs wait
+ time.sleep(1)
+ res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+ assert b'1 watches queued for rechecking.' in res.data
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'890.45' in res.data
+ assert b'unviewed' in res.data
+
+ client.get(url_for("mark_all_viewed"))
+
+ # price changed to something MORE than max (1100.10), SHOULD be a change
+ set_original_response(props_markup=instock_props[0], price='1890.45')
+ client.get(url_for("form_watch_checknow"), follow_redirects=True)
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'1,890.45' or b'1890.45' in res.data
+ assert b'unviewed' in res.data
+
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+
+def test_restock_itemprop_minmax(client, live_server):
+# live_server_setup(live_server)
+ extras = {
+ "restock_settings-follow_price_changes": "y",
+ "restock_settings-price_change_min": 900.0,
+ "restock_settings-price_change_max": 1100.10
+ }
+ _run_test_minmax_limit(client, extra_watch_edit_form=extras)
+
+def test_restock_itemprop_with_tag(client, live_server):
+ #live_server_setup(live_server)
+
+ res = client.post(
+ url_for("tags.form_tag_add"),
+ data={"name": "test-tag"},
+ follow_redirects=True
+ )
+ assert b"Tag added" in res.data
+
+ res = client.post(
+ url_for("tags.form_tag_edit_submit", uuid="first"),
+ data={"name": "test-tag",
+ "restock_settings-follow_price_changes": "y",
+ "restock_settings-price_change_min": 900.0,
+ "restock_settings-price_change_max": 1100.10,
+ "overrides_watch": "y", #overrides_watch should be restock_overrides_watch
+ },
+ follow_redirects=True
+ )
+
+ extras = {
+ "tags": "test-tag"
+ }
+
+ _run_test_minmax_limit(client, extra_watch_edit_form=extras)
+
+
+
+def test_itemprop_percent_threshold(client, live_server):
+ #live_server_setup(live_server)
+
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+ test_url = url_for('test_endpoint', _external=True)
+
+ set_original_response(props_markup=instock_props[0], price="950.95")
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+
+ # A change in price, should trigger a change by default
+ wait_for_all_checks(client)
+
+ res = client.post(
+ url_for("edit_page", uuid="first"),
+ data={"restock_settings-follow_price_changes": "y",
+ "restock_settings-price_change_threshold_percent": 5.0,
+ "url": test_url,
+ "tags": "",
+ "headers": "",
+ 'fetch_backend': "html_requests"
+ },
+ follow_redirects=True
+ )
+ assert b"Updated watch." in res.data
+ wait_for_all_checks(client)
+
+
+ # Basic change should not trigger
+ set_original_response(props_markup=instock_props[0], price='960.45')
+ client.get(url_for("form_watch_checknow"))
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'960.45' in res.data
+ assert b'unviewed' not in res.data
+
+ # Bigger INCREASE change than the threshold should trigger
+ set_original_response(props_markup=instock_props[0], price='1960.45')
+ client.get(url_for("form_watch_checknow"))
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'1,960.45' or b'1960.45' in res.data #depending on locale
+ assert b'unviewed' in res.data
+
+
+ # Small decrease should NOT trigger
+ client.get(url_for("mark_all_viewed"))
+ set_original_response(props_markup=instock_props[0], price='1950.45')
+ client.get(url_for("form_watch_checknow"))
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'1,950.45' or b'1950.45' in res.data #depending on locale
+ assert b'unviewed' not in res.data
+
+
+
+
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+
+
+def test_change_with_notification_values(client, live_server):
+ #live_server_setup(live_server)
+
+ if os.path.isfile("test-datastore/notification.txt"):
+ os.unlink("test-datastore/notification.txt")
+
+ test_url = url_for('test_endpoint', _external=True)
+ set_original_response(props_markup=instock_props[0], price='960.45')
+
+ notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+
+ ######################
+ # You must add a type of 'restock_diff' for its tokens to register as valid in the global settings
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+
+ # A change in price, should trigger a change by default
+ wait_for_all_checks(client)
+
+ # Should see new tokens register
+ res = client.get(url_for("settings_page"))
+ assert b'{{restock.original_price}}' in res.data
+ assert b'Original price at first check' in res.data
+
+ #####################
+ # Set this up for when we remove the notification from the watch, it should fallback with these details
+ res = client.post(
+ url_for("settings_page"),
+ data={"application-notification_urls": notification_url,
+ "application-notification_title": "title new price {{restock.price}}",
+ "application-notification_body": "new price {{restock.price}}",
+ "application-notification_format": default_notification_format,
+ "requests-time_between_check-minutes": 180,
+ 'application-fetch_backend': "html_requests"},
+ follow_redirects=True
+ )
+
+ # check tag accepts without error
+
+ # Check the watches in these modes add the tokens for validating
+ assert b"A variable or function is not defined" not in res.data
+
+ assert b"Settings updated." in res.data
+
+
+ set_original_response(props_markup=instock_props[0], price='960.45')
+ # A change in price, should trigger a change by default
+ set_original_response(props_markup=instock_props[0], price='1950.45')
+ client.get(url_for("form_watch_checknow"))
+ wait_for_all_checks(client)
+ time.sleep(3)
+ assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
+ with open("test-datastore/notification.txt", 'r') as f:
+ notification = f.read()
+ assert "new price 1950.45" in notification
+ assert "title new price 1950.45" in notification
+
+
+
+def test_data_sanity(client, live_server):
+ #live_server_setup(live_server)
+
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+ test_url = url_for('test_endpoint', _external=True)
+ test_url2 = url_for('test_endpoint2', _external=True)
+ set_original_response(props_markup=instock_props[0], price="950.95")
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+
+
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert b'950.95' in res.data
+
+ # Check the restock model object doesnt store the value by mistake and used in a new one
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url2, "tags": 'restock tests', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+ wait_for_all_checks(client)
+ res = client.get(url_for("index"))
+ assert str(res.data.decode()).count("950.95") == 1, "Price should only show once (for the watch added, no other watches yet)"
+
+ ## different test, check the edit page works on an empty request result
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
+
+ client.post(
+ url_for("form_quick_watch_add"),
+ data={"url": test_url2, "tags": 'restock tests', 'processor': 'restock_diff'},
+ follow_redirects=True
+ )
+ wait_for_all_checks(client)
+
+ res = client.get(
+ url_for("edit_page", uuid="first"))
+ assert test_url2.encode('utf-8') in res.data
diff --git a/changedetectionio/tests/test_rss.py b/changedetectionio/tests/test_rss.py
index 7f030482..e249e0ad 100644
--- a/changedetectionio/tests/test_rss.py
+++ b/changedetectionio/tests/test_rss.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -49,10 +49,10 @@ def set_original_cdata_xml():
f.write(test_return_data)
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
live_server_setup(live_server)
-def test_rss_and_token(client, live_server):
+def test_rss_and_token(client, live_server, measure_memory_usage):
# live_server_setup(live_server)
set_original_response()
@@ -69,6 +69,7 @@ def test_rss_and_token(client, live_server):
wait_for_all_checks(client)
set_modified_response()
+ time.sleep(1)
client.get(url_for("form_watch_checknow"), follow_redirects=True)
wait_for_all_checks(client)
@@ -87,9 +88,9 @@ def test_rss_and_token(client, live_server):
assert b"Access denied, bad token" not in res.data
assert b"Random content" in res.data
- res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-def test_basic_cdata_rss_markup(client, live_server):
+def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
set_original_cdata_xml()
@@ -117,7 +118,7 @@ def test_basic_cdata_rss_markup(client, live_server):
assert b'The days of Terminator' in res.data
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-def test_rss_xpath_filtering(client, live_server):
+def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
set_original_cdata_xml()
diff --git a/changedetectionio/tests/test_search.py b/changedetectionio/tests/test_search.py
index 453a0b8a..70dab62f 100644
--- a/changedetectionio/tests/test_search.py
+++ b/changedetectionio/tests/test_search.py
@@ -5,7 +5,7 @@ import time
def test_setup(live_server):
live_server_setup(live_server)
-def test_basic_search(client, live_server):
+def test_basic_search(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
urls = ['https://localhost:12300?first-result=1',
@@ -38,7 +38,7 @@ def test_basic_search(client, live_server):
assert urls[1].encode('utf-8') not in res.data
-def test_search_in_tag_limit(client, live_server):
+def test_search_in_tag_limit(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
urls = ['https://localhost:12300?first-result=1 tag-one',
diff --git a/changedetectionio/tests/test_security.py b/changedetectionio/tests/test_security.py
index c56c84492..af55efb2 100644
--- a/changedetectionio/tests/test_security.py
+++ b/changedetectionio/tests/test_security.py
@@ -1,11 +1,16 @@
+import os
+
from flask import url_for
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
import time
-def test_setup(client, live_server):
+from .. import strtobool
+
+
+def test_setup(client, live_server, measure_memory_usage):
live_server_setup(live_server)
-def test_bad_access(client, live_server):
+def test_bad_access(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
res = client.post(
url_for("import_page"),
@@ -55,19 +60,35 @@ def test_bad_access(client, live_server):
assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
- # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
+def test_file_access(client, live_server, measure_memory_usage):
+ #live_server_setup(live_server)
+
+ test_file_path = "/tmp/test-file.txt"
+
+ # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
client.post(
url_for("form_quick_watch_add"),
- data={"url": 'file:///tasty/disk/drive', "tags": ''},
+ data={"url": f"file://{test_file_path}", "tags": ''},
follow_redirects=True
)
wait_for_all_checks(client)
res = client.get(url_for("index"))
- assert b'file:// type access is denied for security reasons.' in res.data
+ # If it is enabled at test time
+ if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
+ res = client.get(
+ url_for("preview_page", uuid="first"),
+ follow_redirects=True
+ )
+
+ # Should see something (this file added by run_basic_tests.sh)
+ assert b"Hello world" in res.data
+ else:
+ # Default should be here
+ assert b'file:// type access is denied for security reasons.' in res.data
-def test_xss(client, live_server):
+def test_xss(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
from changedetectionio.notification import (
default_notification_format
diff --git a/changedetectionio/tests/test_share_watch.py b/changedetectionio/tests/test_share_watch.py
index bf76fabc..a8e53601 100644
--- a/changedetectionio/tests/test_share_watch.py
+++ b/changedetectionio/tests/test_share_watch.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -9,7 +9,7 @@ import re
sleep_time_for_fetch_thread = 3
-def test_share_watch(client, live_server):
+def test_share_watch(client, live_server, measure_memory_usage):
set_original_response()
live_server_setup(live_server)
diff --git a/changedetectionio/tests/test_source.py b/changedetectionio/tests/test_source.py
index f46e8ad8..cffd1bd6 100644
--- a/changedetectionio/tests/test_source.py
+++ b/changedetectionio/tests/test_source.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -10,7 +10,7 @@ sleep_time_for_fetch_thread = 3
def test_setup(live_server):
live_server_setup(live_server)
-def test_check_basic_change_detection_functionality_source(client, live_server):
+def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage):
set_original_response()
test_url = 'source:'+url_for('test_endpoint', _external=True)
# Add our URL to the import page
@@ -58,7 +58,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
# `subtractive_selectors` should still work in `source:` type requests
-def test_check_ignore_elements(client, live_server):
+def test_check_ignore_elements(client, live_server, measure_memory_usage):
set_original_response()
time.sleep(1)
test_url = 'source:'+url_for('test_endpoint', _external=True)
diff --git a/changedetectionio/tests/test_trigger.py b/changedetectionio/tests/test_trigger.py
index 24e9fabf..d13fe79e 100644
--- a/changedetectionio/tests/test_trigger.py
+++ b/changedetectionio/tests/test_trigger.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -55,7 +55,7 @@ def set_modified_with_trigger_text_response():
f.write(test_return_data)
-def test_trigger_functionality(client, live_server):
+def test_trigger_functionality(client, live_server, measure_memory_usage):
live_server_setup(live_server)
diff --git a/changedetectionio/tests/test_trigger_regex.py b/changedetectionio/tests/test_trigger_regex.py
index 7f070e89..ef465a35 100644
--- a/changedetectionio/tests/test_trigger_regex.py
+++ b/changedetectionio/tests/test_trigger_regex.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -22,7 +22,7 @@ def set_original_ignore_response():
-def test_trigger_regex_functionality(client, live_server):
+def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
live_server_setup(live_server)
diff --git a/changedetectionio/tests/test_trigger_regex_with_filter.py b/changedetectionio/tests/test_trigger_regex_with_filter.py
index 4f3328fc..4e3eba72 100644
--- a/changedetectionio/tests/test_trigger_regex_with_filter.py
+++ b/changedetectionio/tests/test_trigger_regex_with_filter.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -22,7 +22,7 @@ def set_original_ignore_response():
-def test_trigger_regex_functionality_with_filter(client, live_server):
+def test_trigger_regex_functionality_with_filter(client, live_server, measure_memory_usage):
live_server_setup(live_server)
sleep_time_for_fetch_thread = 3
diff --git a/changedetectionio/tests/test_unique_lines.py b/changedetectionio/tests/test_unique_lines.py
index 4eff11fd..65840073 100644
--- a/changedetectionio/tests/test_unique_lines.py
+++ b/changedetectionio/tests/test_unique_lines.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import time
from flask import url_for
@@ -66,10 +66,10 @@ def set_modified_with_trigger_text_response():
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
live_server_setup(live_server)
-def test_unique_lines_functionality(client, live_server):
+def test_unique_lines_functionality(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
@@ -118,7 +118,7 @@ def test_unique_lines_functionality(client, live_server):
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
-def test_sort_lines_functionality(client, live_server):
+def test_sort_lines_functionality(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
set_modified_swapped_lines_with_extra_text_for_sorting()
diff --git a/changedetectionio/tests/test_watch_fields_storage.py b/changedetectionio/tests/test_watch_fields_storage.py
index 7dc3f748..5737c9c7 100644
--- a/changedetectionio/tests/test_watch_fields_storage.py
+++ b/changedetectionio/tests/test_watch_fields_storage.py
@@ -4,7 +4,7 @@ from urllib.request import urlopen
from . util import set_original_response, set_modified_response, live_server_setup
-def test_check_watch_field_storage(client, live_server):
+def test_check_watch_field_storage(client, live_server, measure_memory_usage):
set_original_response()
live_server_setup(live_server)
diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py
index 1a9c5afa..e8b5d855 100644
--- a/changedetectionio/tests/test_xpath_selector.py
+++ b/changedetectionio/tests/test_xpath_selector.py
@@ -49,7 +49,7 @@ def set_modified_response():
# Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
-def test_check_xpath_filter_utf8(client, live_server):
+def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage):
filter = '//item/*[self::description]'
d = '''
@@ -105,7 +105,7 @@ def test_check_xpath_filter_utf8(client, live_server):
# Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
-def test_check_xpath_text_function_utf8(client, live_server):
+def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usage):
filter = '//item/title/text()'
d = '''
@@ -168,7 +168,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
assert b'Deleted' in res.data
-def test_check_markup_xpath_filter_restriction(client, live_server):
+def test_check_markup_xpath_filter_restriction(client, live_server, measure_memory_usage):
xpath_filter = "//*[contains(@class, 'sametext')]"
set_original_response()
@@ -214,7 +214,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
assert b'Deleted' in res.data
-def test_xpath_validation(client, live_server):
+def test_xpath_validation(client, live_server, measure_memory_usage):
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
res = client.post(
@@ -235,7 +235,7 @@ def test_xpath_validation(client, live_server):
assert b'Deleted' in res.data
-def test_xpath23_prefix_validation(client, live_server):
+def test_xpath23_prefix_validation(client, live_server, measure_memory_usage):
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
res = client.post(
@@ -255,7 +255,7 @@ def test_xpath23_prefix_validation(client, live_server):
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
-def test_xpath1_lxml(client, live_server):
+def test_xpath1_lxml(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
d = '''
@@ -319,7 +319,7 @@ def test_xpath1_lxml(client, live_server):
#####
-def test_xpath1_validation(client, live_server):
+def test_xpath1_validation(client, live_server, measure_memory_usage):
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
res = client.post(
@@ -341,7 +341,7 @@ def test_xpath1_validation(client, live_server):
# actually only really used by the distll.io importer, but could be handy too
-def test_check_with_prefix_include_filters(client, live_server):
+def test_check_with_prefix_include_filters(client, live_server, measure_memory_usage):
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
@@ -378,7 +378,7 @@ def test_check_with_prefix_include_filters(client, live_server):
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-def test_various_rules(client, live_server):
+def test_various_rules(client, live_server, measure_memory_usage):
# Just check these don't error
# live_server_setup(live_server)
with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -426,7 +426,7 @@ def test_various_rules(client, live_server):
assert b'Deleted' in res.data
-def test_xpath_20(client, live_server):
+def test_xpath_20(client, live_server, measure_memory_usage):
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("import_page"),
@@ -463,7 +463,7 @@ def test_xpath_20(client, live_server):
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-def test_xpath_20_function_count(client, live_server):
+def test_xpath_20_function_count(client, live_server, measure_memory_usage):
set_original_response()
# Add our URL to the import page
@@ -499,7 +499,7 @@ def test_xpath_20_function_count(client, live_server):
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-def test_xpath_20_function_count2(client, live_server):
+def test_xpath_20_function_count2(client, live_server, measure_memory_usage):
set_original_response()
# Add our URL to the import page
@@ -535,7 +535,7 @@ def test_xpath_20_function_count2(client, live_server):
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-def test_xpath_20_function_string_join_matches(client, live_server):
+def test_xpath_20_function_string_join_matches(client, live_server, measure_memory_usage):
set_original_response()
# Add our URL to the import page
diff --git a/changedetectionio/tests/unit/test_jinja2_security.py b/changedetectionio/tests/unit/test_jinja2_security.py
index eb43db9d..7616bb65 100644
--- a/changedetectionio/tests/unit/test_jinja2_security.py
+++ b/changedetectionio/tests/unit/test_jinja2_security.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# run from dir above changedetectionio/ dir
# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
diff --git a/changedetectionio/tests/unit/test_notification_diff.py b/changedetectionio/tests/unit/test_notification_diff.py
index c7befc20..b106f6a1 100755
--- a/changedetectionio/tests/unit/test_notification_diff.py
+++ b/changedetectionio/tests/unit/test_notification_diff.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# run from dir above changedetectionio/ dir
# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
diff --git a/changedetectionio/tests/unit/test_restock_logic.py b/changedetectionio/tests/unit/test_restock_logic.py
new file mode 100644
index 00000000..46fff2c7
--- /dev/null
+++ b/changedetectionio/tests/unit/test_restock_logic.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_restock_logic
+
+import unittest
+import os
+
+from changedetectionio.processors import restock_diff
+
+# mostly
+class TestDiffBuilder(unittest.TestCase):
+
+ def test_logic(self):
+ assert restock_diff.is_between(number=10, lower=9, upper=11) == True, "Between 9 and 11"
+ assert restock_diff.is_between(number=10, lower=0, upper=11) == True, "Between 9 and 11"
+ assert restock_diff.is_between(number=10, lower=None, upper=11) == True, "Between None and 11"
+ assert not restock_diff.is_between(number=12, lower=None, upper=11) == True, "12 is not between None and 11"
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/changedetectionio/tests/unit/test_watch_model.py b/changedetectionio/tests/unit/test_watch_model.py
index 78ca60b2..a9e3df3c 100644
--- a/changedetectionio/tests/unit/test_watch_model.py
+++ b/changedetectionio/tests/unit/test_watch_model.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# run from dir above changedetectionio/ dir
# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py
index 186fc736..1b0b3e5b 100644
--- a/changedetectionio/tests/util.py
+++ b/changedetectionio/tests/util.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
from flask import make_response, request
from flask import url_for
@@ -121,18 +121,21 @@ def extract_UUID_from_client(client):
return uuid.strip()
def wait_for_all_checks(client):
+ # actually this is not entirely true, it can still be 'processing' but not in the queue
# Loop waiting until done..
attempt=0
- time.sleep(0.1)
+ # because sub-second rechecks are problematic in testing, use lots of delays
+ time.sleep(1)
while attempt < 60:
- time.sleep(1)
res = client.get(url_for("index"))
if not b'Checking now' in res.data:
break
logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))
-
+ time.sleep(1)
attempt += 1
+ time.sleep(1)
+
def live_server_setup(live_server):
@live_server.app.route('/test-random-content-endpoint')
@@ -140,6 +143,9 @@ def live_server_setup(live_server):
import secrets
return "Random content - {}\n".format(secrets.token_hex(64))
+ @live_server.app.route('/test-endpoint2')
+ def test_endpoint2():
+ return "some basic content"
@live_server.app.route('/test-endpoint')
def test_endpoint():
diff --git a/changedetectionio/tests/visualselector/conftest.py b/changedetectionio/tests/visualselector/conftest.py
index 430513d4..9a4bf56b 100644
--- a/changedetectionio/tests/visualselector/conftest.py
+++ b/changedetectionio/tests/visualselector/conftest.py
@@ -1,3 +1,3 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
from .. import conftest
diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py
index 15677f31..de3b9030 100644
--- a/changedetectionio/tests/visualselector/test_fetch_data.py
+++ b/changedetectionio/tests/visualselector/test_fetch_data.py
@@ -1,15 +1,15 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import os
from flask import url_for
from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
live_server_setup(live_server)
# Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_visual_selector_content_ready(client, live_server):
+def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
import os
import json
@@ -79,7 +79,7 @@ def test_visual_selector_content_ready(client, live_server):
follow_redirects=True
)
-def test_basic_browserstep(client, live_server):
+def test_basic_browserstep(client, live_server, measure_memory_usage):
#live_server_setup(live_server)
assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py
index dcb9dbe7..ba183848 100644
--- a/changedetectionio/update_worker.py
+++ b/changedetectionio/update_worker.py
@@ -1,18 +1,19 @@
+from .processors.exceptions import ProcessorException
+import changedetectionio.content_fetchers.exceptions as content_fetchers_exceptions
+from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
+from changedetectionio import html_tools
+
+import importlib
import os
-import threading
import queue
+import threading
import time
-from . import content_fetchers
-from changedetectionio import html_tools
-from .processors.text_json_diff import FilterNotFoundInResponse
-from .processors.restock_diff import UnableToExtractRestockData
# A single update worker
#
# Requests for checking on a single site(watch) from a queue of watches
# (another process inserts watches into the queue that are time-ready for checking)
-import sys
from loguru import logger
class update_worker(threading.Thread):
@@ -26,7 +27,6 @@ class update_worker(threading.Thread):
super().__init__(*args, **kwargs)
def queue_notification_for_watch(self, notification_q, n_object, watch):
-
from changedetectionio import diff
dates = []
trigger_text = ''
@@ -80,6 +80,9 @@ class update_worker(threading.Thread):
'uuid': watch.get('uuid') if watch else None,
'watch_url': watch.get('url') if watch else None,
})
+
+ n_object.update(watch.extra_notification_token_values())
+
logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
logger.debug("Queued notification for sending")
notification_q.put(n_object)
@@ -225,8 +228,6 @@ class update_worker(threading.Thread):
os.unlink(full_path)
def run(self):
-
- from .processors import text_json_diff, restock_diff
now = time.time()
while not self.app.config.exit.is_set():
@@ -245,41 +246,40 @@ class update_worker(threading.Thread):
contents = b''
process_changedetection_results = True
update_obj = {}
- logger.info(f"Processing watch UUID {uuid} "
- f"Priority {queued_item_data.priority} "
- f"URL {self.datastore.data['watching'][uuid]['url']}")
+
+ # Clear last errors (move to preflight func?)
+ self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
+
+ watch = self.datastore.data['watching'].get(uuid)
+
+ logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
now = time.time()
try:
# Processor is what we are using for detecting the "Change"
- processor = self.datastore.data['watching'][uuid].get('processor', 'text_json_diff')
- # if system...
-
+ processor = watch.get('processor', 'text_json_diff')
# Abort processing when the content was the same as the last fetch
skip_when_same_checksum = queued_item_data.item.get('skip_when_checksum_same')
- # @todo some way to switch by name
- # Init a new 'difference_detection_processor'
+ # Init a new 'difference_detection_processor', first look in processors
+ processor_module_name = f"changedetectionio.processors.{processor}.processor"
+ try:
+ processor_module = importlib.import_module(processor_module_name)
+ except ModuleNotFoundError as e:
+ print(f"Processor module '{processor}' not found.")
+ raise e
- if processor == 'restock_diff':
- update_handler = restock_diff.perform_site_check(datastore=self.datastore,
+ update_handler = processor_module.perform_site_check(datastore=self.datastore,
watch_uuid=uuid
)
- else:
- # Used as a default and also by some tests
- update_handler = text_json_diff.perform_site_check(datastore=self.datastore,
- watch_uuid=uuid
- )
-
- # Clear last errors (move to preflight func?)
- self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
update_handler.call_browser()
- changed_detected, update_obj, contents = update_handler.run_changedetection(uuid,
- skip_when_checksum_same=skip_when_same_checksum,
- )
+ changed_detected, update_obj, contents = update_handler.run_changedetection(
+ watch=watch,
+ skip_when_checksum_same=skip_when_same_checksum,
+ )
# Re #342
# In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@@ -290,7 +290,17 @@ class update_worker(threading.Thread):
logger.critical(f"File permission error updating file, watch: {uuid}")
logger.critical(str(e))
process_changedetection_results = False
- except content_fetchers.exceptions.ReplyWithContentButNoText as e:
+
+ # A generic other-exception thrown by processors
+ except ProcessorException as e:
+ if e.screenshot:
+ watch.save_screenshot(screenshot=e.screenshot)
+ if e.xpath_data:
+ watch.save_xpath_data(data=e.xpath_data)
+ self.datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
+ process_changedetection_results = False
+
+ except content_fetchers_exceptions.ReplyWithContentButNoText as e:
# Totally fine, it's by choice - just continue on, nothing more to care about
# Page had elements/content but no renderable text
# Backend (not filters) gave zero output
@@ -309,10 +319,14 @@ class update_worker(threading.Thread):
})
if e.screenshot:
- self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot)
+ watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+
+ if e.xpath_data:
+ watch.save_xpath_data(data=e.xpath_data)
+
process_changedetection_results = False
- except content_fetchers.exceptions.Non200ErrorCodeReceived as e:
+ except content_fetchers_exceptions.Non200ErrorCodeReceived as e:
if e.status_code == 403:
err_text = "Error - 403 (Access denied) received"
elif e.status_code == 404:
@@ -325,11 +339,11 @@ class update_worker(threading.Thread):
err_text = "Error - Request returned a HTTP error code {}".format(str(e.status_code))
if e.screenshot:
- self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
+ watch.save_screenshot(screenshot=e.screenshot, as_error=True)
if e.xpath_data:
- self.datastore.save_xpath_data(watch_uuid=uuid, data=e.xpath_data, as_error=True)
+ watch.save_xpath_data(data=e.xpath_data, as_error=True)
if e.page_text:
- self.datastore.save_error_text(watch_uuid=uuid, contents=e.page_text)
+ watch.save_error_text(contents=e.page_text)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
process_changedetection_results = False
@@ -341,16 +355,23 @@ class update_worker(threading.Thread):
err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+ # Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
+ if e.screenshot:
+ watch.save_screenshot(screenshot=e.screenshot)
+
+ if e.xpath_data:
+ watch.save_xpath_data(data=e.xpath_data)
+
# Only when enabled, send the notification
- if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
- c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
+ if watch.get('filter_failure_notification_send', False):
+ c = watch.get('consecutive_filter_failures', 5)
c += 1
# Send notification if we reached the threshold?
threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
0)
- logger.error(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
+ logger.warning(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
if threshold > 0 and c >= threshold:
- if not self.datastore.data['watching'][uuid].get('notification_muted'):
+ if not watch.get('notification_muted'):
self.send_filter_failure_notification(uuid)
c = 0
@@ -358,24 +379,23 @@ class update_worker(threading.Thread):
process_changedetection_results = False
- except content_fetchers.exceptions.checksumFromPreviousCheckWasTheSame as e:
+ except content_fetchers_exceptions.checksumFromPreviousCheckWasTheSame as e:
# Yes fine, so nothing todo, don't continue to process.
process_changedetection_results = False
changed_detected = False
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
- except content_fetchers.exceptions.BrowserConnectError as e:
+ except content_fetchers_exceptions.BrowserConnectError as e:
self.datastore.update_watch(uuid=uuid,
update_obj={'last_error': e.msg
}
)
process_changedetection_results = False
- except content_fetchers.exceptions.BrowserFetchTimedOut as e:
+ except content_fetchers_exceptions.BrowserFetchTimedOut as e:
self.datastore.update_watch(uuid=uuid,
update_obj={'last_error': e.msg
}
)
process_changedetection_results = False
- except content_fetchers.exceptions.BrowserStepsStepException as e:
+ except content_fetchers_exceptions.BrowserStepsStepException as e:
if not self.datastore.data['watching'].get(uuid):
continue
@@ -401,15 +421,15 @@ class update_worker(threading.Thread):
}
)
- if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
- c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
+ if watch.get('filter_failure_notification_send', False):
+ c = watch.get('consecutive_filter_failures', 5)
c += 1
# Send notification if we reached the threshold?
threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
0)
logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
if threshold > 0 and c >= threshold:
- if not self.datastore.data['watching'][uuid].get('notification_muted'):
+ if not watch.get('notification_muted'):
self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
c = 0
@@ -417,56 +437,48 @@ class update_worker(threading.Thread):
process_changedetection_results = False
- except content_fetchers.exceptions.EmptyReply as e:
+ except content_fetchers_exceptions.EmptyReply as e:
# Some kind of custom to-str handler in the exception handler that does this?
err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
- except content_fetchers.exceptions.ScreenshotUnavailable as e:
+ except content_fetchers_exceptions.ScreenshotUnavailable as e:
err_text = "Screenshot unavailable, page did not render fully in the expected time or page was too long - try increasing 'Wait seconds before extracting text'"
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
- except content_fetchers.exceptions.JSActionExceptions as e:
+ except content_fetchers_exceptions.JSActionExceptions as e:
err_text = "Error running JS Actions - Page request - "+e.message
if e.screenshot:
- self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
+ watch.save_screenshot(screenshot=e.screenshot, as_error=True)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
- except content_fetchers.exceptions.PageUnloadable as e:
+ except content_fetchers_exceptions.PageUnloadable as e:
err_text = "Page request from server didnt respond correctly"
if e.message:
err_text = "{} - {}".format(err_text, e.message)
if e.screenshot:
- self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
+ watch.save_screenshot(screenshot=e.screenshot, as_error=True)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code,
'has_ldjson_price_data': None})
process_changedetection_results = False
- except content_fetchers.exceptions.BrowserStepsInUnsupportedFetcher as e:
+ except content_fetchers_exceptions.BrowserStepsInUnsupportedFetcher as e:
err_text = "This watch has Browser Steps configured and so it cannot run with the 'Basic fast Plaintext/HTTP Client', either remove the Browser Steps or select a Chrome fetcher."
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
process_changedetection_results = False
logger.error(f"Exception (BrowserStepsInUnsupportedFetcher) reached processing watch UUID: {uuid}")
- except UnableToExtractRestockData as e:
- # Usually when fetcher.instock_data returns empty
- logger.error(f"Exception (UnableToExtractRestockData) reached processing watch UUID: {uuid}")
- logger.error(str(e))
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Unable to extract restock data for this page unfortunately. (Got code {e.status_code} from server)"})
- process_changedetection_results = False
except Exception as e:
logger.error(f"Exception reached processing watch UUID: {uuid}")
logger.error(str(e))
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
# Other serious error
process_changedetection_results = False
-# import traceback
-# print(traceback.format_exc())
else:
# Crash protection, the watch entry could have been removed by this point (during a slow chrome fetch etc)
@@ -474,7 +486,7 @@ class update_worker(threading.Thread):
continue
# Mark that we never had any failures
- if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
+ if not watch.get('ignore_status_codes'):
update_obj['consecutive_filter_failures'] = 0
# Everything ran OK, clean off any previous error
@@ -482,28 +494,57 @@ class update_worker(threading.Thread):
self.cleanup_error_artifacts(uuid)
+ if not self.datastore.data['watching'].get(uuid):
+ continue
#
# Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
if process_changedetection_results:
+
+ # Extract
as title if possible/requested.
+ if self.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
+ if not watch['title'] or not len(watch['title']):
+ try:
+ update_obj['title'] = html_tools.extract_element(find='title', html_content=update_handler.fetcher.content)
+ logger.info(f"UUID: {uuid} Extract <title> updated title to '{update_obj['title']}")
+ except Exception as e:
+ logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
+
+ # Now update after running everything
+ timestamp = round(time.time())
try:
- watch = self.datastore.data['watching'].get(uuid)
self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
- # Also save the snapshot on the first time checked
- if changed_detected or not watch['last_checked']:
+
+ # Also save the snapshot on the first time checked, "last checked" will always be updated, so we just check history length.
+ if changed_detected or not watch.history_n:
+
+ if update_handler.screenshot:
+ watch.save_screenshot(screenshot=update_handler.screenshot)
+
+ if update_handler.xpath_data:
+ watch.save_xpath_data(data=update_handler.xpath_data)
+
+ # Small hack so that we sleep just enough to allow 1 second between history snapshots
+ # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
+
+ if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
+ logger.warning(
+ f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+ timestamp = str(int(timestamp) + 1)
+ time.sleep(1)
+
watch.save_history_text(contents=contents,
- timestamp=str(round(time.time())),
+ timestamp=timestamp,
snapshot_id=update_obj.get('previous_md5', 'none'))
- # A change was detected
- if changed_detected:
+ if update_handler.fetcher.content:
+ watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
+
# Notifications should only trigger on the second time (first time, we gather the initial snapshot)
if watch.history_n >= 2:
logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
- if not self.datastore.data['watching'][uuid].get('notification_muted'):
+ if not watch.get('notification_muted'):
self.send_content_changed_notification(watch_uuid=uuid)
- else:
- logger.info(f"Change triggered in UUID {uuid} due to first history saving (no notifications sent) - {watch['url']}")
except Exception as e:
# Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
@@ -511,29 +552,23 @@ class update_worker(threading.Thread):
logger.critical(str(e))
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
- if self.datastore.data['watching'].get(uuid):
- # Always record that we atleast tried
- count = self.datastore.data['watching'][uuid].get('check_count', 0) + 1
- # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
- try:
- server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
- self.datastore.update_watch(uuid=uuid,
- update_obj={'remote_server_reply': server_header}
- )
- except Exception as e:
- pass
-
- self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
- 'last_checked': round(time.time()),
- 'check_count': count
- })
-
- # Always save the screenshot if it's available
- if update_handler.screenshot:
- self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
- if update_handler.xpath_data:
- self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
+ # Always record that we atleast tried
+ count = watch.get('check_count', 0) + 1
+
+ # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
+ try:
+ server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
+ self.datastore.update_watch(uuid=uuid,
+ update_obj={'remote_server_reply': server_header}
+ )
+ except Exception as e:
+ pass
+
+ self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+ 'last_checked': round(time.time()),
+ 'check_count': count
+ })
self.current_uuid = None # Done
diff --git a/docker-compose.yml b/docker-compose.yml
index 4cf17605..2480a339 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -68,9 +68,10 @@ services:
# If WEBDRIVER or PLAYWRIGHT are enabled, changedetection container depends on that
# and must wait before starting (substitute "browser-chrome" with "playwright-chrome" if last one is used)
-# depends_on:
-# browser-chrome:
-# condition: service_started
+# depends_on:
+# playwright-chrome:
+# condition: service_started
+
# Used for fetching pages via Playwright+Chrome where you need Javascript support.
# RECOMMENDED FOR FETCHING PAGES WITH CHROME
diff --git a/docs/restock-overview.png b/docs/restock-overview.png
new file mode 100644
index 00000000..c4a4e78f
Binary files /dev/null and b/docs/restock-overview.png differ
diff --git a/docs/restock-settings.png b/docs/restock-settings.png
new file mode 100644
index 00000000..1e468632
Binary files /dev/null and b/docs/restock-settings.png differ
diff --git a/requirements.txt b/requirements.txt
index 8dc317eb..49a3762e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
# Used by Pyppeteer
pyee
-eventlet==0.33.3 # related to dnspython fixes
+eventlet>=0.36.1 # fixes SSL error on Python 3.12
feedgen~=0.9
flask-compress
# 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -22,31 +22,28 @@ validators~=0.21
# >= 2.26 also adds Brotli support if brotli is installed
brotli~=1.0
requests[socks]
+requests-file
-urllib3==1.26.18
+urllib3==1.26.19
chardet>2.3.0
wtforms~=3.0
jsonpath-ng~=1.5.3
-# Pinned: module 'eventlet.green.select' has no attribute 'epoll'
-# https://github.com/eventlet/eventlet/issues/805#issuecomment-1640463482
-dnspython==2.3.0 # related to eventlet fixes
+dnspython==2.6.1 # related to eventlet fixes
# jq not available on Windows so must be installed manually
# Notification library
-apprise~=1.8.0
+apprise~=1.8.1
# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
# and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible
# use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
paho-mqtt>=1.6.1,<2.0.0
-# This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1"
-# so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found"
-# (introduced once apprise became a dep)
-cryptography~=3.4
+# Requires extra wheel for rPi
+cryptography~=42.0.8
# Used for CSS filtering
beautifulsoup4
@@ -73,8 +70,6 @@ openpyxl
jq~=1.3; python_version >= "3.8" and sys_platform == "darwin"
jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
-# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
-pillow
# playwright is installed at Dockerfile build time because it's not available on all platforms
pyppeteer-ng==2.0.0rc5
@@ -88,3 +83,12 @@ pytest-flask ~=1.2
jsonschema==4.17.3
loguru
+
+# For scraping all possible metadata relating to products so we can do better restock detection
+extruct
+
+# For cleaning up unknown currency formats
+babel
+
+# Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
+greenlet >= 3.0.3
](https://changedetection.io?src=github)
+
+Set price change notification parameters, upper and lower price, price change percentage and more.
+Always know when a product for sale drops in price.
+
+[
](https://changedetection.io?src=github)
+
+
### Example use cases
@@ -272,6 +286,10 @@ I offer commercial support, this software is depended on by network security, ae
[release-link]: https://github.com/dgtlmoon/changedetection.io/releases
[docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io
+## Commercial Licencing
+
+If you are reselling this software either in part or full as part of any commercial arrangement, you must abide by our COMMERCIAL_LICENCE.md found in our code repository, please contact dgtlmoon@gmail.com and contact@changedetection.io .
+
## Third-party licenses
changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)
diff --git a/changedetection.py b/changedetection.py
index 2d0b5d2c..ead2b8c5 100755
--- a/changedetection.py
+++ b/changedetection.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# Only exists for direct CLI usage
diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py
index 6fb76ce4..5ec6f891 100644
--- a/changedetectionio/__init__.py
+++ b/changedetectionio/__init__.py
@@ -1,8 +1,8 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
-__version__ = '0.45.23'
+__version__ = '0.46.02'
from changedetectionio.strtobool import strtobool
from json.decoder import JSONDecodeError
diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py
index 85e2b30e..9b3eb440 100644
--- a/changedetectionio/api/api_v1.py
+++ b/changedetectionio/api/api_v1.py
@@ -12,9 +12,10 @@ import copy
# See docs/README.md for rebuilding the docs/apidoc information
from . import api_schema
+from ..model import watch_base
# Build a JSON Schema atleast partially based on our Watch model
-from changedetectionio.model.Watch import base_config as watch_base_config
+watch_base_config = watch_base()
schema = api_schema.build_watch_json_schema(watch_base_config)
schema_create_watch = copy.deepcopy(schema)
@@ -170,23 +171,33 @@ class WatchSingleHistory(Resource):
curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
@apiName Get single snapshot content
@apiGroup Watch History
+ @apiParam {String} [html] Optional Set to =1 to return the last HTML (only stores last 2 snapshots, use `latest` as timestamp)
@apiSuccess (200) {String} OK
@apiSuccess (404) {String} ERR Not found
"""
watch = self.datastore.data['watching'].get(uuid)
if not watch:
- abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+ abort(404, message=f"No watch exists with the UUID of {uuid}")
if not len(watch.history):
- abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
+ abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
if timestamp == 'latest':
timestamp = list(watch.history.keys())[-1]
- content = watch.get_history_snapshot(timestamp)
+ if request.args.get('html'):
+ content = watch.get_fetched_html(timestamp)
+ if content:
+ response = make_response(content, 200)
+ response.mimetype = "text/html"
+ else:
+ response = make_response("No content found", 404)
+ response.mimetype = "text/plain"
+ else:
+ content = watch.get_history_snapshot(timestamp)
+ response = make_response(content, 200)
+ response.mimetype = "text/plain"
- response = make_response(content, 200)
- response.mimetype = "text/plain"
return response
diff --git a/changedetectionio/blueprint/browser_steps/__init__.py b/changedetectionio/blueprint/browser_steps/__init__.py
index 30797099..f92bf9f8 100644
--- a/changedetectionio/blueprint/browser_steps/__init__.py
+++ b/changedetectionio/blueprint/browser_steps/__init__.py
@@ -187,8 +187,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
if is_last_step and u:
(screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
- datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
- datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
+ watch = datastore.data['watching'].get(uuid)
+ if watch:
+ watch.save_screenshot(screenshot=screenshot)
+ watch.save_xpath_data(data=xpath_data)
# if not this_session.page:
# cleanup_playwright_session()
diff --git a/changedetectionio/blueprint/browser_steps/browser_steps.py b/changedetectionio/blueprint/browser_steps/browser_steps.py
index 76f3d756..6f38be2e 100644
--- a/changedetectionio/blueprint/browser_steps/browser_steps.py
+++ b/changedetectionio/blueprint/browser_steps/browser_steps.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
import os
import time
@@ -255,8 +255,9 @@ class browsersteps_live_ui(steppable_browser_interface):
def get_current_state(self):
"""Return the screenshot and interactive elements mapping, generally always called after action_()"""
- from pkg_resources import resource_string
- xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
+ import importlib.resources
+ xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+
now = time.time()
self.page.wait_for_timeout(1 * 1000)
@@ -287,11 +288,9 @@ class browsersteps_live_ui(steppable_browser_interface):
:param current_include_filters:
:return:
"""
-
+ import importlib.resources
self.page.evaluate("var include_filters=''")
- from pkg_resources import resource_string
- # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
- xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
+ xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
from changedetectionio.content_fetchers import visualselector_xpath_selectors
xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
diff --git a/changedetectionio/blueprint/check_proxies/__init__.py b/changedetectionio/blueprint/check_proxies/__init__.py
index 62a7dab3..8d7df73f 100644
--- a/changedetectionio/blueprint/check_proxies/__init__.py
+++ b/changedetectionio/blueprint/check_proxies/__init__.py
@@ -30,7 +30,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
def long_task(uuid, preferred_proxy):
import time
from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
- from changedetectionio.processors import text_json_diff
+ from changedetectionio.processors.text_json_diff import text_json_diff
from changedetectionio.safe_jinja import render as jinja_render
status = {'status': '', 'length': 0, 'text': ''}
diff --git a/changedetectionio/blueprint/price_data_follower/__init__.py b/changedetectionio/blueprint/price_data_follower/__init__.py
index 89a2fc67..a41552d8 100644
--- a/changedetectionio/blueprint/price_data_follower/__init__.py
+++ b/changedetectionio/blueprint/price_data_follower/__init__.py
@@ -17,6 +17,8 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue
@price_data_follower_blueprint.route("/