diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
index 0397c026..08072500 100644
--- a/.github/workflows/containers.yml
+++ b/.github/workflows/containers.yml
@@ -88,7 +88,7 @@ jobs:
- name: Build and push :dev
id: docker_build
if: ${{ github.ref }} == "refs/heads/master"
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
with:
context: ./
file: ./Dockerfile
@@ -106,7 +106,7 @@ jobs:
- name: Build and push :tag
id: docker_build_tag_release
if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
with:
context: ./
file: ./Dockerfile
diff --git a/.github/workflows/test-container-build.yml b/.github/workflows/test-container-build.yml
index c6fd9efb..3a5f5351 100644
--- a/.github/workflows/test-container-build.yml
+++ b/.github/workflows/test-container-build.yml
@@ -51,7 +51,7 @@ jobs:
# Check we can still build under alpine/musl
- name: Test that the docker containers can build (musl via alpine check)
id: docker_build_musl
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
with:
context: ./
file: ./.github/test/Dockerfile-alpine
@@ -59,7 +59,7 @@ jobs:
- name: Test that the docker containers can build
id: docker_build
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
# https://github.com/docker/build-push-action#customizing
with:
context: ./
diff --git a/Dockerfile b/Dockerfile
index 5e45880c..626759cb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,9 +3,9 @@
# @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
# If you know how to fix it, please do! and test it for both 3.10 and 3.11
-ARG PYTHON_VERSION=3.10
+ARG PYTHON_VERSION=3.11
-FROM python:${PYTHON_VERSION}-slim-bookworm as builder
+FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
# See `cryptography` pin comment in requirements.txt
ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -26,7 +26,8 @@ WORKDIR /install
COPY requirements.txt /requirements.txt
-RUN pip install --target=/dependencies -r /requirements.txt
+# --extra-index-url https://www.piwheels.org/simple is for cryptography module to be prebuilt (or rustc etc needs to be installed)
+RUN pip install --extra-index-url https://www.piwheels.org/simple --target=/dependencies -r /requirements.txt
# Playwright is an alternative to Selenium
# Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py
index e5c86e60..86f26da9 100644
--- a/changedetectionio/__init__.py
+++ b/changedetectionio/__init__.py
@@ -2,7 +2,7 @@
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
-__version__ = '0.45.24'
+__version__ = '0.45.25'
from changedetectionio.strtobool import strtobool
from json.decoder import JSONDecodeError
diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py
index c6014d6c..9b3eb440 100644
--- a/changedetectionio/api/api_v1.py
+++ b/changedetectionio/api/api_v1.py
@@ -171,23 +171,33 @@ class WatchSingleHistory(Resource):
curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
@apiName Get single snapshot content
@apiGroup Watch History
+ @apiParam {String} [html] Optional Set to =1 to return the last HTML (only stores last 2 snapshots, use `latest` as timestamp)
@apiSuccess (200) {String} OK
@apiSuccess (404) {String} ERR Not found
"""
watch = self.datastore.data['watching'].get(uuid)
if not watch:
- abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+ abort(404, message=f"No watch exists with the UUID of {uuid}")
if not len(watch.history):
- abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
+ abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
if timestamp == 'latest':
timestamp = list(watch.history.keys())[-1]
- content = watch.get_history_snapshot(timestamp)
+ if request.args.get('html'):
+ content = watch.get_fetched_html(timestamp)
+ if content:
+ response = make_response(content, 200)
+ response.mimetype = "text/html"
+ else:
+ response = make_response("No content found", 404)
+ response.mimetype = "text/plain"
+ else:
+ content = watch.get_history_snapshot(timestamp)
+ response = make_response(content, 200)
+ response.mimetype = "text/plain"
- response = make_response(content, 200)
- response.mimetype = "text/plain"
return response
diff --git a/changedetectionio/blueprint/browser_steps/__init__.py b/changedetectionio/blueprint/browser_steps/__init__.py
index 30797099..f92bf9f8 100644
--- a/changedetectionio/blueprint/browser_steps/__init__.py
+++ b/changedetectionio/blueprint/browser_steps/__init__.py
@@ -187,8 +187,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
if is_last_step and u:
(screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
- datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
- datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
+ watch = datastore.data['watching'].get(uuid)
+ if watch:
+ watch.save_screenshot(screenshot=screenshot)
+ watch.save_xpath_data(data=xpath_data)
# if not this_session.page:
# cleanup_playwright_session()
diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html
index 1d297c81..9f316c55 100644
--- a/changedetectionio/blueprint/tags/templates/edit-tag.html
+++ b/changedetectionio/blueprint/tags/templates/edit-tag.html
@@ -63,7 +63,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
JSONPath: Prefix with json:, use json:$ to force re-formatting if required, test your JSONPath here.
{% if jq_support %}
-
jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here.
+
jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here. Prefix jqraw: outputs the results as text instead of a JSON list.
{% else %}
jq support not installed
{% endif %}
diff --git a/changedetectionio/content_fetchers/res/stock-not-in-stock.js b/changedetectionio/content_fetchers/res/stock-not-in-stock.js
index b9529152..94c6350d 100644
--- a/changedetectionio/content_fetchers/res/stock-not-in-stock.js
+++ b/changedetectionio/content_fetchers/res/stock-not-in-stock.js
@@ -30,14 +30,21 @@ function isItemInStock() {
'dieser artikel ist bald wieder verfügbar',
'dostępne wkrótce',
'en rupture de stock',
- 'ist derzeit nicht auf lager',
+ 'isn\'t in stock right now',
+ 'isnt in stock right now',
+ 'isn’t in stock right now',
'item is no longer available',
'let me know when it\'s available',
+ 'mail me when available',
'message if back in stock',
'nachricht bei',
'nicht auf lager',
+ 'nicht lagernd',
'nicht lieferbar',
+ 'nicht verfügbar',
+ 'nicht vorrätig',
'nicht zur verfügung',
+ 'nie znaleziono produktów',
'niet beschikbaar',
'niet leverbaar',
'niet op voorraad',
@@ -48,6 +55,7 @@ function isItemInStock() {
'not currently available',
'not in stock',
'notify me when available',
+ 'notify me',
'notify when available',
'não estamos a aceitar encomendas',
'out of stock',
@@ -62,12 +70,16 @@ function isItemInStock() {
'this item is currently unavailable',
'tickets unavailable',
'tijdelijk uitverkocht',
+ 'unavailable nearby',
'unavailable tickets',
+ 'vergriffen',
+ 'vorbestellen',
'vorbestellung ist bald möglich',
'we couldn\'t find any products that match',
'we do not currently have an estimate of when this product will be back in stock.',
'we don\'t know when or if this item will be back in stock.',
'we were not able to find a match',
+ 'when this arrives in stock',
'zur zeit nicht an lager',
'品切れ',
'已售',
diff --git a/changedetectionio/content_fetchers/res/xpath_element_scraper.js b/changedetectionio/content_fetchers/res/xpath_element_scraper.js
index 326889ea..ac9ab095 100644
--- a/changedetectionio/content_fetchers/res/xpath_element_scraper.js
+++ b/changedetectionio/content_fetchers/res/xpath_element_scraper.js
@@ -182,6 +182,7 @@ visibleElementsArray.forEach(function (element) {
// Inject the current one set in the include_filters, which may be a CSS rule
// used for displaying the current one in VisualSelector, where its not one we generated.
if (include_filters.length) {
+ let results;
// Foreach filter, go and find it on the page and add it to the results so we can visualise it again
for (const f of include_filters) {
bbox = false;
@@ -197,10 +198,15 @@ if (include_filters.length) {
if (f.startsWith('/') || f.startsWith('xpath')) {
var qry_f = f.replace(/xpath(:|\d:)/, '')
console.log("[xpath] Scanning for included filter " + qry_f)
- q = document.evaluate(qry_f, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+ let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+ results = [];
+ for (let i = 0; i < xpathResult.snapshotLength; i++) {
+ results.push(xpathResult.snapshotItem(i));
+ }
} else {
console.log("[css] Scanning for included filter " + f)
- q = document.querySelector(f);
+ console.log("[css] Scanning for included filter " + f);
+ results = document.querySelectorAll(f);
}
} catch (e) {
// Maybe catch DOMException and alert?
@@ -208,44 +214,45 @@ if (include_filters.length) {
console.log(e);
}
- if (q) {
- // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
- try {
- if (typeof q.nodeName == 'string' && q.nodeName === '#text') {
- q = q.parentElement
- }
- } catch (e) {
- console.log(e)
- console.log("xpath_element_scraper: #text resolver")
- }
+ if (results.length) {
- // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
- if (typeof q.getBoundingClientRect == 'function') {
- bbox = q.getBoundingClientRect();
- console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
- } else {
+ // Iterate over the results
+ results.forEach(node => {
+ // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
try {
- // Try and see we can find its ownerElement
- bbox = q.ownerElement.getBoundingClientRect();
- console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+ if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
+ node = node.parentElement
+ }
} catch (e) {
console.log(e)
- console.log("xpath_element_scraper: error looking up q.ownerElement")
+ console.log("xpath_element_scraper: #text resolver")
}
- }
- }
- if (!q) {
- console.log("xpath_element_scraper: filter element " + f + " was not found");
- }
+ // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+ if (typeof node.getBoundingClientRect == 'function') {
+ bbox = node.getBoundingClientRect();
+ console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+ } else {
+ try {
+ // Try and see we can find its ownerElement
+ bbox = node.ownerElement.getBoundingClientRect();
+ console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+ } catch (e) {
+ console.log(e)
+ console.log("xpath_element_scraper: error looking up q.ownerElement")
+ }
+ }
- if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
- size_pos.push({
- xpath: f,
- width: parseInt(bbox['width']),
- height: parseInt(bbox['height']),
- left: parseInt(bbox['left']),
- top: parseInt(bbox['top']) + scroll_y
+ if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+ size_pos.push({
+ xpath: f,
+ width: parseInt(bbox['width']),
+ height: parseInt(bbox['height']),
+ left: parseInt(bbox['left']),
+ top: parseInt(bbox['top']) + scroll_y,
+ highlight_as_custom_filter: true
+ });
+ }
});
}
}
diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py
index d0f73a23..49a70196 100644
--- a/changedetectionio/flask_app.py
+++ b/changedetectionio/flask_app.py
@@ -731,7 +731,7 @@ def changedetection_app(config=None, datastore_o=None):
datastore.data['watching'][uuid].update(extra_update_obj)
if request.args.get('unpause_on_save'):
- flash("Updated watch - unpaused!.")
+ flash("Updated watch - unpaused!")
else:
flash("Updated watch.")
diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py
index 9b03ff71..c917a719 100644
--- a/changedetectionio/html_tools.py
+++ b/changedetectionio/html_tools.py
@@ -3,8 +3,6 @@ from bs4 import BeautifulSoup
from inscriptis import get_text
from jsonpath_ng.ext import parse
from typing import List
-from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
-from inscriptis.html_properties import Display
from inscriptis.model.config import ParserConfig
from xml.sax.saxutils import escape as xml_escape
import json
@@ -196,12 +194,12 @@ def extract_element(find='title', html_content=''):
#
def _parse_json(json_data, json_filter):
- if 'json:' in json_filter:
+ if json_filter.startswith("json:"):
jsonpath_expression = parse(json_filter.replace('json:', ''))
match = jsonpath_expression.find(json_data)
return _get_stripped_text_from_json_match(match)
- if 'jq:' in json_filter:
+ if json_filter.startswith("jq:") or json_filter.startswith("jqraw:"):
try:
import jq
@@ -209,10 +207,15 @@ def _parse_json(json_data, json_filter):
# `jq` requires full compilation in windows and so isn't generally available
raise Exception("jq not support not found")
- jq_expression = jq.compile(json_filter.replace('jq:', ''))
- match = jq_expression.input(json_data).all()
+ if json_filter.startswith("jq:"):
+ jq_expression = jq.compile(json_filter.removeprefix("jq:"))
+ match = jq_expression.input(json_data).all()
+ return _get_stripped_text_from_json_match(match)
- return _get_stripped_text_from_json_match(match)
+ if json_filter.startswith("jqraw:"):
+ jq_expression = jq.compile(json_filter.removeprefix("jqraw:"))
+ match = jq_expression.input(json_data).all()
+ return '\n'.join(str(item) for item in match)
def _get_stripped_text_from_json_match(match):
s = []
diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py
index 970e249d..b0dc371f 100644
--- a/changedetectionio/model/Watch.py
+++ b/changedetectionio/model/Watch.py
@@ -169,6 +169,8 @@ class model(watch_base):
if len(tmp_history):
self.__newest_history_key = list(tmp_history.keys())[-1]
+ else:
+ self.__newest_history_key = None
self.__history_n = len(tmp_history)
@@ -266,14 +268,9 @@ class model(watch_base):
def save_history_text(self, contents, timestamp, snapshot_id):
import brotli
- self.ensure_data_dir_exists()
+ logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
- # Small hack so that we sleep just enough to allow 1 second between history snapshots
- # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
- if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
- logger.warning(f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
- timestamp = str(int(timestamp) + 1)
- time.sleep(1)
+ self.ensure_data_dir_exists()
threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))
@@ -466,8 +463,42 @@ class model(watch_base):
# None is set
return False
+ def save_error_text(self, contents):
+ self.ensure_data_dir_exists()
+ target_path = os.path.join(self.watch_data_dir, "last-error.txt")
+ with open(target_path, 'w') as f:
+ f.write(contents)
+
+ def save_xpath_data(self, data, as_error=False):
+ import json
+
+ if as_error:
+ target_path = os.path.join(self.watch_data_dir, "elements-error.json")
+ else:
+ target_path = os.path.join(self.watch_data_dir, "elements.json")
+
+ self.ensure_data_dir_exists()
+
+ with open(target_path, 'w') as f:
+ f.write(json.dumps(data))
+ f.close()
+
+ # Save as PNG, PNG is larger but better for doing visual diff in the future
+ def save_screenshot(self, screenshot: bytes, as_error=False):
+
+ if as_error:
+ target_path = os.path.join(self.watch_data_dir, "last-error-screenshot.png")
+ else:
+ target_path = os.path.join(self.watch_data_dir, "last-screenshot.png")
+
+ self.ensure_data_dir_exists()
+
+ with open(target_path, 'wb') as f:
+ f.write(screenshot)
+ f.close()
- def get_last_fetched_before_filters(self):
+
+ def get_last_fetched_text_before_filters(self):
import brotli
filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
@@ -482,12 +513,56 @@ class model(watch_base):
with open(filepath, 'rb') as f:
return(brotli.decompress(f.read()).decode('utf-8'))
- def save_last_fetched_before_filters(self, contents):
+ def save_last_text_fetched_before_filters(self, contents):
import brotli
filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
with open(filepath, 'wb') as f:
f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
+ def save_last_fetched_html(self, timestamp, contents):
+ import brotli
+
+ self.ensure_data_dir_exists()
+ snapshot_fname = f"{timestamp}.html.br"
+ filepath = os.path.join(self.watch_data_dir, snapshot_fname)
+
+ with open(filepath, 'wb') as f:
+ contents = contents.encode('utf-8') if isinstance(contents, str) else contents
+ try:
+ f.write(brotli.compress(contents))
+ except Exception as e:
+ logger.warning(f"{self.get('uuid')} - Unable to compress snapshot, saving as raw data to {filepath}")
+ logger.warning(e)
+ f.write(contents)
+
+ self._prune_last_fetched_html_snapshots()
+
+ def get_fetched_html(self, timestamp):
+ import brotli
+
+ snapshot_fname = f"{timestamp}.html.br"
+ filepath = os.path.join(self.watch_data_dir, snapshot_fname)
+ if os.path.isfile(filepath):
+ with open(filepath, 'rb') as f:
+ return (brotli.decompress(f.read()).decode('utf-8'))
+
+ return False
+
+
+ def _prune_last_fetched_html_snapshots(self):
+
+ dates = list(self.history.keys())
+ dates.reverse()
+
+ for index, timestamp in enumerate(dates):
+ snapshot_fname = f"{timestamp}.html.br"
+ filepath = os.path.join(self.watch_data_dir, snapshot_fname)
+
+ # Keep only the first 2
+ if index > 1 and os.path.isfile(filepath):
+ os.remove(filepath)
+
+
@property
def get_browsersteps_available_screenshots(self):
"For knowing which screenshots are available to show the user in BrowserSteps UI"
diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py
index 9e4ce6b1..d24c9a9d 100644
--- a/changedetectionio/processors/__init__.py
+++ b/changedetectionio/processors/__init__.py
@@ -1,5 +1,6 @@
from abc import abstractmethod
from changedetectionio.strtobool import strtobool
+from changedetectionio.model import Watch
from copy import deepcopy
from loguru import logger
import hashlib
@@ -138,7 +139,7 @@ class difference_detection_processor():
# After init, call run_changedetection() which will do the actual change-detection
@abstractmethod
- def run_changedetection(self, uuid, skip_when_checksum_same=True):
+ def run_changedetection(self, watch: Watch, skip_when_checksum_same=True):
update_obj = {'last_notification_error': False, 'last_error': False}
some_data = 'xxxxx'
update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
diff --git a/changedetectionio/processors/restock_diff.py b/changedetectionio/processors/restock_diff.py
index 9aba81eb..4bed4938 100644
--- a/changedetectionio/processors/restock_diff.py
+++ b/changedetectionio/processors/restock_diff.py
@@ -1,6 +1,5 @@
from . import difference_detection_processor
from ..model import Restock
-from copy import deepcopy
from loguru import logger
import hashlib
import re
@@ -107,12 +106,7 @@ class perform_site_check(difference_detection_processor):
screenshot = None
xpath_data = None
-
- def run_changedetection(self, uuid, skip_when_checksum_same=True):
-
- # DeepCopy so we can be sure we don't accidently change anything by reference
- watch = deepcopy(self.datastore.data['watching'].get(uuid))
-
+ def run_changedetection(self, watch, skip_when_checksum_same=True):
if not watch:
raise Exception("Watch no longer exists.")
@@ -157,7 +151,14 @@ class perform_site_check(difference_detection_processor):
logger.debug(
f"Restock - using scraped browserdata - Watch UUID {uuid} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
- if not self.fetcher.instock_data:
+ # Main detection method
+ fetched_md5 = None
+ if self.fetcher.instock_data:
+ fetched_md5 = hashlib.md5(self.fetcher.instock_data.encode('utf-8')).hexdigest()
+ # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
+ update_obj["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
+ logger.debug(f"Watch UUID {watch.get('uuid')} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
+ else:
raise UnableToExtractRestockData(status_code=self.fetcher.status_code)
# Main detection method
@@ -165,7 +166,7 @@ class perform_site_check(difference_detection_processor):
# The main thing that all this at the moment comes down to :)
changed_detected = False
- logger.debug(f"Watch UUID {uuid} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+ logger.debug(f"Watch UUID {watch.get('uuid')} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
# out of stock -> back in stock only?
if watch.get('restock') and watch['restock'].get('in_stock') != update_obj['restock'].get('in_stock'):
diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py
index e89e469d..e793de89 100644
--- a/changedetectionio/processors/text_json_diff.py
+++ b/changedetectionio/processors/text_json_diff.py
@@ -10,18 +10,17 @@ from . import difference_detection_processor
from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
from changedetectionio import html_tools, content_fetchers
from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
-import changedetectionio.content_fetchers
-from copy import deepcopy
from loguru import logger
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
name = 'Webpage Text/HTML, JSON and PDF changes'
description = 'Detects all text changes where possible'
-json_filter_prefixes = ['json:', 'jq:']
+json_filter_prefixes = ['json:', 'jq:', 'jqraw:']
class FilterNotFoundInResponse(ValueError):
- def __init__(self, msg):
+ def __init__(self, msg, screenshot=None):
+ self.screenshot = screenshot
ValueError.__init__(self, msg)
@@ -34,14 +33,12 @@ class PDFToHTMLToolNotFound(ValueError):
# (set_proxy_from_list)
class perform_site_check(difference_detection_processor):
- def run_changedetection(self, uuid, skip_when_checksum_same=True):
+ def run_changedetection(self, watch, skip_when_checksum_same=True):
changed_detected = False
html_content = ""
screenshot = False # as bytes
stripped_text_from_html = ""
- # DeepCopy so we can be sure we don't accidently change anything by reference
- watch = deepcopy(self.datastore.data['watching'].get(uuid))
if not watch:
raise Exception("Watch no longer exists.")
@@ -116,12 +113,12 @@ class perform_site_check(difference_detection_processor):
# Better would be if Watch.model could access the global data also
# and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
# https://realpython.com/inherit-python-dict/ instead of doing it procedurely
- include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
+ include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='include_filters')
# 1845 - remove duplicated filters in both group and watch include filter
include_filters_rule = list(dict.fromkeys(watch.get('include_filters', []) + include_filters_from_tags))
- subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
+ subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='subtractive_selectors'),
*watch.get("subtractive_selectors", []),
*self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
]
@@ -188,7 +185,7 @@ class perform_site_check(difference_detection_processor):
append_pretty_line_formatting=not watch.is_source_type_url)
if not html_content.strip():
- raise FilterNotFoundInResponse(include_filters_rule)
+ raise FilterNotFoundInResponse(msg=include_filters_rule, screenshot=self.fetcher.screenshot)
if has_subtractive_selectors:
html_content = html_tools.element_removal(subtractive_selectors, html_content)
@@ -222,7 +219,7 @@ class perform_site_check(difference_detection_processor):
from .. import diff
# needs to not include (added) etc or it may get used twice
# Replace the processed text with the preferred result
- rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_before_filters(),
+ rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_text_before_filters(),
newest_version_file_contents=stripped_text_from_html,
include_equal=False, # not the same lines
include_added=watch.get('filter_text_added', True),
@@ -231,7 +228,7 @@ class perform_site_check(difference_detection_processor):
line_feed_sep="\n",
include_change_type_prefix=False)
- watch.save_last_fetched_before_filters(text_content_before_ignored_filter)
+ watch.save_last_text_fetched_before_filters(text_content_before_ignored_filter)
if not rendered_diff and stripped_text_from_html:
# We had some content, but no differences were found
@@ -344,17 +341,17 @@ class perform_site_check(difference_detection_processor):
if not watch['title'] or not len(watch['title']):
update_obj['title'] = html_tools.extract_element(find='title', html_content=self.fetcher.content)
- logger.debug(f"Watch UUID {uuid} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+ logger.debug(f"Watch UUID {watch.get('uuid')} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
if changed_detected:
if watch.get('check_unique_lines', False):
has_unique_lines = watch.lines_contain_something_unique_compared_to_history(lines=stripped_text_from_html.splitlines())
# One or more lines? unsure?
if not has_unique_lines:
- logger.debug(f"check_unique_lines: UUID {uuid} didnt have anything new setting change_detected=False")
+ logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} didnt have anything new setting change_detected=False")
changed_detected = False
else:
- logger.debug(f"check_unique_lines: UUID {uuid} had unique content")
+ logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
# Always record the new checksum
update_obj["previous_md5"] = fetched_md5
diff --git a/changedetectionio/static/js/browser-steps.js b/changedetectionio/static/js/browser-steps.js
index 4e576bd4..5c5fc52a 100644
--- a/changedetectionio/static/js/browser-steps.js
+++ b/changedetectionio/static/js/browser-steps.js
@@ -1,14 +1,5 @@
$(document).ready(function () {
- // duplicate
- var csrftoken = $('input[name=csrf_token]').val();
- $.ajaxSetup({
- beforeSend: function (xhr, settings) {
- if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
- xhr.setRequestHeader("X-CSRFToken", csrftoken)
- }
- }
- })
var browsersteps_session_id;
var browser_interface_seconds_remaining = 0;
var apply_buttons_disabled = false;
diff --git a/changedetectionio/static/js/csrf.js b/changedetectionio/static/js/csrf.js
new file mode 100644
index 00000000..4e2aca53
--- /dev/null
+++ b/changedetectionio/static/js/csrf.js
@@ -0,0 +1,10 @@
+$(document).ready(function () {
+ $.ajaxSetup({
+ beforeSend: function (xhr, settings) {
+ if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+ xhr.setRequestHeader("X-CSRFToken", csrftoken)
+ }
+ }
+ })
+});
+
diff --git a/changedetectionio/static/js/diff-overview.js b/changedetectionio/static/js/diff-overview.js
index 95e6dd7a..1f501529 100644
--- a/changedetectionio/static/js/diff-overview.js
+++ b/changedetectionio/static/js/diff-overview.js
@@ -1,13 +1,4 @@
$(document).ready(function () {
- var csrftoken = $('input[name=csrf_token]').val();
- $.ajaxSetup({
- beforeSend: function (xhr, settings) {
- if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
- xhr.setRequestHeader("X-CSRFToken", csrftoken)
- }
- }
- })
-
$('.needs-localtime').each(function () {
for (var option of this.options) {
var dateObject = new Date(option.value * 1000);
@@ -48,6 +39,12 @@ $(document).ready(function () {
$("#highlightSnippet").remove();
}
+ // Listen for Escape key press
+ window.addEventListener('keydown', function (e) {
+ if (e.key === 'Escape') {
+ clean();
+ }
+ }, false);
function dragTextHandler(event) {
console.log('mouseupped');
diff --git a/changedetectionio/static/js/notifications.js b/changedetectionio/static/js/notifications.js
index d3a0b81a..95f3eacf 100644
--- a/changedetectionio/static/js/notifications.js
+++ b/changedetectionio/static/js/notifications.js
@@ -13,16 +13,6 @@ $(document).ready(function() {
$('#send-test-notification').click(function (e) {
e.preventDefault();
- // this can be global
- var csrftoken = $('input[name=csrf_token]').val();
- $.ajaxSetup({
- beforeSend: function(xhr, settings) {
- if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
- xhr.setRequestHeader("X-CSRFToken", csrftoken)
- }
- }
- })
-
data = {
notification_body: $('#notification_body').val(),
notification_format: $('#notification_format').val(),
diff --git a/changedetectionio/static/js/visual-selector.js b/changedetectionio/static/js/visual-selector.js
index 9432ae9f..9cde7350 100644
--- a/changedetectionio/static/js/visual-selector.js
+++ b/changedetectionio/static/js/visual-selector.js
@@ -2,250 +2,239 @@
// All rights reserved.
// yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
-$(document).ready(function () {
-
- var current_selected_i;
- var state_clicked = false;
-
- var c;
-
- // greyed out fill context
- var xctx;
- // redline highlight context
- var ctx;
-
- var current_default_xpath = [];
- var x_scale = 1;
- var y_scale = 1;
- var selector_image;
- var selector_image_rect;
- var selector_data;
-
- $('#visualselector-tab').click(function () {
- $("img#selector-background").off('load');
- state_clicked = false;
- current_selected_i = false;
- bootstrap_visualselector();
+let runInClearMode = false;
+
+$(document).ready(() => {
+ let currentSelections = [];
+ let currentSelection = null;
+ let appendToList = false;
+ let c, xctx, ctx;
+ let xScale = 1, yScale = 1;
+ let selectorImage, selectorImageRect, selectorData;
+
+
+ // Global jQuery selectors with "Elem" appended
+ const $selectorCanvasElem = $('#selector-canvas');
+ const $includeFiltersElem = $("#include_filters");
+ const $selectorBackgroundElem = $("img#selector-background");
+ const $selectorCurrentXpathElem = $("#selector-current-xpath span");
+ const $fetchingUpdateNoticeElem = $('.fetching-update-notice');
+ const $selectorWrapperElem = $("#selector-wrapper");
+
+ // Color constants
+ const FILL_STYLE_HIGHLIGHT = 'rgba(205,0,0,0.35)';
+ const FILL_STYLE_GREYED_OUT = 'rgba(205,205,205,0.95)';
+ const STROKE_STYLE_HIGHLIGHT = 'rgba(255,0,0, 0.9)';
+ const FILL_STYLE_REDLINE = 'rgba(255,0,0, 0.1)';
+ const STROKE_STYLE_REDLINE = 'rgba(225,0,0,0.9)';
+
+ $('#visualselector-tab').click(() => {
+ $selectorBackgroundElem.off('load');
+ currentSelections = [];
+ bootstrapVisualSelector();
});
- $(document).on('keydown', function (event) {
- if ($("img#selector-background").is(":visible")) {
- if (event.key == "Escape") {
- state_clicked = false;
- ctx.clearRect(0, 0, c.width, c.height);
- }
+ function clearReset() {
+ ctx.clearRect(0, 0, c.width, c.height);
+ if ($includeFiltersElem.val().length) {
+ alert("Existing filters under the 'Filters & Triggers' tab were cleared.");
}
- });
+ $includeFiltersElem.val('');
+ currentSelections = [];
- // For when the page loads
- if (!window.location.hash || window.location.hash != '#visualselector') {
- $("img#selector-background").attr('src', '');
- return;
+ // Means we ignore the xpaths from the scraper marked as sel.highlight_as_custom_filter (it matched a previous selector)
+ runInClearMode = true;
+
+ highlightCurrentSelected();
+ }
+
+ function splitToList(v) {
+ return v.split('\n').map(line => line.trim()).filter(line => line.length > 0);
+ }
+
+ function sortScrapedElementsBySize() {
+ // Sort the currentSelections array by area (width * height) in descending order
+ selectorData['size_pos'].sort((a, b) => {
+ const areaA = a.width * a.height;
+ const areaB = b.width * b.height;
+ return areaB - areaA;
+ });
}
- // Handle clearing button/link
- $('#clear-selector').on('click', function (event) {
- if (!state_clicked) {
- alert('Oops, Nothing selected!');
+ $(document).on('keydown keyup', (event) => {
+ if (event.code === 'ShiftLeft' || event.code === 'ShiftRight') {
+ appendToList = event.type === 'keydown';
+ }
+
+ if (event.type === 'keydown') {
+ if ($selectorBackgroundElem.is(":visible") && event.key === "Escape") {
+ clearReset();
+ }
}
- state_clicked = false;
- ctx.clearRect(0, 0, c.width, c.height);
- xctx.clearRect(0, 0, c.width, c.height);
- $("#include_filters").val('');
});
+ $('#clear-selector').on('click', () => {
+ clearReset();
+ });
+ // So if they start switching between visualSelector and manual filters, stop it from rendering old filters
+ $('li.tab a').on('click', () => {
+ runInClearMode = true;
+ });
- bootstrap_visualselector();
+ if (!window.location.hash || window.location.hash !== '#visualselector') {
+ $selectorBackgroundElem.attr('src', '');
+ return;
+ }
+ bootstrapVisualSelector();
- function bootstrap_visualselector() {
- if (1) {
- // bootstrap it, this will trigger everything else
- $("img#selector-background").on("error", function () {
- $('.fetching-update-notice').html("Ooops! The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
- $('.fetching-update-notice').css('color','#bb0000');
- $('#selector-current-xpath').hide();
- $('#clear-selector').hide();
- }).bind('load', function () {
+ function bootstrapVisualSelector() {
+ $selectorBackgroundElem
+ .on("error", () => {
+ $fetchingUpdateNoticeElem.html("Ooops! The VisualSelector tool needs at least one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.")
+ .css('color', '#bb0000');
+ $('#selector-current-xpath, #clear-selector').hide();
+ })
+ .on('load', () => {
console.log("Loaded background...");
c = document.getElementById("selector-canvas");
- // greyed out fill context
xctx = c.getContext("2d");
- // redline highlight context
ctx = c.getContext("2d");
- if ($("#include_filters").val().trim().length) {
- current_default_xpath = $("#include_filters").val().split(/\r?\n/g);
- } else {
- current_default_xpath = [];
- }
- fetch_data();
- $('#selector-canvas').off("mousemove mousedown");
- // screenshot_url defined in the edit.html template
- }).attr("src", screenshot_url);
- }
- // Tell visualSelector that the image should update
- var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
- $("img#selector-background").attr('src', s)
+ fetchData();
+ $selectorCanvasElem.off("mousemove mousedown");
+ })
+ .attr("src", screenshot_url);
+
+ let s = `${$selectorBackgroundElem.attr('src')}?${new Date().getTime()}`;
+ $selectorBackgroundElem.attr('src', s);
}
- // This is fired once the img src is loaded in bootstrap_visualselector()
- function fetch_data() {
- // Image is ready
- $('.fetching-update-notice').html("Fetching element data..");
+ function fetchData() {
+ $fetchingUpdateNoticeElem.html("Fetching element data..");
$.ajax({
url: watch_visual_selector_data_url,
context: document.body
- }).done(function (data) {
- $('.fetching-update-notice').html("Rendering..");
- selector_data = data;
+ }).done((data) => {
+ $fetchingUpdateNoticeElem.html("Rendering..");
+ selectorData = data;
+ sortScrapedElementsBySize();
console.log("Reported browser width from backend: " + data['browser_width']);
- state_clicked = false;
- set_scale();
- reflow_selector();
- $('.fetching-update-notice').fadeOut();
+ setScale();
+ reflowSelector();
+ $fetchingUpdateNoticeElem.fadeOut();
});
+ }
+
+ function updateFiltersText() {
+ // Assuming currentSelections is already defined and contains the selections
+ let uniqueSelections = new Set(currentSelections.map(sel => (sel[0] === '/' ? `xpath:${sel.xpath}` : sel.xpath)));
+ // Convert the Set back to an array and join with newline characters
+ let textboxFilterText = Array.from(uniqueSelections).join("\n");
+
+ $includeFiltersElem.val(textboxFilterText);
}
+ function setScale() {
+ $selectorWrapperElem.show();
+ selectorImage = $selectorBackgroundElem[0];
+ selectorImageRect = selectorImage.getBoundingClientRect();
- function set_scale() {
+ $selectorCanvasElem.attr({
+ 'height': selectorImageRect.height,
+ 'width': selectorImageRect.width
+ });
+ $selectorWrapperElem.attr('width', selectorImageRect.width);
+ $('#visual-selector-heading').css('max-width', selectorImageRect.width + "px")
- // some things to check if the scaling doesnt work
- // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
- $("#selector-wrapper").show();
- selector_image = $("img#selector-background")[0];
- selector_image_rect = selector_image.getBoundingClientRect();
+ xScale = selectorImageRect.width / selectorImage.naturalWidth;
+ yScale = selectorImageRect.height / selectorImage.naturalHeight;
- // make the canvas the same size as the image
- $('#selector-canvas').attr('height', selector_image_rect.height);
- $('#selector-canvas').attr('width', selector_image_rect.width);
- $('#selector-wrapper').attr('width', selector_image_rect.width);
- x_scale = selector_image_rect.width / selector_data['browser_width'];
- y_scale = selector_image_rect.height / selector_image.naturalHeight;
- ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
- ctx.fillStyle = 'rgba(255,0,0, 0.1)';
+ ctx.strokeStyle = STROKE_STYLE_HIGHLIGHT;
+ ctx.fillStyle = FILL_STYLE_REDLINE;
ctx.lineWidth = 3;
- console.log("scaling set x: " + x_scale + " by y:" + y_scale);
- $("#selector-current-xpath").css('max-width', selector_image_rect.width);
+ console.log("Scaling set x: " + xScale + " by y:" + yScale);
+ $("#selector-current-xpath").css('max-width', selectorImageRect.width);
}
- function reflow_selector() {
- $(window).resize(function () {
- set_scale();
- highlight_current_selected_i();
+ function reflowSelector() {
+ $(window).resize(() => {
+ setScale();
+ highlightCurrentSelected();
});
- var selector_currnt_xpath_text = $("#selector-current-xpath span");
-
- set_scale();
-
- console.log(selector_data['size_pos'].length + " selectors found");
-
- // highlight the default one if we can find it in the xPath list
- // or the xpath matches the default one
- found = false;
- if (current_default_xpath.length) {
- // Find the first one that matches
- // @todo In the future paint all that match
- for (const c of current_default_xpath) {
- for (var i = selector_data['size_pos'].length; i !== 0; i--) {
- if (selector_data['size_pos'][i - 1].xpath.trim() === c.trim()) {
- console.log("highlighting " + c);
- current_selected_i = i - 1;
- highlight_current_selected_i();
- found = true;
- break;
- }
- }
- if (found) {
- break;
- }
- }
- if (!found) {
- alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
- }
- }
+ setScale();
+
+ console.log(selectorData['size_pos'].length + " selectors found");
+
+ let existingFilters = splitToList($includeFiltersElem.val());
- $('#selector-canvas').bind('mousemove', function (e) {
- if (state_clicked) {
- return;
+ selectorData['size_pos'].forEach(sel => {
+ if ((!runInClearMode && sel.highlight_as_custom_filter) || existingFilters.includes(sel.xpath)) {
+ console.log("highlighting " + c);
+ currentSelections.push(sel);
}
- ctx.clearRect(0, 0, c.width, c.height);
- current_selected_i = null;
+ });
+
+
+ highlightCurrentSelected();
+ updateFiltersText();
- // Add in offset
- if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
- var targetOffset = $(e.target).offset();
+ $selectorCanvasElem.bind('mousemove', handleMouseMove.debounce(5));
+ $selectorCanvasElem.bind('mousedown', handleMouseDown.debounce(5));
+ $selectorCanvasElem.bind('mouseleave', highlightCurrentSelected.debounce(5));
+
+ function handleMouseMove(e) {
+ if (!e.offsetX && !e.offsetY) {
+ const targetOffset = $(e.target).offset();
e.offsetX = e.pageX - targetOffset.left;
e.offsetY = e.pageY - targetOffset.top;
}
- // Reverse order - the most specific one should be deeper/"laster"
- // Basically, find the most 'deepest'
- var found = 0;
- ctx.fillStyle = 'rgba(205,0,0,0.35)';
- // Will be sorted by smallest width*height first
- for (var i = 0; i <= selector_data['size_pos'].length; i++) {
- // draw all of them? let them choose somehow?
- var sel = selector_data['size_pos'][i];
- // If we are in a bounding-box
- if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
- &&
- e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
-
- ) {
-
- // FOUND ONE
- set_current_selected_text(sel.xpath);
- ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
- ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-
- // no need to keep digging
- // @todo or, O to go out/up, I to go in
- // or double click to go up/out the selector?
- current_selected_i = i;
- found += 1;
- break;
+ ctx.fillStyle = FILL_STYLE_HIGHLIGHT;
+
+ selectorData['size_pos'].forEach(sel => {
+ if (e.offsetY > sel.top * yScale && e.offsetY < sel.top * yScale + sel.height * yScale &&
+ e.offsetX > sel.left * yScale && e.offsetX < sel.left * yScale + sel.width * yScale) {
+ setCurrentSelectedText(sel.xpath);
+ drawHighlight(sel);
+ currentSelections.push(sel);
+ currentSelection = sel;
+ highlightCurrentSelected();
+ currentSelections.pop();
}
- }
+ })
+ }
- }.debounce(5));
- function set_current_selected_text(s) {
- selector_currnt_xpath_text[0].innerHTML = s;
+ function setCurrentSelectedText(s) {
+ $selectorCurrentXpathElem[0].innerHTML = s;
}
- function highlight_current_selected_i() {
- if (state_clicked) {
- state_clicked = false;
- xctx.clearRect(0, 0, c.width, c.height);
- return;
- }
-
- var sel = selector_data['size_pos'][current_selected_i];
- if (sel[0] == '/') {
- // @todo - not sure just checking / is right
- $("#include_filters").val('xpath:' + sel.xpath);
- } else {
- $("#include_filters").val(sel.xpath);
- }
- xctx.fillStyle = 'rgba(205,205,205,0.95)';
- xctx.strokeStyle = 'rgba(225,0,0,0.9)';
- xctx.lineWidth = 3;
- xctx.fillRect(0, 0, c.width, c.height);
- // Clear out what only should be seen (make a clear/clean spot)
- xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
- xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
- state_clicked = true;
- set_current_selected_text(sel.xpath);
+ function drawHighlight(sel) {
+ ctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+ ctx.fillRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+ }
+ function handleMouseDown() {
+ // If we are in 'appendToList' mode, grow the list, if not, just 1
+ currentSelections = appendToList ? [...currentSelections, currentSelection] : [currentSelection];
+ highlightCurrentSelected();
+ updateFiltersText();
}
+ }
- $('#selector-canvas').bind('mousedown', function (e) {
- highlight_current_selected_i();
+ function highlightCurrentSelected() {
+ xctx.fillStyle = FILL_STYLE_GREYED_OUT;
+ xctx.strokeStyle = STROKE_STYLE_REDLINE;
+ xctx.lineWidth = 3;
+ xctx.clearRect(0, 0, c.width, c.height);
+
+ currentSelections.forEach(sel => {
+ //xctx.clearRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+ xctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
});
}
-
-});
\ No newline at end of file
+});
diff --git a/changedetectionio/static/styles/scss/parts/_visualselector.scss b/changedetectionio/static/styles/scss/parts/_visualselector.scss
index d0608c0c..17e8a659 100644
--- a/changedetectionio/static/styles/scss/parts/_visualselector.scss
+++ b/changedetectionio/static/styles/scss/parts/_visualselector.scss
@@ -1,6 +1,8 @@
#selector-wrapper {
height: 100%;
+ text-align: center;
+
max-height: 70vh;
overflow-y: scroll;
position: relative;
diff --git a/changedetectionio/static/styles/scss/styles.scss b/changedetectionio/static/styles/scss/styles.scss
index 633b1031..aa805285 100644
--- a/changedetectionio/static/styles/scss/styles.scss
+++ b/changedetectionio/static/styles/scss/styles.scss
@@ -676,14 +676,25 @@ footer {
and also iPads specifically.
*/
.watch-table {
+ /* make headings work on mobile */
+ thead {
+ display: block;
+ tr {
+ th {
+ display: inline-block;
+ }
+ }
+ .empty-cell {
+ display: none;
+ }
+ }
/* Force table to not be like tables anymore */
- thead,
- tbody,
- th,
- td,
- tr {
- display: block;
+ tbody {
+ td,
+ tr {
+ display: block;
+ }
}
.last-checked {
@@ -707,13 +718,6 @@ footer {
display: inline-block;
}
- /* Hide table headers (but not display: none;, for accessibility) */
- thead tr {
- position: absolute;
- top: -9999px;
- left: -9999px;
- }
-
.pure-table td,
.pure-table th {
border: none;
@@ -758,6 +762,7 @@ footer {
thead {
background-color: var(--color-background-table-thead);
color: var(--color-text);
+ border-bottom: 1px solid var(--color-background-table-thead);
}
td,
diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css
index afddd278..b9469fd0 100644
--- a/changedetectionio/static/styles/styles.css
+++ b/changedetectionio/static/styles/styles.css
@@ -866,14 +866,17 @@ footer {
and also iPads specifically.
*/
.watch-table {
+ /* make headings work on mobile */
/* Force table to not be like tables anymore */
- /* Force table to not be like tables anymore */
- /* Hide table headers (but not display: none;, for accessibility) */ }
- .watch-table thead,
- .watch-table tbody,
- .watch-table th,
- .watch-table td,
- .watch-table tr {
+ /* Force table to not be like tables anymore */ }
+ .watch-table thead {
+ display: block; }
+ .watch-table thead tr th {
+ display: inline-block; }
+ .watch-table thead .empty-cell {
+ display: none; }
+ .watch-table tbody td,
+ .watch-table tbody tr {
display: block; }
.watch-table .last-checked > span {
vertical-align: middle; }
@@ -885,10 +888,6 @@ footer {
content: "Last Changed "; }
.watch-table td.inline {
display: inline-block; }
- .watch-table thead tr {
- position: absolute;
- top: -9999px;
- left: -9999px; }
.watch-table .pure-table td,
.watch-table .pure-table th {
border: none; }
@@ -915,7 +914,8 @@ footer {
border-color: var(--color-border-table-cell); }
.pure-table thead {
background-color: var(--color-background-table-thead);
- color: var(--color-text); }
+ color: var(--color-text);
+ border-bottom: 1px solid var(--color-background-table-thead); }
.pure-table td,
.pure-table th {
border-left-color: var(--color-border-table-cell); }
@@ -1068,6 +1068,7 @@ ul {
#selector-wrapper {
height: 100%;
+ text-align: center;
max-height: 70vh;
overflow-y: scroll;
position: relative; }
diff --git a/changedetectionio/store.py b/changedetectionio/store.py
index b9a69efe..ac33c2c1 100644
--- a/changedetectionio/store.py
+++ b/changedetectionio/store.py
@@ -163,7 +163,6 @@ class ChangeDetectionStore:
del (update_obj[dict_key])
self.__data['watching'][uuid].update(update_obj)
-
self.needs_write = True
@property
@@ -243,6 +242,15 @@ class ChangeDetectionStore:
def clear_watch_history(self, uuid):
import pathlib
from .model import Restock
+
+ # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
+ for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
+ unlink(item)
+
+ # Force the attr to recalculate
+ bump = self.__data['watching'][uuid].history
+
+ # Do this last because it will trigger a recheck due to last_checked being zero
self.__data['watching'][uuid].update({
'browser_steps_last_error_step' : None,
'check_count': 0,
@@ -260,13 +268,6 @@ class ChangeDetectionStore:
'restock': Restock()
})
- # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
- for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
- unlink(item)
-
- # Force the attr to recalculate
- bump = self.__data['watching'][uuid].history
-
self.needs_write_urgent = True
def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
@@ -377,46 +378,6 @@ class ChangeDetectionStore:
return False
- # Save as PNG, PNG is larger but better for doing visual diff in the future
- def save_screenshot(self, watch_uuid, screenshot: bytes, as_error=False):
- if not self.data['watching'].get(watch_uuid):
- return
-
- if as_error:
- target_path = os.path.join(self.datastore_path, watch_uuid, "last-error-screenshot.png")
- else:
- target_path = os.path.join(self.datastore_path, watch_uuid, "last-screenshot.png")
-
- self.data['watching'][watch_uuid].ensure_data_dir_exists()
-
- with open(target_path, 'wb') as f:
- f.write(screenshot)
- f.close()
-
-
- def save_error_text(self, watch_uuid, contents):
- if not self.data['watching'].get(watch_uuid):
- return
-
- self.data['watching'][watch_uuid].ensure_data_dir_exists()
- target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
- with open(target_path, 'w') as f:
- f.write(contents)
-
- def save_xpath_data(self, watch_uuid, data, as_error=False):
-
- if not self.data['watching'].get(watch_uuid):
- return
- if as_error:
- target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
- else:
- target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
- self.data['watching'][watch_uuid].ensure_data_dir_exists()
- with open(target_path, 'w') as f:
- f.write(json.dumps(data))
- f.close()
-
-
def sync_to_json(self):
logger.info("Saving JSON..")
try:
@@ -892,3 +853,8 @@ class ChangeDetectionStore:
# Something custom here
self.__data["watching"][uuid]['time_between_check_use_default'] = False
+ # Correctly set datatype for older installs where 'tag' was string and update_12 did not catch it
+ def update_16(self):
+ for uuid, watch in self.data['watching'].items():
+ if isinstance(watch.get('tags'), str):
+ self.data['watching'][uuid]['tags'] = []
diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html
index 87018a7d..27ce8419 100644
--- a/changedetectionio/templates/base.html
+++ b/changedetectionio/templates/base.html
@@ -26,7 +26,11 @@
+
+
diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html
index e4bb243b..3baf4603 100644
--- a/changedetectionio/templates/edit.html
+++ b/changedetectionio/templates/edit.html
@@ -290,7 +290,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
JSONPath: Prefix with json:, use json:$ to force re-formatting if required, test your JSONPath here.
{% if jq_support %}
-
jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here.
+
jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here. Prefix jqraw: outputs the results as text instead of a JSON list.