From 90b357f457747114b51e8039f59c1f0e273c9bfd Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 20 Jun 2024 14:42:17 +0200 Subject: [PATCH 01/19] Upgrade to Python 3.11 from 3.10, add faster prebuilt "wheels" for rPi devices, upgrade cryptography security library --- Dockerfile | 5 +++-- requirements.txt | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5e45880c..f5e061b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py # If you know how to fix it, please do! and test it for both 3.10 and 3.11 -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.11 FROM python:${PYTHON_VERSION}-slim-bookworm as builder @@ -26,7 +26,8 @@ WORKDIR /install COPY requirements.txt /requirements.txt -RUN pip install --target=/dependencies -r /requirements.txt +# --extra-index-url https://www.piwheels.org/simple is for cryptography module to be prebuilt (or rustc etc needs to be installed) +RUN pip install --extra-index-url https://www.piwheels.org/simple --target=/dependencies -r /requirements.txt # Playwright is an alternative to Selenium # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing diff --git a/requirements.txt b/requirements.txt index 3f5f0c50..ae099de4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,10 +41,8 @@ apprise~=1.8.0 # use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814 paho-mqtt>=1.6.1,<2.0.0 -# This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1" -# so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found" -# (introduced once apprise became a dep) -cryptography~=3.4 +# Requires extra wheel for rPi +cryptography~=42.0.8 # Used for CSS filtering beautifulsoup4 From d31fc860cc582b3a02569975777480f4b30baf47 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 20 Jun 2024 15:07:17 +0200 Subject: [PATCH 02/19] Build - fixing build warnings --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f5e061b7..626759cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG PYTHON_VERSION=3.11 -FROM python:${PYTHON_VERSION}-slim-bookworm as builder +FROM python:${PYTHON_VERSION}-slim-bookworm AS builder # See `cryptography` pin comment in requirements.txt ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1 From ffd160ce0eb25597f704000562213ba98ae0fdaf Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 21 Jun 2024 17:01:03 +0530 Subject: [PATCH 03/19] Filters - Implement jqraw: filter (use this to output nicer JSON format when selecting/filtering by JSON) (#2430) --- .../blueprint/tags/templates/edit-tag.html | 2 +- changedetectionio/html_tools.py | 17 +++++---- .../processors/text_json_diff.py | 2 +- changedetectionio/templates/edit.html | 2 +- .../tests/test_jsonpath_jq_selector.py | 36 +++++++++++++++---- 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index 1d297c81..9f316c55 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -63,7 +63,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
  • JSONPath: Prefix with json:, use json:$ to force re-formatting if required, test your JSONPath here.
    • {% if jq_support %} -
  • jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here.
    • +
  • jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here. Prefix jqraw: outputs the results as text instead of a JSON list.
    • {% else %}
  • jq support not installed
    • {% endif %} diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index a03653b9..ff5cedad 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -3,8 +3,6 @@ from bs4 import BeautifulSoup from inscriptis import get_text from jsonpath_ng.ext import parse from typing import List -from inscriptis.css_profiles import CSS_PROFILES, HtmlElement -from inscriptis.html_properties import Display from inscriptis.model.config import ParserConfig from xml.sax.saxutils import escape as xml_escape import json @@ -196,12 +194,12 @@ def extract_element(find='title', html_content=''): # def _parse_json(json_data, json_filter): - if 'json:' in json_filter: + if json_filter.startswith("json:"): jsonpath_expression = parse(json_filter.replace('json:', '')) match = jsonpath_expression.find(json_data) return _get_stripped_text_from_json_match(match) - if 'jq:' in json_filter: + if json_filter.startswith("jq:") or json_filter.startswith("jqraw:"): try: import jq @@ -209,10 +207,15 @@ def _parse_json(json_data, json_filter): # `jq` requires full compilation in windows and so isn't generally available raise Exception("jq not support not found") - jq_expression = jq.compile(json_filter.replace('jq:', '')) - match = jq_expression.input(json_data).all() + if json_filter.startswith("jq:"): + jq_expression = jq.compile(json_filter.removeprefix("jq:")) + match = jq_expression.input(json_data).all() + return _get_stripped_text_from_json_match(match) - return _get_stripped_text_from_json_match(match) + if json_filter.startswith("jqraw:"): + jq_expression = jq.compile(json_filter.removeprefix("jqraw:")) + match = jq_expression.input(json_data).all() + return '\n'.join(str(item) for item in match) def _get_stripped_text_from_json_match(match): s = [] diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index e89e469d..1d60be63 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -18,7 +18,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) name = 'Webpage Text/HTML, JSON and PDF changes' description = 'Detects all text changes where possible' -json_filter_prefixes = ['json:', 'jq:'] +json_filter_prefixes = ['json:', 'jq:', 'jqraw:'] class FilterNotFoundInResponse(ValueError): def __init__(self, msg): diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 6d6d19fc..15cb3fd5 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -292,7 +292,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
    • JSONPath: Prefix with json:, use json:$ to force re-formatting if required, test your JSONPath here.
      • {% if jq_support %} -
    • jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here.
      • +
    • jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here. Prefix jqraw: outputs the results as text instead of a JSON list.
      • {% else %}
    • jq support not installed
      • {% endif %} diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index 1202849f..55f46a0d 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -41,19 +41,26 @@ and it can also be repeated from .. import html_tools # See that we can find the second + diff --git a/changedetectionio/tests/test_ignorehighlighter.py b/changedetectionio/tests/test_ignorehighlighter.py index 88bd0af6..52163845 100644 --- a/changedetectionio/tests/test_ignorehighlighter.py +++ b/changedetectionio/tests/test_ignorehighlighter.py @@ -45,7 +45,6 @@ def test_highlight_ignore(client, live_server): ) res = client.get(url_for("edit_page", uuid=uuid)) - # should be a regex now assert b'/oh\ yeah\ \d+/' in res.data @@ -55,3 +54,7 @@ def test_highlight_ignore(client, live_server): # And it should register in the preview page res = client.get(url_for("preview_page", uuid=uuid)) assert b'
      oh yeah 456' in res.data + + # Should be in base.html + assert b'csrftoken' in res.data + diff --git a/requirements.txt b/requirements.txt index ae099de4..44049fb2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -83,4 +83,4 @@ jsonschema==4.17.3 loguru # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096 -greenlet >= 3.0.3 \ No newline at end of file +greenlet >= 3.0.3 From bed16009bb3d028b4e2b70459cbb41b972614355 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 3 Jul 2024 19:27:23 +0200 Subject: [PATCH 14/19] 0.45.25 --- changedetectionio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index e5c86e60..86f26da9 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.45.24' +__version__ = '0.45.25' from changedetectionio.strtobool import strtobool from json.decoder import JSONDecodeError From 01f910f840616124c93f05450cb266be1941e844 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 4 Jul 2024 15:23:06 +0200 Subject: [PATCH 15/19] Fixing 'tags'' field from old installs (0.43.0+) could have wrong data-type causing crash --- changedetectionio/store.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changedetectionio/store.py b/changedetectionio/store.py index d4a6cb0f..284f3767 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -843,3 +843,8 @@ class ChangeDetectionStore: # Something custom here self.__data["watching"][uuid]['time_between_check_use_default'] = False + # Correctly set datatype for older installs where 'tag' was string and update_12 did not catch it + def update_16(self): + for uuid, watch in self.data['watching'].items(): + if isinstance(watch.get('tags'), str): + self.data['watching'][uuid]['tags'] = [] From a1d04bb37fc1f2210d7ba086ce7cd11f28e64103 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 5 Jul 2024 11:09:31 +0200 Subject: [PATCH 16/19] Snapshot count from history was not updated in watch after using [clear history] (#2459) --- changedetectionio/model/Watch.py | 2 ++ changedetectionio/store.py | 15 ++++++++------- changedetectionio/tests/test_backend.py | 11 ++++++++++- changedetectionio/update_worker.py | 3 +-- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 44157268..553f6227 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -238,6 +238,8 @@ class model(dict): if len(tmp_history): self.__newest_history_key = list(tmp_history.keys())[-1] + else: + self.__newest_history_key = None self.__history_n = len(tmp_history) diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 284f3767..5967091b 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -242,6 +242,14 @@ class ChangeDetectionStore: def clear_watch_history(self, uuid): import pathlib + # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc + for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"): + unlink(item) + + # Force the attr to recalculate + bump = self.__data['watching'][uuid].history + + # Do this last because it will trigger a recheck due to last_checked being zero self.__data['watching'][uuid].update({ 'browser_steps_last_error_step' : None, 'check_count': 0, @@ -258,13 +266,6 @@ class ChangeDetectionStore: 'track_ldjson_price_data': None, }) - # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc - for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"): - unlink(item) - - # Force the attr to recalculate - bump = self.__data['watching'][uuid].history - self.needs_write_urgent = True def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True): diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index 1e1c6496..a6f735b5 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -3,7 +3,8 @@ import time from flask import url_for from urllib.request import urlopen -from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI +from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \ + extract_UUID_from_client sleep_time_for_fetch_thread = 3 @@ -141,6 +142,14 @@ def test_check_basic_change_detection_functionality(client, live_server): assert b'Mark all viewed' not in res.data assert b'unviewed' not in res.data + # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again + uuid = extract_UUID_from_client(client) + client.get(url_for("clear_watch_history", uuid=uuid)) + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + res = client.get(url_for("index")) + assert b'preview/' in res.data + # # Cleanup everything res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 48743f2c..daa0f788 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -250,8 +250,7 @@ class update_worker(threading.Thread): # Clear last errors (move to preflight func?) self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None - # DeepCopy so we can be sure we don't accidently change anything by reference - watch = deepcopy(self.datastore.data['watching'].get(uuid)) + watch = self.datastore.data['watching'].get(uuid) logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}") now = time.time() From 09bc24ff34000e11b6a83b5783fad2a8e22b7c2e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 5 Jul 2024 14:33:13 +0200 Subject: [PATCH 17/19] UI - Visual Selector graphics should be centred --- changedetectionio/static/styles/scss/parts/_visualselector.scss | 2 ++ changedetectionio/static/styles/styles.css | 1 + 2 files changed, 3 insertions(+) diff --git a/changedetectionio/static/styles/scss/parts/_visualselector.scss b/changedetectionio/static/styles/scss/parts/_visualselector.scss index d0608c0c..17e8a659 100644 --- a/changedetectionio/static/styles/scss/parts/_visualselector.scss +++ b/changedetectionio/static/styles/scss/parts/_visualselector.scss @@ -1,6 +1,8 @@ #selector-wrapper { height: 100%; + text-align: center; + max-height: 70vh; overflow-y: scroll; position: relative; diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index f407f40b..da60835b 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -1065,6 +1065,7 @@ ul { #selector-wrapper { height: 100%; + text-align: center; max-height: 70vh; overflow-y: scroll; position: relative; } From e09ee7da97743bc8bdf2e5d582d9d71c2979749f Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 5 Jul 2024 15:20:39 +0200 Subject: [PATCH 18/19] UI - Visual Selector - Show/visualise all/any matching filter elements from all filters in "CSS/JSONPath/JQ/XPath Filters" include filters (#2440) --- .../res/xpath_element_scraper.js | 73 ++++++++++--------- .../static/js/visual-selector.js | 51 ++++++++----- 2 files changed, 74 insertions(+), 50 deletions(-) diff --git a/changedetectionio/content_fetchers/res/xpath_element_scraper.js b/changedetectionio/content_fetchers/res/xpath_element_scraper.js index 326889ea..ac9ab095 100644 --- a/changedetectionio/content_fetchers/res/xpath_element_scraper.js +++ b/changedetectionio/content_fetchers/res/xpath_element_scraper.js @@ -182,6 +182,7 @@ visibleElementsArray.forEach(function (element) { // Inject the current one set in the include_filters, which may be a CSS rule // used for displaying the current one in VisualSelector, where its not one we generated. if (include_filters.length) { + let results; // Foreach filter, go and find it on the page and add it to the results so we can visualise it again for (const f of include_filters) { bbox = false; @@ -197,10 +198,15 @@ if (include_filters.length) { if (f.startsWith('/') || f.startsWith('xpath')) { var qry_f = f.replace(/xpath(:|\d:)/, '') console.log("[xpath] Scanning for included filter " + qry_f) - q = document.evaluate(qry_f, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null); + results = []; + for (let i = 0; i < xpathResult.snapshotLength; i++) { + results.push(xpathResult.snapshotItem(i)); + } } else { console.log("[css] Scanning for included filter " + f) - q = document.querySelector(f); + console.log("[css] Scanning for included filter " + f); + results = document.querySelectorAll(f); } } catch (e) { // Maybe catch DOMException and alert? @@ -208,44 +214,45 @@ if (include_filters.length) { console.log(e); } - if (q) { - // Try to resolve //something/text() back to its /something so we can atleast get the bounding box - try { - if (typeof q.nodeName == 'string' && q.nodeName === '#text') { - q = q.parentElement - } - } catch (e) { - console.log(e) - console.log("xpath_element_scraper: #text resolver") - } + if (results.length) { - // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element. - if (typeof q.getBoundingClientRect == 'function') { - bbox = q.getBoundingClientRect(); - console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y) - } else { + // Iterate over the results + results.forEach(node => { + // Try to resolve //something/text() back to its /something so we can atleast get the bounding box try { - // Try and see we can find its ownerElement - bbox = q.ownerElement.getBoundingClientRect(); - console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y) + if (typeof node.nodeName == 'string' && node.nodeName === '#text') { + node = node.parentElement + } } catch (e) { console.log(e) - console.log("xpath_element_scraper: error looking up q.ownerElement") + console.log("xpath_element_scraper: #text resolver") } - } - } - if (!q) { - console.log("xpath_element_scraper: filter element " + f + " was not found"); - } + // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element. + if (typeof node.getBoundingClientRect == 'function') { + bbox = node.getBoundingClientRect(); + console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y) + } else { + try { + // Try and see we can find its ownerElement + bbox = node.ownerElement.getBoundingClientRect(); + console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y) + } catch (e) { + console.log(e) + console.log("xpath_element_scraper: error looking up q.ownerElement") + } + } - if (bbox && bbox['width'] > 0 && bbox['height'] > 0) { - size_pos.push({ - xpath: f, - width: parseInt(bbox['width']), - height: parseInt(bbox['height']), - left: parseInt(bbox['left']), - top: parseInt(bbox['top']) + scroll_y + if (bbox && bbox['width'] > 0 && bbox['height'] > 0) { + size_pos.push({ + xpath: f, + width: parseInt(bbox['width']), + height: parseInt(bbox['height']), + left: parseInt(bbox['left']), + top: parseInt(bbox['top']) + scroll_y, + highlight_as_custom_filter: true + }); + } }); } } diff --git a/changedetectionio/static/js/visual-selector.js b/changedetectionio/static/js/visual-selector.js index 9432ae9f..24d1ee18 100644 --- a/changedetectionio/static/js/visual-selector.js +++ b/changedetectionio/static/js/visual-selector.js @@ -28,32 +28,34 @@ $(document).ready(function () { bootstrap_visualselector(); }); + function clear_reset() { + state_clicked = false; + ctx.clearRect(0, 0, c.width, c.height); + if($("#include_filters").val().length) { + alert("Existing filters under the 'Filters & Triggers' tab were cleared."); + } + $("#include_filters").val(''); + } + $(document).on('keydown', function (event) { if ($("img#selector-background").is(":visible")) { if (event.key == "Escape") { - state_clicked = false; - ctx.clearRect(0, 0, c.width, c.height); + clear_reset(); } } }); + // Handle clearing button/link + $('#clear-selector').on('click', function (event) { + clear_reset(); + }); + // For when the page loads if (!window.location.hash || window.location.hash != '#visualselector') { $("img#selector-background").attr('src', ''); return; } - // Handle clearing button/link - $('#clear-selector').on('click', function (event) { - if (!state_clicked) { - alert('Oops, Nothing selected!'); - } - state_clicked = false; - ctx.clearRect(0, 0, c.width, c.height); - xctx.clearRect(0, 0, c.width, c.height); - $("#include_filters").val(''); - }); - bootstrap_visualselector(); @@ -117,12 +119,13 @@ $(document).ready(function () { selector_image = $("img#selector-background")[0]; selector_image_rect = selector_image.getBoundingClientRect(); - // make the canvas the same size as the image - $('#selector-canvas').attr('height', selector_image_rect.height); - $('#selector-canvas').attr('width', selector_image_rect.width); + // Make the overlayed canvas the same size as the image + $('#selector-canvas').attr('height', selector_image_rect.height).attr('width', selector_image_rect.width); $('#selector-wrapper').attr('width', selector_image_rect.width); - x_scale = selector_image_rect.width / selector_data['browser_width']; + + x_scale = selector_image_rect.width / selector_image.naturalWidth; y_scale = selector_image_rect.height / selector_image.naturalHeight; + ctx.strokeStyle = 'rgba(255,0,0, 0.9)'; ctx.fillStyle = 'rgba(255,0,0, 0.1)'; ctx.lineWidth = 3; @@ -135,6 +138,7 @@ $(document).ready(function () { set_scale(); highlight_current_selected_i(); }); + var selector_currnt_xpath_text = $("#selector-current-xpath span"); set_scale(); @@ -164,6 +168,7 @@ $(document).ready(function () { if (!found) { alert("Unfortunately your existing CSS/xPath Filter was no longer found!"); } + highlight_matching_filters(); } @@ -243,6 +248,18 @@ $(document).ready(function () { } + function highlight_matching_filters() { + selector_data['size_pos'].forEach(sel => { + if (sel.highlight_as_custom_filter) { + xctx.fillStyle = 'rgba(205,205,205,0.95)'; + xctx.strokeStyle = 'rgba(225,0,0,0.95)'; + xctx.lineWidth = 1; + xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); + xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); + } + }); + } + $('#selector-canvas').bind('mousedown', function (e) { highlight_current_selected_i(); }); From 1af342ef6448b5b177f2527ed8e7a2357a1b3dab Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 5 Jul 2024 20:43:26 +0200 Subject: [PATCH 19/19] UI - Visual Selector now supports Shift+Click for multiple selections! --- .../static/js/visual-selector.js | 380 ++++++++---------- changedetectionio/templates/edit.html | 5 +- 2 files changed, 178 insertions(+), 207 deletions(-) diff --git a/changedetectionio/static/js/visual-selector.js b/changedetectionio/static/js/visual-selector.js index 24d1ee18..9cde7350 100644 --- a/changedetectionio/static/js/visual-selector.js +++ b/changedetectionio/static/js/visual-selector.js @@ -2,267 +2,239 @@ // All rights reserved. // yes - this is really a hack, if you are a front-ender and want to help, please get in touch! -$(document).ready(function () { - - var current_selected_i; - var state_clicked = false; - - var c; - - // greyed out fill context - var xctx; - // redline highlight context - var ctx; - - var current_default_xpath = []; - var x_scale = 1; - var y_scale = 1; - var selector_image; - var selector_image_rect; - var selector_data; - - $('#visualselector-tab').click(function () { - $("img#selector-background").off('load'); - state_clicked = false; - current_selected_i = false; - bootstrap_visualselector(); +let runInClearMode = false; + +$(document).ready(() => { + let currentSelections = []; + let currentSelection = null; + let appendToList = false; + let c, xctx, ctx; + let xScale = 1, yScale = 1; + let selectorImage, selectorImageRect, selectorData; + + + // Global jQuery selectors with "Elem" appended + const $selectorCanvasElem = $('#selector-canvas'); + const $includeFiltersElem = $("#include_filters"); + const $selectorBackgroundElem = $("img#selector-background"); + const $selectorCurrentXpathElem = $("#selector-current-xpath span"); + const $fetchingUpdateNoticeElem = $('.fetching-update-notice'); + const $selectorWrapperElem = $("#selector-wrapper"); + + // Color constants + const FILL_STYLE_HIGHLIGHT = 'rgba(205,0,0,0.35)'; + const FILL_STYLE_GREYED_OUT = 'rgba(205,205,205,0.95)'; + const STROKE_STYLE_HIGHLIGHT = 'rgba(255,0,0, 0.9)'; + const FILL_STYLE_REDLINE = 'rgba(255,0,0, 0.1)'; + const STROKE_STYLE_REDLINE = 'rgba(225,0,0,0.9)'; + + $('#visualselector-tab').click(() => { + $selectorBackgroundElem.off('load'); + currentSelections = []; + bootstrapVisualSelector(); }); - function clear_reset() { - state_clicked = false; + function clearReset() { ctx.clearRect(0, 0, c.width, c.height); - if($("#include_filters").val().length) { + if ($includeFiltersElem.val().length) { alert("Existing filters under the 'Filters & Triggers' tab were cleared."); } - $("#include_filters").val(''); + $includeFiltersElem.val(''); + currentSelections = []; + + // Means we ignore the xpaths from the scraper marked as sel.highlight_as_custom_filter (it matched a previous selector) + runInClearMode = true; + + highlightCurrentSelected(); + } + + function splitToList(v) { + return v.split('\n').map(line => line.trim()).filter(line => line.length > 0); + } + + function sortScrapedElementsBySize() { + // Sort the currentSelections array by area (width * height) in descending order + selectorData['size_pos'].sort((a, b) => { + const areaA = a.width * a.height; + const areaB = b.width * b.height; + return areaB - areaA; + }); } - $(document).on('keydown', function (event) { - if ($("img#selector-background").is(":visible")) { - if (event.key == "Escape") { - clear_reset(); + $(document).on('keydown keyup', (event) => { + if (event.code === 'ShiftLeft' || event.code === 'ShiftRight') { + appendToList = event.type === 'keydown'; + } + + if (event.type === 'keydown') { + if ($selectorBackgroundElem.is(":visible") && event.key === "Escape") { + clearReset(); } } }); - // Handle clearing button/link - $('#clear-selector').on('click', function (event) { - clear_reset(); + $('#clear-selector').on('click', () => { + clearReset(); + }); + // So if they start switching between visualSelector and manual filters, stop it from rendering old filters + $('li.tab a').on('click', () => { + runInClearMode = true; }); - // For when the page loads - if (!window.location.hash || window.location.hash != '#visualselector') { - $("img#selector-background").attr('src', ''); + if (!window.location.hash || window.location.hash !== '#visualselector') { + $selectorBackgroundElem.attr('src', ''); return; } + bootstrapVisualSelector(); - bootstrap_visualselector(); - - - function bootstrap_visualselector() { - if (1) { - // bootstrap it, this will trigger everything else - $("img#selector-background").on("error", function () { - $('.fetching-update-notice').html("Ooops! The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page."); - $('.fetching-update-notice').css('color','#bb0000'); - $('#selector-current-xpath').hide(); - $('#clear-selector').hide(); - }).bind('load', function () { + function bootstrapVisualSelector() { + $selectorBackgroundElem + .on("error", () => { + $fetchingUpdateNoticeElem.html("Ooops! The VisualSelector tool needs at least one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.") + .css('color', '#bb0000'); + $('#selector-current-xpath, #clear-selector').hide(); + }) + .on('load', () => { console.log("Loaded background..."); c = document.getElementById("selector-canvas"); - // greyed out fill context xctx = c.getContext("2d"); - // redline highlight context ctx = c.getContext("2d"); - if ($("#include_filters").val().trim().length) { - current_default_xpath = $("#include_filters").val().split(/\r?\n/g); - } else { - current_default_xpath = []; - } - fetch_data(); - $('#selector-canvas').off("mousemove mousedown"); - // screenshot_url defined in the edit.html template - }).attr("src", screenshot_url); - } - // Tell visualSelector that the image should update - var s = $("img#selector-background").attr('src') + "?" + new Date().getTime(); - $("img#selector-background").attr('src', s) + fetchData(); + $selectorCanvasElem.off("mousemove mousedown"); + }) + .attr("src", screenshot_url); + + let s = `${$selectorBackgroundElem.attr('src')}?${new Date().getTime()}`; + $selectorBackgroundElem.attr('src', s); } - // This is fired once the img src is loaded in bootstrap_visualselector() - function fetch_data() { - // Image is ready - $('.fetching-update-notice').html("Fetching element data.."); + function fetchData() { + $fetchingUpdateNoticeElem.html("Fetching element data.."); $.ajax({ url: watch_visual_selector_data_url, context: document.body - }).done(function (data) { - $('.fetching-update-notice').html("Rendering.."); - selector_data = data; + }).done((data) => { + $fetchingUpdateNoticeElem.html("Rendering.."); + selectorData = data; + sortScrapedElementsBySize(); console.log("Reported browser width from backend: " + data['browser_width']); - state_clicked = false; - set_scale(); - reflow_selector(); - $('.fetching-update-notice').fadeOut(); + setScale(); + reflowSelector(); + $fetchingUpdateNoticeElem.fadeOut(); }); - } + function updateFiltersText() { + // Assuming currentSelections is already defined and contains the selections + let uniqueSelections = new Set(currentSelections.map(sel => (sel[0] === '/' ? `xpath:${sel.xpath}` : sel.xpath))); - function set_scale() { + // Convert the Set back to an array and join with newline characters + let textboxFilterText = Array.from(uniqueSelections).join("\n"); - // some things to check if the scaling doesnt work - // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq - $("#selector-wrapper").show(); - selector_image = $("img#selector-background")[0]; - selector_image_rect = selector_image.getBoundingClientRect(); + $includeFiltersElem.val(textboxFilterText); + } - // Make the overlayed canvas the same size as the image - $('#selector-canvas').attr('height', selector_image_rect.height).attr('width', selector_image_rect.width); - $('#selector-wrapper').attr('width', selector_image_rect.width); + function setScale() { + $selectorWrapperElem.show(); + selectorImage = $selectorBackgroundElem[0]; + selectorImageRect = selectorImage.getBoundingClientRect(); - x_scale = selector_image_rect.width / selector_image.naturalWidth; - y_scale = selector_image_rect.height / selector_image.naturalHeight; + $selectorCanvasElem.attr({ + 'height': selectorImageRect.height, + 'width': selectorImageRect.width + }); + $selectorWrapperElem.attr('width', selectorImageRect.width); + $('#visual-selector-heading').css('max-width', selectorImageRect.width + "px") - ctx.strokeStyle = 'rgba(255,0,0, 0.9)'; - ctx.fillStyle = 'rgba(255,0,0, 0.1)'; + xScale = selectorImageRect.width / selectorImage.naturalWidth; + yScale = selectorImageRect.height / selectorImage.naturalHeight; + + ctx.strokeStyle = STROKE_STYLE_HIGHLIGHT; + ctx.fillStyle = FILL_STYLE_REDLINE; ctx.lineWidth = 3; - console.log("scaling set x: " + x_scale + " by y:" + y_scale); - $("#selector-current-xpath").css('max-width', selector_image_rect.width); + console.log("Scaling set x: " + xScale + " by y:" + yScale); + $("#selector-current-xpath").css('max-width', selectorImageRect.width); } - function reflow_selector() { - $(window).resize(function () { - set_scale(); - highlight_current_selected_i(); + function reflowSelector() { + $(window).resize(() => { + setScale(); + highlightCurrentSelected(); }); - - var selector_currnt_xpath_text = $("#selector-current-xpath span"); - - set_scale(); - - console.log(selector_data['size_pos'].length + " selectors found"); - - // highlight the default one if we can find it in the xPath list - // or the xpath matches the default one - found = false; - if (current_default_xpath.length) { - // Find the first one that matches - // @todo In the future paint all that match - for (const c of current_default_xpath) { - for (var i = selector_data['size_pos'].length; i !== 0; i--) { - if (selector_data['size_pos'][i - 1].xpath.trim() === c.trim()) { - console.log("highlighting " + c); - current_selected_i = i - 1; - highlight_current_selected_i(); - found = true; - break; - } - } - if (found) { - break; - } - } - if (!found) { - alert("Unfortunately your existing CSS/xPath Filter was no longer found!"); - } - highlight_matching_filters(); - } + setScale(); + + console.log(selectorData['size_pos'].length + " selectors found"); - $('#selector-canvas').bind('mousemove', function (e) { - if (state_clicked) { - return; + let existingFilters = splitToList($includeFiltersElem.val()); + + selectorData['size_pos'].forEach(sel => { + if ((!runInClearMode && sel.highlight_as_custom_filter) || existingFilters.includes(sel.xpath)) { + console.log("highlighting " + c); + currentSelections.push(sel); } - ctx.clearRect(0, 0, c.width, c.height); - current_selected_i = null; + }); + + + highlightCurrentSelected(); + updateFiltersText(); + + $selectorCanvasElem.bind('mousemove', handleMouseMove.debounce(5)); + $selectorCanvasElem.bind('mousedown', handleMouseDown.debounce(5)); + $selectorCanvasElem.bind('mouseleave', highlightCurrentSelected.debounce(5)); - // Add in offset - if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) { - var targetOffset = $(e.target).offset(); + function handleMouseMove(e) { + if (!e.offsetX && !e.offsetY) { + const targetOffset = $(e.target).offset(); e.offsetX = e.pageX - targetOffset.left; e.offsetY = e.pageY - targetOffset.top; } - // Reverse order - the most specific one should be deeper/"laster" - // Basically, find the most 'deepest' - var found = 0; - ctx.fillStyle = 'rgba(205,0,0,0.35)'; - // Will be sorted by smallest width*height first - for (var i = 0; i <= selector_data['size_pos'].length; i++) { - // draw all of them? let them choose somehow? - var sel = selector_data['size_pos'][i]; - // If we are in a bounding-box - if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale - && - e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale - - ) { - - // FOUND ONE - set_current_selected_text(sel.xpath); - ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); - ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); - - // no need to keep digging - // @todo or, O to go out/up, I to go in - // or double click to go up/out the selector? - current_selected_i = i; - found += 1; - break; + ctx.fillStyle = FILL_STYLE_HIGHLIGHT; + + selectorData['size_pos'].forEach(sel => { + if (e.offsetY > sel.top * yScale && e.offsetY < sel.top * yScale + sel.height * yScale && + e.offsetX > sel.left * yScale && e.offsetX < sel.left * yScale + sel.width * yScale) { + setCurrentSelectedText(sel.xpath); + drawHighlight(sel); + currentSelections.push(sel); + currentSelection = sel; + highlightCurrentSelected(); + currentSelections.pop(); } - } + }) + } - }.debounce(5)); - function set_current_selected_text(s) { - selector_currnt_xpath_text[0].innerHTML = s; + function setCurrentSelectedText(s) { + $selectorCurrentXpathElem[0].innerHTML = s; } - function highlight_current_selected_i() { - if (state_clicked) { - state_clicked = false; - xctx.clearRect(0, 0, c.width, c.height); - return; - } - - var sel = selector_data['size_pos'][current_selected_i]; - if (sel[0] == '/') { - // @todo - not sure just checking / is right - $("#include_filters").val('xpath:' + sel.xpath); - } else { - $("#include_filters").val(sel.xpath); - } - xctx.fillStyle = 'rgba(205,205,205,0.95)'; - xctx.strokeStyle = 'rgba(225,0,0,0.9)'; - xctx.lineWidth = 3; - xctx.fillRect(0, 0, c.width, c.height); - // Clear out what only should be seen (make a clear/clean spot) - xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); - xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); - state_clicked = true; - set_current_selected_text(sel.xpath); + function drawHighlight(sel) { + ctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale); + ctx.fillRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale); + } + function handleMouseDown() { + // If we are in 'appendToList' mode, grow the list, if not, just 1 + currentSelections = appendToList ? [...currentSelections, currentSelection] : [currentSelection]; + highlightCurrentSelected(); + updateFiltersText(); } + } - function highlight_matching_filters() { - selector_data['size_pos'].forEach(sel => { - if (sel.highlight_as_custom_filter) { - xctx.fillStyle = 'rgba(205,205,205,0.95)'; - xctx.strokeStyle = 'rgba(225,0,0,0.95)'; - xctx.lineWidth = 1; - xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); - xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); - } - }); - } + function highlightCurrentSelected() { + xctx.fillStyle = FILL_STYLE_GREYED_OUT; + xctx.strokeStyle = STROKE_STYLE_REDLINE; + xctx.lineWidth = 3; + xctx.clearRect(0, 0, c.width, c.height); - $('#selector-canvas').bind('mousedown', function (e) { - highlight_current_selected_i(); + currentSelections.forEach(sel => { + //xctx.clearRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale); + xctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale); }); } - -}); \ No newline at end of file +}); diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 15cb3fd5..69fed0aa 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -432,9 +432,8 @@ Unavailable") }}
      {% if visualselector_enabled %} - - The Visual Selector tool lets you select the text elements that will be used for the change detection ‐ after the Browser Steps has completed.
      - This tool is a helper to manage filters in the "CSS/JSONPath/JQ/XPath Filters" box of the Filters & Triggers tab. + + The Visual Selector tool lets you select the text elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the Filters & Triggers tab. Use Shift+Click to select multiple items.