From 338b4dacd0b2418407699a3bed76e080488e402e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 13 Sep 2022 15:09:53 +0200 Subject: [PATCH] WIP --- changedetectionio/content_fetcher.py | 12 +++++++++--- changedetectionio/fetch_processor/image.py | 16 +++++++++++++--- changedetectionio/image_diff.py | 4 ++++ changedetectionio/templates/watch-overview.html | 8 ++++---- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index d077081a..fd67bbdf 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -21,7 +21,6 @@ class Non200ErrorCodeReceived(Exception): self.page_text = html_tools.html_to_text(page_html) return - class JSActionExceptions(Exception): def __init__(self, status_code, url, screenshot, message=''): self.status_code = status_code @@ -203,6 +202,7 @@ class Fetcher(): # Will be needed in the future by the VisualSelector, always get this where possible. screenshot = False + element_screenshot = None system_http_proxy = os.getenv('HTTP_PROXY') system_https_proxy = os.getenv('HTTPS_PROXY') @@ -311,7 +311,8 @@ class base_html_playwright(Fetcher): request_body, request_method, ignore_status_codes=False, - current_css_filter=None): + current_css_filter=None + ): from playwright.sync_api import sync_playwright import playwright._impl._api_types @@ -405,8 +406,13 @@ class base_html_playwright(Fetcher): self.status_code = response.status self.headers = response.all_headers() - if current_css_filter is not None: + if current_css_filter is not None and len(current_css_filter): page.evaluate("var css_filter={}".format(json.dumps(current_css_filter))) + + el = page.locator(current_css_filter) + if el: + el.scroll_into_view_if_needed() + self.element_screenshot = el.screenshot() else: page.evaluate("var css_filter=''") diff --git a/changedetectionio/fetch_processor/image.py b/changedetectionio/fetch_processor/image.py index e4dbd1b7..32c0a106 100644 --- a/changedetectionio/fetch_processor/image.py +++ b/changedetectionio/fetch_processor/image.py @@ -77,7 +77,16 @@ class perform_site_check(fetch_processor): proxy_args = self.set_proxy_from_list(watch) fetcher = klass(proxy_override=proxy_args) - fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes) + fetcher.run( + ignore_status_codes=ignore_status_codes, + request_body=request_body, + request_headers=request_headers, + request_method=request_method, + current_css_filter=watch.get('css_filter'), + timeout=timeout, + url=url + ) + fetcher.quit() # if not image/foobar in mimetype @@ -89,18 +98,19 @@ class perform_site_check(fetch_processor): if 'image' in fetcher.headers['content-type']: self.contents = fetcher.raw_content else: - # should be element if the filter is set, no? - self.contents = fetcher.screenshot + self.contents = fetcher.element_screenshot if fetcher.element_screenshot else fetcher.screenshot # Used for visual-selector self.xpath_data = fetcher.xpath_data self.screenshot = fetcher.screenshot + now = time.time() image = Image.open(io.BytesIO(self.contents)) # @todo different choice? # https://github.com/JohannesBuchner/imagehash#references fetched_hash = str(imagehash.average_hash(image)) + print(uuid, "Time to image hash", time.time() - now) # The main thing that all this at the moment comes down to :) if watch['previous_md5'] != fetched_hash: diff --git a/changedetectionio/image_diff.py b/changedetectionio/image_diff.py index 77ecb494..ab718223 100644 --- a/changedetectionio/image_diff.py +++ b/changedetectionio/image_diff.py @@ -6,6 +6,9 @@ import cv2 # From https://www.pyimagesearch.com/2017/06/19/image-difference-with-opencv-and-python/ def render_diff(fpath_imageA, fpath_imageB): + import time + now = time.time() + imageA = cv2.imread(fpath_imageA) imageB = cv2.imread(fpath_imageB) @@ -37,4 +40,5 @@ def render_diff(fpath_imageA, fpath_imageB): cv2.rectangle(imageB, (x, y), (x + w, y + h), (0, 0, 255), 1) #return cv2.imencode('.jpg', imageB)[1].tobytes() + print ("Image comparison processing time", time.time()-now) return cv2.imencode('.jpg', imageA)[1].tobytes() diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index cc2629f9..1a5f31ce 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -95,11 +95,11 @@ Mute notifications {{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}} - - + + Share - {%if watch.fetch_backend == "html_webdriver" %}{% endif %} - {%if watch.fetch_processor == "image" %}{% endif %} + {%if watch.fetch_backend == "html_webdriver" %}Fetching with Chrome{% endif %} + {%if watch.fetch_processor == "image" %}Comparing graphically{% endif %} {% if watch.last_error is defined and watch.last_error != False %}
{{ watch.last_error }}
{% endif %}