From af4311a68c805fe71d3ccb5fdf2911dea9e91bb9 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 9 Dec 2023 14:56:00 +0100 Subject: [PATCH 01/27] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e9be7f8..cc7c2f52 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,7 +90,7 @@ services: # # Used for fetching pages via Playwright+Chrome where you need Javascript support. - # Note: works well but is deprecated, doesnt fetch full page screenshots and other issues + # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues # browser-chrome: # hostname: browser-chrome # image: selenium/standalone-chrome:4 From 0dce3f4fecc994c147ad9ba815a0f5014f327e88 Mon Sep 17 00:00:00 2001 From: Constantin Hong Date: Tue, 19 Dec 2023 19:10:51 +0900 Subject: [PATCH 02/27] Testing: Improve application signal handling test coverage (#2052) --- .github/workflows/test-only.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index d6962098..666fe051 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -118,7 +118,8 @@ jobs: sleep 3 # invert the check (it should be not 0/not running) docker ps - docker logs sig-test + # check signal catch(STDOUT) log + docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1 test -z "`docker ps|grep sig-test`" if [ $? -ne 0 ] then @@ -138,7 +139,7 @@ jobs: sleep 3 # invert the check (it should be not 0/not running) docker ps - docker logs sig-test + docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1 test -z "`docker ps|grep sig-test`" if [ $? -ne 0 ] then From 5528b7c4b36530f6a1489f9a3ec9305e330a1458 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 20 Dec 2023 15:28:43 +0100 Subject: [PATCH 03/27] Restock detection - Update stock-not-in-stock.js strings (Dutch translations) --- changedetectionio/res/stock-not-in-stock.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changedetectionio/res/stock-not-in-stock.js b/changedetectionio/res/stock-not-in-stock.js index d2d870b5..9cefe40c 100644 --- a/changedetectionio/res/stock-not-in-stock.js +++ b/changedetectionio/res/stock-not-in-stock.js @@ -28,6 +28,8 @@ function isItemInStock() { 'nicht auf lager', 'nicht lieferbar', 'nicht zur verfügung', + 'niet leverbaar', + 'niet beschikbaar', 'no disponible temporalmente', 'no longer in stock', 'no tickets available', @@ -44,6 +46,7 @@ function isItemInStock() { 'temporarily out of stock', 'temporarily unavailable', 'tickets unavailable', + 'tijdelijk uitverkocht', 'unavailable tickets', 'we do not currently have an estimate of when this product will be back in stock.', 'zur zeit nicht an lager', From 3d1e1025d2eea3085c6247d6b88905ffe7b84a40 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 20 Dec 2023 15:30:58 +0100 Subject: [PATCH 04/27] 0.45.9 --- changedetectionio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 99a659a5..42bd672d 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.45.8.1' +__version__ = '0.45.9' from distutils.util import strtobool from json.decoder import JSONDecodeError From 273bd45ad797d2e517b705fd2c85d2643027d7f0 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 1 Jan 2024 16:40:24 +0100 Subject: [PATCH 05/27] Fetching - Custom browser on experimental/puppeteer fetcher - Don't switch to custom puppeteer mode if external browser URL is active (#2068) --- changedetectionio/content_fetcher.py | 33 ++++++++++++++---------- changedetectionio/processors/__init__.py | 6 ++--- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index be426c88..e9477270 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -91,19 +91,20 @@ class ReplyWithContentButNoText(Exception): class Fetcher(): + browser_connection_is_custom = None + browser_connection_url = None browser_steps = None browser_steps_screenshot_path = None content = None error = None fetcher_description = "No description" - browser_connection_url = None headers = {} + instock_data = None + instock_data_js = "" status_code = None webdriver_js_execute_code = None xpath_data = None xpath_element_js = "" - instock_data = None - instock_data_js = "" # Will be needed in the future by the VisualSelector, always get this where possible. screenshot = False @@ -252,16 +253,19 @@ class base_html_playwright(Fetcher): proxy = None - def __init__(self, proxy_override=None, browser_connection_url=None): + def __init__(self, proxy_override=None, custom_browser_connection_url=None): super().__init__() self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"') - # .strip('"') is going to save someone a lot of time when they accidently wrap the env value - if not browser_connection_url: - self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"') + if custom_browser_connection_url: + self.browser_connection_is_custom = True + self.browser_connection_url = custom_browser_connection_url else: - self.browser_connection_url = browser_connection_url + # Fallback to fetching from system + # .strip('"') is going to save someone a lot of time when they accidently wrap the env value + self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"') + # If any proxy settings are enabled, then we should setup the proxy object proxy_args = {} @@ -421,8 +425,10 @@ class base_html_playwright(Fetcher): current_include_filters=None, is_binary=False): + # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!) - if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'): + # browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case) + if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'): if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')): # Temporary backup solution until we rewrite the playwright code return self.run_fetch_browserless_puppeteer( @@ -569,15 +575,16 @@ class base_html_webdriver(Fetcher): 'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword'] proxy = None - def __init__(self, proxy_override=None, browser_connection_url=None): + def __init__(self, proxy_override=None, custom_browser_connection_url=None): super().__init__() from selenium.webdriver.common.proxy import Proxy as SeleniumProxy # .strip('"') is going to save someone a lot of time when they accidently wrap the env value - if not browser_connection_url: + if not custom_browser_connection_url: self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"') else: - self.browser_connection_url = browser_connection_url + self.browser_connection_is_custom = True + self.browser_connection_url = custom_browser_connection_url # If any proxy settings are enabled, then we should setup the proxy object proxy_args = {} @@ -674,7 +681,7 @@ class base_html_webdriver(Fetcher): class html_requests(Fetcher): fetcher_description = "Basic fast Plaintext/HTTP Client" - def __init__(self, proxy_override=None, browser_connection_url=None): + def __init__(self, proxy_override=None, custom_browser_connection_url=None): super().__init__() self.proxy_override = proxy_override # browser_connection_url is none because its always 'launched locally' diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index efccea49..7aa8994a 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -43,14 +43,14 @@ class difference_detection_processor(): # In the case that the preferred fetcher was a browser config with custom connection URL.. # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..) - browser_connection_url = None + custom_browser_connection_url = None if prefer_fetch_backend.startswith('extra_browser_'): (t, key) = prefer_fetch_backend.split('extra_browser_') connection = list( filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', []))) if connection: prefer_fetch_backend = 'base_html_playwright' - browser_connection_url = connection[0].get('browser_connection_url') + custom_browser_connection_url = connection[0].get('browser_connection_url') # PDF should be html_requests because playwright will serve it up (so far) in a embedded page # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019 @@ -74,7 +74,7 @@ class difference_detection_processor(): # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need. # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc) self.fetcher = fetcher_obj(proxy_override=proxy_url, - browser_connection_url=browser_connection_url + custom_browser_connection_url=custom_browser_connection_url ) if self.watch.has_browser_steps: From eda23678aac47c442c7b2578d1ceeb2093b40b18 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 1 Jan 2024 16:43:34 +0100 Subject: [PATCH 06/27] Restock detection - updating texts --- changedetectionio/res/stock-not-in-stock.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/changedetectionio/res/stock-not-in-stock.js b/changedetectionio/res/stock-not-in-stock.js index 9cefe40c..12ba53b3 100644 --- a/changedetectionio/res/stock-not-in-stock.js +++ b/changedetectionio/res/stock-not-in-stock.js @@ -4,6 +4,7 @@ function isItemInStock() { ' أخبرني عندما يتوفر', '0 in stock', 'agotado', + 'article épuisé', 'artikel zurzeit vergriffen', 'as soon as stock is available', 'ausverkauft', // sold out @@ -17,10 +18,8 @@ function isItemInStock() { 'currently have any tickets for this', 'currently unavailable', 'dostępne wkrótce', - 'dostępne wkrótce', 'en rupture de stock', 'ist derzeit nicht auf lager', - 'ist derzeit nicht auf lager', 'item is no longer available', 'let me know when it\'s available', 'message if back in stock', @@ -28,8 +27,8 @@ function isItemInStock() { 'nicht auf lager', 'nicht lieferbar', 'nicht zur verfügung', - 'niet leverbaar', 'niet beschikbaar', + 'niet leverbaar', 'no disponible temporalmente', 'no longer in stock', 'no tickets available', From 946a556fb6ef3eccfdbf4e6ee1446c9a74c40489 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 1 Jan 2024 17:10:27 +0100 Subject: [PATCH 07/27] Restock detection - "In stock" should be None/"Not yet checked" by default (#2069) --- changedetectionio/model/Watch.py | 1 + changedetectionio/store.py | 1 + changedetectionio/templates/watch-overview.html | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index a4774efc..f785c815 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -38,6 +38,7 @@ base_config = { 'track_ldjson_price_data': None, 'headers': {}, # Extra headers to send 'ignore_text': [], # List of text to ignore when calculating the comparison checksum + 'in_stock' : None, 'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock 'include_filters': [], 'last_checked': 0, diff --git a/changedetectionio/store.py b/changedetectionio/store.py index be2546e4..d4214184 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -248,6 +248,7 @@ class ChangeDetectionStore: 'check_count': 0, 'fetch_time' : 0.0, 'has_ldjson_price_data': None, + 'in_stock': None, 'last_checked': 0, 'last_error': False, 'last_notification_error': False, diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 9b3e9b64..f6643df5 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -141,7 +141,7 @@ {% if watch['processor'] == 'restock_diff' %} - {% if watch['last_checked'] %} + {% if watch['last_checked'] and watch['in_stock'] != None %} {% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %} {% else %} Not yet checked From 2db04e42118ddb6eec1dcad19c0c779377823712 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 3 Jan 2024 11:16:09 +0100 Subject: [PATCH 08/27] Notifications upgrade - Upgrade to Apprise 1.7.1 - Emojis support, Telegram topics support, Discord support for user and role @ping support. (#2075) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3599c50e..cbd1e365 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ dnspython~=2.4 # related to eventlet fixes # jq not available on Windows so must be installed manually # Notification library -apprise~=1.6.0 +apprise~=1.7.1 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315 paho-mqtt From d115b2c858e3336f888cfd7e6a2fc56611d1d956 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 4 Jan 2024 17:02:31 +0100 Subject: [PATCH 09/27] UI - [Send test notification] - Refactor to use all tokens like a real watch and Notification Body+Title from UI value (#2079) --- .../blueprint/tags/templates/edit-tag.html | 2 +- changedetectionio/flask_app.py | 18 +++++-- changedetectionio/notification.py | 5 +- changedetectionio/static/js/notifications.js | 17 ++++--- changedetectionio/templates/edit.html | 2 +- changedetectionio/templates/settings.html | 2 +- changedetectionio/update_worker.py | 50 ++++++++++++------- 7 files changed, 61 insertions(+), 35 deletions(-) diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index 449ba382..9834f566 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -3,7 +3,7 @@ {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} {% from '_common_fields.jinja' import render_common_settings_form %} diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 9345eb9a..21d2c68f 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -485,14 +485,18 @@ def changedetection_app(config=None, datastore_o=None): # AJAX endpoint for sending a test + @app.route("/notification/send-test/", methods=['POST']) @app.route("/notification/send-test", methods=['POST']) + @app.route("/notification/send-test/", methods=['POST']) @login_optionally_required - def ajax_callback_send_notification_test(): + def ajax_callback_send_notification_test(watch_uuid=None): + # Watch_uuid could be unsuet in the case its used in tag editor, global setings import apprise from .apprise_asset import asset apobj = apprise.Apprise(asset=asset) + watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None # validate URLS if not len(request.form['notification_urls'].strip()): @@ -505,9 +509,11 @@ def changedetection_app(config=None, datastore_o=None): return make_response({'error': message}, 400) try: - n_object = {'watch_url': request.form['window_url'], - 'notification_urls': request.form['notification_urls'].splitlines() - } + # use the same as when it is triggered, but then override it with the form test values + n_object = { + 'watch_url': request.form['window_url'], + 'notification_urls': request.form['notification_urls'].splitlines() + } # Only use if present, if not set in n_object it should use the default system value if 'notification_format' in request.form and request.form['notification_format'].strip(): @@ -519,7 +525,9 @@ def changedetection_app(config=None, datastore_o=None): if 'notification_body' in request.form and request.form['notification_body'].strip(): n_object['notification_body'] = request.form.get('notification_body', '').strip() - notification_q.put(n_object) + from . import update_worker + new_worker = update_worker.update_worker(update_q, notification_q, app, datastore) + new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch) except Exception as e: return make_response({'error': str(e)}, 400) diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 93cd304e..c97412d8 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -221,13 +221,14 @@ def process_notification(n_object, datastore): # Notification title + body content parameters get created here. +# ( Where we prepare the tokens in the notification to be replaced with actual values ) def create_notification_parameters(n_object, datastore): from copy import deepcopy # in the case we send a test notification from the main settings, there is no UUID. uuid = n_object['uuid'] if 'uuid' in n_object else '' - if uuid != '': + if uuid: watch_title = datastore.data['watching'][uuid].get('title', '') tag_list = [] tags = datastore.get_all_tags_for_watch(uuid) @@ -255,7 +256,7 @@ def create_notification_parameters(n_object, datastore): tokens.update( { 'base_url': base_url, - 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '', + 'current_snapshot': n_object.get('current_snapshot', ''), 'diff': n_object.get('diff', ''), # Null default in the case we use a test 'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test 'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test diff --git a/changedetectionio/static/js/notifications.js b/changedetectionio/static/js/notifications.js index 6b855b18..046b645c 100644 --- a/changedetectionio/static/js/notifications.js +++ b/changedetectionio/static/js/notifications.js @@ -24,14 +24,17 @@ $(document).ready(function() { }) data = { - window_url : window.location.href, - notification_urls : $('.notification-urls').val(), + notification_body: $('#notification_body').val(), + notification_format: $('#notification_format').val(), + notification_title: $('#notification_title').val(), + notification_urls: $('.notification-urls').val(), + window_url: window.location.href, } - for (key in data) { - if (!data[key].length) { - alert(key+" is empty, cannot send test.") - return; - } + + + if (!data['notification_urls'].length) { + alert("Notification URL list is empty, cannot send test.") + return; } $.ajax({ diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 103f57af..d43ed666 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -14,7 +14,7 @@ {% if emailprefix %} const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}'); {% endif %} - const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}"; + const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}"; const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %}; const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}"; const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}"; diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index ef93069e..508f49b2 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -4,7 +4,7 @@ {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} {% from '_common_fields.jinja' import render_common_settings_form %}