diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 00000000..f9839aad --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,72 @@ +name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI + +on: push +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + + test-pypi-package: + name: Test the built 📦 package works basically. + runs-on: ubuntu-latest + needs: + - build + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Test that the basic pip built package runs without error + run: | + set -e + pip3 install dist/changedetection.io*.whl + changedetection.io -d /tmp -p 10000 & + sleep 3 + curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null + curl http://127.0.0.1:10000/ >/dev/null + killall changedetection.io + + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - test-pypi-package + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/p/changedetection.io + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index d6962098..666fe051 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -118,7 +118,8 @@ jobs: sleep 3 # invert the check (it should be not 0/not running) docker ps - docker logs sig-test + # check signal catch(STDOUT) log + docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1 test -z "`docker ps|grep sig-test`" if [ $? -ne 0 ] then @@ -138,7 +139,7 @@ jobs: sleep 3 # invert the check (it should be not 0/not running) docker ps - docker logs sig-test + docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1 test -z "`docker ps|grep sig-test`" if [ $? -ne 0 ] then diff --git a/.github/workflows/test-pip-build.yml b/.github/workflows/test-pip-build.yml deleted file mode 100644 index e5e0bd30..00000000 --- a/.github/workflows/test-pip-build.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: ChangeDetection.io PIP package test - -# Triggers the workflow on push or pull request events - -# This line doesnt work, even tho it is the documented one -on: [push, pull_request] - - # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing - # @todo: some kind of path filter for requirements.txt and Dockerfile -jobs: - test-pip-build-basics: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - - - name: Test that the basic pip built package runs without error - run: | - set -e - mkdir dist - pip3 install wheel - python3 setup.py bdist_wheel - pip3 install -r requirements.txt - rm ./changedetection.py - rm -rf changedetectio - - pip3 install dist/changedetection.io*.whl - changedetection.io -d /tmp -p 10000 & - sleep 3 - curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null - killall -9 changedetection.io diff --git a/MANIFEST.in b/MANIFEST.in index f7393309..b06768fc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,6 +10,8 @@ prune changedetectionio/static/package-lock.json prune changedetectionio/static/styles/node_modules prune changedetectionio/static/styles/package-lock.json include changedetection.py +include requirements.txt +include README-pip.md global-exclude *.pyc global-exclude node_modules global-exclude venv diff --git a/Procfile b/Procfile deleted file mode 100644 index 116f3f1a..00000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: python3 ./changedetection.py -C -d ./datastore -p $PORT diff --git a/README.md b/README.md index fc259a30..424bdfa1 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ _Live your data-life pro-actively._ - Nothing to install, access via browser login after signup. - Super fast, no registration needed setup. - Get started watching and receiving website change notifications straight away. - +- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials) ### Target specific parts of the webpage using the Visual Selector tool. @@ -98,7 +98,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl With Docker composer, just clone this repository and.. ```bash -$ docker-compose up -d +$ docker compose up -d ``` Docker standalone @@ -137,10 +137,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q) docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io ``` -### docker-compose +### docker compose ```bash -docker-compose pull && docker-compose up -d +docker compose pull && docker compose up -d ``` See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki @@ -249,7 +249,7 @@ Supports managing the website watch list [via our API](https://changedetection.i Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you. -Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!) +Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!) Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ) diff --git a/app.json b/app.json deleted file mode 100644 index a9249e88..00000000 --- a/app.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "ChangeDetection.io", - "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.", - "keywords": [ - "changedetection", - "website monitoring" - ], - "repository": "https://github.com/dgtlmoon/changedetection.io", - "success_url": "/", - "scripts": { - }, - "env": { - }, - "formation": { - "web": { - "quantity": 1, - "size": "free" - } - }, - "image": "heroku/python" -} diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 99a659a5..c88c0eab 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.45.8.1' +__version__ = '0.45.12' from distutils.util import strtobool from json.decoder import JSONDecodeError diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py index d51f47ed..190bd449 100644 --- a/changedetectionio/api/api_v1.py +++ b/changedetectionio/api/api_v1.py @@ -30,7 +30,7 @@ class Watch(Resource): self.update_q = kwargs['update_q'] # Get information about a single watch, excluding the history list (can be large) - # curl http://localhost:4000/api/v1/watch/ + # curl http://localhost:5000/api/v1/watch/ # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK" # ?recheck=true @auth.check_token @@ -39,9 +39,9 @@ class Watch(Resource): @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute. @apiDescription Retrieve watch information and set muted/paused status @apiExample {curl} Example usage: - curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45" - curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted" -H"x-api-key:813031b16330fe25e3780cf0325daa45" - curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused" -H"x-api-key:813031b16330fe25e3780cf0325daa45" + curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45" + curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted" -H"x-api-key:813031b16330fe25e3780cf0325daa45" + curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused" -H"x-api-key:813031b16330fe25e3780cf0325daa45" @apiName Watch @apiGroup Watch @apiParam {uuid} uuid Watch unique ID. @@ -84,7 +84,7 @@ class Watch(Resource): """ @api {delete} /api/v1/watch/:uuid Delete a watch and related history @apiExample {curl} Example usage: - curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45" + curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45" @apiParam {uuid} uuid Watch unique ID. @apiName Delete @apiGroup Watch @@ -103,7 +103,7 @@ class Watch(Resource): @api {put} /api/v1/watch/:uuid Update watch information @apiExample {curl} Example usage: Update (PUT) - curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}' + curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}' @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in get single watch information @apiParam {uuid} uuid Watch unique ID. @@ -132,13 +132,13 @@ class WatchHistory(Resource): self.datastore = kwargs['datastore'] # Get a list of available history for a watch by UUID - # curl http://localhost:4000/api/v1/watch//history + # curl http://localhost:5000/api/v1/watch//history def get(self, uuid): """ @api {get} /api/v1/watch//history Get a list of all historical snapshots available for a watch @apiDescription Requires `uuid`, returns list @apiExample {curl} Example usage: - curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" + curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" { "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt", "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt", @@ -166,7 +166,7 @@ class WatchSingleHistory(Resource): @api {get} /api/v1/watch//history/ Get single snapshot from watch @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or use the list returned here @apiExample {curl} Example usage: - curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" + curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" @apiName Get single snapshot content @apiGroup Watch History @apiSuccess (200) {String} OK @@ -202,7 +202,7 @@ class CreateWatch(Resource): @api {post} /api/v1/watch Create a single watch @apiDescription Requires atleast `url` set, can accept the same structure as get single watch information to create. @apiExample {curl} Example usage: - curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}' + curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}' @apiName Create @apiGroup Watch @apiSuccess (200) {String} OK Was created @@ -245,7 +245,7 @@ class CreateWatch(Resource): @api {get} /api/v1/watch List watches @apiDescription Return concise list of available watches and some very basic info @apiExample {curl} Example usage: - curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" + curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" { "6a4b7d5c-fee4-4616-9f43-4ac97046b595": { "last_changed": 1677103794, @@ -363,7 +363,7 @@ class SystemInfo(Resource): @api {get} /api/v1/systeminfo Return system info @apiDescription Return some info about the current system state @apiExample {curl} Example usage: - curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45" + curl http://localhost:5000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45" HTTP/1.0 200 { 'queue_size': 10 , diff --git a/changedetectionio/blueprint/browser_steps/__init__.py b/changedetectionio/blueprint/browser_steps/__init__.py index b58eedee..227c8ec2 100644 --- a/changedetectionio/blueprint/browser_steps/__init__.py +++ b/changedetectionio/blueprint/browser_steps/__init__.py @@ -58,7 +58,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): io_interface_context = io_interface_context.start() keepalive_ms = ((keepalive_seconds + 3) * 1000) - base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '') + base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"') a = "?" if not '?' in base_url else '&' base_url += a + f"timeout={keepalive_ms}" diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index 449ba382..9834f566 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -3,7 +3,7 @@ {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} {% from '_common_fields.jinja' import render_common_settings_form %} diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index be426c88..4c8a382c 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -43,9 +43,11 @@ class JSActionExceptions(Exception): return -class BrowserStepsStepTimout(Exception): - def __init__(self, step_n): +class BrowserStepsStepException(Exception): + def __init__(self, step_n, original_e): self.step_n = step_n + self.original_e = original_e + print(f"Browser Steps exception at step {self.step_n}", str(original_e)) return @@ -91,19 +93,20 @@ class ReplyWithContentButNoText(Exception): class Fetcher(): + browser_connection_is_custom = None + browser_connection_url = None browser_steps = None browser_steps_screenshot_path = None content = None error = None fetcher_description = "No description" - browser_connection_url = None headers = {} + instock_data = None + instock_data_js = "" status_code = None webdriver_js_execute_code = None xpath_data = None xpath_element_js = "" - instock_data = None - instock_data_js = "" # Will be needed in the future by the VisualSelector, always get this where possible. screenshot = False @@ -172,7 +175,7 @@ class Fetcher(): def iterate_browser_steps(self): from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface - from playwright._impl._errors import TimeoutError + from playwright._impl._errors import TimeoutError, Error from jinja2 import Environment jinja2_env = Environment(extensions=['jinja2_time.TimeExtension']) @@ -202,10 +205,10 @@ class Fetcher(): optional_value=optional_value) self.screenshot_step(step_n) self.save_step_html(step_n) - except TimeoutError as e: - print(str(e)) + + except (Error, TimeoutError) as e: # Stop processing here - raise BrowserStepsStepTimout(step_n=step_n) + raise BrowserStepsStepException(step_n=step_n, original_e=e) # It's always good to reset these def delete_browser_steps_screenshots(self): @@ -252,16 +255,19 @@ class base_html_playwright(Fetcher): proxy = None - def __init__(self, proxy_override=None, browser_connection_url=None): + def __init__(self, proxy_override=None, custom_browser_connection_url=None): super().__init__() self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"') - # .strip('"') is going to save someone a lot of time when they accidently wrap the env value - if not browser_connection_url: - self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"') + if custom_browser_connection_url: + self.browser_connection_is_custom = True + self.browser_connection_url = custom_browser_connection_url else: - self.browser_connection_url = browser_connection_url + # Fallback to fetching from system + # .strip('"') is going to save someone a lot of time when they accidently wrap the env value + self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"') + # If any proxy settings are enabled, then we should setup the proxy object proxy_args = {} @@ -421,8 +427,10 @@ class base_html_playwright(Fetcher): current_include_filters=None, is_binary=False): + # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!) - if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'): + # browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case) + if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'): if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')): # Temporary backup solution until we rewrite the playwright code return self.run_fetch_browserless_puppeteer( @@ -569,15 +577,16 @@ class base_html_webdriver(Fetcher): 'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword'] proxy = None - def __init__(self, proxy_override=None, browser_connection_url=None): + def __init__(self, proxy_override=None, custom_browser_connection_url=None): super().__init__() from selenium.webdriver.common.proxy import Proxy as SeleniumProxy # .strip('"') is going to save someone a lot of time when they accidently wrap the env value - if not browser_connection_url: + if not custom_browser_connection_url: self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"') else: - self.browser_connection_url = browser_connection_url + self.browser_connection_is_custom = True + self.browser_connection_url = custom_browser_connection_url # If any proxy settings are enabled, then we should setup the proxy object proxy_args = {} @@ -674,7 +683,7 @@ class base_html_webdriver(Fetcher): class html_requests(Fetcher): fetcher_description = "Basic fast Plaintext/HTTP Client" - def __init__(self, proxy_override=None, browser_connection_url=None): + def __init__(self, proxy_override=None, custom_browser_connection_url=None): super().__init__() self.proxy_override = proxy_override # browser_connection_url is none because its always 'launched locally' diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 9345eb9a..f077e688 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -485,14 +485,18 @@ def changedetection_app(config=None, datastore_o=None): # AJAX endpoint for sending a test + @app.route("/notification/send-test/", methods=['POST']) @app.route("/notification/send-test", methods=['POST']) + @app.route("/notification/send-test/", methods=['POST']) @login_optionally_required - def ajax_callback_send_notification_test(): + def ajax_callback_send_notification_test(watch_uuid=None): + # Watch_uuid could be unsuet in the case its used in tag editor, global setings import apprise from .apprise_asset import asset apobj = apprise.Apprise(asset=asset) + watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None # validate URLS if not len(request.form['notification_urls'].strip()): @@ -505,9 +509,11 @@ def changedetection_app(config=None, datastore_o=None): return make_response({'error': message}, 400) try: - n_object = {'watch_url': request.form['window_url'], - 'notification_urls': request.form['notification_urls'].splitlines() - } + # use the same as when it is triggered, but then override it with the form test values + n_object = { + 'watch_url': request.form['window_url'], + 'notification_urls': request.form['notification_urls'].splitlines() + } # Only use if present, if not set in n_object it should use the default system value if 'notification_format' in request.form and request.form['notification_format'].strip(): @@ -519,7 +525,9 @@ def changedetection_app(config=None, datastore_o=None): if 'notification_body' in request.form and request.form['notification_body'].strip(): n_object['notification_body'] = request.form.get('notification_body', '').strip() - notification_q.put(n_object) + from . import update_worker + new_worker = update_worker.update_worker(update_q, notification_q, app, datastore) + new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch) except Exception as e: return make_response({'error': str(e)}, 400) @@ -1584,6 +1592,15 @@ def notification_runner(): try: from changedetectionio import notification + # Fallback to system config if not set + if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'): + n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body') + + if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'): + n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title') + + if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'): + n_object['notification_title'] = datastore.data['settings']['application'].get('notification_format') sent_obj = notification.process_notification(n_object, datastore) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index d8646305..9f72a748 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -43,6 +43,7 @@ valid_method = { 'PUT', 'PATCH', 'DELETE', + 'OPTIONS', } default_method = 'GET' diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index a4774efc..f785c815 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -38,6 +38,7 @@ base_config = { 'track_ldjson_price_data': None, 'headers': {}, # Extra headers to send 'ignore_text': [], # List of text to ignore when calculating the comparison checksum + 'in_stock' : None, 'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock 'include_filters': [], 'last_checked': 0, diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 93cd304e..436d08c6 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -1,4 +1,5 @@ import apprise +import time from jinja2 import Environment, BaseLoader from apprise import NotifyFormat import json @@ -119,8 +120,8 @@ def process_notification(n_object, datastore): # Get the notification body from datastore jinja2_env = Environment(loader=BaseLoader) - n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters) - n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters) + n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters) + n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters) n_format = valid_notification_formats.get( n_object.get('notification_format', default_notification_format), valid_notification_formats[default_notification_format], @@ -131,103 +132,108 @@ def process_notification(n_object, datastore): # Initially text or whatever n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]) - # https://github.com/caronc/apprise/wiki/Development_LogCapture # Anything higher than or equal to WARNING (which covers things like Connection errors) # raise it as an exception - apobjs=[] - sent_objs=[] + + sent_objs = [] from .apprise_asset import asset - for url in n_object['notification_urls']: - url = jinja2_env.from_string(url).render(**notification_parameters) - apobj = apprise.Apprise(debug=True, asset=asset) - url = url.strip() - if len(url): + apobj = apprise.Apprise(debug=True, asset=asset) + + if not n_object.get('notification_urls'): + return None + + with apprise.LogCapture(level=apprise.logging.DEBUG) as logs: + for url in n_object['notification_urls']: + url = url.strip() print(">> Process Notification: AppRise notifying {}".format(url)) - with apprise.LogCapture(level=apprise.logging.DEBUG) as logs: - # Re 323 - Limit discord length to their 2000 char limit total or it wont send. - # Because different notifications may require different pre-processing, run each sequentially :( - # 2000 bytes minus - - # 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers - # Length of URL - Incase they specify a longer custom avatar_url - - # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload - k = '?' if not '?' in url else '&' - if not 'avatar_url' in url \ - and not url.startswith('mail') \ - and not url.startswith('post') \ - and not url.startswith('get') \ - and not url.startswith('delete') \ - and not url.startswith('put'): - url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png' - - if url.startswith('tgram://'): - # Telegram only supports a limit subset of HTML, remove the '
' we place in. - # re https://github.com/dgtlmoon/changedetection.io/issues/555 - # @todo re-use an existing library we have already imported to strip all non-allowed tags - n_body = n_body.replace('
', '\n') - n_body = n_body.replace('
', '\n') - # real limit is 4096, but minus some for extra metadata - payload_max_size = 3600 - body_limit = max(0, payload_max_size - len(n_title)) - n_title = n_title[0:payload_max_size] - n_body = n_body[0:body_limit] - - elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith('https://discord.com/api'): - # real limit is 2000, but minus some for extra metadata - payload_max_size = 1700 - body_limit = max(0, payload_max_size - len(n_title)) - n_title = n_title[0:payload_max_size] - n_body = n_body[0:body_limit] - - elif url.startswith('mailto'): - # Apprise will default to HTML, so we need to override it - # So that whats' generated in n_body is in line with what is going to be sent. - # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321 - if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'): - prefix = '?' if not '?' in url else '&' - # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633 - n_format = n_format.tolower() - url = "{}{}format={}".format(url, prefix, n_format) - # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only - - apobj.add(url) - - apobj.notify( - title=n_title, - body=n_body, - body_format=n_format, - # False is not an option for AppRise, must be type None - attach=n_object.get('screenshot', None) - ) - - apobj.clear() - - # Incase it needs to exist in memory for a while after to process(?) - apobjs.append(apobj) - - # Returns empty string if nothing found, multi-line string otherwise - log_value = logs.getvalue() - if log_value and 'WARNING' in log_value or 'ERROR' in log_value: - raise Exception(log_value) - - sent_objs.append({'title': n_title, - 'body': n_body, - 'url' : url, - 'body_format': n_format}) + url = jinja2_env.from_string(url).render(**notification_parameters) + + # Re 323 - Limit discord length to their 2000 char limit total or it wont send. + # Because different notifications may require different pre-processing, run each sequentially :( + # 2000 bytes minus - + # 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers + # Length of URL - Incase they specify a longer custom avatar_url + + # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload + k = '?' if not '?' in url else '&' + if not 'avatar_url' in url \ + and not url.startswith('mail') \ + and not url.startswith('post') \ + and not url.startswith('get') \ + and not url.startswith('delete') \ + and not url.startswith('put'): + url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png' + + if url.startswith('tgram://'): + # Telegram only supports a limit subset of HTML, remove the '
' we place in. + # re https://github.com/dgtlmoon/changedetection.io/issues/555 + # @todo re-use an existing library we have already imported to strip all non-allowed tags + n_body = n_body.replace('
', '\n') + n_body = n_body.replace('
', '\n') + # real limit is 4096, but minus some for extra metadata + payload_max_size = 3600 + body_limit = max(0, payload_max_size - len(n_title)) + n_title = n_title[0:payload_max_size] + n_body = n_body[0:body_limit] + + elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith( + 'https://discord.com/api'): + # real limit is 2000, but minus some for extra metadata + payload_max_size = 1700 + body_limit = max(0, payload_max_size - len(n_title)) + n_title = n_title[0:payload_max_size] + n_body = n_body[0:body_limit] + + elif url.startswith('mailto'): + # Apprise will default to HTML, so we need to override it + # So that whats' generated in n_body is in line with what is going to be sent. + # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321 + if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'): + prefix = '?' if not '?' in url else '&' + # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633 + n_format = n_format.lower() + url = f"{url}{prefix}format={n_format}" + # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only + + apobj.add(url) + + sent_objs.append({'title': n_title, + 'body': n_body, + 'url': url, + 'body_format': n_format}) + + # Blast off the notifications tht are set in .add() + apobj.notify( + title=n_title, + body=n_body, + body_format=n_format, + # False is not an option for AppRise, must be type None + attach=n_object.get('screenshot', None) + ) + + # Give apprise time to register an error + time.sleep(3) + + # Returns empty string if nothing found, multi-line string otherwise + log_value = logs.getvalue() + + if log_value and 'WARNING' in log_value or 'ERROR' in log_value: + raise Exception(log_value) # Return what was sent for better logging - after the for loop return sent_objs # Notification title + body content parameters get created here. +# ( Where we prepare the tokens in the notification to be replaced with actual values ) def create_notification_parameters(n_object, datastore): from copy import deepcopy # in the case we send a test notification from the main settings, there is no UUID. uuid = n_object['uuid'] if 'uuid' in n_object else '' - if uuid != '': + if uuid: watch_title = datastore.data['watching'][uuid].get('title', '') tag_list = [] tags = datastore.get_all_tags_for_watch(uuid) @@ -255,7 +261,7 @@ def create_notification_parameters(n_object, datastore): tokens.update( { 'base_url': base_url, - 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '', + 'current_snapshot': n_object.get('current_snapshot', ''), 'diff': n_object.get('diff', ''), # Null default in the case we use a test 'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test 'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index efccea49..7aa8994a 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -43,14 +43,14 @@ class difference_detection_processor(): # In the case that the preferred fetcher was a browser config with custom connection URL.. # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..) - browser_connection_url = None + custom_browser_connection_url = None if prefer_fetch_backend.startswith('extra_browser_'): (t, key) = prefer_fetch_backend.split('extra_browser_') connection = list( filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', []))) if connection: prefer_fetch_backend = 'base_html_playwright' - browser_connection_url = connection[0].get('browser_connection_url') + custom_browser_connection_url = connection[0].get('browser_connection_url') # PDF should be html_requests because playwright will serve it up (so far) in a embedded page # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019 @@ -74,7 +74,7 @@ class difference_detection_processor(): # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need. # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc) self.fetcher = fetcher_obj(proxy_override=proxy_url, - browser_connection_url=browser_connection_url + custom_browser_connection_url=custom_browser_connection_url ) if self.watch.has_browser_steps: diff --git a/changedetectionio/processors/restock_diff.py b/changedetectionio/processors/restock_diff.py index 86f646ff..dc7e248c 100644 --- a/changedetectionio/processors/restock_diff.py +++ b/changedetectionio/processors/restock_diff.py @@ -109,4 +109,8 @@ class perform_site_check(difference_detection_processor): # All cases changed_detected = True - return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8') + # Always record the new checksum + update_obj["previous_md5"] = fetched_md5 + + return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8').strip() + diff --git a/changedetectionio/res/stock-not-in-stock.js b/changedetectionio/res/stock-not-in-stock.js index d2d870b5..681cbd58 100644 --- a/changedetectionio/res/stock-not-in-stock.js +++ b/changedetectionio/res/stock-not-in-stock.js @@ -1,115 +1,131 @@ function isItemInStock() { - // @todo Pass these in so the same list can be used in non-JS fetchers - const outOfStockTexts = [ - ' أخبرني عندما يتوفر', - '0 in stock', - 'agotado', - 'artikel zurzeit vergriffen', - 'as soon as stock is available', - 'ausverkauft', // sold out - 'available for back order', - 'back-order or out of stock', - 'backordered', - 'benachrichtigt mich', // notify me - 'brak na stanie', - 'brak w magazynie', - 'coming soon', - 'currently have any tickets for this', - 'currently unavailable', - 'dostępne wkrótce', - 'dostępne wkrótce', - 'en rupture de stock', - 'ist derzeit nicht auf lager', - 'ist derzeit nicht auf lager', - 'item is no longer available', - 'let me know when it\'s available', - 'message if back in stock', - 'nachricht bei', - 'nicht auf lager', - 'nicht lieferbar', - 'nicht zur verfügung', - 'no disponible temporalmente', - 'no longer in stock', - 'no tickets available', - 'not available', - 'not currently available', - 'not in stock', - 'notify me when available', - 'não estamos a aceitar encomendas', - 'out of stock', - 'out-of-stock', - 'produkt niedostępny', - 'sold out', - 'sold-out', - 'temporarily out of stock', - 'temporarily unavailable', - 'tickets unavailable', - 'unavailable tickets', - 'we do not currently have an estimate of when this product will be back in stock.', - 'zur zeit nicht an lager', - '品切れ', - '已售完', - '품절' - ]; + // @todo Pass these in so the same list can be used in non-JS fetchers + const outOfStockTexts = [ + ' أخبرني عندما يتوفر', + '0 in stock', + 'agotado', + 'article épuisé', + 'artikel zurzeit vergriffen', + 'as soon as stock is available', + 'ausverkauft', // sold out + 'available for back order', + 'back-order or out of stock', + 'backordered', + 'benachrichtigt mich', // notify me + 'brak na stanie', + 'brak w magazynie', + 'coming soon', + 'currently have any tickets for this', + 'currently unavailable', + 'dostępne wkrótce', + 'en rupture de stock', + 'ist derzeit nicht auf lager', + 'item is no longer available', + 'let me know when it\'s available', + 'message if back in stock', + 'nachricht bei', + 'nicht auf lager', + 'nicht lieferbar', + 'nicht zur verfügung', + 'niet beschikbaar', + 'niet leverbaar', + 'no disponible temporalmente', + 'no longer in stock', + 'no tickets available', + 'not available', + 'not currently available', + 'not in stock', + 'notify me when available', + 'não estamos a aceitar encomendas', + 'out of stock', + 'out-of-stock', + 'produkt niedostępny', + 'sold out', + 'sold-out', + 'temporarily out of stock', + 'temporarily unavailable', + 'tickets unavailable', + 'tijdelijk uitverkocht', + 'unavailable tickets', + 'we do not currently have an estimate of when this product will be back in stock.', + 'we don\'t know when or if this item will be back in stock.', + 'zur zeit nicht an lager', + '品切れ', + '已售完', + '품절' + ]; + function getElementBaseText(element) { + // .textContent can include text from children which may give the wrong results + // scan only immediate TEXT_NODEs, which will be a child of the element + var text = ""; + for (var i = 0; i < element.childNodes.length; ++i) + if (element.childNodes[i].nodeType === Node.TEXT_NODE) + text += element.childNodes[i].textContent; + return text.toLowerCase().trim(); + } - const negateOutOfStockRegexs = [ - '[0-9] in stock' - ] - var negateOutOfStockRegexs_r = []; - for (let i = 0; i < negateOutOfStockRegexs.length; i++) { - negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g')); - } + const negateOutOfStockRegexs = [ + '[0-9] in stock' + ] + var negateOutOfStockRegexs_r = []; + for (let i = 0; i < negateOutOfStockRegexs.length; i++) { + negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g')); + } + // The out-of-stock or in-stock-text is generally always above-the-fold + // and often below-the-fold is a list of related products that may or may not contain trigger text + // so it's good to filter to just the 'above the fold' elements + // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist + const elementsToScan = Array.from(document.getElementsByTagName('*')).filter(element => element.getBoundingClientRect().top + window.scrollY <= window.innerHeight && element.getBoundingClientRect().top + window.scrollY >= 100); - const elementsWithZeroChildren = Array.from(document.getElementsByTagName('*')).filter(element => element.children.length === 0); + var elementText = ""; - // REGEXS THAT REALLY MEAN IT'S IN STOCK - for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) { - const element = elementsWithZeroChildren[i]; - if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) { - var elementText=""; - if (element.tagName.toLowerCase() === "input") { - elementText = element.value.toLowerCase(); - } else { - elementText = element.textContent.toLowerCase(); - } + // REGEXS THAT REALLY MEAN IT'S IN STOCK + for (let i = elementsToScan.length - 1; i >= 0; i--) { + const element = elementsToScan[i]; + elementText = ""; + if (element.tagName.toLowerCase() === "input") { + elementText = element.value.toLowerCase(); + } else { + elementText = getElementBaseText(element); + } - if (elementText.length) { - // try which ones could mean its in stock - for (let i = 0; i < negateOutOfStockRegexs.length; i++) { - if (negateOutOfStockRegexs_r[i].test(elementText)) { - return 'Possibly in stock'; - } + if (elementText.length) { + // try which ones could mean its in stock + for (let i = 0; i < negateOutOfStockRegexs.length; i++) { + if (negateOutOfStockRegexs_r[i].test(elementText)) { + return 'Possibly in stock'; + } + } } - } } - } - // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK - for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) { - const element = elementsWithZeroChildren[i]; - if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) { - var elementText=""; - if (element.tagName.toLowerCase() === "input") { - elementText = element.value.toLowerCase(); - } else { - elementText = element.textContent.toLowerCase(); - } + // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK + for (let i = elementsToScan.length - 1; i >= 0; i--) { + const element = elementsToScan[i]; + if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) { + elementText = ""; + if (element.tagName.toLowerCase() === "input") { + elementText = element.value.toLowerCase(); + } else { + elementText = getElementBaseText(element); + } - if (elementText.length) { - // and these mean its out of stock - for (const outOfStockText of outOfStockTexts) { - if (elementText.includes(outOfStockText)) { - return elementText; // item is out of stock - } + if (elementText.length) { + // and these mean its out of stock + for (const outOfStockText of outOfStockTexts) { + if (elementText.includes(outOfStockText)) { + return outOfStockText; // item is out of stock + } + } + } } - } } - } - return 'Possibly in stock'; // possibly in stock, cant decide otherwise. + return 'Possibly in stock'; // possibly in stock, cant decide otherwise. } // returns the element text that makes it think it's out of stock -return isItemInStock(); +return isItemInStock().trim() + diff --git a/changedetectionio/static/js/notifications.js b/changedetectionio/static/js/notifications.js index 6b855b18..046b645c 100644 --- a/changedetectionio/static/js/notifications.js +++ b/changedetectionio/static/js/notifications.js @@ -24,14 +24,17 @@ $(document).ready(function() { }) data = { - window_url : window.location.href, - notification_urls : $('.notification-urls').val(), + notification_body: $('#notification_body').val(), + notification_format: $('#notification_format').val(), + notification_title: $('#notification_title').val(), + notification_urls: $('.notification-urls').val(), + window_url: window.location.href, } - for (key in data) { - if (!data[key].length) { - alert(key+" is empty, cannot send test.") - return; - } + + + if (!data['notification_urls'].length) { + alert("Notification URL list is empty, cannot send test.") + return; } $.ajax({ diff --git a/changedetectionio/store.py b/changedetectionio/store.py index be2546e4..d4214184 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -248,6 +248,7 @@ class ChangeDetectionStore: 'check_count': 0, 'fetch_time' : 0.0, 'has_ldjson_price_data': None, + 'in_stock': None, 'last_checked': 0, 'last_error': False, 'last_notification_error': False, diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 103f57af..d43ed666 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -14,7 +14,7 @@ {% if emailprefix %} const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}'); {% endif %} - const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}"; + const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}"; const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %}; const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}"; const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}"; diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index ef93069e..508f49b2 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -4,7 +4,7 @@ {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} {% from '_common_fields.jinja' import render_common_settings_form %}