Self-Hosted
/
changedetection.io
mirror of https://github.com/dgtlmoon/changedetection.io.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into 2039-restock-use-itemprop

Browse Source
piwheels-rpi-crypto
dgtlmoon 10 months ago committed by GitHub
parent 61185b5514 c3d825f38c
commit b71b457c24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 457 additions and 333 deletions
Show Stats Download Patch File Download Diff File

72
.github/workflows/pypi-release.yml vendored
Unescape View File

@ -0,0 +1,72 @@
name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI
on: push
jobs:
build:
name: Build distribution 📦
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Install pypa/build
run: >-
python3 -m
pip install
build
--user
- name: Build a binary wheel and a source tarball
run: python3 -m build
- name: Store the distribution packages
uses: actions/upload-artifact@v3
with:
name: python-package-distributions
path: dist/
test-pypi-package:
name: Test the built 📦 package works basically.
runs-on: ubuntu-latest
needs:
- build
steps:
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: Test that the basic pip built package runs without error
run: |
set -e
pip3 install dist/changedetection.io*.whl
changedetection.io -d /tmp -p 10000 &
sleep 3
curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
curl http://127.0.0.1:10000/ >/dev/null
killall changedetection.io
publish-to-pypi:
name: >-
Publish Python 🐍 distribution 📦 to PyPI
if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
needs:
- test-pypi-package
runs-on: ubuntu-latest
environment:
name: release
url: https://pypi.org/p/changedetection.io
permissions:
id-token: write # IMPORTANT: mandatory for trusted publishing
steps:
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: Publish distribution 📦 to PyPI
uses: pypa/gh-action-pypi-publish@release/v1

5
.github/workflows/test-only.yml vendored
Unescape View File

@ -118,7 +118,8 @@ jobs:
sleep 3 sleep 3
# invert the check (it should be not 0/not running) # invert the check (it should be not 0/not running)
docker ps docker ps
docker logs sig-test # check signal catch(STDOUT) log
docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
test -z "`docker ps|grep sig-test`" test -z "`docker ps|grep sig-test`"
if [ $? -ne 0 ] if [ $? -ne 0 ]
then then
@ -138,7 +139,7 @@ jobs:
sleep 3 sleep 3
# invert the check (it should be not 0/not running) # invert the check (it should be not 0/not running)
docker ps docker ps
docker logs sig-test docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
test -z "`docker ps|grep sig-test`" test -z "`docker ps|grep sig-test`"
if [ $? -ne 0 ] if [ $? -ne 0 ]
then then

36
.github/workflows/test-pip-build.yml vendored
Unescape View File

@ -1,36 +0,0 @@
name: ChangeDetection.io PIP package test
# Triggers the workflow on push or pull request events
# This line doesnt work, even tho it is the documented one
on: [push, pull_request]
# Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
# @todo: some kind of path filter for requirements.txt and Dockerfile
jobs:
test-pip-build-basics:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Test that the basic pip built package runs without error
run: |
set -e
mkdir dist
pip3 install wheel
python3 setup.py bdist_wheel
pip3 install -r requirements.txt
rm ./changedetection.py
rm -rf changedetectio
pip3 install dist/changedetection.io*.whl
changedetection.io -d /tmp -p 10000 &
sleep 3
curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
killall -9 changedetection.io

2
MANIFEST.in
Unescape View File

@ -10,6 +10,8 @@ prune changedetectionio/static/package-lock.json
prune changedetectionio/static/styles/node_modules prune changedetectionio/static/styles/node_modules
prune changedetectionio/static/styles/package-lock.json prune changedetectionio/static/styles/package-lock.json
include changedetection.py include changedetection.py
include requirements.txt
include README-pip.md
global-exclude *.pyc global-exclude *.pyc
global-exclude node_modules global-exclude node_modules
global-exclude venv global-exclude venv

1
Procfile
Unescape View File

@ -1 +0,0 @@
web: python3 ./changedetection.py -C -d ./datastore -p $PORT

10
README.md
Unescape View File

@ -17,7 +17,7 @@ _Live your data-life pro-actively._
- Nothing to install, access via browser login after signup. - Nothing to install, access via browser login after signup.
- Super fast, no registration needed setup. - Super fast, no registration needed setup.
- Get started watching and receiving website change notifications straight away. - Get started watching and receiving website change notifications straight away.
- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials)
### Target specific parts of the webpage using the Visual Selector tool. ### Target specific parts of the webpage using the Visual Selector tool.
@ -98,7 +98,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl
With Docker composer, just clone this repository and.. With Docker composer, just clone this repository and..
```bash ```bash
$ docker-compose up -d $ docker compose up -d
``` ```
Docker standalone Docker standalone
@ -137,10 +137,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q)
docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
``` ```
### docker-compose ### docker compose
```bash ```bash
docker-compose pull && docker-compose up -d docker compose pull && docker compose up -d
``` ```
See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
@ -249,7 +249,7 @@ Supports managing the website watch list [via our API](https://changedetection.i
Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you. Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!) Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ) Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)

21
app.json
Unescape View File

@ -1,21 +0,0 @@
{
"name": "ChangeDetection.io",
"description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
"keywords": [
"changedetection",
"website monitoring"
],
"repository": "https://github.com/dgtlmoon/changedetection.io",
"success_url": "/",
"scripts": {
},
"env": {
},
"formation": {
"web": {
"quantity": 1,
"size": "free"
}
},
"image": "heroku/python"
}

2
changedetectionio/__init__.py
Unescape View File

@ -2,7 +2,7 @@
# Read more https://github.com/dgtlmoon/changedetection.io/wiki # Read more https://github.com/dgtlmoon/changedetection.io/wiki
__version__ = '0.45.8.1' __version__ = '0.45.12'
from distutils.util import strtobool from distutils.util import strtobool
from json.decoder import JSONDecodeError from json.decoder import JSONDecodeError

24
changedetectionio/api/api_v1.py
Unescape View File

@ -30,7 +30,7 @@ class Watch(Resource):
self.update_q = kwargs['update_q'] self.update_q = kwargs['update_q']
# Get information about a single watch, excluding the history list (can be large) # Get information about a single watch, excluding the history list (can be large)
# curl http://localhost:4000/api/v1/watch/<string:uuid> # curl http://localhost:5000/api/v1/watch/<string:uuid>
# @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK" # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
# ?recheck=true # ?recheck=true
@auth.check_token @auth.check_token
@ -39,9 +39,9 @@ class Watch(Resource):
@api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute. @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
@apiDescription Retrieve watch information and set muted/paused status @apiDescription Retrieve watch information and set muted/paused status
@apiExample {curl} Example usage: @apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45" curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted" -H"x-api-key:813031b16330fe25e3780cf0325daa45" curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused" -H"x-api-key:813031b16330fe25e3780cf0325daa45" curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@apiName Watch @apiName Watch
@apiGroup Watch @apiGroup Watch
@apiParam {uuid} uuid Watch unique ID. @apiParam {uuid} uuid Watch unique ID.
@ -84,7 +84,7 @@ class Watch(Resource):
""" """
@api {delete} /api/v1/watch/:uuid Delete a watch and related history @api {delete} /api/v1/watch/:uuid Delete a watch and related history
@apiExample {curl} Example usage: @apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45" curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@apiParam {uuid} uuid Watch unique ID. @apiParam {uuid} uuid Watch unique ID.
@apiName Delete @apiName Delete
@apiGroup Watch @apiGroup Watch
@ -103,7 +103,7 @@ class Watch(Resource):
@api {put} /api/v1/watch/:uuid Update watch information @api {put} /api/v1/watch/:uuid Update watch information
@apiExample {curl} Example usage: @apiExample {curl} Example usage:
Update (PUT) Update (PUT)
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}' curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
@apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a> @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
@apiParam {uuid} uuid Watch unique ID. @apiParam {uuid} uuid Watch unique ID.
@ -132,13 +132,13 @@ class WatchHistory(Resource):
self.datastore = kwargs['datastore'] self.datastore = kwargs['datastore']
# Get a list of available history for a watch by UUID # Get a list of available history for a watch by UUID
# curl http://localhost:4000/api/v1/watch/<string:uuid>/history # curl http://localhost:5000/api/v1/watch/<string:uuid>/history
def get(self, uuid): def get(self, uuid):
""" """
@api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
@apiDescription Requires `uuid`, returns list @apiDescription Requires `uuid`, returns list
@apiExample {curl} Example usage: @apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
{ {
"1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt", "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
"1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt", "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
@ -166,7 +166,7 @@ class WatchSingleHistory(Resource):
@api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch @api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
@apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a> @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
@apiExample {curl} Example usage: @apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
@apiName Get single snapshot content @apiName Get single snapshot content
@apiGroup Watch History @apiGroup Watch History
@apiSuccess (200) {String} OK @apiSuccess (200) {String} OK
@ -202,7 +202,7 @@ class CreateWatch(Resource):
@api {post} /api/v1/watch Create a single watch @api {post} /api/v1/watch Create a single watch
@apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create. @apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
@apiExample {curl} Example usage: @apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}' curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
@apiName Create @apiName Create
@apiGroup Watch @apiGroup Watch
@apiSuccess (200) {String} OK Was created @apiSuccess (200) {String} OK Was created
@ -245,7 +245,7 @@ class CreateWatch(Resource):
@api {get} /api/v1/watch List watches @api {get} /api/v1/watch List watches
@apiDescription Return concise list of available watches and some very basic info @apiDescription Return concise list of available watches and some very basic info
@apiExample {curl} Example usage: @apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
{ {
"6a4b7d5c-fee4-4616-9f43-4ac97046b595": { "6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
"last_changed": 1677103794, "last_changed": 1677103794,
@ -363,7 +363,7 @@ class SystemInfo(Resource):
@api {get} /api/v1/systeminfo Return system info @api {get} /api/v1/systeminfo Return system info
@apiDescription Return some info about the current system state @apiDescription Return some info about the current system state
@apiExample {curl} Example usage: @apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45" curl http://localhost:5000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
HTTP/1.0 200 HTTP/1.0 200
{ {
'queue_size': 10 , 'queue_size': 10 ,

2
changedetectionio/blueprint/browser_steps/__init__.py
Unescape View File

@ -58,7 +58,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
io_interface_context = io_interface_context.start() io_interface_context = io_interface_context.start()
keepalive_ms = ((keepalive_seconds + 3) * 1000) keepalive_ms = ((keepalive_seconds + 3) * 1000)
base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '') base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"')
a = "?" if not '?' in base_url else '&' a = "?" if not '?' in base_url else '&'
base_url += a + f"timeout={keepalive_ms}" base_url += a + f"timeout={keepalive_ms}"

2
changedetectionio/blueprint/tags/templates/edit-tag.html
Unescape View File

@ -3,7 +3,7 @@
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.jinja' import render_common_settings_form %} {% from '_common_fields.jinja' import render_common_settings_form %}
<script> <script>
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}"; const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
</script> </script>
<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script> <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

47
changedetectionio/content_fetcher.py
Unescape View File

@ -43,9 +43,11 @@ class JSActionExceptions(Exception):
return return
class BrowserStepsStepTimout(Exception): class BrowserStepsStepException(Exception):
def __init__(self, step_n): def __init__(self, step_n, original_e):
self.step_n = step_n self.step_n = step_n
self.original_e = original_e
print(f"Browser Steps exception at step {self.step_n}", str(original_e))
return return
@ -91,19 +93,20 @@ class ReplyWithContentButNoText(Exception):
class Fetcher(): class Fetcher():
browser_connection_is_custom = None
browser_connection_url = None
browser_steps = None browser_steps = None
browser_steps_screenshot_path = None browser_steps_screenshot_path = None
content = None content = None
error = None error = None
fetcher_description = "No description" fetcher_description = "No description"
browser_connection_url = None
headers = {} headers = {}
instock_data = None
instock_data_js = ""
status_code = None status_code = None
webdriver_js_execute_code = None webdriver_js_execute_code = None
xpath_data = None xpath_data = None
xpath_element_js = "" xpath_element_js = ""
instock_data = None
instock_data_js = ""
# Will be needed in the future by the VisualSelector, always get this where possible. # Will be needed in the future by the VisualSelector, always get this where possible.
screenshot = False screenshot = False
@ -172,7 +175,7 @@ class Fetcher():
def iterate_browser_steps(self): def iterate_browser_steps(self):
from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
from playwright._impl._errors import TimeoutError from playwright._impl._errors import TimeoutError, Error
from jinja2 import Environment from jinja2 import Environment
jinja2_env = Environment(extensions=['jinja2_time.TimeExtension']) jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
@ -202,10 +205,10 @@ class Fetcher():
optional_value=optional_value) optional_value=optional_value)
self.screenshot_step(step_n) self.screenshot_step(step_n)
self.save_step_html(step_n) self.save_step_html(step_n)
except TimeoutError as e:
print(str(e)) except (Error, TimeoutError) as e:
# Stop processing here # Stop processing here
raise BrowserStepsStepTimout(step_n=step_n) raise BrowserStepsStepException(step_n=step_n, original_e=e)
# It's always good to reset these # It's always good to reset these
def delete_browser_steps_screenshots(self): def delete_browser_steps_screenshots(self):
@ -252,16 +255,19 @@ class base_html_playwright(Fetcher):
proxy = None proxy = None
def __init__(self, proxy_override=None, browser_connection_url=None): def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__() super().__init__()
self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"') self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value if custom_browser_connection_url:
if not browser_connection_url: self.browser_connection_is_custom = True
self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"') self.browser_connection_url = custom_browser_connection_url
else: else:
self.browser_connection_url = browser_connection_url # Fallback to fetching from system
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
# If any proxy settings are enabled, then we should setup the proxy object # If any proxy settings are enabled, then we should setup the proxy object
proxy_args = {} proxy_args = {}
@ -421,8 +427,10 @@ class base_html_playwright(Fetcher):
current_include_filters=None, current_include_filters=None,
is_binary=False): is_binary=False):
# For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!) # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'): # browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')): if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
# Temporary backup solution until we rewrite the playwright code # Temporary backup solution until we rewrite the playwright code
return self.run_fetch_browserless_puppeteer( return self.run_fetch_browserless_puppeteer(
@ -569,15 +577,16 @@ class base_html_webdriver(Fetcher):
'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword'] 'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
proxy = None proxy = None
def __init__(self, proxy_override=None, browser_connection_url=None): def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__() super().__init__()
from selenium.webdriver.common.proxy import Proxy as SeleniumProxy from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
if not browser_connection_url: if not custom_browser_connection_url:
self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"') self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
else: else:
self.browser_connection_url = browser_connection_url self.browser_connection_is_custom = True
self.browser_connection_url = custom_browser_connection_url
# If any proxy settings are enabled, then we should setup the proxy object # If any proxy settings are enabled, then we should setup the proxy object
proxy_args = {} proxy_args = {}
@ -674,7 +683,7 @@ class base_html_webdriver(Fetcher):
class html_requests(Fetcher): class html_requests(Fetcher):
fetcher_description = "Basic fast Plaintext/HTTP Client" fetcher_description = "Basic fast Plaintext/HTTP Client"
def __init__(self, proxy_override=None, browser_connection_url=None): def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__() super().__init__()
self.proxy_override = proxy_override self.proxy_override = proxy_override
# browser_connection_url is none because its always 'launched locally' # browser_connection_url is none because its always 'launched locally'

27
changedetectionio/flask_app.py
Unescape View File

@ -485,14 +485,18 @@ def changedetection_app(config=None, datastore_o=None):
# AJAX endpoint for sending a test # AJAX endpoint for sending a test
@app.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
@app.route("/notification/send-test", methods=['POST']) @app.route("/notification/send-test", methods=['POST'])
@app.route("/notification/send-test/", methods=['POST'])
@login_optionally_required @login_optionally_required
def ajax_callback_send_notification_test(): def ajax_callback_send_notification_test(watch_uuid=None):
# Watch_uuid could be unsuet in the case its used in tag editor, global setings
import apprise import apprise
from .apprise_asset import asset from .apprise_asset import asset
apobj = apprise.Apprise(asset=asset) apobj = apprise.Apprise(asset=asset)
watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None
# validate URLS # validate URLS
if not len(request.form['notification_urls'].strip()): if not len(request.form['notification_urls'].strip()):
@ -505,9 +509,11 @@ def changedetection_app(config=None, datastore_o=None):
return make_response({'error': message}, 400) return make_response({'error': message}, 400)
try: try:
n_object = {'watch_url': request.form['window_url'], # use the same as when it is triggered, but then override it with the form test values
'notification_urls': request.form['notification_urls'].splitlines() n_object = {
} 'watch_url': request.form['window_url'],
'notification_urls': request.form['notification_urls'].splitlines()
}
# Only use if present, if not set in n_object it should use the default system value # Only use if present, if not set in n_object it should use the default system value
if 'notification_format' in request.form and request.form['notification_format'].strip(): if 'notification_format' in request.form and request.form['notification_format'].strip():
@ -519,7 +525,9 @@ def changedetection_app(config=None, datastore_o=None):
if 'notification_body' in request.form and request.form['notification_body'].strip(): if 'notification_body' in request.form and request.form['notification_body'].strip():
n_object['notification_body'] = request.form.get('notification_body', '').strip() n_object['notification_body'] = request.form.get('notification_body', '').strip()
notification_q.put(n_object) from . import update_worker
new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
except Exception as e: except Exception as e:
return make_response({'error': str(e)}, 400) return make_response({'error': str(e)}, 400)
@ -1584,6 +1592,15 @@ def notification_runner():
try: try:
from changedetectionio import notification from changedetectionio import notification
# Fallback to system config if not set
if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
n_object['notification_title'] = datastore.data['settings']['application'].get('notification_format')
sent_obj = notification.process_notification(n_object, datastore) sent_obj = notification.process_notification(n_object, datastore)

1
changedetectionio/forms.py
Unescape View File

@ -43,6 +43,7 @@ valid_method = {
'PUT', 'PUT',
'PATCH', 'PATCH',
'DELETE', 'DELETE',
'OPTIONS',
} }
default_method = 'GET' default_method = 'GET'

1
changedetectionio/model/Watch.py
Unescape View File

@ -38,6 +38,7 @@ base_config = {
'track_ldjson_price_data': None, 'track_ldjson_price_data': None,
'headers': {}, # Extra headers to send 'headers': {}, # Extra headers to send
'ignore_text': [], # List of text to ignore when calculating the comparison checksum 'ignore_text': [], # List of text to ignore when calculating the comparison checksum
'in_stock' : None,
'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock 'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
'include_filters': [], 'include_filters': [],
'last_checked': 0, 'last_checked': 0,

172
changedetectionio/notification.py
Unescape View File

@ -1,4 +1,5 @@
import apprise import apprise
import time
from jinja2 import Environment, BaseLoader from jinja2 import Environment, BaseLoader
from apprise import NotifyFormat from apprise import NotifyFormat
import json import json
@ -119,8 +120,8 @@ def process_notification(n_object, datastore):
# Get the notification body from datastore # Get the notification body from datastore
jinja2_env = Environment(loader=BaseLoader) jinja2_env = Environment(loader=BaseLoader)
n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters) n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters) n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
n_format = valid_notification_formats.get( n_format = valid_notification_formats.get(
n_object.get('notification_format', default_notification_format), n_object.get('notification_format', default_notification_format),
valid_notification_formats[default_notification_format], valid_notification_formats[default_notification_format],
@ -131,103 +132,108 @@ def process_notification(n_object, datastore):
# Initially text or whatever # Initially text or whatever
n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]) n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
# https://github.com/caronc/apprise/wiki/Development_LogCapture # https://github.com/caronc/apprise/wiki/Development_LogCapture
# Anything higher than or equal to WARNING (which covers things like Connection errors) # Anything higher than or equal to WARNING (which covers things like Connection errors)
# raise it as an exception # raise it as an exception
apobjs=[]
sent_objs=[] sent_objs = []
from .apprise_asset import asset from .apprise_asset import asset
for url in n_object['notification_urls']: apobj = apprise.Apprise(debug=True, asset=asset)
url = jinja2_env.from_string(url).render(**notification_parameters)
apobj = apprise.Apprise(debug=True, asset=asset) if not n_object.get('notification_urls'):
url = url.strip() return None
if len(url):
with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
for url in n_object['notification_urls']:
url = url.strip()
print(">> Process Notification: AppRise notifying {}".format(url)) print(">> Process Notification: AppRise notifying {}".format(url))
with apprise.LogCapture(level=apprise.logging.DEBUG) as logs: url = jinja2_env.from_string(url).render(**notification_parameters)
# Re 323 - Limit discord length to their 2000 char limit total or it wont send.
# Because different notifications may require different pre-processing, run each sequentially :( # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
# 2000 bytes minus - # Because different notifications may require different pre-processing, run each sequentially :(
# 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers # 2000 bytes minus -
# Length of URL - Incase they specify a longer custom avatar_url # 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
# Length of URL - Incase they specify a longer custom avatar_url
# So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
k = '?' if not '?' in url else '&' # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
if not 'avatar_url' in url \ k = '?' if not '?' in url else '&'
and not url.startswith('mail') \ if not 'avatar_url' in url \
and not url.startswith('post') \ and not url.startswith('mail') \
and not url.startswith('get') \ and not url.startswith('post') \
and not url.startswith('delete') \ and not url.startswith('get') \
and not url.startswith('put'): and not url.startswith('delete') \
url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png' and not url.startswith('put'):
url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
if url.startswith('tgram://'):
# Telegram only supports a limit subset of HTML, remove the '<br>' we place in. if url.startswith('tgram://'):
# re https://github.com/dgtlmoon/changedetection.io/issues/555 # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
# @todo re-use an existing library we have already imported to strip all non-allowed tags # re https://github.com/dgtlmoon/changedetection.io/issues/555
n_body = n_body.replace('<br>', '\n') # @todo re-use an existing library we have already imported to strip all non-allowed tags
n_body = n_body.replace('</br>', '\n') n_body = n_body.replace('<br>', '\n')
# real limit is 4096, but minus some for extra metadata n_body = n_body.replace('</br>', '\n')
payload_max_size = 3600 # real limit is 4096, but minus some for extra metadata
body_limit = max(0, payload_max_size - len(n_title)) payload_max_size = 3600
n_title = n_title[0:payload_max_size] body_limit = max(0, payload_max_size - len(n_title))
n_body = n_body[0:body_limit] n_title = n_title[0:payload_max_size]
n_body = n_body[0:body_limit]
elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith('https://discord.com/api'):
# real limit is 2000, but minus some for extra metadata elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
payload_max_size = 1700 'https://discord.com/api'):
body_limit = max(0, payload_max_size - len(n_title)) # real limit is 2000, but minus some for extra metadata
n_title = n_title[0:payload_max_size] payload_max_size = 1700
n_body = n_body[0:body_limit] body_limit = max(0, payload_max_size - len(n_title))
n_title = n_title[0:payload_max_size]
elif url.startswith('mailto'): n_body = n_body[0:body_limit]
# Apprise will default to HTML, so we need to override it
# So that whats' generated in n_body is in line with what is going to be sent. elif url.startswith('mailto'):
# https://github.com/caronc/apprise/issues/633#issuecomment-1191449321 # Apprise will default to HTML, so we need to override it
if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'): # So that whats' generated in n_body is in line with what is going to be sent.
prefix = '?' if not '?' in url else '&' # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
# Apprise format is lowercase text https://github.com/caronc/apprise/issues/633 if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
n_format = n_format.tolower() prefix = '?' if not '?' in url else '&'
url = "{}{}format={}".format(url, prefix, n_format) # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
# If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only n_format = n_format.lower()
url = f"{url}{prefix}format={n_format}"
apobj.add(url) # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
apobj.notify( apobj.add(url)
title=n_title,
body=n_body, sent_objs.append({'title': n_title,
body_format=n_format, 'body': n_body,
# False is not an option for AppRise, must be type None 'url': url,
attach=n_object.get('screenshot', None) 'body_format': n_format})
)
# Blast off the notifications tht are set in .add()
apobj.clear() apobj.notify(
title=n_title,
# Incase it needs to exist in memory for a while after to process(?) body=n_body,
apobjs.append(apobj) body_format=n_format,
# False is not an option for AppRise, must be type None
# Returns empty string if nothing found, multi-line string otherwise attach=n_object.get('screenshot', None)
log_value = logs.getvalue() )
if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
raise Exception(log_value) # Give apprise time to register an error
time.sleep(3)
sent_objs.append({'title': n_title,
'body': n_body, # Returns empty string if nothing found, multi-line string otherwise
'url' : url, log_value = logs.getvalue()
'body_format': n_format})
if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
raise Exception(log_value)
# Return what was sent for better logging - after the for loop # Return what was sent for better logging - after the for loop
return sent_objs return sent_objs
# Notification title + body content parameters get created here. # Notification title + body content parameters get created here.
# ( Where we prepare the tokens in the notification to be replaced with actual values )
def create_notification_parameters(n_object, datastore): def create_notification_parameters(n_object, datastore):
from copy import deepcopy from copy import deepcopy
# in the case we send a test notification from the main settings, there is no UUID. # in the case we send a test notification from the main settings, there is no UUID.
uuid = n_object['uuid'] if 'uuid' in n_object else '' uuid = n_object['uuid'] if 'uuid' in n_object else ''
if uuid != '': if uuid:
watch_title = datastore.data['watching'][uuid].get('title', '') watch_title = datastore.data['watching'][uuid].get('title', '')
tag_list = [] tag_list = []
tags = datastore.get_all_tags_for_watch(uuid) tags = datastore.get_all_tags_for_watch(uuid)
@ -255,7 +261,7 @@ def create_notification_parameters(n_object, datastore):
tokens.update( tokens.update(
{ {
'base_url': base_url, 'base_url': base_url,
'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '', 'current_snapshot': n_object.get('current_snapshot', ''),
'diff': n_object.get('diff', ''), # Null default in the case we use a test 'diff': n_object.get('diff', ''), # Null default in the case we use a test
'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test 'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test
'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test 'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test

6
changedetectionio/processors/__init__.py
Unescape View File

@ -43,14 +43,14 @@ class difference_detection_processor():
# In the case that the preferred fetcher was a browser config with custom connection URL.. # In the case that the preferred fetcher was a browser config with custom connection URL..
# @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..) # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
browser_connection_url = None custom_browser_connection_url = None
if prefer_fetch_backend.startswith('extra_browser_'): if prefer_fetch_backend.startswith('extra_browser_'):
(t, key) = prefer_fetch_backend.split('extra_browser_') (t, key) = prefer_fetch_backend.split('extra_browser_')
connection = list( connection = list(
filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', []))) filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
if connection: if connection:
prefer_fetch_backend = 'base_html_playwright' prefer_fetch_backend = 'base_html_playwright'
browser_connection_url = connection[0].get('browser_connection_url') custom_browser_connection_url = connection[0].get('browser_connection_url')
# PDF should be html_requests because playwright will serve it up (so far) in a embedded page # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
# @todo https://github.com/dgtlmoon/changedetection.io/issues/2019 # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
@ -74,7 +74,7 @@ class difference_detection_processor():
# Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need. # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
# When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc) # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
self.fetcher = fetcher_obj(proxy_override=proxy_url, self.fetcher = fetcher_obj(proxy_override=proxy_url,
browser_connection_url=browser_connection_url custom_browser_connection_url=custom_browser_connection_url
) )
if self.watch.has_browser_steps: if self.watch.has_browser_steps:

6
changedetectionio/processors/restock_diff.py
Unescape View File

@ -109,4 +109,8 @@ class perform_site_check(difference_detection_processor):
# All cases # All cases
changed_detected = True changed_detected = True
return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8') # Always record the new checksum
update_obj["previous_md5"] = fetched_md5
return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8').strip()

212
changedetectionio/res/stock-not-in-stock.js
Unescape View File

@ -1,115 +1,131 @@
function isItemInStock() { function isItemInStock() {
// @todo Pass these in so the same list can be used in non-JS fetchers // @todo Pass these in so the same list can be used in non-JS fetchers
const outOfStockTexts = [ const outOfStockTexts = [
' أخبرني عندما يتوفر', ' أخبرني عندما يتوفر',
'0 in stock', '0 in stock',
'agotado', 'agotado',
'artikel zurzeit vergriffen', 'article épuisé',
'as soon as stock is available', 'artikel zurzeit vergriffen',
'ausverkauft', // sold out 'as soon as stock is available',
'available for back order', 'ausverkauft', // sold out
'back-order or out of stock', 'available for back order',
'backordered', 'back-order or out of stock',
'benachrichtigt mich', // notify me 'backordered',
'brak na stanie', 'benachrichtigt mich', // notify me
'brak w magazynie', 'brak na stanie',
'coming soon', 'brak w magazynie',
'currently have any tickets for this', 'coming soon',
'currently unavailable', 'currently have any tickets for this',
'dostępne wkrótce', 'currently unavailable',
'dostępne wkrótce', 'dostępne wkrótce',
'en rupture de stock', 'en rupture de stock',
'ist derzeit nicht auf lager', 'ist derzeit nicht auf lager',
'ist derzeit nicht auf lager', 'item is no longer available',
'item is no longer available', 'let me know when it\'s available',
'let me know when it\'s available', 'message if back in stock',
'message if back in stock', 'nachricht bei',
'nachricht bei', 'nicht auf lager',
'nicht auf lager', 'nicht lieferbar',
'nicht lieferbar', 'nicht zur verfügung',
'nicht zur verfügung', 'niet beschikbaar',
'no disponible temporalmente', 'niet leverbaar',
'no longer in stock', 'no disponible temporalmente',
'no tickets available', 'no longer in stock',
'not available', 'no tickets available',
'not currently available', 'not available',
'not in stock', 'not currently available',
'notify me when available', 'not in stock',
'não estamos a aceitar encomendas', 'notify me when available',
'out of stock', 'não estamos a aceitar encomendas',
'out-of-stock', 'out of stock',
'produkt niedostępny', 'out-of-stock',
'sold out', 'produkt niedostępny',
'sold-out', 'sold out',
'temporarily out of stock', 'sold-out',
'temporarily unavailable', 'temporarily out of stock',
'tickets unavailable', 'temporarily unavailable',
'unavailable tickets', 'tickets unavailable',
'we do not currently have an estimate of when this product will be back in stock.', 'tijdelijk uitverkocht',
'zur zeit nicht an lager', 'unavailable tickets',
'品切れ', 'we do not currently have an estimate of when this product will be back in stock.',
'已售完', 'we don\'t know when or if this item will be back in stock.',
'품절' 'zur zeit nicht an lager',
]; '品切れ',
'已售完',
'품절'
];
function getElementBaseText(element) {
// .textContent can include text from children which may give the wrong results
// scan only immediate TEXT_NODEs, which will be a child of the element
var text = "";
for (var i = 0; i < element.childNodes.length; ++i)
if (element.childNodes[i].nodeType === Node.TEXT_NODE)
text += element.childNodes[i].textContent;
return text.toLowerCase().trim();
}
const negateOutOfStockRegexs = [ const negateOutOfStockRegexs = [
'[0-9] in stock' '[0-9] in stock'
] ]
var negateOutOfStockRegexs_r = []; var negateOutOfStockRegexs_r = [];
for (let i = 0; i < negateOutOfStockRegexs.length; i++) { for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g')); negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g'));
} }
// The out-of-stock or in-stock-text is generally always above-the-fold
// and often below-the-fold is a list of related products that may or may not contain trigger text
// so it's good to filter to just the 'above the fold' elements
// and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
const elementsToScan = Array.from(document.getElementsByTagName('*')).filter(element => element.getBoundingClientRect().top + window.scrollY <= window.innerHeight && element.getBoundingClientRect().top + window.scrollY >= 100);
const elementsWithZeroChildren = Array.from(document.getElementsByTagName('*')).filter(element => element.children.length === 0); var elementText = "";
// REGEXS THAT REALLY MEAN IT'S IN STOCK // REGEXS THAT REALLY MEAN IT'S IN STOCK
for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) { for (let i = elementsToScan.length - 1; i >= 0; i--) {
const element = elementsWithZeroChildren[i]; const element = elementsToScan[i];
if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) { elementText = "";
var elementText=""; if (element.tagName.toLowerCase() === "input") {
if (element.tagName.toLowerCase() === "input") { elementText = element.value.toLowerCase();
elementText = element.value.toLowerCase(); } else {
} else { elementText = getElementBaseText(element);
elementText = element.textContent.toLowerCase(); }
}
if (elementText.length) { if (elementText.length) {
// try which ones could mean its in stock // try which ones could mean its in stock
for (let i = 0; i < negateOutOfStockRegexs.length; i++) { for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
if (negateOutOfStockRegexs_r[i].test(elementText)) { if (negateOutOfStockRegexs_r[i].test(elementText)) {
return 'Possibly in stock'; return 'Possibly in stock';
} }
}
} }
}
} }
}
// OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) { for (let i = elementsToScan.length - 1; i >= 0; i--) {
const element = elementsWithZeroChildren[i]; const element = elementsToScan[i];
if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) { if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
var elementText=""; elementText = "";
if (element.tagName.toLowerCase() === "input") { if (element.tagName.toLowerCase() === "input") {
elementText = element.value.toLowerCase(); elementText = element.value.toLowerCase();
} else { } else {
elementText = element.textContent.toLowerCase(); elementText = getElementBaseText(element);
} }
if (elementText.length) { if (elementText.length) {
// and these mean its out of stock // and these mean its out of stock
for (const outOfStockText of outOfStockTexts) { for (const outOfStockText of outOfStockTexts) {
if (elementText.includes(outOfStockText)) { if (elementText.includes(outOfStockText)) {
return elementText; // item is out of stock return outOfStockText; // item is out of stock
} }
}
}
} }
}
} }
}
return 'Possibly in stock'; // possibly in stock, cant decide otherwise. return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
} }
// returns the element text that makes it think it's out of stock // returns the element text that makes it think it's out of stock
return isItemInStock(); return isItemInStock().trim()

17
changedetectionio/static/js/notifications.js
Unescape View File

@ -24,14 +24,17 @@ $(document).ready(function() {
}) })
data = { data = {
window_url : window.location.href, notification_body: $('#notification_body').val(),
notification_urls : $('.notification-urls').val(), notification_format: $('#notification_format').val(),
notification_title: $('#notification_title').val(),
notification_urls: $('.notification-urls').val(),
window_url: window.location.href,
} }
for (key in data) {
if (!data[key].length) {
alert(key+" is empty, cannot send test.") if (!data['notification_urls'].length) {
return; alert("Notification URL list is empty, cannot send test.")
} return;
} }
$.ajax({ $.ajax({

1
changedetectionio/store.py
Unescape View File

@ -248,6 +248,7 @@ class ChangeDetectionStore:
'check_count': 0, 'check_count': 0,
'fetch_time' : 0.0, 'fetch_time' : 0.0,
'has_ldjson_price_data': None, 'has_ldjson_price_data': None,
'in_stock': None,
'last_checked': 0, 'last_checked': 0,
'last_error': False, 'last_error': False,
'last_notification_error': False, 'last_notification_error': False,

2
changedetectionio/templates/edit.html
Unescape View File

@ -14,7 +14,7 @@
{% if emailprefix %} {% if emailprefix %}
const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}'); const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
{% endif %} {% endif %}
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}"; const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %}; const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}"; const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}"; const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";

2
changedetectionio/templates/settings.html
Unescape View File

@ -4,7 +4,7 @@
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.jinja' import render_common_settings_form %} {% from '_common_fields.jinja' import render_common_settings_form %}
<script> <script>
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}"; const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
{% if emailprefix %} {% if emailprefix %}
const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}'); const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
{% endif %} {% endif %}

18
changedetectionio/tests/smtp/test_notification_smtp.py
Unescape View File

@ -97,6 +97,17 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
set_original_response() set_original_response()
global smtp_test_server global smtp_test_server
notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
notification_body = f"""<!DOCTYPE html>
<html lang="en">
<head>
<title>My Webpage</title>
</head>
<body>
<h1>Test</h1>
{default_notification_body}
</body>
</html>
"""
##################### #####################
# Set this up for when we remove the notification from the watch, it should fallback with these details # Set this up for when we remove the notification from the watch, it should fallback with these details
@ -104,7 +115,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
url_for("settings_page"), url_for("settings_page"),
data={"application-notification_urls": notification_url, data={"application-notification_urls": notification_url,
"application-notification_title": "fallback-title " + default_notification_title, "application-notification_title": "fallback-title " + default_notification_title,
"application-notification_body": default_notification_body, "application-notification_body": notification_body,
"application-notification_format": 'Text', "application-notification_format": 'Text',
"requests-time_between_check-minutes": 180, "requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_requests"}, 'application-fetch_backend': "html_requests"},
@ -161,5 +172,10 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
assert 'Content-Type: text/html' in msg assert 'Content-Type: text/html' in msg
assert '(removed) So let\'s see what happens.<br>' in msg # the html part assert '(removed) So let\'s see what happens.<br>' in msg # the html part
# https://github.com/dgtlmoon/changedetection.io/issues/2103
assert '<h1>Test</h1>' in msg
assert '&lt;' not in msg
assert 'Content-Type: text/html' in msg
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data assert b'Deleted' in res.data

19
changedetectionio/tests/test_notification_errors.py
Unescape View File

@ -1,8 +1,7 @@
import os import os
import time import time
import re
from flask import url_for from flask import url_for
from . util import set_original_response, set_modified_response, live_server_setup from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
import logging import logging
def test_check_notification_error_handling(client, live_server): def test_check_notification_error_handling(client, live_server):
@ -11,7 +10,7 @@ def test_check_notification_error_handling(client, live_server):
set_original_response() set_original_response()
# Give the endpoint time to spin up # Give the endpoint time to spin up
time.sleep(2) time.sleep(1)
# Set a URL and fetch it, then set a notification URL which is going to give errors # Set a URL and fetch it, then set a notification URL which is going to give errors
test_url = url_for('test_endpoint', _external=True) test_url = url_for('test_endpoint', _external=True)
@ -22,12 +21,16 @@ def test_check_notification_error_handling(client, live_server):
) )
assert b"Watch added" in res.data assert b"Watch added" in res.data
time.sleep(2) wait_for_all_checks(client)
set_modified_response() set_modified_response()
working_notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
broken_notification_url = "jsons://broken-url-xxxxxxxx123/test"
res = client.post( res = client.post(
url_for("edit_page", uuid="first"), url_for("edit_page", uuid="first"),
data={"notification_urls": "jsons://broken-url-xxxxxxxx123/test", # A URL with errors should not block the one that is working
data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}",
"notification_title": "xxx", "notification_title": "xxx",
"notification_body": "xxxxx", "notification_body": "xxxxx",
"notification_format": "Text", "notification_format": "Text",
@ -63,4 +66,10 @@ def test_check_notification_error_handling(client, live_server):
found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
assert found_name_resolution_error assert found_name_resolution_error
# And the working one, which is after the 'broken' one should still have fired
with open("test-datastore/notification.txt", "r") as f:
notification_submission = f.read()
os.unlink("test-datastore/notification.txt")
assert 'xxxxx' in notification_submission
client.get(url_for("form_delete", uuid="all"), follow_redirects=True) client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

72
changedetectionio/update_worker.py
Unescape View File

@ -26,47 +26,61 @@ class update_worker(threading.Thread):
self.datastore = datastore self.datastore = datastore
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def queue_notification_for_watch(self, n_object, watch): def queue_notification_for_watch(self, notification_q, n_object, watch):
from changedetectionio import diff from changedetectionio import diff
dates = []
trigger_text = ''
if watch:
watch_history = watch.history
dates = list(watch_history.keys())
trigger_text = watch.get('trigger_text', [])
watch_history = watch.history
dates = list(watch_history.keys())
# Add text that was triggered # Add text that was triggered
snapshot_contents = watch.get_history_snapshot(dates[-1]) if len(dates):
snapshot_contents = watch.get_history_snapshot(dates[-1])
else:
snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
# HTML needs linebreak, but MarkDown and Text can use a linefeed # HTML needs linebreak, but MarkDown and Text can use a linefeed
if n_object['notification_format'] == 'HTML': if n_object.get('notification_format') == 'HTML':
line_feed_sep = "<br>" line_feed_sep = "<br>"
# Snapshot will be plaintext on the disk, convert to some kind of HTML # Snapshot will be plaintext on the disk, convert to some kind of HTML
snapshot_contents = snapshot_contents.replace('\n', line_feed_sep) snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
else: else:
line_feed_sep = "\n" line_feed_sep = "\n"
trigger_text = watch.get('trigger_text', [])
triggered_text = '' triggered_text = ''
if len(trigger_text): if len(trigger_text):
from . import html_tools from . import html_tools
triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text) triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
if triggered_text: if triggered_text:
triggered_text = line_feed_sep.join(triggered_text) triggered_text = line_feed_sep.join(triggered_text)
# Could be called as a 'test notification' with only 1 snapshot available
prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
if len(dates) > 1:
prev_snapshot = watch.get_history_snapshot(dates[-2])
current_snapshot = watch.get_history_snapshot(dates[-1])
n_object.update({ n_object.update({
'current_snapshot': snapshot_contents, 'current_snapshot': snapshot_contents,
'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep), 'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep), 'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep), 'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True), 'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep), 'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None, 'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
'triggered_text': triggered_text, 'triggered_text': triggered_text,
'uuid': watch.get('uuid'), 'uuid': watch.get('uuid') if watch else None,
'watch_url': watch.get('url'), 'watch_url': watch.get('url') if watch else None,
}) })
logging.info (">> SENDING NOTIFICATION") logging.info(">> SENDING NOTIFICATION")
self.notification_q.put(n_object)
notification_q.put(n_object)
# Prefer - Individual watch settings > Tag settings > Global settings (in that order) # Prefer - Individual watch settings > Tag settings > Global settings (in that order)
def _check_cascading_vars(self, var_name, watch): def _check_cascading_vars(self, var_name, watch):
@ -134,7 +148,7 @@ class update_worker(threading.Thread):
queued = False queued = False
if n_object and n_object.get('notification_urls'): if n_object and n_object.get('notification_urls'):
queued = True queued = True
self.queue_notification_for_watch(n_object, watch) self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
return queued return queued
@ -174,9 +188,9 @@ class update_worker(threading.Thread):
return return
threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts') threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1), n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
'notification_body': "Your configured browser step at position {} for {{watch['url']}} " 'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
"did not appear on the page after {} attempts, did the page change layout? " "did not appear on the page after {} attempts, did the page change layout? "
"Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n" "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
"Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold), "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
'notification_format': 'text'} 'notification_format': 'text'}
@ -340,20 +354,32 @@ class update_worker(threading.Thread):
changed_detected = False changed_detected = False
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False}) self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
except content_fetcher.BrowserStepsStepTimout as e: except content_fetcher.BrowserStepsStepException as e:
if not self.datastore.data['watching'].get(uuid): if not self.datastore.data['watching'].get(uuid):
continue continue
error_step = e.step_n + 1 error_step = e.step_n + 1
err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step" from playwright._impl._errors import TimeoutError, Error
# Generally enough info for TimeoutError (couldnt locate the element after default seconds)
err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
if e.original_e.name == "TimeoutError":
# Just the first line is enough, the rest is the stack trace
err_text += " Could not find the target."
else:
# Other Error, more info is good.
err_text += " " + str(e.original_e).splitlines()[0]
print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
self.datastore.update_watch(uuid=uuid, self.datastore.update_watch(uuid=uuid,
update_obj={'last_error': err_text, update_obj={'last_error': err_text,
'browser_steps_last_error_step': error_step 'browser_steps_last_error_step': error_step
} }
) )
if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False): if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5) c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
c += 1 c += 1

2
docker-compose.yml
Unescape View File

@ -90,7 +90,7 @@ services:
# #
# Used for fetching pages via Playwright+Chrome where you need Javascript support. # Used for fetching pages via Playwright+Chrome where you need Javascript support.
# Note: works well but is deprecated, doesnt fetch full page screenshots and other issues # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
# browser-chrome: # browser-chrome:
# hostname: browser-chrome # hostname: browser-chrome
# image: selenium/standalone-chrome:4 # image: selenium/standalone-chrome:4

2
heroku.yml
Unescape View File

@ -1,2 +0,0 @@
run:
changedetection: python3 ./changedetection.py -C -d ./datastore -p $PORT

2
requirements.txt
Unescape View File

@ -30,7 +30,7 @@ dnspython~=2.4 # related to eventlet fixes
# jq not available on Windows so must be installed manually # jq not available on Windows so must be installed manually
# Notification library # Notification library
apprise~=1.6.0 apprise~=1.7.1
# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
paho-mqtt paho-mqtt

4
setup.py
Unescape View File

@ -27,7 +27,7 @@ install_requires = open('requirements.txt').readlines()
setup( setup(
name='changedetection.io', name='changedetection.io',
version=find_version("changedetectionio", "__init__.py"), version=find_version("changedetectionio", "__init__.py"),
description='Website change detection and monitoring service', description='Website change detection and monitoring service, detect changes to web pages and send alerts/notifications.',
long_description=open('README-pip.md').read(), long_description=open('README-pip.md').read(),
long_description_content_type='text/markdown', long_description_content_type='text/markdown',
keywords='website change monitor for changes notification change detection ' keywords='website change monitor for changes notification change detection '
@ -41,7 +41,7 @@ setup(
include_package_data=True, include_package_data=True,
install_requires=install_requires, install_requires=install_requires,
license="Apache License 2.0", license="Apache License 2.0",
python_requires=">= 3.7", python_requires=">= 3.10",
classifiers=['Intended Audience :: Customer Service', classifiers=['Intended Audience :: Customer Service',
'Intended Audience :: Developers', 'Intended Audience :: Developers',
'Intended Audience :: Education', 'Intended Audience :: Education',

Write Preview
Loading…
Cancel
Save