From bce7eb68fbb58b593eb4804eb30d109bd944376c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 29 Jan 2024 14:20:39 +0100 Subject: [PATCH 1/3] Notifications - skip empty notification URLs from being processed (#2138) --- changedetectionio/notification.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index b2046004..21bda720 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -152,6 +152,10 @@ def process_notification(n_object, datastore): with apprise.LogCapture(level=apprise.logging.DEBUG) as logs: for url in n_object['notification_urls']: url = url.strip() + if not url: + logger.warning(f"Process Notification: skipping empty notification URL.") + continue + logger.info(">> Process Notification: AppRise notifying {}".format(url)) url = jinja2_env.from_string(url).render(**notification_parameters) From 66e2dfcead0b5b427af01dce5a33213410b6dba6 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 29 Jan 2024 16:26:14 +0100 Subject: [PATCH 2/3] RSS - Include link to the watched URL in the feed (#2139 #2131 and #327) --- changedetectionio/flask_app.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index e0ee8b83..058651dd 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -13,7 +13,6 @@ from threading import Event import datetime import flask_login from loguru import logger -import sys import os import pytz import queue @@ -317,6 +316,8 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/rss", methods=['GET']) def rss(): + from jinja2 import Environment, BaseLoader + jinja2_env = Environment(loader=BaseLoader) now = time.time() # Always requires token set app_rss_token = datastore.data['settings']['application'].get('rss_access_token') @@ -381,8 +382,12 @@ def changedetection_app(config=None, datastore_o=None): include_equal=False, line_feed_sep="
") - fe.content(content="

{}

{}".format(watch_title, html_diff), - type='CDATA') + # @todo Make this configurable and also consider html-colored markup + # @todo User could decide if goes to the diff page, or to the watch link + rss_template = "\n

{{watch_title}}

\n

{{html_diff}}

\n\n" + content = jinja2_env.from_string(rss_template).render(watch_title=watch_title, html_diff=html_diff, watch_url=watch.link) + + fe.content(content=content, type='CDATA') fe.guid(guid, permalink=False) dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key)) From d8fbf4fbdae604f03fbb15695306f374d435c9d6 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 31 Jan 2024 14:23:23 +0100 Subject: [PATCH 3/3] General test improvements --- .github/workflows/pypi-release.yml | 6 +++--- .github/workflows/test-only.yml | 28 ++++++++++++++++++++-------- changedetectionio/tests/test_api.py | 1 + 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 0f38941f..1b2ffb76 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -40,12 +40,12 @@ jobs: path: dist/ - name: Test that the basic pip built package runs without error run: | - set -e + set -ex pip3 install dist/changedetection.io*.whl changedetection.io -d /tmp -p 10000 & sleep 3 - curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null - curl http://127.0.0.1:10000/ >/dev/null + curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null + curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null killall changedetection.io diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index 6f762437..f3fde056 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -27,7 +27,7 @@ jobs: run: | docker network create changedet-network - + # Selenium+browserless docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4 docker run --network changedet-network -d --name browserless --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.60-chrome-stable @@ -47,7 +47,7 @@ jobs: # Debug SMTP server/echo message back server docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' - - name: Test built container with pytest + - name: Test built container with Pytest (generally as requests/plaintext fetching) run: | # Unit tests echo "run test with unittest" @@ -61,20 +61,32 @@ jobs: # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG' docker run --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh' - - name: Test built container selenium+browserless/playwright + - name: Specific tests in built container for Selenium run: | # Selenium fetch docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py' - + + - name: Specific tests in built container for Playwright + run: | # Playwright/Browserless fetch docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' - + + - name: Specific tests in built container for headers and requests checks with Playwright + run: | # Settings headers playwright tests - Call back in from Browserless, check headers docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' + + - name: Specific tests in built container for headers and requests checks with Selenium + run: | docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' + + - name: Specific tests in built container with Playwright as Puppeteer experimental fetcher + run: | docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' - + + - name: Test built container restock detection via Playwright + run: | # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py' @@ -106,10 +118,10 @@ jobs: docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio sleep 3 # Should return 0 (no error) when grep finds it - curl -s http://localhost:5556 |grep -q checkbox-uuid + curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid # and IPv6 - curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid + curl --retry-connrefused --retry 6 -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid # Check whether TRACE log is enabled. # Also, check whether TRACE is came from STDERR diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index d83ababa..5be55ec2 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -163,6 +163,7 @@ def test_api_simple(client, live_server): # Loading the most recent snapshot should force viewed to become true client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True) + time.sleep(3) # Fetch the whole watch again, viewed should be true res = client.get( url_for("watch", uuid=watch_uuid),