diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml
index 5483e6f6..69e42cba 100644
--- a/.github/workflows/test-only.yml
+++ b/.github/workflows/test-only.yml
@@ -4,17 +4,10 @@ name: ChangeDetection.io App Test
on: [push, pull_request]
jobs:
- test-application:
+ lint-code:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
-
- # Mainly just for link/flake8
- - name: Set up Python 3.11
- uses: actions/setup-python@v5
- with:
- python-version: '3.11'
-
- name: Lint with flake8
run: |
pip3 install flake8
@@ -23,202 +16,24 @@ jobs:
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- - name: Spin up ancillary testable services
- run: |
-
- docker network create changedet-network
-
- # Selenium
- docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
-
- # SocketPuppetBrowser + Extra for custom browser test
- docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
- docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
-
- - name: Build changedetection.io container for testing
- run: |
- # Build a changedetection.io container and start testing inside
- docker build --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
- # Debug info
- docker run test-changedetectionio bash -c 'pip list'
-
- - name: Spin up ancillary SMTP+Echo message test server
- run: |
- # Debug SMTP server/echo message back server
- docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py'
-
- - name: Show docker container state and other debug info
- run: |
- set -x
- echo "Running processes in docker..."
- docker ps
-
- - name: Test built container with Pytest (generally as requests/plaintext fetching)
- run: |
- # Unit tests
- echo "run test with unittest"
- docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
- docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
- docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-
- # All tests
- echo "run test with pytest"
- # The default pytest logger_level is TRACE
- # To change logger_level for pytest(test/conftest.py),
- # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
- docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'
-
-# PLAYWRIGHT/NODE-> CDP
- - name: Playwright and SocketPuppetBrowser - Specific tests in built container
- run: |
- # Playwright via Sockpuppetbrowser fetch
- # tests/visualselector/test_fetch_data.py will do browser steps
- docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
- docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
- docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
- docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-
- - name: Playwright and SocketPuppetBrowser - Headers and requests
- run: |
- # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
- docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
-
- - name: Playwright and SocketPuppetBrowser - Restock detection
- run: |
- # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
- docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# STRAIGHT TO CDP
- - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
- run: |
- # Playwright via Sockpuppetbrowser fetch
- docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
- docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
- docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
- docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
- - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
- run: |
- # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
- docker run --name "changedet" --hostname changedet --rm -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
+ test-application-3-10:
+ needs: lint-code
+ uses: ./.github/workflows/test-stack-reusable-workflow.yml
+ with:
+ python-version: '3.10'
- - name: Pyppeteer and SocketPuppetBrowser - Restock detection
- run: |
- # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
- docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-# SELENIUM
- - name: Specific tests in built container for Selenium
- run: |
- # Selenium fetch
- docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-
- - name: Specific tests in built container for headers and requests checks with Selenium
- run: |
- docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
-
-# OTHER STUFF
- - name: Test SMTP notification mime types
- run: |
- # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
- docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
- # @todo Add a test via playwright/puppeteer
- # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
- - name: Test proxy squid style interaction
- run: |
- cd changedetectionio
- ./run_proxy_tests.sh
- cd ..
-
- - name: Test proxy SOCKS5 style interaction
- run: |
- cd changedetectionio
- ./run_socks_proxy_tests.sh
- cd ..
+ test-application-3-11:
+ needs: lint-code
+ uses: ./.github/workflows/test-stack-reusable-workflow.yml
+ with:
+ python-version: '3.11'
+ skip-pypuppeteer: true
- - name: Test custom browser URL
- run: |
- cd changedetectionio
- ./run_custom_browser_url_tests.sh
- cd ..
-
- - name: Test changedetection.io container starts+runs basically without error
- run: |
- docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio
- sleep 3
- # Should return 0 (no error) when grep finds it
- curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid
-
- # and IPv6
- curl --retry-connrefused --retry 6 -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-
- # Check whether TRACE log is enabled.
- # Also, check whether TRACE is came from STDERR
- docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
- # Check whether DEBUG is came from STDOUT
- docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
-
- docker kill test-changedetectionio
-
- - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
- run: |
-
- echo SIGINT Shutdown request test
- docker run --name sig-test -d test-changedetectionio
- sleep 3
- echo ">>> Sending SIGINT to sig-test container"
- docker kill --signal=SIGINT sig-test
- sleep 3
- # invert the check (it should be not 0/not running)
- docker ps
- # check signal catch(STDERR) log. Because of
- # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
- docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
- test -z "`docker ps|grep sig-test`"
- if [ $? -ne 0 ]
- then
- echo "Looks like container was running when it shouldnt be"
- docker ps
- exit 1
- fi
-
- # @todo - scan the container log to see the right "graceful shutdown" text exists
- docker rm sig-test
-
- echo SIGTERM Shutdown request test
- docker run --name sig-test -d test-changedetectionio
- sleep 3
- echo ">>> Sending SIGTERM to sig-test container"
- docker kill --signal=SIGTERM sig-test
- sleep 3
- # invert the check (it should be not 0/not running)
- docker ps
- # check signal catch(STDERR) log. Because of
- # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
- docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
- test -z "`docker ps|grep sig-test`"
- if [ $? -ne 0 ]
- then
- echo "Looks like container was running when it shouldnt be"
- docker ps
- exit 1
- fi
-
- # @todo - scan the container log to see the right "graceful shutdown" text exists
- docker rm sig-test
-
- - name: Dump container log
- if: always()
- run: |
- mkdir output-logs
- docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout.txt
- docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr.txt
+ test-application-3-12:
+ needs: lint-code
+ uses: ./.github/workflows/test-stack-reusable-workflow.yml
+ with:
+ python-version: '3.12'
+ skip-pypuppeteer: true
- - name: Store container log
- if: always()
- uses: actions/upload-artifact@v4
- with:
- name: test-cdio-basic-tests-output
- path: output-logs
diff --git a/.github/workflows/test-stack-reusable-workflow.yml b/.github/workflows/test-stack-reusable-workflow.yml
new file mode 100644
index 00000000..a4c7b87c
--- /dev/null
+++ b/.github/workflows/test-stack-reusable-workflow.yml
@@ -0,0 +1,239 @@
+name: ChangeDetection.io App Test
+
+on:
+ workflow_call:
+ inputs:
+ python-version:
+ description: 'Python version to use'
+ required: true
+ type: string
+ default: '3.10'
+ skip-pypuppeteer:
+ description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
+ required: false
+ type: boolean
+ default: false
+
+jobs:
+ test-application:
+ runs-on: ubuntu-latest
+ env:
+ PYTHON_VERSION: ${{ inputs.python-version }}
+ steps:
+ - uses: actions/checkout@v4
+
+ # Mainly just for link/flake8
+ - name: Set up Python ${{ env.PYTHON_VERSION }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
+ run: |
+ echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
+ # Build a changedetection.io container and start testing inside
+ docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
+ # Debug info
+ docker run test-changedetectionio bash -c 'pip list'
+
+ - name: We should be Python ${{ env.PYTHON_VERSION }} ...
+ run: |
+ docker run test-changedetectionio bash -c 'python3 --version'
+
+ - name: Spin up ancillary testable services
+ run: |
+
+ docker network create changedet-network
+
+ # Selenium
+ docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
+
+ # SocketPuppetBrowser + Extra for custom browser test
+ docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+ docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+ - name: Spin up ancillary SMTP+Echo message test server
+ run: |
+ # Debug SMTP server/echo message back server
+ docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
+ docker ps
+
+ - name: Show docker container state and other debug info
+ run: |
+ set -x
+ echo "Running processes in docker..."
+ docker ps
+
+ - name: Test built container with Pytest (generally as requests/plaintext fetching)
+ run: |
+ # Unit tests
+ echo "run test with unittest"
+ docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+ docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+ docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
+
+ # All tests
+ echo "run test with pytest"
+ # The default pytest logger_level is TRACE
+ # To change logger_level for pytest(test/conftest.py),
+ # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
+ docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+
+# PLAYWRIGHT/NODE-> CDP
+ - name: Playwright and SocketPuppetBrowser - Specific tests in built container
+ run: |
+ # Playwright via Sockpuppetbrowser fetch
+ # tests/visualselector/test_fetch_data.py will do browser steps
+ docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+ docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+ docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+ docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+
+ - name: Playwright and SocketPuppetBrowser - Headers and requests
+ run: |
+ # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+ docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
+
+ - name: Playwright and SocketPuppetBrowser - Restock detection
+ run: |
+ # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+ docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# STRAIGHT TO CDP
+ - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
+ if: ${{ inputs.skip-pypuppeteer == false }}
+ run: |
+ # Playwright via Sockpuppetbrowser fetch
+ docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+ docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+ docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+ docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+ - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
+ if: ${{ inputs.skip-pypuppeteer == false }}
+ run: |
+ # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+ docker run --name "changedet" --hostname changedet --rm -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
+
+ - name: Pyppeteer and SocketPuppetBrowser - Restock detection
+ if: ${{ inputs.skip-pypuppeteer == false }}
+ run: |
+ # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+ docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# SELENIUM
+ - name: Specific tests in built container for Selenium
+ run: |
+ # Selenium fetch
+ docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+
+ - name: Specific tests in built container for headers and requests checks with Selenium
+ run: |
+ docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
+
+# OTHER STUFF
+ - name: Test SMTP notification mime types
+ run: |
+ # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+ # "mailserver" hostname defined above
+ docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
+ # @todo Add a test via playwright/puppeteer
+ # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
+ - name: Test proxy squid style interaction
+ run: |
+ cd changedetectionio
+ ./run_proxy_tests.sh
+ cd ..
+
+ - name: Test proxy SOCKS5 style interaction
+ run: |
+ cd changedetectionio
+ ./run_socks_proxy_tests.sh
+ cd ..
+
+ - name: Test custom browser URL
+ run: |
+ cd changedetectionio
+ ./run_custom_browser_url_tests.sh
+ cd ..
+
+ - name: Test changedetection.io container starts+runs basically without error
+ run: |
+ docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio
+ sleep 3
+ # Should return 0 (no error) when grep finds it
+ curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid
+
+ # and IPv6
+ curl --retry-connrefused --retry 6 -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+
+ # Check whether TRACE log is enabled.
+ # Also, check whether TRACE is came from STDERR
+ docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
+ # Check whether DEBUG is came from STDOUT
+ docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+
+ docker kill test-changedetectionio
+
+ - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+ run: |
+
+ echo SIGINT Shutdown request test
+ docker run --name sig-test -d test-changedetectionio
+ sleep 3
+ echo ">>> Sending SIGINT to sig-test container"
+ docker kill --signal=SIGINT sig-test
+ sleep 3
+ # invert the check (it should be not 0/not running)
+ docker ps
+ # check signal catch(STDERR) log. Because of
+ # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+ docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+ test -z "`docker ps|grep sig-test`"
+ if [ $? -ne 0 ]
+ then
+ echo "Looks like container was running when it shouldnt be"
+ docker ps
+ exit 1
+ fi
+
+ # @todo - scan the container log to see the right "graceful shutdown" text exists
+ docker rm sig-test
+
+ echo SIGTERM Shutdown request test
+ docker run --name sig-test -d test-changedetectionio
+ sleep 3
+ echo ">>> Sending SIGTERM to sig-test container"
+ docker kill --signal=SIGTERM sig-test
+ sleep 3
+ # invert the check (it should be not 0/not running)
+ docker ps
+ # check signal catch(STDERR) log. Because of
+ # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+ docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+ test -z "`docker ps|grep sig-test`"
+ if [ $? -ne 0 ]
+ then
+ echo "Looks like container was running when it shouldnt be"
+ docker ps
+ exit 1
+ fi
+
+ # @todo - scan the container log to see the right "graceful shutdown" text exists
+ docker rm sig-test
+
+ - name: Dump container log
+ if: always()
+ run: |
+ mkdir output-logs
+ docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
+ docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
+
+ - name: Store container log
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
+ path: output-logs
diff --git a/Dockerfile b/Dockerfile
index e592c9bb..5e45880c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,10 @@
# @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
# If you know how to fix it, please do! and test it for both 3.10 and 3.11
-FROM python:3.10-slim-bookworm as builder
+
+ARG PYTHON_VERSION=3.10
+
+FROM python:${PYTHON_VERSION}-slim-bookworm as builder
# See `cryptography` pin comment in requirements.txt
ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -32,7 +35,7 @@ RUN pip install --target=/dependencies playwright~=1.41.2 \
|| echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
# Final image stage
-FROM python:3.10-slim-bookworm
+FROM python:${PYTHON_VERSION}-slim-bookworm
RUN apt-get update && apt-get install -y --no-install-recommends \
libxslt1.1 \
diff --git a/changedetectionio/tests/smtp/smtp-test-server.py b/changedetectionio/tests/smtp/smtp-test-server.py
index 3481ce7e..e294cf01 100755
--- a/changedetectionio/tests/smtp/smtp-test-server.py
+++ b/changedetectionio/tests/smtp/smtp-test-server.py
@@ -1,42 +1,51 @@
#!/usr/bin/python3
-import smtpd
-import asyncore
+import asyncio
+from aiosmtpd.controller import Controller
+from aiosmtpd.smtp import SMTP
# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
last_received_message = b"Nothing"
-class CustomSMTPServer(smtpd.SMTPServer):
-
- def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
+class CustomSMTPHandler:
+ async def handle_DATA(self, server, session, envelope):
global last_received_message
- last_received_message = data
- print('Receiving message from:', peer)
- print('Message addressed from:', mailfrom)
- print('Message addressed to :', rcpttos)
- print('Message length :', len(data))
- print(data.decode('utf8'))
- return
-
-
-# Just print out the last message received on plain TCP socket server
-class EchoServer(asyncore.dispatcher):
-
- def __init__(self, host, port):
- asyncore.dispatcher.__init__(self)
- self.create_socket()
- self.set_reuse_addr()
- self.bind((host, port))
- self.listen(5)
-
- def handle_accepted(self, sock, addr):
+ last_received_message = envelope.content
+ print('Receiving message from:', session.peer)
+ print('Message addressed from:', envelope.mail_from)
+ print('Message addressed to :', envelope.rcpt_tos)
+ print('Message length :', len(envelope.content))
+ print(envelope.content.decode('utf8'))
+ return '250 Message accepted for delivery'
+
+
+class EchoServerProtocol(asyncio.Protocol):
+ def connection_made(self, transport):
global last_received_message
- print('Incoming connection from %s' % repr(addr))
- sock.send(last_received_message)
+ self.transport = transport
+ peername = transport.get_extra_info('peername')
+ print('Incoming connection from {}'.format(peername))
+ self.transport.write(last_received_message)
+
last_received_message = b''
+ self.transport.close()
+
+
+async def main():
+ # Start the SMTP server
+ controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)
+ controller.start()
+
+ # Start the TCP Echo server
+ loop = asyncio.get_running_loop()
+ server = await loop.create_server(
+ lambda: EchoServerProtocol(),
+ '0.0.0.0', 11080
+ )
+ async with server:
+ await server.serve_forever()
-server = CustomSMTPServer(('0.0.0.0', 11025), None) # SMTP mail goes here
-server2 = EchoServer('0.0.0.0', 11080) # Echo back last message received
-asyncore.loop()
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py
index 2ab92b4f..c69299fe 100644
--- a/changedetectionio/tests/smtp/test_notification_smtp.py
+++ b/changedetectionio/tests/smtp/test_notification_smtp.py
@@ -32,6 +32,8 @@ def get_last_message_from_smtp_server():
client_socket.connect((smtp_test_server, port)) # connect to the server
data = client_socket.recv(50024).decode() # receive response
+ logging.info("get_last_message_from_smtp_server..")
+ logging.info(data)
client_socket.close() # close the connection
return data
@@ -83,7 +85,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
# The email should have two bodies, and the text/html part should be
assert 'Content-Type: text/plain' in msg
- assert '(added) So let\'s see what happens.\n' in msg # The plaintext part with \n
+ assert '(added) So let\'s see what happens.\r\n' in msg # The plaintext part with \r\n
assert 'Content-Type: text/html' in msg
assert '(added) So let\'s see what happens.
' in msg # the html part
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
@@ -150,7 +152,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
# The email should not have two bodies, should be TEXT only
assert 'Content-Type: text/plain' in msg
- assert '(added) So let\'s see what happens.\n' in msg # The plaintext part with \n
+ assert '(added) So let\'s see what happens.\r\n' in msg # The plaintext part with \r\n
set_original_response()
# Now override as HTML format
@@ -171,7 +173,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
# The email should have two bodies, and the text/html part should be
assert 'Content-Type: text/plain' in msg
- assert '(removed) So let\'s see what happens.\n' in msg # The plaintext part with \n
+ assert '(removed) So let\'s see what happens.\r\n' in msg # The plaintext part with \n
assert 'Content-Type: text/html' in msg
assert '(removed) So let\'s see what happens.
' in msg # the html part
diff --git a/requirements.txt b/requirements.txt
index 9fc485e8..a13bc8d0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,9 +29,7 @@ chardet>2.3.0
wtforms~=3.0
jsonpath-ng~=1.5.3
-# Pinned: module 'eventlet.green.select' has no attribute 'epoll'
-# https://github.com/eventlet/eventlet/issues/805#issuecomment-1640463482
-dnspython==2.3.0 # related to eventlet fixes
+dnspython==2.6.1
# jq not available on Windows so must be installed manually
@@ -86,3 +84,5 @@ pytest-flask ~=1.2
jsonschema==4.17.3
loguru
+# Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
+greenlet >= 3.0.3
\ No newline at end of file