From e8e176f3bd8c4ca21118061dccfcac3af890a4e3 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 19 Dec 2022 14:41:34 +0100 Subject: [PATCH] Testing - Run test as fully built docker container (#1245) --- .github/workflows/containers.yml | 1 - .github/workflows/pypi.yml | 6 - .github/workflows/test-only.yml | 64 ++++++++--- CONTRIBUTING.md | 6 - Dockerfile | 1 + changedetectionio/run_all_tests.sh | 104 ------------------ changedetectionio/run_basic_tests.sh | 30 +++++ changedetectionio/run_proxy_tests.sh | 34 ++++++ changedetectionio/static/styles/.dockerignore | 3 + .../tests/proxy_list/proxies.json-example | 8 +- requirements-dev.txt | 2 - requirements.txt | 4 + 12 files changed, 126 insertions(+), 137 deletions(-) delete mode 100755 changedetectionio/run_all_tests.sh create mode 100755 changedetectionio/run_basic_tests.sh create mode 100755 changedetectionio/run_proxy_tests.sh create mode 100644 changedetectionio/static/styles/.dockerignore delete mode 100644 requirements-dev.txt diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index d079a699..e1ed892f 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -50,7 +50,6 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Create release metadata run: | diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 47b8791f..f792bf50 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -19,12 +19,6 @@ jobs: with: python-version: 3.9 -# - name: Install dependencies -# run: | -# python -m pip install --upgrade pip -# pip install flake8 pytest -# if [ -f requirements.txt ]; then pip install -r requirements.txt; fi -# if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Test that pip builds without error run: | diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index aac97335..93c6c108 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -8,32 +8,68 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + + # Mainly just for link/flake8 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.9 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + python-version: '3.10' - name: Lint with flake8 run: | + pip3 install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Unit tests + - name: Spin up ancillary testable services + run: | + + docker network create changedet-network + + # Selenium+browserless + docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome-debug:3.141.59 + docker run --network changedet-network -d --hostname browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable + + - name: Build changedetection.io container for testing + run: | + # Build a changedetection.io container and start testing inside + docker build . -t test-changedetectionio + + - name: Test built container with pytest run: | - python3 -m unittest changedetectionio.tests.unit.test_notification_diff + + # Unit tests + docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff' + + # All tests + docker run --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh' - - name: Test with pytest + - name: Test built container selenium+browserless/playwright run: | - # Each test is totally isolated and performs its own cleanup/reset - cd changedetectionio; ./run_all_tests.sh + + # Selenium fetch + docker run -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py' + + # Playwright/Browserless fetch + docker run -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' + - name: Test proxy interaction + run: | + cd changedetectionio + ./run_proxy_tests.sh + cd .. + + + - name: Test changedetection.io container starts+runs basically without error + run: | + docker run -p 5556:5000 -d test-changedetectionio + sleep 3 + # Should return 0 (no error) when grep finds it + curl -s http://localhost:5556 |grep -q checkbox-uuid + curl -s http://localhost:5556/rss|grep -q rss-specification +#export WEBDRIVER_URL=http://localhost:4444/wd/hub +#pytest tests/fetchers/test_content.py +#pytest tests/test_errorhandling.py \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8478a7ab..4e6a0c98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,9 +7,3 @@ Otherwise, it's always best to PR into the `dev` branch. Please be sure that all new functionality has a matching test! Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example - -``` -pip3 install -r requirements-dev -``` - -this is from https://github.com/dgtlmoon/changedetection.io/blob/master/requirements-dev.txt diff --git a/Dockerfile b/Dockerfile index b13ce427..ab2d1e98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,6 +65,7 @@ COPY changedetectionio /app/changedetectionio # The eventlet server wrapper COPY changedetection.py /app/changedetection.py + WORKDIR /app CMD [ "python", "./changedetection.py" , "-d", "/datastore"] diff --git a/changedetectionio/run_all_tests.sh b/changedetectionio/run_all_tests.sh deleted file mode 100755 index 55bb3dc1..00000000 --- a/changedetectionio/run_all_tests.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - - -# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions -# and I like to restart the server for each test (and have the test cleanup after each test) -# merge request welcome :) - - -# exit when any command fails -set -e - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -find tests/test_*py -type f|while read test_name -do - echo "TEST RUNNING $test_name" - pytest $test_name -done - -echo "RUNNING WITH BASE_URL SET" - -# Now re-run some tests with BASE_URL enabled -# Re #65 - Ability to include a link back to the installation, in the notification. -export BASE_URL="https://really-unique-domain.io" -pytest tests/test_notification.py - - -# Re-run with HIDE_REFERER set - could affect login -export HIDE_REFERER=True -pytest tests/test_access_control.py - - -# Now for the selenium and playwright/browserless fetchers -# Note - this is not UI functional tests - just checking that each one can fetch the content - -echo "TESTING WEBDRIVER FETCH > SELENIUM/WEBDRIVER..." -docker run -d --name $$-test_selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome-debug:3.141.59 -# takes a while to spin up -sleep 5 -export WEBDRIVER_URL=http://localhost:4444/wd/hub -pytest tests/fetchers/test_content.py -pytest tests/test_errorhandling.py -unset WEBDRIVER_URL -docker kill $$-test_selenium - -echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..." -# Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt -PLAYWRIGHT_VERSION=$(grep -i -E "RUN pip install.+" "$SCRIPT_DIR/../Dockerfile" | grep --only-matching -i -E "playwright[=><~+]+[0-9\.]+") -echo "using $PLAYWRIGHT_VERSION" -pip3 install "$PLAYWRIGHT_VERSION" -docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable -# takes a while to spin up -sleep 5 -export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 -pytest tests/fetchers/test_content.py -pytest tests/test_errorhandling.py -pytest tests/visualselector/test_fetch_data.py - -unset PLAYWRIGHT_DRIVER_URL -docker kill $$-test_browserless - -# Test proxy list handling, starting two squids on different ports -# Each squid adds a different header to the response, which is the main thing we test for. -docker run -d --name $$-squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3128:3128 ubuntu/squid:4.13-21.10_edge -docker run -d --name $$-squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3129:3128 ubuntu/squid:4.13-21.10_edge - - -# So, basic HTTP as env var test -export HTTP_PROXY=http://localhost:3128 -export HTTPS_PROXY=http://localhost:3128 -pytest tests/proxy_list/test_proxy.py -docker logs $$-squid-one 2>/dev/null|grep one.changedetection.io -if [ $? -ne 0 ] -then - echo "Did not see a request to one.changedetection.io in the squid logs (while checking env vars HTTP_PROXY/HTTPS_PROXY)" -fi -unset HTTP_PROXY -unset HTTPS_PROXY - - -# 2nd test actually choose the preferred proxy from proxies.json -cp tests/proxy_list/proxies.json-example ./test-datastore/proxies.json -# Makes a watch use a preferred proxy -pytest tests/proxy_list/test_multiple_proxy.py - -# Should be a request in the default "first" squid -docker logs $$-squid-one 2>/dev/null|grep chosen.changedetection.io -if [ $? -ne 0 ] -then - echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)" -fi - -# And one in the 'second' squid (user selects this as preferred) -docker logs $$-squid-two 2>/dev/null|grep chosen.changedetection.io -if [ $? -ne 0 ] -then - echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)" -fi - -# @todo - test system override proxy selection and watch defaults, setup a 3rd squid? -docker kill $$-squid-one -docker kill $$-squid-two - - diff --git a/changedetectionio/run_basic_tests.sh b/changedetectionio/run_basic_tests.sh new file mode 100755 index 00000000..9803d78d --- /dev/null +++ b/changedetectionio/run_basic_tests.sh @@ -0,0 +1,30 @@ +#!/bin/bash + + +# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions +# and I like to restart the server for each test (and have the test cleanup after each test) +# merge request welcome :) + + +# exit when any command fails +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +find tests/test_*py -type f|while read test_name +do + echo "TEST RUNNING $test_name" + pytest $test_name +done + +echo "RUNNING WITH BASE_URL SET" + +# Now re-run some tests with BASE_URL enabled +# Re #65 - Ability to include a link back to the installation, in the notification. +export BASE_URL="https://really-unique-domain.io" +pytest tests/test_notification.py + + +# Re-run with HIDE_REFERER set - could affect login +export HIDE_REFERER=True +pytest tests/test_access_control.py diff --git a/changedetectionio/run_proxy_tests.sh b/changedetectionio/run_proxy_tests.sh new file mode 100755 index 00000000..506d0068 --- /dev/null +++ b/changedetectionio/run_proxy_tests.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# exit when any command fails +set -e + +# Test proxy list handling, starting two squids on different ports +# Each squid adds a different header to the response, which is the main thing we test for. +docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge +docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge + + +## 2nd test actually choose the preferred proxy from proxies.json + +docker run --network changedet-network \ + -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \ + test-changedetectionio \ + bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py' + + +## Should be a request in the default "first" squid +docker logs squid-one 2>/dev/null|grep chosen.changedetection.io +if [ $? -ne 0 ] +then + echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid one)" + exit 1 +fi + +# And one in the 'second' squid (user selects this as preferred) +docker logs squid-two 2>/dev/null|grep chosen.changedetection.io +if [ $? -ne 0 ] +then + echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid two)" + exit 1 +fi diff --git a/changedetectionio/static/styles/.dockerignore b/changedetectionio/static/styles/.dockerignore new file mode 100644 index 00000000..099ae147 --- /dev/null +++ b/changedetectionio/static/styles/.dockerignore @@ -0,0 +1,3 @@ +node_modules +package-lock.json + diff --git a/changedetectionio/tests/proxy_list/proxies.json-example b/changedetectionio/tests/proxy_list/proxies.json-example index 0ae2178c..b3ddfe8a 100644 --- a/changedetectionio/tests/proxy_list/proxies.json-example +++ b/changedetectionio/tests/proxy_list/proxies.json-example @@ -1,10 +1,10 @@ { "proxy-one": { - "label": "One", - "url": "http://127.0.0.1:3128" + "label": "Proxy One", + "url": "http://squid-one:3128" }, "proxy-two": { - "label": "two", - "url": "http://127.0.0.1:3129" + "label": "Proxy Two", + "url": "http://squid-two:3128" } } diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 146e954c..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest ~=6.2 -pytest-flask ~=1.2 diff --git a/requirements.txt b/requirements.txt index dbc9837f..21101413 100644 --- a/requirements.txt +++ b/requirements.txt @@ -59,3 +59,7 @@ jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux" # Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future pillow # playwright is installed at Dockerfile build time because it's not available on all platforms + +# Include pytest, so if theres a support issue we can ask them to run these tests on their setup +pytest ~=6.2 +pytest-flask ~=1.2 \ No newline at end of file