dgtlmoon
6 months ago
committed by
GitHub
parent
bce02f9c82
commit
434a1b242e
@ -0,0 +1,239 @@
|
|||||||
|
name: ChangeDetection.io App Test
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
python-version:
|
||||||
|
description: 'Python version to use'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: '3.10'
|
||||||
|
skip-pypuppeteer:
|
||||||
|
description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
|
||||||
|
required: false
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test-application:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
PYTHON_VERSION: ${{ inputs.python-version }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
# Mainly just for link/flake8
|
||||||
|
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: ${{ env.PYTHON_VERSION }}
|
||||||
|
|
||||||
|
- name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
|
||||||
|
run: |
|
||||||
|
echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
|
||||||
|
# Build a changedetection.io container and start testing inside
|
||||||
|
docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
|
||||||
|
# Debug info
|
||||||
|
docker run test-changedetectionio bash -c 'pip list'
|
||||||
|
|
||||||
|
- name: We should be Python ${{ env.PYTHON_VERSION }} ...
|
||||||
|
run: |
|
||||||
|
docker run test-changedetectionio bash -c 'python3 --version'
|
||||||
|
|
||||||
|
- name: Spin up ancillary testable services
|
||||||
|
run: |
|
||||||
|
|
||||||
|
docker network create changedet-network
|
||||||
|
|
||||||
|
# Selenium
|
||||||
|
docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
|
||||||
|
|
||||||
|
# SocketPuppetBrowser + Extra for custom browser test
|
||||||
|
docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
|
||||||
|
docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
|
||||||
|
|
||||||
|
- name: Spin up ancillary SMTP+Echo message test server
|
||||||
|
run: |
|
||||||
|
# Debug SMTP server/echo message back server
|
||||||
|
docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
|
||||||
|
docker ps
|
||||||
|
|
||||||
|
- name: Show docker container state and other debug info
|
||||||
|
run: |
|
||||||
|
set -x
|
||||||
|
echo "Running processes in docker..."
|
||||||
|
docker ps
|
||||||
|
|
||||||
|
- name: Test built container with Pytest (generally as requests/plaintext fetching)
|
||||||
|
run: |
|
||||||
|
# Unit tests
|
||||||
|
echo "run test with unittest"
|
||||||
|
docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
|
||||||
|
docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
|
||||||
|
docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
|
||||||
|
|
||||||
|
# All tests
|
||||||
|
echo "run test with pytest"
|
||||||
|
# The default pytest logger_level is TRACE
|
||||||
|
# To change logger_level for pytest(test/conftest.py),
|
||||||
|
# append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
|
||||||
|
docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'
|
||||||
|
|
||||||
|
# PLAYWRIGHT/NODE-> CDP
|
||||||
|
- name: Playwright and SocketPuppetBrowser - Specific tests in built container
|
||||||
|
run: |
|
||||||
|
# Playwright via Sockpuppetbrowser fetch
|
||||||
|
# tests/visualselector/test_fetch_data.py will do browser steps
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
|
||||||
|
|
||||||
|
|
||||||
|
- name: Playwright and SocketPuppetBrowser - Headers and requests
|
||||||
|
run: |
|
||||||
|
# Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
|
||||||
|
docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
|
||||||
|
|
||||||
|
- name: Playwright and SocketPuppetBrowser - Restock detection
|
||||||
|
run: |
|
||||||
|
# restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
|
||||||
|
docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
|
||||||
|
|
||||||
|
# STRAIGHT TO CDP
|
||||||
|
- name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
|
||||||
|
if: ${{ inputs.skip-pypuppeteer == false }}
|
||||||
|
run: |
|
||||||
|
# Playwright via Sockpuppetbrowser fetch
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
|
||||||
|
|
||||||
|
- name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
|
||||||
|
if: ${{ inputs.skip-pypuppeteer == false }}
|
||||||
|
run: |
|
||||||
|
# Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
|
||||||
|
docker run --name "changedet" --hostname changedet --rm -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
|
||||||
|
|
||||||
|
- name: Pyppeteer and SocketPuppetBrowser - Restock detection
|
||||||
|
if: ${{ inputs.skip-pypuppeteer == false }}
|
||||||
|
run: |
|
||||||
|
# restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
|
||||||
|
docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
|
||||||
|
|
||||||
|
# SELENIUM
|
||||||
|
- name: Specific tests in built container for Selenium
|
||||||
|
run: |
|
||||||
|
# Selenium fetch
|
||||||
|
docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
|
||||||
|
|
||||||
|
- name: Specific tests in built container for headers and requests checks with Selenium
|
||||||
|
run: |
|
||||||
|
docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
|
||||||
|
|
||||||
|
# OTHER STUFF
|
||||||
|
- name: Test SMTP notification mime types
|
||||||
|
run: |
|
||||||
|
# SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
|
||||||
|
# "mailserver" hostname defined above
|
||||||
|
docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
|
||||||
|
|
||||||
|
# @todo Add a test via playwright/puppeteer
|
||||||
|
# squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
|
||||||
|
- name: Test proxy squid style interaction
|
||||||
|
run: |
|
||||||
|
cd changedetectionio
|
||||||
|
./run_proxy_tests.sh
|
||||||
|
cd ..
|
||||||
|
|
||||||
|
- name: Test proxy SOCKS5 style interaction
|
||||||
|
run: |
|
||||||
|
cd changedetectionio
|
||||||
|
./run_socks_proxy_tests.sh
|
||||||
|
cd ..
|
||||||
|
|
||||||
|
- name: Test custom browser URL
|
||||||
|
run: |
|
||||||
|
cd changedetectionio
|
||||||
|
./run_custom_browser_url_tests.sh
|
||||||
|
cd ..
|
||||||
|
|
||||||
|
- name: Test changedetection.io container starts+runs basically without error
|
||||||
|
run: |
|
||||||
|
docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio
|
||||||
|
sleep 3
|
||||||
|
# Should return 0 (no error) when grep finds it
|
||||||
|
curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid
|
||||||
|
|
||||||
|
# and IPv6
|
||||||
|
curl --retry-connrefused --retry 6 -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
|
||||||
|
|
||||||
|
# Check whether TRACE log is enabled.
|
||||||
|
# Also, check whether TRACE is came from STDERR
|
||||||
|
docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
|
||||||
|
# Check whether DEBUG is came from STDOUT
|
||||||
|
docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
|
||||||
|
|
||||||
|
docker kill test-changedetectionio
|
||||||
|
|
||||||
|
- name: Test changedetection.io SIGTERM and SIGINT signal shutdown
|
||||||
|
run: |
|
||||||
|
|
||||||
|
echo SIGINT Shutdown request test
|
||||||
|
docker run --name sig-test -d test-changedetectionio
|
||||||
|
sleep 3
|
||||||
|
echo ">>> Sending SIGINT to sig-test container"
|
||||||
|
docker kill --signal=SIGINT sig-test
|
||||||
|
sleep 3
|
||||||
|
# invert the check (it should be not 0/not running)
|
||||||
|
docker ps
|
||||||
|
# check signal catch(STDERR) log. Because of
|
||||||
|
# changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
|
||||||
|
docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
|
||||||
|
test -z "`docker ps|grep sig-test`"
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "Looks like container was running when it shouldnt be"
|
||||||
|
docker ps
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# @todo - scan the container log to see the right "graceful shutdown" text exists
|
||||||
|
docker rm sig-test
|
||||||
|
|
||||||
|
echo SIGTERM Shutdown request test
|
||||||
|
docker run --name sig-test -d test-changedetectionio
|
||||||
|
sleep 3
|
||||||
|
echo ">>> Sending SIGTERM to sig-test container"
|
||||||
|
docker kill --signal=SIGTERM sig-test
|
||||||
|
sleep 3
|
||||||
|
# invert the check (it should be not 0/not running)
|
||||||
|
docker ps
|
||||||
|
# check signal catch(STDERR) log. Because of
|
||||||
|
# changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
|
||||||
|
docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
|
||||||
|
test -z "`docker ps|grep sig-test`"
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "Looks like container was running when it shouldnt be"
|
||||||
|
docker ps
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# @todo - scan the container log to see the right "graceful shutdown" text exists
|
||||||
|
docker rm sig-test
|
||||||
|
|
||||||
|
- name: Dump container log
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
mkdir output-logs
|
||||||
|
docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
|
||||||
|
docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
|
||||||
|
|
||||||
|
- name: Store container log
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
|
||||||
|
path: output-logs
|
||||||
@ -1,42 +1,51 @@
|
|||||||
#!/usr/bin/python3
|
#!/usr/bin/python3
|
||||||
import smtpd
|
import asyncio
|
||||||
import asyncore
|
from aiosmtpd.controller import Controller
|
||||||
|
from aiosmtpd.smtp import SMTP
|
||||||
|
|
||||||
# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
|
# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
|
||||||
|
|
||||||
last_received_message = b"Nothing"
|
last_received_message = b"Nothing"
|
||||||
|
|
||||||
|
|
||||||
class CustomSMTPServer(smtpd.SMTPServer):
|
class CustomSMTPHandler:
|
||||||
|
async def handle_DATA(self, server, session, envelope):
|
||||||
def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
|
|
||||||
global last_received_message
|
global last_received_message
|
||||||
last_received_message = data
|
last_received_message = envelope.content
|
||||||
print('Receiving message from:', peer)
|
print('Receiving message from:', session.peer)
|
||||||
print('Message addressed from:', mailfrom)
|
print('Message addressed from:', envelope.mail_from)
|
||||||
print('Message addressed to :', rcpttos)
|
print('Message addressed to :', envelope.rcpt_tos)
|
||||||
print('Message length :', len(data))
|
print('Message length :', len(envelope.content))
|
||||||
print(data.decode('utf8'))
|
print(envelope.content.decode('utf8'))
|
||||||
return
|
return '250 Message accepted for delivery'
|
||||||
|
|
||||||
|
|
||||||
# Just print out the last message received on plain TCP socket server
|
class EchoServerProtocol(asyncio.Protocol):
|
||||||
class EchoServer(asyncore.dispatcher):
|
def connection_made(self, transport):
|
||||||
|
|
||||||
def __init__(self, host, port):
|
|
||||||
asyncore.dispatcher.__init__(self)
|
|
||||||
self.create_socket()
|
|
||||||
self.set_reuse_addr()
|
|
||||||
self.bind((host, port))
|
|
||||||
self.listen(5)
|
|
||||||
|
|
||||||
def handle_accepted(self, sock, addr):
|
|
||||||
global last_received_message
|
global last_received_message
|
||||||
print('Incoming connection from %s' % repr(addr))
|
self.transport = transport
|
||||||
sock.send(last_received_message)
|
peername = transport.get_extra_info('peername')
|
||||||
|
print('Incoming connection from {}'.format(peername))
|
||||||
|
self.transport.write(last_received_message)
|
||||||
|
|
||||||
last_received_message = b''
|
last_received_message = b''
|
||||||
|
self.transport.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
# Start the SMTP server
|
||||||
|
controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)
|
||||||
|
controller.start()
|
||||||
|
|
||||||
|
# Start the TCP Echo server
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
server = await loop.create_server(
|
||||||
|
lambda: EchoServerProtocol(),
|
||||||
|
'0.0.0.0', 11080
|
||||||
|
)
|
||||||
|
async with server:
|
||||||
|
await server.serve_forever()
|
||||||
|
|
||||||
|
|
||||||
server = CustomSMTPServer(('0.0.0.0', 11025), None) # SMTP mail goes here
|
if __name__ == "__main__":
|
||||||
server2 = EchoServer('0.0.0.0', 11080) # Echo back last message received
|
asyncio.run(main())
|
||||||
asyncore.loop()
|
|
||||||
|
|||||||
Loading…
Reference in new issue