dgtlmoon
5 months ago
commit
7aa1e5cc00
@ -0,0 +1,239 @@
|
|||||||
|
name: ChangeDetection.io App Test
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
python-version:
|
||||||
|
description: 'Python version to use'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: '3.10'
|
||||||
|
skip-pypuppeteer:
|
||||||
|
description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
|
||||||
|
required: false
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test-application:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
PYTHON_VERSION: ${{ inputs.python-version }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
# Mainly just for link/flake8
|
||||||
|
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: ${{ env.PYTHON_VERSION }}
|
||||||
|
|
||||||
|
- name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
|
||||||
|
run: |
|
||||||
|
echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
|
||||||
|
# Build a changedetection.io container and start testing inside
|
||||||
|
docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
|
||||||
|
# Debug info
|
||||||
|
docker run test-changedetectionio bash -c 'pip list'
|
||||||
|
|
||||||
|
- name: We should be Python ${{ env.PYTHON_VERSION }} ...
|
||||||
|
run: |
|
||||||
|
docker run test-changedetectionio bash -c 'python3 --version'
|
||||||
|
|
||||||
|
- name: Spin up ancillary testable services
|
||||||
|
run: |
|
||||||
|
|
||||||
|
docker network create changedet-network
|
||||||
|
|
||||||
|
# Selenium
|
||||||
|
docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
|
||||||
|
|
||||||
|
# SocketPuppetBrowser + Extra for custom browser test
|
||||||
|
docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
|
||||||
|
docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
|
||||||
|
|
||||||
|
- name: Spin up ancillary SMTP+Echo message test server
|
||||||
|
run: |
|
||||||
|
# Debug SMTP server/echo message back server
|
||||||
|
docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
|
||||||
|
docker ps
|
||||||
|
|
||||||
|
- name: Show docker container state and other debug info
|
||||||
|
run: |
|
||||||
|
set -x
|
||||||
|
echo "Running processes in docker..."
|
||||||
|
docker ps
|
||||||
|
|
||||||
|
- name: Test built container with Pytest (generally as requests/plaintext fetching)
|
||||||
|
run: |
|
||||||
|
# Unit tests
|
||||||
|
echo "run test with unittest"
|
||||||
|
docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
|
||||||
|
docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
|
||||||
|
docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
|
||||||
|
|
||||||
|
# All tests
|
||||||
|
echo "run test with pytest"
|
||||||
|
# The default pytest logger_level is TRACE
|
||||||
|
# To change logger_level for pytest(test/conftest.py),
|
||||||
|
# append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
|
||||||
|
docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'
|
||||||
|
|
||||||
|
# PLAYWRIGHT/NODE-> CDP
|
||||||
|
- name: Playwright and SocketPuppetBrowser - Specific tests in built container
|
||||||
|
run: |
|
||||||
|
# Playwright via Sockpuppetbrowser fetch
|
||||||
|
# tests/visualselector/test_fetch_data.py will do browser steps
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
|
||||||
|
|
||||||
|
|
||||||
|
- name: Playwright and SocketPuppetBrowser - Headers and requests
|
||||||
|
run: |
|
||||||
|
# Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
|
||||||
|
docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
|
||||||
|
|
||||||
|
- name: Playwright and SocketPuppetBrowser - Restock detection
|
||||||
|
run: |
|
||||||
|
# restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
|
||||||
|
docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
|
||||||
|
|
||||||
|
# STRAIGHT TO CDP
|
||||||
|
- name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
|
||||||
|
if: ${{ inputs.skip-pypuppeteer == false }}
|
||||||
|
run: |
|
||||||
|
# Playwright via Sockpuppetbrowser fetch
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
|
||||||
|
docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
|
||||||
|
|
||||||
|
- name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
|
||||||
|
if: ${{ inputs.skip-pypuppeteer == false }}
|
||||||
|
run: |
|
||||||
|
# Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
|
||||||
|
docker run --name "changedet" --hostname changedet --rm -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
|
||||||
|
|
||||||
|
- name: Pyppeteer and SocketPuppetBrowser - Restock detection
|
||||||
|
if: ${{ inputs.skip-pypuppeteer == false }}
|
||||||
|
run: |
|
||||||
|
# restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
|
||||||
|
docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
|
||||||
|
|
||||||
|
# SELENIUM
|
||||||
|
- name: Specific tests in built container for Selenium
|
||||||
|
run: |
|
||||||
|
# Selenium fetch
|
||||||
|
docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
|
||||||
|
|
||||||
|
- name: Specific tests in built container for headers and requests checks with Selenium
|
||||||
|
run: |
|
||||||
|
docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
|
||||||
|
|
||||||
|
# OTHER STUFF
|
||||||
|
- name: Test SMTP notification mime types
|
||||||
|
run: |
|
||||||
|
# SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
|
||||||
|
# "mailserver" hostname defined above
|
||||||
|
docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
|
||||||
|
|
||||||
|
# @todo Add a test via playwright/puppeteer
|
||||||
|
# squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
|
||||||
|
- name: Test proxy squid style interaction
|
||||||
|
run: |
|
||||||
|
cd changedetectionio
|
||||||
|
./run_proxy_tests.sh
|
||||||
|
cd ..
|
||||||
|
|
||||||
|
- name: Test proxy SOCKS5 style interaction
|
||||||
|
run: |
|
||||||
|
cd changedetectionio
|
||||||
|
./run_socks_proxy_tests.sh
|
||||||
|
cd ..
|
||||||
|
|
||||||
|
- name: Test custom browser URL
|
||||||
|
run: |
|
||||||
|
cd changedetectionio
|
||||||
|
./run_custom_browser_url_tests.sh
|
||||||
|
cd ..
|
||||||
|
|
||||||
|
- name: Test changedetection.io container starts+runs basically without error
|
||||||
|
run: |
|
||||||
|
docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio
|
||||||
|
sleep 3
|
||||||
|
# Should return 0 (no error) when grep finds it
|
||||||
|
curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid
|
||||||
|
|
||||||
|
# and IPv6
|
||||||
|
curl --retry-connrefused --retry 6 -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
|
||||||
|
|
||||||
|
# Check whether TRACE log is enabled.
|
||||||
|
# Also, check whether TRACE is came from STDERR
|
||||||
|
docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
|
||||||
|
# Check whether DEBUG is came from STDOUT
|
||||||
|
docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
|
||||||
|
|
||||||
|
docker kill test-changedetectionio
|
||||||
|
|
||||||
|
- name: Test changedetection.io SIGTERM and SIGINT signal shutdown
|
||||||
|
run: |
|
||||||
|
|
||||||
|
echo SIGINT Shutdown request test
|
||||||
|
docker run --name sig-test -d test-changedetectionio
|
||||||
|
sleep 3
|
||||||
|
echo ">>> Sending SIGINT to sig-test container"
|
||||||
|
docker kill --signal=SIGINT sig-test
|
||||||
|
sleep 3
|
||||||
|
# invert the check (it should be not 0/not running)
|
||||||
|
docker ps
|
||||||
|
# check signal catch(STDERR) log. Because of
|
||||||
|
# changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
|
||||||
|
docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
|
||||||
|
test -z "`docker ps|grep sig-test`"
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "Looks like container was running when it shouldnt be"
|
||||||
|
docker ps
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# @todo - scan the container log to see the right "graceful shutdown" text exists
|
||||||
|
docker rm sig-test
|
||||||
|
|
||||||
|
echo SIGTERM Shutdown request test
|
||||||
|
docker run --name sig-test -d test-changedetectionio
|
||||||
|
sleep 3
|
||||||
|
echo ">>> Sending SIGTERM to sig-test container"
|
||||||
|
docker kill --signal=SIGTERM sig-test
|
||||||
|
sleep 3
|
||||||
|
# invert the check (it should be not 0/not running)
|
||||||
|
docker ps
|
||||||
|
# check signal catch(STDERR) log. Because of
|
||||||
|
# changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
|
||||||
|
docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
|
||||||
|
test -z "`docker ps|grep sig-test`"
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "Looks like container was running when it shouldnt be"
|
||||||
|
docker ps
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# @todo - scan the container log to see the right "graceful shutdown" text exists
|
||||||
|
docker rm sig-test
|
||||||
|
|
||||||
|
- name: Dump container log
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
mkdir output-logs
|
||||||
|
docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
|
||||||
|
docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
|
||||||
|
|
||||||
|
- name: Store container log
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
|
||||||
|
path: output-logs
|
||||||
@ -1,42 +1,51 @@
|
|||||||
#!/usr/bin/python3
|
#!/usr/bin/python3
|
||||||
import smtpd
|
import asyncio
|
||||||
import asyncore
|
from aiosmtpd.controller import Controller
|
||||||
|
from aiosmtpd.smtp import SMTP
|
||||||
|
|
||||||
# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
|
# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
|
||||||
|
|
||||||
last_received_message = b"Nothing"
|
last_received_message = b"Nothing"
|
||||||
|
|
||||||
|
|
||||||
class CustomSMTPServer(smtpd.SMTPServer):
|
class CustomSMTPHandler:
|
||||||
|
async def handle_DATA(self, server, session, envelope):
|
||||||
def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
|
|
||||||
global last_received_message
|
global last_received_message
|
||||||
last_received_message = data
|
last_received_message = envelope.content
|
||||||
print('Receiving message from:', peer)
|
print('Receiving message from:', session.peer)
|
||||||
print('Message addressed from:', mailfrom)
|
print('Message addressed from:', envelope.mail_from)
|
||||||
print('Message addressed to :', rcpttos)
|
print('Message addressed to :', envelope.rcpt_tos)
|
||||||
print('Message length :', len(data))
|
print('Message length :', len(envelope.content))
|
||||||
print(data.decode('utf8'))
|
print(envelope.content.decode('utf8'))
|
||||||
return
|
return '250 Message accepted for delivery'
|
||||||
|
|
||||||
|
|
||||||
# Just print out the last message received on plain TCP socket server
|
class EchoServerProtocol(asyncio.Protocol):
|
||||||
class EchoServer(asyncore.dispatcher):
|
def connection_made(self, transport):
|
||||||
|
|
||||||
def __init__(self, host, port):
|
|
||||||
asyncore.dispatcher.__init__(self)
|
|
||||||
self.create_socket()
|
|
||||||
self.set_reuse_addr()
|
|
||||||
self.bind((host, port))
|
|
||||||
self.listen(5)
|
|
||||||
|
|
||||||
def handle_accepted(self, sock, addr):
|
|
||||||
global last_received_message
|
global last_received_message
|
||||||
print('Incoming connection from %s' % repr(addr))
|
self.transport = transport
|
||||||
sock.send(last_received_message)
|
peername = transport.get_extra_info('peername')
|
||||||
|
print('Incoming connection from {}'.format(peername))
|
||||||
|
self.transport.write(last_received_message)
|
||||||
|
|
||||||
last_received_message = b''
|
last_received_message = b''
|
||||||
|
self.transport.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
# Start the SMTP server
|
||||||
|
controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)
|
||||||
|
controller.start()
|
||||||
|
|
||||||
|
# Start the TCP Echo server
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
server = await loop.create_server(
|
||||||
|
lambda: EchoServerProtocol(),
|
||||||
|
'0.0.0.0', 11080
|
||||||
|
)
|
||||||
|
async with server:
|
||||||
|
await server.serve_forever()
|
||||||
|
|
||||||
|
|
||||||
server = CustomSMTPServer(('0.0.0.0', 11025), None) # SMTP mail goes here
|
if __name__ == "__main__":
|
||||||
server2 = EchoServer('0.0.0.0', 11080) # Echo back last message received
|
asyncio.run(main())
|
||||||
asyncore.loop()
|
|
||||||
|
|||||||
@ -0,0 +1,235 @@
|
|||||||
|
#!/usr/bin/python3
|
||||||
|
|
||||||
|
import time
|
||||||
|
from flask import url_for
|
||||||
|
from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI, wait_for_all_checks
|
||||||
|
|
||||||
|
|
||||||
|
def set_response_with_ldjson():
|
||||||
|
test_return_data = """<html>
|
||||||
|
<body>
|
||||||
|
Some initial text<br>
|
||||||
|
<p>Which is across multiple lines</p>
|
||||||
|
<br>
|
||||||
|
So let's see what happens. <br>
|
||||||
|
<div class="sametext">Some text thats the same</div>
|
||||||
|
<div class="changetext">Some text that will change</div>
|
||||||
|
<script type="application/ld+json">
|
||||||
|
{
|
||||||
|
"@context":"https://schema.org/",
|
||||||
|
"@type":"Product",
|
||||||
|
"@id":"https://www.some-virtual-phone-shop.com/celular-iphone-14/p",
|
||||||
|
"name":"Celular Iphone 14 Pro Max 256Gb E Sim A16 Bionic",
|
||||||
|
"brand":{
|
||||||
|
"@type":"Brand",
|
||||||
|
"name":"APPLE"
|
||||||
|
},
|
||||||
|
"image":"https://www.some-virtual-phone-shop.com/15509426/image.jpg",
|
||||||
|
"description":"You dont need it",
|
||||||
|
"mpn":"111111",
|
||||||
|
"sku":"22222",
|
||||||
|
"Offers":{
|
||||||
|
"@type":"AggregateOffer",
|
||||||
|
"lowPrice":8097000,
|
||||||
|
"highPrice":8099900,
|
||||||
|
"priceCurrency":"COP",
|
||||||
|
"offers":[
|
||||||
|
{
|
||||||
|
"@type":"Offer",
|
||||||
|
"price":8097000,
|
||||||
|
"priceCurrency":"COP",
|
||||||
|
"availability":"http://schema.org/InStock",
|
||||||
|
"sku":"102375961",
|
||||||
|
"itemCondition":"http://schema.org/NewCondition",
|
||||||
|
"seller":{
|
||||||
|
"@type":"Organization",
|
||||||
|
"name":"ajax"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"offerCount":1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||||
|
f.write(test_return_data)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set_response_without_ldjson():
|
||||||
|
test_return_data = """<html>
|
||||||
|
<body>
|
||||||
|
Some initial text<br>
|
||||||
|
<p>Which is across multiple lines</p>
|
||||||
|
<br>
|
||||||
|
So let's see what happens. <br>
|
||||||
|
<div class="sametext">Some text thats the same</div>
|
||||||
|
<div class="changetext">Some text that will change</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||||
|
f.write(test_return_data)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def test_setup(client, live_server):
|
||||||
|
live_server_setup(live_server)
|
||||||
|
|
||||||
|
# actually only really used by the distll.io importer, but could be handy too
|
||||||
|
def test_check_ldjson_price_autodetect(client, live_server):
|
||||||
|
|
||||||
|
set_response_with_ldjson()
|
||||||
|
|
||||||
|
# Add our URL to the import page
|
||||||
|
test_url = url_for('test_endpoint', _external=True)
|
||||||
|
res = client.post(
|
||||||
|
url_for("import_page"),
|
||||||
|
data={"urls": test_url},
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
assert b"1 Imported" in res.data
|
||||||
|
wait_for_all_checks(client)
|
||||||
|
|
||||||
|
# Should get a notice that it's available
|
||||||
|
res = client.get(url_for("index"))
|
||||||
|
assert b'ldjson-price-track-offer' in res.data
|
||||||
|
|
||||||
|
# Accept it
|
||||||
|
uuid = extract_UUID_from_client(client)
|
||||||
|
time.sleep(1)
|
||||||
|
client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
|
||||||
|
wait_for_all_checks(client)
|
||||||
|
|
||||||
|
# Trigger a check
|
||||||
|
time.sleep(1)
|
||||||
|
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||||
|
wait_for_all_checks(client)
|
||||||
|
# Offer should be gone
|
||||||
|
res = client.get(url_for("index"))
|
||||||
|
assert b'Embedded price data' not in res.data
|
||||||
|
assert b'tracking-ldjson-price-data' in res.data
|
||||||
|
|
||||||
|
# and last snapshop (via API) should be just the price
|
||||||
|
api_key = extract_api_key_from_UI(client)
|
||||||
|
res = client.get(
|
||||||
|
url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
|
||||||
|
headers={'x-api-key': api_key},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Should see this (dont know where the whitespace came from)
|
||||||
|
assert b'"highPrice": 8099900' in res.data
|
||||||
|
# And not this cause its not the ld-json
|
||||||
|
assert b"So let's see what happens" not in res.data
|
||||||
|
|
||||||
|
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||||
|
|
||||||
|
##########################################################################################
|
||||||
|
# And we shouldnt see the offer
|
||||||
|
set_response_without_ldjson()
|
||||||
|
|
||||||
|
# Add our URL to the import page
|
||||||
|
test_url = url_for('test_endpoint', _external=True)
|
||||||
|
res = client.post(
|
||||||
|
url_for("import_page"),
|
||||||
|
data={"urls": test_url},
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
assert b"1 Imported" in res.data
|
||||||
|
wait_for_all_checks(client)
|
||||||
|
res = client.get(url_for("index"))
|
||||||
|
assert b'ldjson-price-track-offer' not in res.data
|
||||||
|
|
||||||
|
##########################################################################################
|
||||||
|
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_data):
|
||||||
|
|
||||||
|
test_url = url_for('test_endpoint', _external=True)
|
||||||
|
res = client.post(
|
||||||
|
url_for("import_page"),
|
||||||
|
data={"urls": test_url},
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
assert b"1 Imported" in res.data
|
||||||
|
wait_for_all_checks(client)
|
||||||
|
|
||||||
|
for k,v in client.application.config.get('DATASTORE').data['watching'].items():
|
||||||
|
assert v.get('last_error') == False
|
||||||
|
assert v.get('has_ldjson_price_data') == has_ldjson_price_data
|
||||||
|
|
||||||
|
|
||||||
|
##########################################################################################
|
||||||
|
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bad_ldjson_is_correctly_ignored(client, live_server):
|
||||||
|
#live_server_setup(live_server)
|
||||||
|
test_return_data = """
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<script type="application/ld+json">
|
||||||
|
{
|
||||||
|
"@context": "http://schema.org",
|
||||||
|
"@type": ["Product", "SubType"],
|
||||||
|
"name": "My test product",
|
||||||
|
"description": "",
|
||||||
|
"offers": {
|
||||||
|
"note" : "You can see the case-insensitive OffERS key, it should work",
|
||||||
|
"@type": "Offer",
|
||||||
|
"offeredBy": {
|
||||||
|
"@type": "Organization",
|
||||||
|
"name":"Person",
|
||||||
|
"telephone":"+1 999 999 999"
|
||||||
|
},
|
||||||
|
"price": "1",
|
||||||
|
"priceCurrency": "EUR",
|
||||||
|
"url": "/some/url"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="yes">Some extra stuff</div>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||||
|
f.write(test_return_data)
|
||||||
|
|
||||||
|
_test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True)
|
||||||
|
test_return_data = """
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<script type="application/ld+json">
|
||||||
|
{
|
||||||
|
"@context": "http://schema.org",
|
||||||
|
"@type": ["Product", "SubType"],
|
||||||
|
"name": "My test product",
|
||||||
|
"description": "",
|
||||||
|
"BrokenOffers": {
|
||||||
|
"@type": "Offer",
|
||||||
|
"offeredBy": {
|
||||||
|
"@type": "Organization",
|
||||||
|
"name":"Person",
|
||||||
|
"telephone":"+1 999 999 999"
|
||||||
|
},
|
||||||
|
"price": "1",
|
||||||
|
"priceCurrency": "EUR",
|
||||||
|
"url": "/some/url"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="yes">Some extra stuff</div>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||||
|
f.write(test_return_data)
|
||||||
|
|
||||||
|
_test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False)
|
||||||
|
|
||||||
Loading…
Reference in new issue