From bf5b1143e33b091f1a9eb7069be2245462730141 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 23 Aug 2024 18:31:54 +0200 Subject: [PATCH] Adding test --- .../test-stack-reusable-workflow.yml | 9 ++ .../test_scrape_price_element.py | 124 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 changedetectionio/tests/ai-price-scraper/test_scrape_price_element.py diff --git a/.github/workflows/test-stack-reusable-workflow.yml b/.github/workflows/test-stack-reusable-workflow.yml index f2864680..4e35995f 100644 --- a/.github/workflows/test-stack-reusable-workflow.yml +++ b/.github/workflows/test-stack-reusable-workflow.yml @@ -52,6 +52,10 @@ jobs: docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest + # CDIO AI Element scraper for prices + # Run CDIO with PRICE_SCRAPER_ML_ENDPOINT=http://cdio-ai-price-element:5005/price-element + docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --name cdio-ai-price-element --hostname cdio-ai-price-element -p 5005:5005 --rm dgtlmoon/dgtlmoon/changedetection.io-ai:latest + - name: Spin up ancillary SMTP+Echo message test server run: | # Debug SMTP server/echo message back server @@ -95,6 +99,11 @@ jobs: # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py; pwd;find .' +# PLAYWRIGHT/NODE-> CDP + - name: ML/AI Price element scraper via Playwright+dgtlmoon/changedetection.io-ai + run: | + docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" -e "PLAYWRIGHT_DRIVER_URL=ws://PRICE_SCRAPER_ML_ENDPOINT:5005/price-element" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/ai-price-scraper/test_scrape_price_element.py' + - name: Playwright and SocketPuppetBrowser - Restock detection run: | # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it diff --git a/changedetectionio/tests/ai-price-scraper/test_scrape_price_element.py b/changedetectionio/tests/ai-price-scraper/test_scrape_price_element.py new file mode 100644 index 00000000..1ad8581f --- /dev/null +++ b/changedetectionio/tests/ai-price-scraper/test_scrape_price_element.py @@ -0,0 +1,124 @@ + +#!/usr/bin/env python3 +import os +import time + +from flask import url_for +from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client + + +# No semantic data just some text, we should be able to find the product price. +def set_response(price="121.95"): + html_content = f""" + + + + + + Ajax Widget + + + +
+
+

Ajax Widget

+

The Ajax Widget is the ultimate solution for all your widget needs. Crafted with precision and using the latest technology, this widget offers unmatched performance and durability. Whether you're using it for personal or professional purposes, the Ajax Widget will not disappoint. It's easy to use, reliable, and comes with a sleek design that complements any setup. Don't settle for less; get the best with the Ajax Widget today!

+
+
+ ${price} +
+ Buy Now
+ IN STOCK +
+
+ + + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(html_content) + time.sleep(1) + return None + + + + +def test_restock_itemprop_basic(client, live_server): + + # needs to be set and something like 'ws://127.0.0.1:3000' + assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test" + assert os.getenv('PRICE_SCRAPER_ML_ENDPOINT'), "Needs PRICE_SCRAPER_ML_ENDPOINT set for this test" + + + live_server_setup(live_server) + + set_response(price="123.99") + + test_url = url_for('test_endpoint', _external=True) + + client.post( + url_for("form_quick_watch_add"), + data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'}, + follow_redirects=True + ) + wait_for_all_checks(client) + res = client.get(url_for("index")) + + assert b'123.99' in res.data + assert b' in-stock' in res.data + assert b' not-in-stock' not in res.data + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data