From e79bb8ce6946e28a8a321243511b2f20111c64a0 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 29 May 2023 17:08:47 +0200 Subject: [PATCH] Fix up test and import --- .github/workflows/test-only.yml | 5 ++++- changedetectionio/importer.py | 3 ++- changedetectionio/tests/test_request.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index 1bcb5d2a..02b96783 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -54,7 +54,10 @@ jobs: # Playwright/Browserless fetch docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' - docker run --hostname cdio --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/test_request.py' + + # Settings headers playwright tests - Call back in from Browserless, check headers + docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' + docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py' diff --git a/changedetectionio/importer.py b/changedetectionio/importer.py index d49706a1..c39b7a68 100644 --- a/changedetectionio/importer.py +++ b/changedetectionio/importer.py @@ -52,7 +52,8 @@ class import_url_list(Importer): # Flask wtform validators wont work with basic auth, use validators package # Up to 5000 per batch so we dont flood the server - if len(url) and validators.url(url.replace('source:', '')) and good < 5000: + # @todo validators.url failed on local hostnames (such as referring to ourself when using browserless) + if len(url) and 'http' in url.lower() and good < 5000: extras = None if processor: extras = {'processor': processor} diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 2615659e..51cc3970 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -15,7 +15,7 @@ def test_headers_in_request(client, live_server): test_url = url_for('test_headers', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): # Because its no longer calling back to localhost but from browserless, set in test-only.yml - test_url = test_url.replace('localhost', 'cdio') + test_url = test_url.replace('localhost', 'changedet') # Add the test URL twice, we will check res = client.post(