Self-Hosted
/
changedetection.io
mirror of https://github.com/dgtlmoon/changedetection.io.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Prefer to use SockPuppetbrowser

Browse Source
pull/2163/head
dgtlmoon 11 months ago
parent c90b27823a
commit 03976cd0e8
14 changed files with 47 additions and 69 deletions
Show Stats Download Patch File Download Diff File

24
.github/workflows/test-only.yml vendored
Unescape View File

@ -28,12 +28,12 @@ jobs:
docker network create changedet-network docker network create changedet-network
# Selenium+browserless # Selenium and sockpuppetbrowser
docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4 docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
docker run --network changedet-network -d --name browserless --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.60-chrome-stable docker run --network changedet-network -d --name sockpuppetbrowser --hostname sockpuppetbrowser -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" dgtlmoon/sockpuppetbrowser
# For accessing custom browser tests # For accessing custom browser tests
docker run --network changedet-network -d --name browserless-custom-url --hostname browserless-custom-url -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm --shm-size="2g" browserless/chrome:1.60-chrome-stable docker run --network changedet-network -d --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm --shm-size="2g" dgtlmoon/sockpuppetbrowser
- name: Build changedetection.io container for testing - name: Build changedetection.io container for testing
run: | run: |
@ -69,13 +69,13 @@ jobs:
- name: Specific tests in built container for Playwright - name: Specific tests in built container for Playwright
run: | run: |
# Playwright/Browserless fetch # Playwright via Sockpuppetbrowser fetch
docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
- name: Specific tests in built container for headers and requests checks with Playwright - name: Specific tests in built container for headers and requests checks with Playwright
run: | run: |
# Settings headers playwright tests - Call back in from Browserless, check headers # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
- name: Specific tests in built container for headers and requests checks with Selenium - name: Specific tests in built container for headers and requests checks with Selenium
run: | run: |
@ -83,12 +83,12 @@ jobs:
- name: Specific tests in built container with Playwright as Puppeteer experimental fetcher - name: Specific tests in built container with Playwright as Puppeteer experimental fetcher
run: | run: |
docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
- name: Test built container restock detection via Playwright - name: Test built container restock detection via Playwright
run: | run: |
# restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py' docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
- name: Test SMTP notification mime types - name: Test SMTP notification mime types
run: | run: |
@ -97,8 +97,8 @@ jobs:
- name: Test with puppeteer fetcher and disk cache - name: Test with puppeteer fetcher and disk cache
run: | run: |
docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
# Browserless would have had -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" added above # Required the browser having FS and CRYPTO enabled
- name: Test proxy interaction - name: Test proxy interaction
run: | run: |

11
changedetectionio/blueprint/browser_steps/__init__.py
Unescape View File

@ -4,22 +4,13 @@
# Why? # Why?
# `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async() # `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()
# - this flask app is not async() # - this flask app is not async()
# - browserless has a single timeout/keepalive which applies to the session made at .connect_over_cdp() # - A single timeout/keepalive which applies to the session made at .connect_over_cdp()
# #
# So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run # So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run
# and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user # and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user
# that their time is up, insert another coin. (reload) # that their time is up, insert another coin. (reload)
# #
# Bigger picture
# - It's horrible that we have this click+wait deal, some nice socket.io solution using something similar
# to what the browserless debug UI already gives us would be smarter..
# #
# OR
# - Some API call that should be hacked into browserless or playwright that we can "/api/bump-keepalive/{session_id}/60"
# So we can tell it that we need more time (run this on each action)
#
# OR
# - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes)
from distutils.util import strtobool from distutils.util import strtobool
from flask import Blueprint, request, make_response from flask import Blueprint, request, make_response

2
changedetectionio/blueprint/browser_steps/browser_steps.py
Unescape View File

@ -169,7 +169,7 @@ class steppable_browser_interface():
self.page.locator(selector, timeout=1000).uncheck(timeout=1000) self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
# Responsible for maintaining a live 'context' with browserless # Responsible for maintaining a live 'context' with the chrome CDP
# @todo - how long do contexts live for anyway? # @todo - how long do contexts live for anyway?
class browsersteps_live_ui(steppable_browser_interface): class browsersteps_live_ui(steppable_browser_interface):
context = None context = None

12
changedetectionio/content_fetcher.py
Unescape View File

@ -332,7 +332,7 @@ class base_html_playwright(Fetcher):
code = code.replace('%instock_scrape_code%', self.instock_data_js) code = code.replace('%instock_scrape_code%', self.instock_data_js)
from requests.exceptions import ConnectTimeout, ReadTimeout from requests.exceptions import ConnectTimeout, ReadTimeout
wait_browserless_seconds = 240 wait_browser_seconds = 240
browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL') browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
from urllib.parse import urlparse from urllib.parse import urlparse
@ -386,14 +386,14 @@ class base_html_playwright(Fetcher):
}, },
# @todo /function needs adding ws:// to http:// rebuild this # @todo /function needs adding ws:// to http:// rebuild this
url=browserless_function_url+f"{amp}--disable-features=AudioServiceOutOfProcess&dumpio=true&--disable-remote-fonts", url=browserless_function_url+f"{amp}--disable-features=AudioServiceOutOfProcess&dumpio=true&--disable-remote-fonts",
timeout=wait_browserless_seconds) timeout=wait_browser_seconds)
except ReadTimeout: except ReadTimeout:
raise PageUnloadable(url=url, status_code=None, message=f"No response from browserless in {wait_browserless_seconds}s") raise PageUnloadable(url=url, status_code=None, message=f"No response from browser in {wait_browser_seconds}s")
except ConnectTimeout: except ConnectTimeout:
raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browserless, retrying..") raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browser, retrying..")
else: else:
# 200 Here means that the communication to browserless worked only, not the page state # 200 Here means that the communication to the browser worked only, not the page state
try: try:
x = response.json() x = response.json()
except Exception as e: except Exception as e:
@ -528,7 +528,7 @@ class base_html_playwright(Fetcher):
self.status_code = response.status self.status_code = response.status
except Exception as e: except Exception as e:
# https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962 # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
logger.critical(f"Response from browserless/playwright did not have a status_code! Response follows.") logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
logger.critical(response) logger.critical(response)
raise PageUnloadable(url=url, status_code=None, message=str(e)) raise PageUnloadable(url=url, status_code=None, message=str(e))

2
changedetectionio/importer.py
Unescape View File

@ -57,7 +57,7 @@ class import_url_list(Importer):
# Flask wtform validators wont work with basic auth, use validators package # Flask wtform validators wont work with basic auth, use validators package
# Up to 5000 per batch so we dont flood the server # Up to 5000 per batch so we dont flood the server
# @todo validators.url failed on local hostnames (such as referring to ourself when using browserless) # @todo validators.url will fail when you add your own IP etc
if len(url) and 'http' in url.lower() and good < 5000: if len(url) and 'http' in url.lower() and good < 5000:
extras = None extras = None
if processor: if processor:

2
changedetectionio/res/puppeteer_fetch.js
Unescape View File

@ -146,7 +146,7 @@ module.exports = async ({page, context}) => {
var xpath_data; var xpath_data;
var instock_data; var instock_data;
try { try {
// Not sure the best way here, in the future this should be a new package added to npm then run in browserless // Not sure the best way here, in the future this should be a new package added to npm then run in evaluatedCode
// (Once the old playwright is removed) // (Once the old playwright is removed)
xpath_data = await page.evaluate((include_filters) => {%xpath_scrape_code%}, include_filters); xpath_data = await page.evaluate((include_filters) => {%xpath_scrape_code%}, include_filters);
instock_data = await page.evaluate(() => {%instock_scrape_code%}); instock_data = await page.evaluate(() => {%instock_scrape_code%});

16
changedetectionio/run_custom_browser_url_tests.sh
Unescape View File

@ -6,16 +6,16 @@
set -x set -x
# A extra browser is configured, but we never chose to use it, so it should NOT show in the logs # A extra browser is configured, but we never chose to use it, so it should NOT show in the logs
docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url' docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'
docker logs browserless-custom-url &>log.txt docker logs sockpuppetbrowser-custom-url &>log.txt
grep 'custom-browser-search-string=1' log.txt grep 'custom-browser-search-string=1' log.txt
if [ $? -ne 1 ] if [ $? -ne 1 ]
then then
echo "Saw a request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should not" echo "Saw a request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should not"
exit 1 exit 1
fi fi
docker logs browserless &>log.txt docker logs sockpuppetbrowser &>log.txt
grep 'custom-browser-search-string=1' log.txt grep 'custom-browser-search-string=1' log.txt
if [ $? -ne 1 ] if [ $? -ne 1 ]
then then
@ -24,16 +24,16 @@ then
fi fi
# Special connect string should appear in the custom-url container, but not in the 'default' one # Special connect string should appear in the custom-url container, but not in the 'default' one
docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url' docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url'
docker logs browserless-custom-url &>log.txt docker logs sockpuppetbrowser-custom-url &>log.txt
grep 'custom-browser-search-string=1' log.txt grep 'custom-browser-search-string=1' log.txt
if [ $? -ne 0 ] if [ $? -ne 0 ]
then then
echo "Did not see request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should" echo "Did not see request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should"
exit 1 exit 1
fi fi
docker logs browserless &>log.txt docker logs sockpuppetbrowser &>log.txt
grep 'custom-browser-search-string=1' log.txt grep 'custom-browser-search-string=1' log.txt
if [ $? -ne 1 ] if [ $? -ne 1 ]
then then

2
changedetectionio/run_proxy_tests.sh
Unescape View File

@ -35,7 +35,7 @@ docker run --network changedet-network \
docker run --network changedet-network \ docker run --network changedet-network \
-e "SOCKSTEST=manual-playwright" \ -e "SOCKSTEST=manual-playwright" \
-v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \ -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
-e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" \ -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
--rm \ --rm \
test-changedetectionio \ test-changedetectionio \
bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py' bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'

14
changedetectionio/static/js/browser-steps.js
Unescape View File

@ -10,7 +10,7 @@ $(document).ready(function () {
} }
}) })
var browsersteps_session_id; var browsersteps_session_id;
var browserless_seconds_remaining = 0; var browser_interface_seconds_remaining = 0;
var apply_buttons_disabled = false; var apply_buttons_disabled = false;
var include_text_elements = $("#include_text_elements"); var include_text_elements = $("#include_text_elements");
var xpath_data = false; var xpath_data = false;
@ -49,7 +49,7 @@ $(document).ready(function () {
$('#browsersteps-img').removeAttr('src'); $('#browsersteps-img').removeAttr('src');
$("#browsersteps-click-start").show(); $("#browsersteps-click-start").show();
$("#browsersteps-selector-wrapper .spinner").hide(); $("#browsersteps-selector-wrapper .spinner").hide();
browserless_seconds_remaining = 0; browser_interface_seconds_remaining = 0;
browsersteps_session_id = false; browsersteps_session_id = false;
apply_buttons_disabled = false; apply_buttons_disabled = false;
ctx.clearRect(0, 0, c.width, c.height); ctx.clearRect(0, 0, c.width, c.height);
@ -61,12 +61,12 @@ $(document).ready(function () {
$('#browser_steps >li:first-child').css('opacity', '0.5'); $('#browser_steps >li:first-child').css('opacity', '0.5');
} }
// Show seconds remaining until playwright/browserless needs to restart the session // Show seconds remaining until the browser interface needs to restart the session
// (See comment at the top of changedetectionio/blueprint/browser_steps/__init__.py ) // (See comment at the top of changedetectionio/blueprint/browser_steps/__init__.py )
setInterval(() => { setInterval(() => {
if (browserless_seconds_remaining >= 1) { if (browser_interface_seconds_remaining >= 1) {
document.getElementById('browserless-seconds-remaining').innerText = browserless_seconds_remaining + " seconds remaining in session"; document.getElementById('browser-seconds-remaining').innerText = browser_interface_seconds_remaining + " seconds remaining in session";
browserless_seconds_remaining -= 1; browser_interface_seconds_remaining -= 1;
} }
}, "1000") }, "1000")
@ -261,7 +261,7 @@ $(document).ready(function () {
// This should trigger 'Goto site' // This should trigger 'Goto site'
console.log("Got startup response, requesting Goto-Site (first) step fake click"); console.log("Got startup response, requesting Goto-Site (first) step fake click");
$('#browser_steps >li:first-child .apply').click(); $('#browser_steps >li:first-child .apply').click();
browserless_seconds_remaining = 500; browser_interface_seconds_remaining = 500;
set_first_gotosite_disabled(); set_first_gotosite_disabled();
}).fail(function (data) { }).fail(function (data) {
console.log(data); console.log(data);

2
changedetectionio/templates/edit.html
Unescape View File

@ -228,7 +228,7 @@ User-Agent: wonderbra 1.0") }}
</div> </div>
</div> </div>
<div id="browser-steps-fieldlist" style="padding-left: 1em; width: 350px; font-size: 80%;" > <div id="browser-steps-fieldlist" style="padding-left: 1em; width: 350px; font-size: 80%;" >
<span id="browserless-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span> <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
{{ render_field(form.browser_steps) }} {{ render_field(form.browser_steps) }}
</div> </div>
</div> </div>

3
changedetectionio/tests/custom_browser_url/test_custom_browser_url.py
Unescape View File

@ -19,9 +19,8 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
data={"application-empty_pages_are_a_change": "", data={"application-empty_pages_are_a_change": "",
"requests-time_between_check-minutes": 180, "requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_webdriver", 'application-fetch_backend': "html_webdriver",
# browserless-custom-url is setup in .github/workflows/test-only.yml
# the test script run_custom_browser_url_test.sh will look for 'custom-browser-search-string' in the container logs # the test script run_custom_browser_url_test.sh will look for 'custom-browser-search-string' in the container logs
'requests-extra_browsers-0-browser_connection_url': 'ws://browserless-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1', 'requests-extra_browsers-0-browser_connection_url': 'ws://sockpuppetbrowser-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1',
'requests-extra_browsers-0-browser_name': custom_browser_name 'requests-extra_browsers-0-browser_name': custom_browser_name
}, },
follow_redirects=True follow_redirects=True

2
changedetectionio/tests/test_jsonpath_jq_selector.py
Unescape View File

@ -456,7 +456,7 @@ def test_ignore_json_order(client, live_server):
def test_correct_header_detect(client, live_server): def test_correct_header_detect(client, live_server):
# Like in https://github.com/dgtlmoon/changedetection.io/pull/1593 # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
# Specify extra html that JSON is sometimes wrapped in - when using Browserless/Puppeteer etc # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
with open("test-datastore/endpoint-content.txt", "w") as f: with open("test-datastore/endpoint-content.txt", "w") as f:
f.write('<html><body>{"hello" : 123, "world": 123}') f.write('<html><body>{"hello" : 123, "world": 123}')

8
changedetectionio/tests/test_request.py
Unescape View File

@ -14,7 +14,7 @@ def test_headers_in_request(client, live_server):
# Add our URL to the import page # Add our URL to the import page
test_url = url_for('test_headers', _external=True) test_url = url_for('test_headers', _external=True)
if os.getenv('PLAYWRIGHT_DRIVER_URL'): if os.getenv('PLAYWRIGHT_DRIVER_URL'):
# Because its no longer calling back to localhost but from browserless, set in test-only.yml # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
test_url = test_url.replace('localhost', 'changedet') test_url = test_url.replace('localhost', 'changedet')
# Add the test URL twice, we will check # Add the test URL twice, we will check
@ -89,7 +89,7 @@ def test_body_in_request(client, live_server):
# Add our URL to the import page # Add our URL to the import page
test_url = url_for('test_body', _external=True) test_url = url_for('test_body', _external=True)
if os.getenv('PLAYWRIGHT_DRIVER_URL'): if os.getenv('PLAYWRIGHT_DRIVER_URL'):
# Because its no longer calling back to localhost but from browserless, set in test-only.yml # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
test_url = test_url.replace('localhost', 'cdio') test_url = test_url.replace('localhost', 'cdio')
res = client.post( res = client.post(
@ -181,7 +181,7 @@ def test_method_in_request(client, live_server):
# Add our URL to the import page # Add our URL to the import page
test_url = url_for('test_method', _external=True) test_url = url_for('test_method', _external=True)
if os.getenv('PLAYWRIGHT_DRIVER_URL'): if os.getenv('PLAYWRIGHT_DRIVER_URL'):
# Because its no longer calling back to localhost but from browserless, set in test-only.yml # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
test_url = test_url.replace('localhost', 'cdio') test_url = test_url.replace('localhost', 'cdio')
# Add the test URL twice, we will check # Add the test URL twice, we will check
@ -258,7 +258,7 @@ def test_headers_textfile_in_request(client, live_server):
# Add our URL to the import page # Add our URL to the import page
test_url = url_for('test_headers', _external=True) test_url = url_for('test_headers', _external=True)
if os.getenv('PLAYWRIGHT_DRIVER_URL'): if os.getenv('PLAYWRIGHT_DRIVER_URL'):
# Because its no longer calling back to localhost but from browserless, set in test-only.yml # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
test_url = test_url.replace('localhost', 'cdio') test_url = test_url.replace('localhost', 'cdio')
print ("TEST URL IS ",test_url) print ("TEST URL IS ",test_url)

16
docker-compose.yml
Unescape View File

@ -71,32 +71,20 @@ services:
# condition: service_started # condition: service_started
# Used for fetching pages via Playwright+Chrome where you need Javascript support. # Used for fetching pages via Playwright+Chrome where you need Javascript support.
# Note: Playwright/browserless not supported on ARM type devices (rPi etc)
# RECOMMENDED FOR FETCHING PAGES WITH CHROME # RECOMMENDED FOR FETCHING PAGES WITH CHROME
# playwright-chrome: # playwright-chrome:
# hostname: playwright-chrome # hostname: playwright-chrome
# image: browserless/chrome:1.60-chrome-stable # image: dgtlmoon/sockpuppetbrowser:latest
# restart: unless-stopped # restart: unless-stopped
# environment: # environment:
# - SCREEN_WIDTH=1920 # - SCREEN_WIDTH=1920
# - SCREEN_HEIGHT=1024 # - SCREEN_HEIGHT=1024
# - SCREEN_DEPTH=16 # - SCREEN_DEPTH=16
# - ENABLE_DEBUGGER=false # - MAX_CONCURRENT_CHROME_PROCESSES=10
# - PREBOOT_CHROME=true
# - CONNECTION_TIMEOUT=300000
# - MAX_CONCURRENT_SESSIONS=10
# - CHROME_REFRESH_TIME=600000
# - DEFAULT_BLOCK_ADS=true
# - DEFAULT_STEALTH=true
#
# Ignore HTTPS errors, like for self-signed certs
# - DEFAULT_IGNORE_HTTPS_ERRORS=true
#
# Used for fetching pages via Playwright+Chrome where you need Javascript support. # Used for fetching pages via Playwright+Chrome where you need Javascript support.
# Note: Works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) # Note: Works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector)
# Does not report status codes (200, 404, 403) and other issues # Does not report status codes (200, 404, 403) and other issues
# More information about the advantages of playwright/browserless https://www.browserless.io/blog/2023/12/13/migrating-selenium-to-playwright/
# browser-chrome: # browser-chrome:
# hostname: browser-chrome # hostname: browser-chrome
# image: selenium/standalone-chrome:4 # image: selenium/standalone-chrome:4

Write Preview
Loading…
Cancel
Save