New functionanlity - Selectable browser / ability to add extra browser connections (good for using "scraping browsers"/ etc) (#1943)
dgtlmoon
6 months ago
committed by
GitHub
parent
5a306aa78c
commit
5229094e44
@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
|
||||
# run some tests and look if the 'custom-browser-search-string=1' connect string appeared in the correct containers
|
||||
|
||||
# enable debug
|
||||
set -x
|
||||
|
||||
# A extra browser is configured, but we never chose to use it, so it should NOT show in the logs
|
||||
docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'
|
||||
docker logs browserless-custom-url &>log.txt
|
||||
grep 'custom-browser-search-string=1' log.txt
|
||||
if [ $? -ne 1 ]
|
||||
then
|
||||
echo "Saw a request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should not"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker logs browserless &>log.txt
|
||||
grep 'custom-browser-search-string=1' log.txt
|
||||
if [ $? -ne 1 ]
|
||||
then
|
||||
echo "Saw a request in 'browser' container with 'custom-browser-search-string=1' when I should not"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Special connect string should appear in the custom-url container, but not in the 'default' one
|
||||
docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url'
|
||||
docker logs browserless-custom-url &>log.txt
|
||||
grep 'custom-browser-search-string=1' log.txt
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo "Did not see request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker logs browserless &>log.txt
|
||||
grep 'custom-browser-search-string=1' log.txt
|
||||
if [ $? -ne 1 ]
|
||||
then
|
||||
echo "Saw a request in 'browser' container with 'custom-browser-search-string=1' when I should not"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@ -0,0 +1,24 @@
|
||||
ul#requests-extra_browsers {
|
||||
list-style: none;
|
||||
/* tidy up the table to look more "inline" */
|
||||
li {
|
||||
> label {
|
||||
display: none;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/* each proxy entry is a `table` */
|
||||
table {
|
||||
tr {
|
||||
display: inline;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#extra-browsers-setting {
|
||||
border: 1px solid var(--color-grey-800);
|
||||
border-radius: 4px;
|
||||
margin: 1em;
|
||||
padding: 1em;
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
# placeholder
|
||||
@ -0,0 +1,89 @@
|
||||
# !/usr/bin/python3
|
||||
import os
|
||||
|
||||
from flask import url_for
|
||||
from ..util import live_server_setup, wait_for_all_checks
|
||||
|
||||
def do_test(client, live_server, make_test_use_extra_browser=False):
|
||||
|
||||
# Grep for this string in the logs?
|
||||
test_url = f"https://changedetection.io/ci-test.html"
|
||||
custom_browser_name = 'custom browser URL'
|
||||
|
||||
# needs to be set and something like 'ws://127.0.0.1:3000?stealth=1&--disable-web-security=true'
|
||||
assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
|
||||
|
||||
#####################
|
||||
res = client.post(
|
||||
url_for("settings_page"),
|
||||
data={"application-empty_pages_are_a_change": "",
|
||||
"requests-time_between_check-minutes": 180,
|
||||
'application-fetch_backend': "html_webdriver",
|
||||
# browserless-custom-url is setup in .github/workflows/test-only.yml
|
||||
# the test script run_custom_browser_url_test.sh will look for 'custom-browser-search-string' in the container logs
|
||||
'requests-extra_browsers-0-browser_connection_url': 'ws://browserless-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1',
|
||||
'requests-extra_browsers-0-browser_name': custom_browser_name
|
||||
},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Settings updated." in res.data
|
||||
|
||||
# Add our URL to the import page
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
if make_test_use_extra_browser:
|
||||
|
||||
# So the name should appear in the edit page under "Request" > "Fetch Method"
|
||||
res = client.get(
|
||||
url_for("edit_page", uuid="first"),
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b'custom browser URL' in res.data
|
||||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': f"extra_browser_{custom_browser_name}",
|
||||
'webdriver_js_execute_code': ''
|
||||
},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Updated watch." in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Force recheck
|
||||
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
assert b'1 watches queued for rechecking.' in res.data
|
||||
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(
|
||||
url_for("preview_page", uuid="first"),
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b'cool it works' in res.data
|
||||
|
||||
|
||||
# Requires playwright to be installed
|
||||
def test_request_via_custom_browser_url(client, live_server):
|
||||
live_server_setup(live_server)
|
||||
# We do this so we can grep the logs of the custom container and see if the request actually went through that container
|
||||
do_test(client, live_server, make_test_use_extra_browser=True)
|
||||
|
||||
|
||||
def test_request_not_via_custom_browser_url(client, live_server):
|
||||
live_server_setup(live_server)
|
||||
# We do this so we can grep the logs of the custom container and see if the request actually went through that container
|
||||
do_test(client, live_server, make_test_use_extra_browser=False)
|
||||
Loading…
Reference in new issue