New functionanlity - Selectable browser / ability to add extra browser connections (good for using "scraping browsers"/ etc) (#1943)
dgtlmoon
1 year ago
committed by
GitHub
parent
5a306aa78c
commit
5229094e44
@ -0,0 +1,44 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# run some tests and look if the 'custom-browser-search-string=1' connect string appeared in the correct containers
|
||||||
|
|
||||||
|
# enable debug
|
||||||
|
set -x
|
||||||
|
|
||||||
|
# A extra browser is configured, but we never chose to use it, so it should NOT show in the logs
|
||||||
|
docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'
|
||||||
|
docker logs browserless-custom-url &>log.txt
|
||||||
|
grep 'custom-browser-search-string=1' log.txt
|
||||||
|
if [ $? -ne 1 ]
|
||||||
|
then
|
||||||
|
echo "Saw a request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should not"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker logs browserless &>log.txt
|
||||||
|
grep 'custom-browser-search-string=1' log.txt
|
||||||
|
if [ $? -ne 1 ]
|
||||||
|
then
|
||||||
|
echo "Saw a request in 'browser' container with 'custom-browser-search-string=1' when I should not"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Special connect string should appear in the custom-url container, but not in the 'default' one
|
||||||
|
docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url'
|
||||||
|
docker logs browserless-custom-url &>log.txt
|
||||||
|
grep 'custom-browser-search-string=1' log.txt
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "Did not see request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker logs browserless &>log.txt
|
||||||
|
grep 'custom-browser-search-string=1' log.txt
|
||||||
|
if [ $? -ne 1 ]
|
||||||
|
then
|
||||||
|
echo "Saw a request in 'browser' container with 'custom-browser-search-string=1' when I should not"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,24 @@
|
|||||||
|
ul#requests-extra_browsers {
|
||||||
|
list-style: none;
|
||||||
|
/* tidy up the table to look more "inline" */
|
||||||
|
li {
|
||||||
|
> label {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/* each proxy entry is a `table` */
|
||||||
|
table {
|
||||||
|
tr {
|
||||||
|
display: inline;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#extra-browsers-setting {
|
||||||
|
border: 1px solid var(--color-grey-800);
|
||||||
|
border-radius: 4px;
|
||||||
|
margin: 1em;
|
||||||
|
padding: 1em;
|
||||||
|
}
|
||||||
@ -0,0 +1 @@
|
|||||||
|
# placeholder
|
||||||
@ -0,0 +1,89 @@
|
|||||||
|
# !/usr/bin/python3
|
||||||
|
import os
|
||||||
|
|
||||||
|
from flask import url_for
|
||||||
|
from ..util import live_server_setup, wait_for_all_checks
|
||||||
|
|
||||||
|
def do_test(client, live_server, make_test_use_extra_browser=False):
|
||||||
|
|
||||||
|
# Grep for this string in the logs?
|
||||||
|
test_url = f"https://changedetection.io/ci-test.html"
|
||||||
|
custom_browser_name = 'custom browser URL'
|
||||||
|
|
||||||
|
# needs to be set and something like 'ws://127.0.0.1:3000?stealth=1&--disable-web-security=true'
|
||||||
|
assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
|
||||||
|
|
||||||
|
#####################
|
||||||
|
res = client.post(
|
||||||
|
url_for("settings_page"),
|
||||||
|
data={"application-empty_pages_are_a_change": "",
|
||||||
|
"requests-time_between_check-minutes": 180,
|
||||||
|
'application-fetch_backend': "html_webdriver",
|
||||||
|
# browserless-custom-url is setup in .github/workflows/test-only.yml
|
||||||
|
# the test script run_custom_browser_url_test.sh will look for 'custom-browser-search-string' in the container logs
|
||||||
|
'requests-extra_browsers-0-browser_connection_url': 'ws://browserless-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1',
|
||||||
|
'requests-extra_browsers-0-browser_name': custom_browser_name
|
||||||
|
},
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
|
||||||
|
assert b"Settings updated." in res.data
|
||||||
|
|
||||||
|
# Add our URL to the import page
|
||||||
|
res = client.post(
|
||||||
|
url_for("import_page"),
|
||||||
|
data={"urls": test_url},
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
|
||||||
|
assert b"1 Imported" in res.data
|
||||||
|
wait_for_all_checks(client)
|
||||||
|
|
||||||
|
if make_test_use_extra_browser:
|
||||||
|
|
||||||
|
# So the name should appear in the edit page under "Request" > "Fetch Method"
|
||||||
|
res = client.get(
|
||||||
|
url_for("edit_page", uuid="first"),
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
assert b'custom browser URL' in res.data
|
||||||
|
|
||||||
|
res = client.post(
|
||||||
|
url_for("edit_page", uuid="first"),
|
||||||
|
data={
|
||||||
|
"url": test_url,
|
||||||
|
"tags": "",
|
||||||
|
"headers": "",
|
||||||
|
'fetch_backend': f"extra_browser_{custom_browser_name}",
|
||||||
|
'webdriver_js_execute_code': ''
|
||||||
|
},
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
|
||||||
|
assert b"Updated watch." in res.data
|
||||||
|
wait_for_all_checks(client)
|
||||||
|
|
||||||
|
# Force recheck
|
||||||
|
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||||
|
assert b'1 watches queued for rechecking.' in res.data
|
||||||
|
|
||||||
|
wait_for_all_checks(client)
|
||||||
|
|
||||||
|
res = client.get(
|
||||||
|
url_for("preview_page", uuid="first"),
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
assert b'cool it works' in res.data
|
||||||
|
|
||||||
|
|
||||||
|
# Requires playwright to be installed
|
||||||
|
def test_request_via_custom_browser_url(client, live_server):
|
||||||
|
live_server_setup(live_server)
|
||||||
|
# We do this so we can grep the logs of the custom container and see if the request actually went through that container
|
||||||
|
do_test(client, live_server, make_test_use_extra_browser=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_not_via_custom_browser_url(client, live_server):
|
||||||
|
live_server_setup(live_server)
|
||||||
|
# We do this so we can grep the logs of the custom container and see if the request actually went through that container
|
||||||
|
do_test(client, live_server, make_test_use_extra_browser=False)
|
||||||
Loading…
Reference in new issue