diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index f3fde056..ca17e81b 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -28,12 +28,12 @@ jobs: docker network create changedet-network - # Selenium+browserless + # Selenium and sockpuppetbrowser docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4 - docker run --network changedet-network -d --name browserless --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.60-chrome-stable + docker run --network changedet-network -d --name sockpuppetbrowser --hostname sockpuppetbrowser -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" dgtlmoon/sockpuppetbrowser # For accessing custom browser tests - docker run --network changedet-network -d --name browserless-custom-url --hostname browserless-custom-url -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm --shm-size="2g" browserless/chrome:1.60-chrome-stable + docker run --network changedet-network -d --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm --shm-size="2g" dgtlmoon/sockpuppetbrowser - name: Build changedetection.io container for testing run: | @@ -69,13 +69,13 @@ jobs: - name: Specific tests in built container for Playwright run: | - # Playwright/Browserless fetch - docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' + # Playwright via Sockpuppetbrowser fetch + docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' - name: Specific tests in built container for headers and requests checks with Playwright run: | - # Settings headers playwright tests - Call back in from Browserless, check headers - docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' + # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers + docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' - name: Specific tests in built container for headers and requests checks with Selenium run: | @@ -83,12 +83,12 @@ jobs: - name: Specific tests in built container with Playwright as Puppeteer experimental fetcher run: | - docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' + docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' - name: Test built container restock detection via Playwright run: | - # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it - docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py' + # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it + docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py' - name: Test SMTP notification mime types run: | @@ -97,8 +97,8 @@ jobs: - name: Test with puppeteer fetcher and disk cache run: | - docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' - # Browserless would have had -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" added above + docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' + # Required the browser having FS and CRYPTO enabled - name: Test proxy interaction run: | diff --git a/changedetectionio/blueprint/browser_steps/__init__.py b/changedetectionio/blueprint/browser_steps/__init__.py index 999b81bc..2ee0d44f 100644 --- a/changedetectionio/blueprint/browser_steps/__init__.py +++ b/changedetectionio/blueprint/browser_steps/__init__.py @@ -4,22 +4,13 @@ # Why? # `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async() # - this flask app is not async() -# - browserless has a single timeout/keepalive which applies to the session made at .connect_over_cdp() +# - A single timeout/keepalive which applies to the session made at .connect_over_cdp() # # So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run # and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user # that their time is up, insert another coin. (reload) # -# Bigger picture -# - It's horrible that we have this click+wait deal, some nice socket.io solution using something similar -# to what the browserless debug UI already gives us would be smarter.. # -# OR -# - Some API call that should be hacked into browserless or playwright that we can "/api/bump-keepalive/{session_id}/60" -# So we can tell it that we need more time (run this on each action) -# -# OR -# - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes) from distutils.util import strtobool from flask import Blueprint, request, make_response diff --git a/changedetectionio/blueprint/browser_steps/browser_steps.py b/changedetectionio/blueprint/browser_steps/browser_steps.py index 40925a3f..cfc96a20 100644 --- a/changedetectionio/blueprint/browser_steps/browser_steps.py +++ b/changedetectionio/blueprint/browser_steps/browser_steps.py @@ -169,7 +169,7 @@ class steppable_browser_interface(): self.page.locator(selector, timeout=1000).uncheck(timeout=1000) -# Responsible for maintaining a live 'context' with browserless +# Responsible for maintaining a live 'context' with the chrome CDP # @todo - how long do contexts live for anyway? class browsersteps_live_ui(steppable_browser_interface): context = None diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index 9544a950..3ef867fe 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -332,7 +332,7 @@ class base_html_playwright(Fetcher): code = code.replace('%instock_scrape_code%', self.instock_data_js) from requests.exceptions import ConnectTimeout, ReadTimeout - wait_browserless_seconds = 240 + wait_browser_seconds = 240 browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL') from urllib.parse import urlparse @@ -386,14 +386,14 @@ class base_html_playwright(Fetcher): }, # @todo /function needs adding ws:// to http:// rebuild this url=browserless_function_url+f"{amp}--disable-features=AudioServiceOutOfProcess&dumpio=true&--disable-remote-fonts", - timeout=wait_browserless_seconds) + timeout=wait_browser_seconds) except ReadTimeout: - raise PageUnloadable(url=url, status_code=None, message=f"No response from browserless in {wait_browserless_seconds}s") + raise PageUnloadable(url=url, status_code=None, message=f"No response from browser in {wait_browser_seconds}s") except ConnectTimeout: - raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browserless, retrying..") + raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browser, retrying..") else: - # 200 Here means that the communication to browserless worked only, not the page state + # 200 Here means that the communication to the browser worked only, not the page state try: x = response.json() except Exception as e: @@ -528,7 +528,7 @@ class base_html_playwright(Fetcher): self.status_code = response.status except Exception as e: # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962 - logger.critical(f"Response from browserless/playwright did not have a status_code! Response follows.") + logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.") logger.critical(response) raise PageUnloadable(url=url, status_code=None, message=str(e)) diff --git a/changedetectionio/importer.py b/changedetectionio/importer.py index ba7b1139..42a062be 100644 --- a/changedetectionio/importer.py +++ b/changedetectionio/importer.py @@ -57,7 +57,7 @@ class import_url_list(Importer): # Flask wtform validators wont work with basic auth, use validators package # Up to 5000 per batch so we dont flood the server - # @todo validators.url failed on local hostnames (such as referring to ourself when using browserless) + # @todo validators.url will fail when you add your own IP etc if len(url) and 'http' in url.lower() and good < 5000: extras = None if processor: diff --git a/changedetectionio/res/puppeteer_fetch.js b/changedetectionio/res/puppeteer_fetch.js index 7bb9e17b..21c5abc8 100644 --- a/changedetectionio/res/puppeteer_fetch.js +++ b/changedetectionio/res/puppeteer_fetch.js @@ -146,7 +146,7 @@ module.exports = async ({page, context}) => { var xpath_data; var instock_data; try { - // Not sure the best way here, in the future this should be a new package added to npm then run in browserless + // Not sure the best way here, in the future this should be a new package added to npm then run in evaluatedCode // (Once the old playwright is removed) xpath_data = await page.evaluate((include_filters) => {%xpath_scrape_code%}, include_filters); instock_data = await page.evaluate(() => {%instock_scrape_code%}); diff --git a/changedetectionio/run_custom_browser_url_tests.sh b/changedetectionio/run_custom_browser_url_tests.sh index 10cea9c5..aef8b533 100755 --- a/changedetectionio/run_custom_browser_url_tests.sh +++ b/changedetectionio/run_custom_browser_url_tests.sh @@ -6,16 +6,16 @@ set -x # A extra browser is configured, but we never chose to use it, so it should NOT show in the logs -docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url' -docker logs browserless-custom-url &>log.txt +docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url' +docker logs sockpuppetbrowser-custom-url &>log.txt grep 'custom-browser-search-string=1' log.txt if [ $? -ne 1 ] then - echo "Saw a request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should not" + echo "Saw a request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should not" exit 1 fi -docker logs browserless &>log.txt +docker logs sockpuppetbrowser &>log.txt grep 'custom-browser-search-string=1' log.txt if [ $? -ne 1 ] then @@ -24,16 +24,16 @@ then fi # Special connect string should appear in the custom-url container, but not in the 'default' one -docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url' -docker logs browserless-custom-url &>log.txt +docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url' +docker logs sockpuppetbrowser-custom-url &>log.txt grep 'custom-browser-search-string=1' log.txt if [ $? -ne 0 ] then - echo "Did not see request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should" + echo "Did not see request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should" exit 1 fi -docker logs browserless &>log.txt +docker logs sockpuppetbrowser &>log.txt grep 'custom-browser-search-string=1' log.txt if [ $? -ne 1 ] then diff --git a/changedetectionio/run_proxy_tests.sh b/changedetectionio/run_proxy_tests.sh index 97f81ed5..279cd85c 100755 --- a/changedetectionio/run_proxy_tests.sh +++ b/changedetectionio/run_proxy_tests.sh @@ -35,7 +35,7 @@ docker run --network changedet-network \ docker run --network changedet-network \ -e "SOCKSTEST=manual-playwright" \ -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \ - -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" \ + -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \ --rm \ test-changedetectionio \ bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py' diff --git a/changedetectionio/static/js/browser-steps.js b/changedetectionio/static/js/browser-steps.js index 6da56b6a..90198d32 100644 --- a/changedetectionio/static/js/browser-steps.js +++ b/changedetectionio/static/js/browser-steps.js @@ -10,7 +10,7 @@ $(document).ready(function () { } }) var browsersteps_session_id; - var browserless_seconds_remaining = 0; + var browser_interface_seconds_remaining = 0; var apply_buttons_disabled = false; var include_text_elements = $("#include_text_elements"); var xpath_data = false; @@ -49,7 +49,7 @@ $(document).ready(function () { $('#browsersteps-img').removeAttr('src'); $("#browsersteps-click-start").show(); $("#browsersteps-selector-wrapper .spinner").hide(); - browserless_seconds_remaining = 0; + browser_interface_seconds_remaining = 0; browsersteps_session_id = false; apply_buttons_disabled = false; ctx.clearRect(0, 0, c.width, c.height); @@ -61,12 +61,12 @@ $(document).ready(function () { $('#browser_steps >li:first-child').css('opacity', '0.5'); } - // Show seconds remaining until playwright/browserless needs to restart the session + // Show seconds remaining until the browser interface needs to restart the session // (See comment at the top of changedetectionio/blueprint/browser_steps/__init__.py ) setInterval(() => { - if (browserless_seconds_remaining >= 1) { - document.getElementById('browserless-seconds-remaining').innerText = browserless_seconds_remaining + " seconds remaining in session"; - browserless_seconds_remaining -= 1; + if (browser_interface_seconds_remaining >= 1) { + document.getElementById('browser-seconds-remaining').innerText = browser_interface_seconds_remaining + " seconds remaining in session"; + browser_interface_seconds_remaining -= 1; } }, "1000") @@ -261,7 +261,7 @@ $(document).ready(function () { // This should trigger 'Goto site' console.log("Got startup response, requesting Goto-Site (first) step fake click"); $('#browser_steps >li:first-child .apply').click(); - browserless_seconds_remaining = 500; + browser_interface_seconds_remaining = 500; set_first_gotosite_disabled(); }).fail(function (data) { console.log(data); diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index a5acd213..ddb1229b 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -228,7 +228,7 @@ User-Agent: wonderbra 1.0") }}
- Loading (?) + Loading (?) {{ render_field(form.browser_steps) }}
diff --git a/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py b/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py index bfc2c95e..c0627fcf 100644 --- a/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py +++ b/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py @@ -19,9 +19,8 @@ def do_test(client, live_server, make_test_use_extra_browser=False): data={"application-empty_pages_are_a_change": "", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_webdriver", - # browserless-custom-url is setup in .github/workflows/test-only.yml # the test script run_custom_browser_url_test.sh will look for 'custom-browser-search-string' in the container logs - 'requests-extra_browsers-0-browser_connection_url': 'ws://browserless-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1', + 'requests-extra_browsers-0-browser_connection_url': 'ws://sockpuppetbrowser-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1', 'requests-extra_browsers-0-browser_name': custom_browser_name }, follow_redirects=True diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index 7dc4d68f..5dfdfef2 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -456,7 +456,7 @@ def test_ignore_json_order(client, live_server): def test_correct_header_detect(client, live_server): # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593 - # Specify extra html that JSON is sometimes wrapped in - when using Browserless/Puppeteer etc + # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc with open("test-datastore/endpoint-content.txt", "w") as f: f.write('{"hello" : 123, "world": 123}') diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 5deaafa8..869ea349 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -14,7 +14,7 @@ def test_headers_in_request(client, live_server): # Add our URL to the import page test_url = url_for('test_headers', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): - # Because its no longer calling back to localhost but from browserless, set in test-only.yml + # Because its no longer calling back to localhost but from the browser container, set in test-only.yml test_url = test_url.replace('localhost', 'changedet') # Add the test URL twice, we will check @@ -89,7 +89,7 @@ def test_body_in_request(client, live_server): # Add our URL to the import page test_url = url_for('test_body', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): - # Because its no longer calling back to localhost but from browserless, set in test-only.yml + # Because its no longer calling back to localhost but from the browser container, set in test-only.yml test_url = test_url.replace('localhost', 'cdio') res = client.post( @@ -181,7 +181,7 @@ def test_method_in_request(client, live_server): # Add our URL to the import page test_url = url_for('test_method', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): - # Because its no longer calling back to localhost but from browserless, set in test-only.yml + # Because its no longer calling back to localhost but from the browser container, set in test-only.yml test_url = test_url.replace('localhost', 'cdio') # Add the test URL twice, we will check @@ -258,7 +258,7 @@ def test_headers_textfile_in_request(client, live_server): # Add our URL to the import page test_url = url_for('test_headers', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): - # Because its no longer calling back to localhost but from browserless, set in test-only.yml + # Because its no longer calling back to localhost but from the browser container, set in test-only.yml test_url = test_url.replace('localhost', 'cdio') print ("TEST URL IS ",test_url) diff --git a/docker-compose.yml b/docker-compose.yml index e9bdf85e..7b7acac5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,32 +71,20 @@ services: # condition: service_started # Used for fetching pages via Playwright+Chrome where you need Javascript support. - # Note: Playwright/browserless not supported on ARM type devices (rPi etc) # RECOMMENDED FOR FETCHING PAGES WITH CHROME # playwright-chrome: # hostname: playwright-chrome -# image: browserless/chrome:1.60-chrome-stable +# image: dgtlmoon/sockpuppetbrowser:latest # restart: unless-stopped # environment: # - SCREEN_WIDTH=1920 # - SCREEN_HEIGHT=1024 # - SCREEN_DEPTH=16 -# - ENABLE_DEBUGGER=false -# - PREBOOT_CHROME=true -# - CONNECTION_TIMEOUT=300000 -# - MAX_CONCURRENT_SESSIONS=10 -# - CHROME_REFRESH_TIME=600000 -# - DEFAULT_BLOCK_ADS=true -# - DEFAULT_STEALTH=true -# -# Ignore HTTPS errors, like for self-signed certs -# - DEFAULT_IGNORE_HTTPS_ERRORS=true -# +# - MAX_CONCURRENT_CHROME_PROCESSES=10 # Used for fetching pages via Playwright+Chrome where you need Javascript support. # Note: Works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) # Does not report status codes (200, 404, 403) and other issues - # More information about the advantages of playwright/browserless https://www.browserless.io/blog/2023/12/13/migrating-selenium-to-playwright/ # browser-chrome: # hostname: browser-chrome # image: selenium/standalone-chrome:4