diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index 9229c9e2..5f40dafb 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -29,7 +29,7 @@ jobs: docker network create changedet-network # Selenium+browserless - docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome-debug:3.141.59 + docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:117 docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable - name: Build changedetection.io container for testing diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index dab956a5..10e7ee1b 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -597,6 +597,24 @@ class base_html_webdriver(Fetcher): proxy_args['httpProxy'] = proxy_override if proxy_args: + + # Convert socks5:// to selenium SOCKS5 correct config + o = urlparse(proxy_args.get('httpProxy', '')) + # https://github.com/tebeka/selenium/issues/167 + # https://github.com/SeleniumHQ/selenium/issues/6917 + if o.scheme == 'socks5': + if proxy_args.get('httpProxy'): + del(proxy_args['httpProxy']) + path = "/" + o.path if o.path else '' + if not o.port: + raise Exception("No Port number specific in this proxy config URL") + proxy_args['socksProxy'] = f"{o.hostname}:{o.port}{path}" + proxy_args['socksVersion'] = 5 + proxy_args['proxyType'] = 'MANUAL' + if o.username: + proxy_args['socksUsername'] = o.username + proxy_args['socksPassword'] = o.password + self.proxy = SeleniumProxy(raw=proxy_args) def run(self, diff --git a/requirements.txt b/requirements.txt index 87dff7b9..3289327c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ beautifulsoup4 lxml # 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0 -selenium~=4.1.0 +selenium~=4.6.1 # https://stackoverflow.com/questions/71652965/importerror-cannot-import-name-safe-str-cmp-from-werkzeug-security/71653849#71653849 # ImportError: cannot import name 'safe_str_cmp' from 'werkzeug.security'