From e1f19a3265e124960f412a9d467e566bdb14b92b Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 23 Sep 2023 14:34:56 +0200 Subject: [PATCH] Dont set user agent default if none is set --- changedetectionio/content_fetcher.py | 4 ++-- changedetectionio/res/puppeteer_fetch.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index db9e23dc..4e899b5c 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -343,7 +343,7 @@ class base_html_playwright(Fetcher): 'req_headers': request_headers, 'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)), 'url': url, - 'user_agent': request_headers.get('User-Agent', 'Mozilla/5.0'), + 'user_agent': request_headers.get('User-Agent', None), 'proxy_username': self.proxy.get('username','') if self.proxy else False, 'proxy_password': self.proxy.get('password', '') if self.proxy else False, 'no_cache_list': [ @@ -443,7 +443,7 @@ class base_html_playwright(Fetcher): # Set user agent to prevent Cloudflare from blocking the browser # Use the default one configured in the App.py model that's passed from fetch_site_status.py context = browser.new_context( - user_agent=request_headers.get('User-Agent', 'Mozilla/5.0'), + user_agent=request_headers.get('User-Agent', None), proxy=self.proxy, # This is needed to enable JavaScript execution on GitHub and others bypass_csp=True, diff --git a/changedetectionio/res/puppeteer_fetch.js b/changedetectionio/res/puppeteer_fetch.js index d376729f..6015fbe8 100644 --- a/changedetectionio/res/puppeteer_fetch.js +++ b/changedetectionio/res/puppeteer_fetch.js @@ -18,7 +18,9 @@ module.exports = async ({page, context}) => { await page.setBypassCSP(true) await page.setExtraHTTPHeaders(req_headers); - await page.setUserAgent(user_agent); + if (user_agent) { + await page.setUserAgent(user_agent); + } // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded await page.setDefaultNavigationTimeout(0);