Dont set user agent default if none is set

changedetectionio/content_fetcher.py

@@ -343,7 +343,7 @@ class base_html_playwright(Fetcher):
                         'req_headers': request_headers,
                         'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
                         'url': url,
-                        'user_agent': request_headers.get('User-Agent', 'Mozilla/5.0'),
+                        'user_agent': request_headers.get('User-Agent', None),
                         'proxy_username': self.proxy.get('username','') if self.proxy else False,
                         'proxy_password': self.proxy.get('password', '') if self.proxy else False,
                         'no_cache_list': [
@@ -443,7 +443,7 @@ class base_html_playwright(Fetcher):
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
-                user_agent=request_headers.get('User-Agent', 'Mozilla/5.0'),
+                user_agent=request_headers.get('User-Agent', None),
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,

changedetectionio/res/puppeteer_fetch.js

@@ -18,7 +18,9 @@ module.exports = async ({page, context}) => {
 
     await page.setBypassCSP(true)
     await page.setExtraHTTPHeaders(req_headers);
+    if (user_agent) {
         await page.setUserAgent(user_agent);
+    }
     // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded
 
     await page.setDefaultNavigationTimeout(0);