only-use-default-headers-for-plain-requests
dgtlmoon 2 years ago
parent b72d6f8dec
commit 8578cc3582

@ -575,15 +575,10 @@ class html_requests(Fetcher):
ignore_status_codes=False, ignore_status_codes=False,
current_css_filter=None): current_css_filter=None):
if not 'Accept-Encoding' in request_headers:
# Some sites will assume Requests can use Brotli, but Brotli is not yet supported
# So just announce that we only accept gzip and deflate.
request_headers['Accept-Encoding'] = 'gzip, deflate',
# Make requests use a more modern looking user-agent # Make requests use a more modern looking user-agent
if not 'User-Agent' in request_headers: if not 'User-Agent' in request_headers:
request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'), 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
proxies = {} proxies = {}

Loading…
Cancel
Save