@ -61,6 +61,10 @@ class PageUnloadable(Exception):
self . message = message
self . message = message
return
return
class BrowserStepsInUnsupportedFetcher ( Exception ) :
def __init__ ( self , url ) :
self . url = url
return
class EmptyReply ( Exception ) :
class EmptyReply ( Exception ) :
def __init__ ( self , status_code , url , screenshot = None ) :
def __init__ ( self , status_code , url , screenshot = None ) :
@ -712,6 +716,9 @@ class html_requests(Fetcher):
current_include_filters = None ,
current_include_filters = None ,
is_binary = False ) :
is_binary = False ) :
if self . browser_steps_get_valid_steps ( ) :
raise BrowserStepsInUnsupportedFetcher ( url = url )
# Make requests use a more modern looking user-agent
# Make requests use a more modern looking user-agent
if not { k . lower ( ) : v for k , v in request_headers . items ( ) } . get ( ' user-agent ' , None ) :
if not { k . lower ( ) : v for k , v in request_headers . items ( ) } . get ( ' user-agent ' , None ) :
request_headers [ ' User-Agent ' ] = os . getenv ( " DEFAULT_SETTINGS_HEADERS_USERAGENT " ,
request_headers [ ' User-Agent ' ] = os . getenv ( " DEFAULT_SETTINGS_HEADERS_USERAGENT " ,