Let the fetcher throw an exception which will be caught and handed to the operator anyway

pull/187/head
dgtlmoon 3 years ago
parent 6d3962acb6
commit b9ed7e2d20

@ -109,24 +109,14 @@ class html_requests(Fetcher):
def run(self, url, timeout, request_headers):
import requests
try:
r = requests.get(url,
headers=request_headers,
timeout=timeout,
verify=False)
html = r.text
# Usually from networkIO/requests level
except (
requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout,
requests.exceptions.MissingSchema) as e:
self.error = str(e)
return None
except Exception as e:
self.error = "Other exception" + str(e)
return None
r = requests.get(url,
headers=request_headers,
timeout=timeout,
verify=False)
html = r.text
# @todo test this
if not r or not html or not len(html):

Loading…
Cancel
Save