Woops!

changedetectionio/content_fetcher.py

@@ -748,6 +748,8 @@ class html_requests(Fetcher):
                 if encoding:
                     r.encoding = encoding
 
+        self.headers = r.headers
+
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
 
@@ -764,7 +766,7 @@ class html_requests(Fetcher):
         else:
             self.content = r.text
 
-        self.headers = r.headers
+
         self.raw_content = r.content
 
 

changedetectionio/tests/test_request.py

@@ -72,18 +72,17 @@ def test_headers_in_request(client, live_server):
 
     # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
     watches_with_headers = 0
-    with open('test-datastore/url-watches.json') as f:
+    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
-        app_struct = json.load(f)
+            if (len(watch['headers'])):
-        for uuid in app_struct['watching']:
-            if (len(app_struct['watching'][uuid]['headers'])):
                 watches_with_headers += 1
+    # Should be only one with headers set
+    assert watches_with_headers == 1
 
+    # 'server' http header was automatically recorded
     for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
         assert 'werkzeug' in watch.get('remote_server_reply')
         assert 'custom' in watch.get('remote_server_reply') # added in util.py, it should append it with a , to the original one
 
-    # Should be only one with headers set
-    assert watches_with_headers==1
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/update_worker.py

@@ -494,7 +494,7 @@ class update_worker(threading.Thread):
 
                         # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
                         try:
-                            server_header = update_handler.fetcher.headers.get('server', '').trim().lower()[:255]
+                            server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
                             self.datastore.update_watch(uuid=uuid,
                                                         update_obj={'remote_server_reply': server_header}
                                                         )