Data storage - Don't recreate DB if its corrupt, exit with error cleanly so operator can look into the problem (#1296)

changedetection.py

@@ -7,7 +7,7 @@
 
 from changedetectionio import changedetection
 import multiprocessing
-import signal
+import sys
 import os
 
 def sigchld_handler(_signo, _stack_frame):
@@ -35,6 +35,9 @@ if __name__ == '__main__':
     try:
         while True:
             time.sleep(1)
+            if not parse_process.is_alive():
+                # Process died/crashed for some reason, exit with error set
+                sys.exit(1)
 
     except KeyboardInterrupt:
         #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway

changedetectionio/changedetection.py

@@ -3,13 +3,15 @@
 # Launch as a eventlet.wsgi server instance.
 
 from distutils.util import strtobool
+from json.decoder import JSONDecodeError
+
 import eventlet
 import eventlet.wsgi
 import getopt
 import os
 import signal
-import sys
 import socket
+import sys
 
 from . import store, changedetection_app, content_fetcher
 from . import __version__
@@ -84,8 +86,14 @@ def main():
                 "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
             sys.exit(2)
 
+    try:
+        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+    except JSONDecodeError as e:
+        # Dont' start if the JSON DB looks corrupt
+        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
+        print(str(e))
+        return
 
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
     app = changedetection_app(app_config, datastore)
 
     signal.signal(signal.SIGTERM, sigterm_handler)

changedetectionio/store.py

@@ -77,10 +77,10 @@ class ChangeDetectionStore:
                     self.__data['watching'][uuid] = Watch.model(datastore_path=self.datastore_path, default=watch)
                     print("Watching:", uuid, self.__data['watching'][uuid]['url'])
 
-        # First time ran, doesnt exist.
-        except (FileNotFoundError, json.decoder.JSONDecodeError):
+        # First time ran, Create the datastore.
+        except (FileNotFoundError):
             if include_default_watches:
-                print("Creating JSON store at", self.datastore_path)
+                print("No JSON DB found at {}, creating JSON store at {}".format(self.json_store_path, self.datastore_path))
                 self.add_watch(url='https://news.ycombinator.com/',
                                tag='Tech news',
                                extras={'fetch_backend': 'html_requests'})
@@ -88,9 +88,11 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://changedetection.io/CHANGELOG.txt',
                                tag='changedetection.io',
                                extras={'fetch_backend': 'html_requests'})
-
         self.__data['version_tag'] = version_tag
 
+        # Just to test that proxies.json if it exists, doesnt throw a parsing error on startup
+        test_list = self.proxy_list
+
         # Helper to remove password protection
         password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
         if path.isfile(password_reset_lockfile):