Workon threads

.gitignore

@@ -2,3 +2,4 @@ __pycache__
 .idea
 *.pyc
 datastore/url-watches.json
+datastore/*

backend/backend.py

@@ -9,13 +9,21 @@ import os
 import getopt
 import sys
 import datetime
+
+import threading
+
 from flask import Flask, render_template, request, send_file, send_from_directory, safe_join, abort, redirect, url_for
 
 # Local
 import store
+import fetch_site_status
+
+ticker_thread = None
 
 datastore = store.ChangeDetectionStore()
 messages = []
+running_update_threads={}
+
 app = Flask(__name__, static_url_path='/static')
 app.config['STATIC_RESOURCES'] = "/app/static"
 
@@ -24,6 +32,26 @@ app.config['STATIC_RESOURCES'] = "/app/static"
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 
+# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
+# running or something similar.
+@app.template_filter('format_last_checked_time')
+def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
+
+    global running_update_threads
+    if watch_obj['uuid'] in running_update_threads:
+        if running_update_threads[watch_obj['uuid']].is_alive():
+            return "Checking now.."
+
+    if watch_obj['last_checked'] == 0:
+        return 'Never'
+
+    return datetime.datetime.utcfromtimestamp(int(watch_obj['last_checked'])).strftime(format)
+
+@app.template_filter('format_timestamp')
+def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
+    if timestamp == 0:
+        return 'Never'
+    return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
 
 @app.route("/", methods=['GET'])
 def main_page():
@@ -50,17 +78,53 @@ def api_watch_add():
     #@todo add_watch should throw a custom Exception for validation etc
     datastore.add_watch(url=request.form.get('url'), tag=request.form.get('tag'))
     messages.append({'class':'ok', 'message': 'Saved'})
+    launch_checks()
+    return redirect(url_for('main_page'))
+
+
+@app.route("/api/checknow", methods=['GET'])
+def api_watch_checknow():
+    global messages
+
+    uuid=request.args.get('uuid')
+
+    # dict would be better, this is a simple safety catch.
+    for watch in datastore.data['watching']:
+        if watch['uuid'] == uuid:
+            # @todo cancel if already running?
+            running_update_threads[uuid] = fetch_site_status.perform_site_check(uuid=uuid,
+                                                                                         datastore=datastore)
+            running_update_threads[uuid].start()
 
     return redirect(url_for('main_page'))
-    # datastore.add_watch
 
 
+# Can be used whenever, launch threads that need launching to update the stored information
+def launch_checks():
+    import fetch_site_status
+    global running_update_threads
+
+    for watch in datastore.data['watching']:
+        if watch['last_checked'] <= time.time() - 20:
+            running_update_threads[watch['uuid']] = fetch_site_status.perform_site_check(uuid = watch['uuid'], datastore=datastore)
+            running_update_threads[watch['uuid']].start()
+
+def ticker_thread_check_time_launch_checks():
+
+    while True:
+        print ("lanching")
+        launch_checks()
+        time.sleep(60)
 
 
 def main(argv):
     ssl_mode = False
     port = 5000
 
+    #@todo handle ctrl break
+    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
+
+
     try:
         opts, args = getopt.getopt(argv, "sp:")
     except getopt.GetoptError:

backend/fetch_site_status.py

@@ -0,0 +1,54 @@
+from threading import Thread
+import time
+import requests
+
+import hashlib
+
+# Hmm Polymorphism datastore, thread, etc
+class perform_site_check(Thread):
+    def __init__(self, *args, uuid=False, datastore, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.timestamp = int(time.time()) # used for storage etc too
+        self.uuid = uuid
+        self.datastore = datastore
+        self.url = datastore.get_val(uuid, 'url')
+        self.current_md5 = datastore.get_val(uuid, 'previous_md5')
+
+    def save_firefox_screenshot(self, uuid, output):
+        #@todo call selenium or whatever
+        return
+
+    def save_response_output(self, output):
+        # @todo maybe record a history.json, [timestamp, md5, filename]
+        import os
+        path = "/datastore/{}".format(self.uuid)
+        try:
+            os.stat(path)
+        except:
+            os.mkdir(path)
+
+        with open("{}/{}.txt".format(path, self.timestamp), 'w') as f:
+            f.write(output)
+            f.close()
+
+
+    def run(self):
+        try:
+            r = requests.get(self.url)
+        except requests.exceptions.ConnectionError as e:
+            self.datastore.update_watch(self.uuid, 'last_error', str(e))
+
+            print (str(e))
+        else:
+            self.datastore.update_watch(self.uuid, 'last_error', False)
+            self.datastore.update_watch(self.uuid, 'last_check_status', r.status_code)
+
+            fetched_md5=hashlib.md5(r.text.encode('utf-8')).hexdigest()
+
+            if self.current_md5 != fetched_md5:
+                self.datastore.update_watch(self.uuid, 'previous_md5', fetched_md5)
+                self.save_response_output(r.text)
+                self.datastore.update_watch(self.uuid, 'last_changed', self.timestamp)
+
+        self.datastore.update_watch(self.uuid, 'last_checked', int(time.time()))
+        pass

backend/store.py

@@ -25,19 +25,50 @@ class ChangeDetectionStore:
                 'url': 'https://changedetection.io',
                 'tag': 'general',
                 'last_checked': 0,
+                'last_changed' : 0,
                 'uuid': str(uuid.uuid4())
             })
+            self.data['watching'].append({
+                'url': 'http://www.quotationspage.com/random.php',
+                'tag': 'test',
+                'last_checked': 0,
+                'last_changed' : 0,
+                'uuid': str(uuid.uuid4())
+            })
+
 
             with open('/datastore/url-watches.json', 'w') as json_file:
                 json.dump(self.data, json_file)
 
+    def update_watch(self, uuid, val, var):
+        # Probably their should be dict...
+        for watch in self.data['watching']:
+            if watch['uuid'] == uuid:
+                watch[val] = var
+                # print("Updated..", val)
+                self.sync_to_json()
+
+
+    def get_val(self, uuid, val):
+        # Probably their should be dict...
+        for watch in self.data['watching']:
+            if watch['uuid'] == uuid:
+                if val in watch:
+                    return watch[val]
+                else:
+                    return None
+
+        return None
 
     def add_watch(self, url, tag):
         validators.url(url)
 
+        # @todo use a common generic version of this
         self.data['watching'].append({
             'url': url,
             'tag': tag,
+            'last_checked':0,
+            'last_changed': 0,
             'uuid': str(uuid.uuid4())
         })
         self.sync_to_json()

backend/templates/base.html

@@ -3,8 +3,8 @@
 <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <meta name="description" content="A layout example that shows off a responsive email layout.">
+    <meta name="description" content="Self hosted website change detection.">
-    <title>Email &ndash; Layout Examples &ndash; Pure</title>
+    <title>Change Detection</title>
     <link rel="stylesheet" href="/static/css/pure-min.css">
     <link rel="stylesheet" href="/static/css/styles.css">
 </head>

backend/templates/watch-overview.html

@@ -10,15 +10,19 @@
         <input type="text" placeholder="tag" size="10" name="tag"/>
         <button type="submit" class="pure-button pure-button-primary">Save</button>
     </fieldset>
+    <!-- add extra stuff, like do a http POST and send headers -->
+    <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
 </form>
 
+<!-- make a nice list of tags here to click on -->
+
 <table class="pure-table pure-table-striped">
     <thead>
         <tr>
             <th>#</th>
             <th>URL</th>
             <th>Last Checked</th>
-            <th>Status</th>
+            <th>Last Changed</th>
             <th>op</th>
         </tr>
     </thead>
@@ -29,9 +33,9 @@
         <tr class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
             <td>{{ loop.index }}</td>
             <td>{{ watch.url }}</td>
-            <td>2021/2/2 14:00:00</td>
+            <td>{{watch|format_last_checked_time}}</td>
-            <td>No Change</td>
+            <td>{{watch.last_changed|format_timestamp}}</td>
-            <td><button type="submit" class="pure-button pure-button-primary">Recheck</button> <button type="submit" class="pure-button pure-button-primary">Delete</button></td>
+            <td><a href="/api/checknow?uuid={{ watch.uuid}}" class="pure-button pure-button-primary">Recheck</a> <button type="submit" class="pure-button pure-button-primary">Delete</button></td>
         </tr>
   {% endfor %}