diff --git a/.gitignore b/.gitignore index ef0e2d73..715c22f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ .idea *.pyc datastore/url-watches.json +datastore/* \ No newline at end of file diff --git a/backend/backend.py b/backend/backend.py index 898c0b76..26e52ce6 100644 --- a/backend/backend.py +++ b/backend/backend.py @@ -9,13 +9,21 @@ import os import getopt import sys import datetime + +import threading + from flask import Flask, render_template, request, send_file, send_from_directory, safe_join, abort, redirect, url_for # Local import store +import fetch_site_status + +ticker_thread = None datastore = store.ChangeDetectionStore() messages = [] +running_update_threads={} + app = Flask(__name__, static_url_path='/static') app.config['STATIC_RESOURCES'] = "/app/static" @@ -24,6 +32,26 @@ app.config['STATIC_RESOURCES'] = "/app/static" # Disables caching of the templates app.config['TEMPLATES_AUTO_RELOAD'] = True +# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread +# running or something similar. +@app.template_filter('format_last_checked_time') +def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"): + + global running_update_threads + if watch_obj['uuid'] in running_update_threads: + if running_update_threads[watch_obj['uuid']].is_alive(): + return "Checking now.." + + if watch_obj['last_checked'] == 0: + return 'Never' + + return datetime.datetime.utcfromtimestamp(int(watch_obj['last_checked'])).strftime(format) + +@app.template_filter('format_timestamp') +def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"): + if timestamp == 0: + return 'Never' + return datetime.datetime.utcfromtimestamp(timestamp).strftime(format) @app.route("/", methods=['GET']) def main_page(): @@ -50,17 +78,53 @@ def api_watch_add(): #@todo add_watch should throw a custom Exception for validation etc datastore.add_watch(url=request.form.get('url'), tag=request.form.get('tag')) messages.append({'class':'ok', 'message': 'Saved'}) + launch_checks() + return redirect(url_for('main_page')) + + +@app.route("/api/checknow", methods=['GET']) +def api_watch_checknow(): + global messages + + uuid=request.args.get('uuid') + + # dict would be better, this is a simple safety catch. + for watch in datastore.data['watching']: + if watch['uuid'] == uuid: + # @todo cancel if already running? + running_update_threads[uuid] = fetch_site_status.perform_site_check(uuid=uuid, + datastore=datastore) + running_update_threads[uuid].start() return redirect(url_for('main_page')) - # datastore.add_watch +# Can be used whenever, launch threads that need launching to update the stored information +def launch_checks(): + import fetch_site_status + global running_update_threads + + for watch in datastore.data['watching']: + if watch['last_checked'] <= time.time() - 20: + running_update_threads[watch['uuid']] = fetch_site_status.perform_site_check(uuid = watch['uuid'], datastore=datastore) + running_update_threads[watch['uuid']].start() + +def ticker_thread_check_time_launch_checks(): + + while True: + print ("lanching") + launch_checks() + time.sleep(60) def main(argv): ssl_mode = False port = 5000 + #@todo handle ctrl break + ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() + + try: opts, args = getopt.getopt(argv, "sp:") except getopt.GetoptError: diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py new file mode 100644 index 00000000..9d0a4223 --- /dev/null +++ b/backend/fetch_site_status.py @@ -0,0 +1,54 @@ +from threading import Thread +import time +import requests + +import hashlib + +# Hmm Polymorphism datastore, thread, etc +class perform_site_check(Thread): + def __init__(self, *args, uuid=False, datastore, **kwargs): + super().__init__(*args, **kwargs) + self.timestamp = int(time.time()) # used for storage etc too + self.uuid = uuid + self.datastore = datastore + self.url = datastore.get_val(uuid, 'url') + self.current_md5 = datastore.get_val(uuid, 'previous_md5') + + def save_firefox_screenshot(self, uuid, output): + #@todo call selenium or whatever + return + + def save_response_output(self, output): + # @todo maybe record a history.json, [timestamp, md5, filename] + import os + path = "/datastore/{}".format(self.uuid) + try: + os.stat(path) + except: + os.mkdir(path) + + with open("{}/{}.txt".format(path, self.timestamp), 'w') as f: + f.write(output) + f.close() + + + def run(self): + try: + r = requests.get(self.url) + except requests.exceptions.ConnectionError as e: + self.datastore.update_watch(self.uuid, 'last_error', str(e)) + + print (str(e)) + else: + self.datastore.update_watch(self.uuid, 'last_error', False) + self.datastore.update_watch(self.uuid, 'last_check_status', r.status_code) + + fetched_md5=hashlib.md5(r.text.encode('utf-8')).hexdigest() + + if self.current_md5 != fetched_md5: + self.datastore.update_watch(self.uuid, 'previous_md5', fetched_md5) + self.save_response_output(r.text) + self.datastore.update_watch(self.uuid, 'last_changed', self.timestamp) + + self.datastore.update_watch(self.uuid, 'last_checked', int(time.time())) + pass diff --git a/backend/store.py b/backend/store.py index 5ff89424..794d0148 100644 --- a/backend/store.py +++ b/backend/store.py @@ -25,19 +25,50 @@ class ChangeDetectionStore: 'url': 'https://changedetection.io', 'tag': 'general', 'last_checked': 0, + 'last_changed' : 0, + 'uuid': str(uuid.uuid4()) + }) + self.data['watching'].append({ + 'url': 'http://www.quotationspage.com/random.php', + 'tag': 'test', + 'last_checked': 0, + 'last_changed' : 0, 'uuid': str(uuid.uuid4()) }) + with open('/datastore/url-watches.json', 'w') as json_file: json.dump(self.data, json_file) + def update_watch(self, uuid, val, var): + # Probably their should be dict... + for watch in self.data['watching']: + if watch['uuid'] == uuid: + watch[val] = var + # print("Updated..", val) + self.sync_to_json() + + + def get_val(self, uuid, val): + # Probably their should be dict... + for watch in self.data['watching']: + if watch['uuid'] == uuid: + if val in watch: + return watch[val] + else: + return None + + return None def add_watch(self, url, tag): validators.url(url) + # @todo use a common generic version of this self.data['watching'].append({ 'url': url, 'tag': tag, + 'last_checked':0, + 'last_changed': 0, 'uuid': str(uuid.uuid4()) }) self.sync_to_json() diff --git a/backend/templates/base.html b/backend/templates/base.html index 79a0ba1b..f4604023 100644 --- a/backend/templates/base.html +++ b/backend/templates/base.html @@ -3,8 +3,8 @@ - - Email – Layout Examples – Pure + + Change Detection diff --git a/backend/templates/watch-overview.html b/backend/templates/watch-overview.html index ff71239c..bed00980 100644 --- a/backend/templates/watch-overview.html +++ b/backend/templates/watch-overview.html @@ -10,15 +10,19 @@ + + + + - + @@ -29,9 +33,9 @@ - - - + + + {% endfor %}
# URL Last CheckedStatusLast Changed op
{{ loop.index }} {{ watch.url }}2021/2/2 14:00:00No Change {{watch|format_last_checked_time}}{{watch.last_changed|format_timestamp}}Recheck