Self-Hosted
/
changedetection.io
mirror of https://github.com/dgtlmoon/changedetection.io.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Linting fixups

Browse Source
pull/11/head
Leigh Morresi 4 years ago
parent 22bc8fabd1
commit 63eea2d6db
4 changed files with 32 additions and 51 deletions
Show Stats Download Patch File Download Diff File

35
backend/__init__.py
Unescape View File

@ -13,7 +13,7 @@
# https://distill.io/features # https://distill.io/features
# proxy per check # proxy per check
# - flask_cors, itsdangerous,MarkupSafe # - flask_cors, itsdangerous,MarkupSafe
import json
import time import time
import os import os
import timeago import timeago
@ -21,8 +21,7 @@ import timeago
import threading import threading
import queue import queue
from flask import Flask, render_template, request, send_file, send_from_directory, abort, redirect, url_for
from flask import Flask, render_template, request, send_file, send_from_directory, safe_join, abort, redirect, url_for
datastore = None datastore = None
@ -35,7 +34,6 @@ extra_stylesheets = []
update_q = queue.Queue() update_q = queue.Queue()
app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static") app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
# Stop browser caching of assets # Stop browser caching of assets
@ -74,15 +72,14 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
# return timeago.format(timestamp, time.time()) # return timeago.format(timestamp, time.time())
# return datetime.datetime.utcfromtimestamp(timestamp).strftime(format) # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
def changedetection_app(config=None, datastore_o=None):
def changedetection_app(config=None, datastore_o=None):
global datastore global datastore
datastore = datastore_o datastore = datastore_o
# Hmm # Hmm
app.config.update(dict(DEBUG=True)) app.config.update(dict(DEBUG=True))
app.config.update(config or {}) app.config.update(config or {})
# Setup cors headers to allow all domains # Setup cors headers to allow all domains
# https://flask-cors.readthedocs.io/en/latest/ # https://flask-cors.readthedocs.io/en/latest/
# CORS(app) # CORS(app)
@ -100,7 +97,6 @@ def changedetection_app(config=None, datastore_o=None):
sorted_watches = [] sorted_watches = []
for uuid, watch in datastore.data['watching'].items(): for uuid, watch in datastore.data['watching'].items():
if limit_tag != None: if limit_tag != None:
# Support for comma separated list of tags. # Support for comma separated list of tags.
for tag_in_watch in watch['tag'].split(','): for tag_in_watch in watch['tag'].split(','):
@ -113,7 +109,6 @@ def changedetection_app(config=None, datastore_o=None):
watch['uuid'] = uuid watch['uuid'] = uuid
sorted_watches.append(watch) sorted_watches.append(watch)
sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True) sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
existing_tags = datastore.get_all_tags() existing_tags = datastore.get_all_tags()
@ -156,7 +151,6 @@ def changedetection_app(config=None, datastore_o=None):
return render_template("scrub.html") return render_template("scrub.html")
@app.route("/edit", methods=['GET', 'POST']) @app.route("/edit", methods=['GET', 'POST'])
def edit_page(): def edit_page():
global messages global messages
@ -193,7 +187,6 @@ def changedetection_app(config=None, datastore_o=None):
return output return output
@app.route("/settings", methods=['GET', "POST"]) @app.route("/settings", methods=['GET', "POST"])
def settings_page(): def settings_page():
global messages global messages
@ -210,9 +203,11 @@ def changedetection_app(config=None, datastore_o=None):
messages.append({'class': 'ok', 'message': "Updated"}) messages.append({'class': 'ok', 'message': "Updated"})
else: else:
messages.append({'class': 'error', 'message': "Must be equal to or greater than 5 and less than 600 minutes"}) messages.append(
{'class': 'error', 'message': "Must be equal to or greater than 5 and less than 600 minutes"})
output = render_template("settings.html", messages=messages, minutes=datastore.data['settings']['requests']['minutes_between_check']) output = render_template("settings.html", messages=messages,
minutes=datastore.data['settings']['requests']['minutes_between_check'])
messages = [] messages = []
return output return output
@ -250,7 +245,6 @@ def changedetection_app(config=None, datastore_o=None):
messages = [] messages = []
return output return output
@app.route("/diff/<string:uuid>", methods=['GET']) @app.route("/diff/<string:uuid>", methods=['GET'])
def diff_history_page(uuid): def diff_history_page(uuid):
global messages global messages
@ -299,7 +293,6 @@ def changedetection_app(config=None, datastore_o=None):
def favicon(): def favicon():
return send_from_directory("/app/static/images", filename="favicon.ico") return send_from_directory("/app/static/images", filename="favicon.ico")
# We're good but backups are even better! # We're good but backups are even better!
@app.route("/backup", methods=['GET']) @app.route("/backup", methods=['GET'])
def get_backup(): def get_backup():
@ -313,7 +306,8 @@ def changedetection_app(config=None, datastore_o=None):
# We only care about UUIDS from the current index file # We only care about UUIDS from the current index file
uuids = list(datastore.data['watching'].keys()) uuids = list(datastore.data['watching'].keys())
with zipfile.ZipFile(os.path.join(app.config['datastore_path'], backupname), 'w', compression=zipfile.ZIP_DEFLATED, with zipfile.ZipFile(os.path.join(app.config['datastore_path'], backupname), 'w',
compression=zipfile.ZIP_DEFLATED,
compresslevel=6) as zipObj: compresslevel=6) as zipObj:
# Be sure we're written fresh # Be sure we're written fresh
@ -332,7 +326,6 @@ def changedetection_app(config=None, datastore_o=None):
mimetype="application/zip", mimetype="application/zip",
attachment_filename=backupname) attachment_filename=backupname)
@app.route("/static/<string:group>/<string:filename>", methods=['GET']) @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
def static_content(group, filename): def static_content(group, filename):
# These files should be in our subdirectory # These files should be in our subdirectory
@ -344,7 +337,6 @@ def changedetection_app(config=None, datastore_o=None):
except FileNotFoundError: except FileNotFoundError:
abort(404) abort(404)
@app.route("/api/add", methods=['POST']) @app.route("/api/add", methods=['POST'])
def api_watch_add(): def api_watch_add():
global messages global messages
@ -357,7 +349,6 @@ def changedetection_app(config=None, datastore_o=None):
messages.append({'class': 'ok', 'message': 'Watch added.'}) messages.append({'class': 'ok', 'message': 'Watch added.'})
return redirect(url_for('index')) return redirect(url_for('index'))
@app.route("/api/delete", methods=['GET']) @app.route("/api/delete", methods=['GET'])
def api_delete(): def api_delete():
global messages global messages
@ -367,7 +358,6 @@ def changedetection_app(config=None, datastore_o=None):
return redirect(url_for('index')) return redirect(url_for('index'))
@app.route("/api/checknow", methods=['GET']) @app.route("/api/checknow", methods=['GET'])
def api_watch_checknow(): def api_watch_checknow():
@ -405,20 +395,16 @@ def changedetection_app(config=None, datastore_o=None):
messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)}) messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
return redirect(url_for('index', tag=tag)) return redirect(url_for('index', tag=tag))
# @todo handle ctrl break # @todo handle ctrl break
ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
return app return app
# Requests for checking on the site use a pool of thread Workers managed by a Queue. # Requests for checking on the site use a pool of thread Workers managed by a Queue.
class Worker(threading.Thread): class Worker(threading.Thread):
current_uuid = None current_uuid = None
def __init__(self, q, *args, **kwargs): def __init__(self, q, *args, **kwargs):
self.q = q self.q = q
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@ -461,14 +447,12 @@ class Worker(threading.Thread):
# No change # No change
x = 1 x = 1
self.current_uuid = None # Done self.current_uuid = None # Done
self.q.task_done() self.q.task_done()
# Thread runner to check every minute, look for new watches to feed into the Queue. # Thread runner to check every minute, look for new watches to feed into the Queue.
def ticker_thread_check_time_launch_checks(): def ticker_thread_check_time_launch_checks():
# Spin up Workers. # Spin up Workers.
for _ in range(datastore.data['settings']['requests']['workers']): for _ in range(datastore.data['settings']['requests']['workers']):
new_worker = Worker(update_q) new_worker = Worker(update_q)
@ -496,4 +480,3 @@ def ticker_thread_check_time_launch_checks():
# Should be low so we can break this out in testing # Should be low so we can break this out in testing
time.sleep(1) time.sleep(1)

4
backend/fetch_site_status.py
Unescape View File

@ -1,8 +1,6 @@
import time import time
import requests import requests
import hashlib import hashlib
import os
import re
from inscriptis import get_text from inscriptis import get_text
@ -13,7 +11,6 @@ class perform_site_check():
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.datastore = datastore self.datastore = datastore
def run(self, uuid): def run(self, uuid):
timestamp = int(time.time()) # used for storage etc too timestamp = int(time.time()) # used for storage etc too
stripped_text_from_html = False stripped_text_from_html = False
@ -88,7 +85,6 @@ class perform_site_check():
if self.datastore.get_val(uuid, 'previous_md5'): if self.datastore.get_val(uuid, 'previous_md5'):
update_obj["last_changed"] = timestamp update_obj["last_changed"] = timestamp
update_obj["previous_md5"] = fetched_md5 update_obj["previous_md5"] = fetched_md5
return update_obj, stripped_text_from_html return update_obj, stripped_text_from_html

16
backend/store.py
Unescape View File

@ -11,13 +11,14 @@ import logging
import time import time
import threading import threading
# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods? # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
# Open a github issue if you know something :) # Open a github issue if you know something :)
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
class ChangeDetectionStore: class ChangeDetectionStore:
lock = Lock() lock = Lock()
def __init__(self, datastore_path="/datastore"): def __init__(self, datastore_path="/datastore", include_default_watches=True):
self.needs_write = False self.needs_write = False
self.datastore_path = datastore_path self.datastore_path = datastore_path
self.json_store_path = "{}/url-watches.json".format(self.datastore_path) self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
@ -90,11 +91,13 @@ class ChangeDetectionStore:
# First time ran, doesnt exist. # First time ran, doesnt exist.
except (FileNotFoundError, json.decoder.JSONDecodeError): except (FileNotFoundError, json.decoder.JSONDecodeError):
if include_default_watches:
print("Creating JSON store at", self.datastore_path) print("Creating JSON store at", self.datastore_path)
#self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
#self.add_watch(url='https://news.ycombinator.com/', tag='Tech news') self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
#self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid') self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
#self.add_watch(url='https://changedetection.io', tag='Tech news') self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
self.add_watch(url='https://changedetection.io', tag='Tech news')
# Finally start the thread that will manage periodic data saves to JSON # Finally start the thread that will manage periodic data saves to JSON
save_data_thread = threading.Thread(target=self.save_datastore).start() save_data_thread = threading.Thread(target=self.save_datastore).start()
@ -146,7 +149,7 @@ class ChangeDetectionStore:
# Support for comma separated list of tags. # Support for comma separated list of tags.
for tag in watch['tag'].split(','): for tag in watch['tag'].split(','):
tag = tag.strip() tag = tag.strip()
if not tag in tags: if tag not in tags:
tags.append(tag) tags.append(tag)
tags.sort() tags.sort()
@ -229,5 +232,4 @@ class ChangeDetectionStore:
self.sync_to_json() self.sync_to_json()
time.sleep(1) time.sleep(1)
# body of the constructor # body of the constructor

2
backend/tests/conftest.py
Unescape View File

@ -33,7 +33,7 @@ def app(request):
app_config = {'datastore_path': datastore_path} app_config = {'datastore_path': datastore_path}
datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path']) datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
app = changedetection_app(app_config, datastore) app = changedetection_app(app_config, datastore)
# Establish an application context before running the tests. # Establish an application context before running the tests.

Write Preview
Loading…
Cancel
Save