@ -9,7 +9,6 @@ from copy import deepcopy, copy
from os import path , unlink
from os import path , unlink
from threading import Lock
from threading import Lock
import json
import json
import logging
import os
import os
import re
import re
import requests
import requests
@ -17,6 +16,7 @@ import secrets
import threading
import threading
import time
import time
import uuid as uuid_builder
import uuid as uuid_builder
from loguru import logger
# Because the server will run as a daemon and wont know the URL for notification links when firing off a notification
# Because the server will run as a daemon and wont know the URL for notification links when firing off a notification
BASE_URL_NOT_SET_TEXT = ' ( " Base URL " not set - see settings - notifications) '
BASE_URL_NOT_SET_TEXT = ' ( " Base URL " not set - see settings - notifications) '
@ -42,7 +42,7 @@ class ChangeDetectionStore:
self . __data = App . model ( )
self . __data = App . model ( )
self . datastore_path = datastore_path
self . datastore_path = datastore_path
self . json_store_path = " {} /url-watches.json " . format ( self . datastore_path )
self . json_store_path = " {} /url-watches.json " . format ( self . datastore_path )
print ( " >>> Datastore path is " , self . json_store_path )
logger . info ( f " Datastore path is ' { self . json_store_path } ' " )
self . needs_write = False
self . needs_write = False
self . start_time = time . time ( )
self . start_time = time . time ( )
self . stop_thread = False
self . stop_thread = False
@ -83,12 +83,12 @@ class ChangeDetectionStore:
for uuid , watch in self . __data [ ' watching ' ] . items ( ) :
for uuid , watch in self . __data [ ' watching ' ] . items ( ) :
watch [ ' uuid ' ] = uuid
watch [ ' uuid ' ] = uuid
self . __data [ ' watching ' ] [ uuid ] = Watch . model ( datastore_path = self . datastore_path , default = watch )
self . __data [ ' watching ' ] [ uuid ] = Watch . model ( datastore_path = self . datastore_path , default = watch )
print ( " Watching: " , uuid , self . __data [ ' watching ' ] [ uuid ] [ ' url ' ] )
logger . debug ( f " Watching: { uuid } { self . __data [ ' watching ' ] [ uuid ] [ ' url ' ] } " )
# First time ran, Create the datastore.
# First time ran, Create the datastore.
except ( FileNotFoundError ) :
except ( FileNotFoundError ) :
if include_default_watches :
if include_default_watches :
print ( " No JSON DB found at {} , creating JSON store at {} " . format ( self . json_store_path , self . datastore_path ) )
logger . critical ( f " No JSON DB found at { self . json_store_path } , creating JSON store at { self . datastore_path } " )
self . add_watch ( url = ' https://news.ycombinator.com/ ' ,
self . add_watch ( url = ' https://news.ycombinator.com/ ' ,
tag = ' Tech news ' ,
tag = ' Tech news ' ,
extras = { ' fetch_backend ' : ' html_requests ' } )
extras = { ' fetch_backend ' : ' html_requests ' } )
@ -139,7 +139,7 @@ class ChangeDetectionStore:
save_data_thread = threading . Thread ( target = self . save_datastore ) . start ( )
save_data_thread = threading . Thread ( target = self . save_datastore ) . start ( )
def set_last_viewed ( self , uuid , timestamp ) :
def set_last_viewed ( self , uuid , timestamp ) :
logg ing. debug ( " Setting watch UUID: {} last viewed to {} " . format ( uuid , int ( timestamp ) ) )
logg er. debug ( f " Setting watch UUID: { uuid } last viewed to { int ( timestamp ) } " )
self . data [ ' watching ' ] [ uuid ] . update ( { ' last_viewed ' : int ( timestamp ) } )
self . data [ ' watching ' ] [ uuid ] . update ( { ' last_viewed ' : int ( timestamp ) } )
self . needs_write = True
self . needs_write = True
@ -316,7 +316,7 @@ class ChangeDetectionStore:
apply_extras [ ' include_filters ' ] = [ res [ ' css_filter ' ] ]
apply_extras [ ' include_filters ' ] = [ res [ ' css_filter ' ] ]
except Exception as e :
except Exception as e :
logg ing. error ( " Error fetching metadata for shared watch link " , url , str ( e ) )
logg er. error ( f " Error fetching metadata for shared watch link { url } { str ( e ) } " )
flash ( " Error fetching metadata for {} " . format ( url ) , ' error ' )
flash ( " Error fetching metadata for {} " . format ( url ) , ' error ' )
return False
return False
from . model . Watch import is_safe_url
from . model . Watch import is_safe_url
@ -345,7 +345,7 @@ class ChangeDetectionStore:
new_uuid = new_watch . get ( ' uuid ' )
new_uuid = new_watch . get ( ' uuid ' )
logg ing. debug ( " Added URL {} - {} " . format ( url , new_uuid ) )
logg er. debug ( f " Adding URL { url } - { new_uuid } " )
for k in [ ' uuid ' , ' history ' , ' last_checked ' , ' last_changed ' , ' newest_history_key ' , ' previous_md5 ' , ' viewed ' ] :
for k in [ ' uuid ' , ' history ' , ' last_checked ' , ' last_changed ' , ' newest_history_key ' , ' previous_md5 ' , ' viewed ' ] :
if k in apply_extras :
if k in apply_extras :
@ -362,7 +362,7 @@ class ChangeDetectionStore:
if write_to_disk_now :
if write_to_disk_now :
self . sync_to_json ( )
self . sync_to_json ( )
print ( " added " , url )
logger . debug ( f " Added ' { url } ' " )
return new_uuid
return new_uuid
@ -416,14 +416,13 @@ class ChangeDetectionStore:
def sync_to_json ( self ) :
def sync_to_json ( self ) :
logging . info ( " Saving JSON.. " )
logger . info ( " Saving JSON.. " )
print ( " Saving JSON.. " )
try :
try :
data = deepcopy ( self . __data )
data = deepcopy ( self . __data )
except RuntimeError as e :
except RuntimeError as e :
# Try again in 15 seconds
# Try again in 15 seconds
time . sleep ( 15 )
time . sleep ( 15 )
logg ing. error ( " ! Data changed when writing to JSON, trying again.. %s " , str ( e ) )
logg er. error ( f " ! Data changed when writing to JSON, trying again.. { str ( e ) } " )
self . sync_to_json ( )
self . sync_to_json ( )
return
return
else :
else :
@ -436,7 +435,7 @@ class ChangeDetectionStore:
json . dump ( data , json_file , indent = 4 )
json . dump ( data , json_file , indent = 4 )
os . replace ( self . json_store_path + " .tmp " , self . json_store_path )
os . replace ( self . json_store_path + " .tmp " , self . json_store_path )
except Exception as e :
except Exception as e :
logg ing. error ( " Error writing JSON!! (Main JSON file save was skipped) : %s " , str ( e ) )
logg er. error ( f " Error writing JSON!! (Main JSON file save was skipped) : { str ( e ) } " )
self . needs_write = False
self . needs_write = False
self . needs_write_urgent = False
self . needs_write_urgent = False
@ -447,7 +446,16 @@ class ChangeDetectionStore:
while True :
while True :
if self . stop_thread :
if self . stop_thread :
print ( " Shutting down datastore thread " )
# Suppressing "Logging error in Loguru Handler #0" during CICD.
# Not a meaningful difference for a real use-case just for CICD.
# the side effect is a "Shutting down datastore thread" message
# at the end of each test.
# But still more looking better.
import sys
logger . remove ( )
logger . add ( sys . stderr )
logger . critical ( " Shutting down datastore thread " )
return
return
if self . needs_write or self . needs_write_urgent :
if self . needs_write or self . needs_write_urgent :
@ -463,7 +471,7 @@ class ChangeDetectionStore:
# Go through the datastore path and remove any snapshots that are not mentioned in the index
# Go through the datastore path and remove any snapshots that are not mentioned in the index
# This usually is not used, but can be handy.
# This usually is not used, but can be handy.
def remove_unused_snapshots ( self ) :
def remove_unused_snapshots ( self ) :
print ( " Removing snapshots from datastore that are not in the index.. " )
logger . info ( " Removing snapshots from datastore that are not in the index.. " )
index = [ ]
index = [ ]
for uuid in self . data [ ' watching ' ] :
for uuid in self . data [ ' watching ' ] :
@ -476,7 +484,7 @@ class ChangeDetectionStore:
for uuid in self . data [ ' watching ' ] :
for uuid in self . data [ ' watching ' ] :
for item in pathlib . Path ( self . datastore_path ) . rglob ( uuid + " /*.txt " ) :
for item in pathlib . Path ( self . datastore_path ) . rglob ( uuid + " /*.txt " ) :
if not str ( item ) in index :
if not str ( item ) in index :
print ( " Removing " , item )
logger . info ( f " Removing { item } " )
unlink ( item )
unlink ( item )
@property
@property
@ -562,7 +570,7 @@ class ChangeDetectionStore:
if os . path . isfile ( filepath ) :
if os . path . isfile ( filepath ) :
headers . update ( parse_headers_from_text_file ( filepath ) )
headers . update ( parse_headers_from_text_file ( filepath ) )
except Exception as e :
except Exception as e :
print ( f " ERROR reading headers.txt at { filepath } ", str ( e ) )
logger . error ( f " ERROR reading headers.txt at { filepath } { str ( e ) } " )
watch = self . data [ ' watching ' ] . get ( uuid )
watch = self . data [ ' watching ' ] . get ( uuid )
if watch :
if watch :
@ -573,7 +581,7 @@ class ChangeDetectionStore:
if os . path . isfile ( filepath ) :
if os . path . isfile ( filepath ) :
headers . update ( parse_headers_from_text_file ( filepath ) )
headers . update ( parse_headers_from_text_file ( filepath ) )
except Exception as e :
except Exception as e :
print ( f " ERROR reading headers.txt at { filepath } ", str ( e ) )
logger . error ( f " ERROR reading headers.txt at { filepath } { str ( e ) } " )
# In /datastore/tag-name.txt
# In /datastore/tag-name.txt
tags = self . get_all_tags_for_watch ( uuid = uuid )
tags = self . get_all_tags_for_watch ( uuid = uuid )
@ -584,7 +592,7 @@ class ChangeDetectionStore:
if os . path . isfile ( filepath ) :
if os . path . isfile ( filepath ) :
headers . update ( parse_headers_from_text_file ( filepath ) )
headers . update ( parse_headers_from_text_file ( filepath ) )
except Exception as e :
except Exception as e :
print ( f " ERROR reading headers.txt at { filepath } ", str ( e ) )
logger . error ( f " ERROR reading headers.txt at { filepath } { str ( e ) } " )
return headers
return headers
@ -602,13 +610,13 @@ class ChangeDetectionStore:
def add_tag ( self , name ) :
def add_tag ( self , name ) :
# If name exists, return that
# If name exists, return that
n = name . strip ( ) . lower ( )
n = name . strip ( ) . lower ( )
print ( f " >>> Adding new tag - ' { n } ' " )
logger . debug ( f " >>> Adding new tag - ' { n } ' " )
if not n :
if not n :
return False
return False
for uuid , tag in self . __data [ ' settings ' ] [ ' application ' ] . get ( ' tags ' , { } ) . items ( ) :
for uuid , tag in self . __data [ ' settings ' ] [ ' application ' ] . get ( ' tags ' , { } ) . items ( ) :
if n == tag . get ( ' title ' , ' ' ) . lower ( ) . strip ( ) :
if n == tag . get ( ' title ' , ' ' ) . lower ( ) . strip ( ) :
print ( f " >>> Tag { name } already exists " )
logger . error ( f " >>> Tag { name } already exists " )
return uuid
return uuid
# Eventually almost everything todo with a watch will apply as a Tag
# Eventually almost everything todo with a watch will apply as a Tag
@ -670,7 +678,7 @@ class ChangeDetectionStore:
updates_available = self . get_updates_available ( )
updates_available = self . get_updates_available ( )
for update_n in updates_available :
for update_n in updates_available :
if update_n > self . __data [ ' settings ' ] [ ' application ' ] [ ' schema_version ' ] :
if update_n > self . __data [ ' settings ' ] [ ' application ' ] [ ' schema_version ' ] :
print ( " Applying update_ {} " . format ( ( update_n ) ) )
logger . critical ( f " Applying update_ { update_n } " )
# Wont exist on fresh installs
# Wont exist on fresh installs
if os . path . exists ( self . json_store_path ) :
if os . path . exists ( self . json_store_path ) :
shutil . copyfile ( self . json_store_path , self . datastore_path + " /url-watches-before- {} .json " . format ( update_n ) )
shutil . copyfile ( self . json_store_path , self . datastore_path + " /url-watches-before- {} .json " . format ( update_n ) )
@ -678,8 +686,8 @@ class ChangeDetectionStore:
try :
try :
update_method = getattr ( self , " update_ {} " . format ( update_n ) ) ( )
update_method = getattr ( self , " update_ {} " . format ( update_n ) ) ( )
except Exception as e :
except Exception as e :
print ( " Error while trying update_ {} " . format ( ( update_n ) ) )
logger . error ( f " Error while trying update_ { update_n } " )
print ( e )
logger . error ( e )
# Don't run any more updates
# Don't run any more updates
return
return
else :
else :
@ -717,7 +725,7 @@ class ChangeDetectionStore:
with open ( os . path . join ( target_path , " history.txt " ) , " w " ) as f :
with open ( os . path . join ( target_path , " history.txt " ) , " w " ) as f :
f . writelines ( history )
f . writelines ( history )
else :
else :
logg ing. warning ( " Datastore history directory { } does not exist, skipping history import. " . format ( target_path ) )
logg er. warning ( f " Datastore history directory { target_path } does not exist, skipping history import. " )
# No longer needed, dynamically pulled from the disk when needed.
# No longer needed, dynamically pulled from the disk when needed.
# But we should set it back to a empty dict so we don't break if this schema runs on an earlier version.
# But we should set it back to a empty dict so we don't break if this schema runs on an earlier version.