@ -2,7 +2,12 @@ import threading
import queue import queue
import time import time
# Requests for checking on the site use a pool of thread Workers managed by a Queue. # A single update worker
#
# Requests for checking on a single site(watch) from a queue of watches
# (another process inserts watches into the queue that are time-ready for checking)
class update_worker ( threading . Thread ) : class update_worker ( threading . Thread ) :
current_uuid = None
current_uuid = None
@ -34,92 +39,108 @@ class update_worker(threading.Thread):
changed_detected = False
changed_detected = False
contents = " "
contents = " "
update_obj = { }
update_obj = { }
now = time . time ( )
try :
try :
now = time . time ( )
changed_detected , update_obj , contents = update_handler . run ( uuid )
changed_detected , update_obj , contents = update_handler . run ( uuid )
# Always record that we atleast tried
# Re #342
self . datastore . update_watch ( uuid = uuid , update_obj = { ' fetch_time ' : round ( time . time ( ) - now , 3 ) } )
# In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
# We then convert/.decode('utf-8') for the notification etc
if not isinstance ( contents , ( bytes , bytearray ) ) :
raise Exception ( " Error - returned data from the fetch handler SHOULD be bytes " )
except PermissionError as e :
except PermissionError as e :
self . app . logger . error ( " File permission error updating " , uuid , str ( e ) )
self . app . logger . error ( " File permission error updating " , uuid , str ( e ) )
except content_fetcher . EmptyReply as e :
except content_fetcher . EmptyReply as e :
self . datastore . update_watch ( uuid = uuid , update_obj = { ' last_error ' : str ( e ) } )
# Some kind of custom to-str handler in the exception handler that does this?
err_text = " EmptyReply: Status Code {} " . format ( e . status_code )
self . datastore . update_watch ( uuid = uuid , update_obj = { ' last_error ' : err_text ,
' last_check_status ' : e . status_code } )
except Exception as e :
except Exception as e :
self . app . logger . error ( " Exception reached processing watch UUID: %s - %s " , uuid , str ( e ) )
self . app . logger . error ( " Exception reached processing watch UUID: %s - %s " , uuid , str ( e ) )
self . datastore . update_watch ( uuid = uuid , update_obj = { ' last_error ' : str ( e ) } )
self . datastore . update_watch ( uuid = uuid , update_obj = { ' last_error ' : str ( e ) } )
else :
else :
if update_obj :
try :
try :
watch = self . datastore . data [ ' watching ' ] [ uuid ]
self . datastore . update_watch ( uuid = uuid , update_obj = update_obj )
fname = " " # Saved history text filename
if changed_detected :
n_object = { }
# For the FIRST time we check a site, or a change detected, save the snapshot.
# A change was detected
if changed_detected or not watch [ ' last_checked ' ] :
fname = self . datastore . save_history_text ( watch_uuid = uuid , contents = contents )
# A change was detected
fname = self . datastore . save_history_text ( watch_uuid = uuid , contents = contents )
# Update history with the stripped text for future reference, this will also mean we save the first
# Should always be keyed by string(timestamp)
# Should always be keyed by string(timestamp)
self . datastore . update_watch ( uuid , { " history " : { str ( round ( time . time ( ) ) ) : fname } } )
self . datastore . update_watch ( uuid , { " history " : { str ( update_obj [ " last_checked " ] ) : fname } } )
# Generally update anything interesting returned
watch = self . datastore . data [ ' watching ' ] [ uuid ]
self . datastore . update_watch ( uuid = uuid , update_obj = update_obj )
print ( " >> Change detected in UUID {} - {} " . format ( uuid , watch [ ' url ' ] ) )
# A change was detected
if changed_detected :
# Notifications should only trigger on the second time (first time, we gather the initial snapshot)
n_object = { }
if len ( watch [ ' history ' ] ) > 1 :
print ( " >> Change detected in UUID {} - {} " . format ( uuid , watch [ ' url ' ] ) )
dates = list ( watch [ ' history ' ] . keys ( ) )
# Notifications should only trigger on the second time (first time, we gather the initial snapshot)
# Convert to int, sort and back to str again
if len ( watch [ ' history ' ] ) > 1 :
# @todo replace datastore getter that does this automatically
dates = [ int ( i ) for i in dates ]
dates = list ( watch [ ' history ' ] . keys ( ) )
dates . sort ( reverse = True )
# Convert to int, sort and back to str again
dates = [ str ( i ) for i in dates ]
# @todo replace datastore getter that does this automatically
dates = [ int ( i ) for i in dates ]
prev_fname = watch [ ' history ' ] [ dates [ 1 ] ]
dates . sort ( reverse = True )
dates = [ str ( i ) for i in dates ]
# Did it have any notification alerts to hit?
prev_fname = watch [ ' history ' ] [ dates [ 1 ] ]
if len ( watch [ ' notification_urls ' ] ) :
print ( " >>> Notifications queued for UUID from watch {} " . format ( uuid ) )
n_object [ ' notification_urls ' ] = watch [ ' notification_urls ' ]
# Did it have any notification alerts to hit?
n_object [ ' notification_title ' ] = watch [ ' notification_title ' ]
if len ( watch [ ' notification_urls ' ] ) :
n_object [ ' notification_body ' ] = watch [ ' notification_body ' ]
print ( " >>> Notifications queued for UUID from watch {} " . format ( uuid ) )
n_object [ ' notification_format ' ] = watch [ ' notification_format ' ]
n_object [ ' notification_urls ' ] = watch [ ' notification_urls ' ]
n_object [ ' notification_title ' ] = watch [ ' notification_title ' ]
# No? maybe theres a global setting, queue them all
n_object [ ' notification_body ' ] = watch [ ' notification_body ' ]
elif len ( self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_urls ' ] ) :
n_object [ ' notification_format ' ] = watch [ ' notification_format ' ]
print ( " >>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {} " . format ( uuid ) )
n_object [ ' notification_urls ' ] = self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_urls ' ]
# No? maybe theres a global setting, queue them all
n_object [ ' notification_title ' ] = self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_title ' ]
elif len ( self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_urls ' ] ) :
n_object [ ' notification_body ' ] = self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_body ' ]
print ( " >>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {} " . format ( uuid ) )
n_object [ ' notification_format ' ] = self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_format ' ]
n_object [ ' notification_urls ' ] = self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_urls ' ]
n_object [ ' notification_title ' ] = self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_title ' ]
n_object [ ' notification_body ' ] = self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_body ' ]
n_object [ ' notification_format ' ] = self . datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_format ' ]
else :
print ( " >>> NO notifications queued, watch and global notification URLs were empty. " )
# Only prepare to notify if the rules above matched
if ' notification_urls ' in n_object :
# HTML needs linebreak, but MarkDown and Text can use a linefeed
if n_object [ ' notification_format ' ] == ' HTML ' :
line_feed_sep = " </br> "
else :
else :
print ( " >>> NO notifications queued, watch and global notification URLs were empty. " )
line_feed_sep = " \n "
# Only prepare to notify if the rules above matched
from changedetectionio import diff
if ' notification_urls ' in n_object :
n_object . update ( {
# HTML needs linebreak, but MarkDown and Text can use a linefeed
' watch_url ' : watch [ ' url ' ] ,
if n_object [ ' notification_format ' ] == ' HTML ' :
' uuid ' : uuid ,
line_feed_sep = " </br> "
' current_snapshot ' : contents . decode ( ' utf-8 ' ) ,
else :
' diff_full ' : diff . render_diff ( prev_fname , fname , line_feed_sep = line_feed_sep ) ,
line_feed_sep = " \n "
' diff ' : diff . render_diff ( prev_fname , fname , True , line_feed_sep = line_feed_sep )
} )
from changedetectionio import diff
n_object . update ( {
self . notification_q . put ( n_object )
' watch_url ' : watch [ ' url ' ] ,
' uuid ' : uuid ,
except Exception as e :
' current_snapshot ' : str ( contents ) ,
# Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
' diff_full ' : diff . render_diff ( prev_fname , fname , line_feed_sep = line_feed_sep ) ,
print ( " !!!! Exception in update_worker !!! \n " , e )
' diff ' : diff . render_diff ( prev_fname , fname , True , line_feed_sep = line_feed_sep )
} )
finally :
# Always record that we atleast tried
self . notification_q . put ( n_object )
self . datastore . update_watch ( uuid = uuid , update_obj = { ' fetch_time ' : round ( time . time ( ) - now , 3 ) ,
' last_checked ' : round ( time . time ( ) ) } )
except Exception as e :
print ( " !!!! Exception in update_worker !!! \n " , e )
self . current_uuid = None # Done
self . current_uuid = None # Done
self . q . task_done ( )
self . q . task_done ( )
ntmmfts
3 years ago