@ -5,11 +5,11 @@ import os
import queue
import threading
import time
from copy import deepcopy
from .safe_jinja import render as jinja_render
from changedetectionio . strtobool import strtobool
from copy import deepcopy
from functools import wraps
from threading import Event
import flask_login
import pytz
import timeago
@ -319,8 +319,6 @@ def changedetection_app(config=None, datastore_o=None):
@app.route ( " /rss " , methods = [ ' GET ' ] )
def rss ( ) :
from jinja2 import Environment , BaseLoader
jinja2_env = Environment ( loader = BaseLoader )
now = time . time ( )
# Always requires token set
app_rss_token = datastore . data [ ' settings ' ] [ ' application ' ] . get ( ' rss_access_token ' )
@ -388,7 +386,7 @@ def changedetection_app(config=None, datastore_o=None):
# @todo Make this configurable and also consider html-colored markup
# @todo User could decide if <link> goes to the diff page, or to the watch link
rss_template = " <html><body> \n <h4><a href= \" {{ watch_url}} \" > {{ watch_title}}</a></h4> \n <p> {{ html_diff}}</p> \n </body></html> \n "
content = jinja 2_env. from_string ( rss_template ) . render ( watch_title = watch_title , html_diff = html_diff , watch_url = watch . link )
content = jinja _render( template_str = rss_template , watch_title = watch_title , html_diff = html_diff , watch_url = watch . link )
fe . content ( content = content , type = ' CDATA ' )
@ -452,6 +450,8 @@ def changedetection_app(config=None, datastore_o=None):
if search_q :
if ( watch . get ( ' title ' ) and search_q in watch . get ( ' title ' ) . lower ( ) ) or search_q in watch . get ( ' url ' , ' ' ) . lower ( ) :
sorted_watches . append ( watch )
elif watch . get ( ' last_error ' ) and search_q in watch . get ( ' last_error ' ) . lower ( ) :
sorted_watches . append ( watch )
else :
sorted_watches . append ( watch )
@ -516,21 +516,38 @@ def changedetection_app(config=None, datastore_o=None):
watch = datastore . data [ ' watching ' ] . get ( watch_uuid ) if watch_uuid else None
# validate URLS
if not len ( request . form [ ' notification_urls ' ] . strip ( ) ) :
return make_response ( { ' error ' : ' No Notification URLs set ' } , 400 )
notification_urls = request . form [ ' notification_urls ' ] . strip ( ) . splitlines ( )
if not notification_urls :
logger . debug ( " Test notification - Trying by group/tag in the edit form if available " )
# On an edit page, we should also fire off to the tags if they have notifications
if request . form . get ( ' tags ' ) and request . form [ ' tags ' ] . strip ( ) :
for k in request . form [ ' tags ' ] . split ( ' , ' ) :
tag = datastore . tag_exists_by_name ( k . strip ( ) )
notification_urls = tag . get ( ' notifications_urls ' ) if tag and tag . get ( ' notifications_urls ' ) else None
for server_url in request . form [ ' notification_urls ' ] . splitlines ( ) :
if len ( server_url . strip ( ) ) :
if not apobj . add ( server_url ) :
message = ' {} is not a valid AppRise URL. ' . format ( server_url )
return make_response ( { ' error ' : message } , 400 )
is_global_settings_form = request . args . get ( ' mode ' , ' ' ) == ' global-settings '
is_group_settings_form = request . args . get ( ' mode ' , ' ' ) == ' group-settings '
if not notification_urls and not is_global_settings_form and not is_group_settings_form :
# In the global settings, use only what is typed currently in the text box
logger . debug ( " Test notification - Trying by global system settings notifications " )
if datastore . data [ ' settings ' ] [ ' application ' ] . get ( ' notification_urls ' ) :
notification_urls = datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_urls ' ]
if not notification_urls :
return ' No Notification URLs set/found '
for n_url in notification_urls :
if len ( n_url . strip ( ) ) :
if not apobj . add ( n_url ) :
return f ' Error - { n_url } is not a valid AppRise URL. '
try :
# use the same as when it is triggered, but then override it with the form test values
n_object = {
' watch_url ' : request . form [ ' window_url ' ] ,
' notification_urls ' : request . form [ ' notification_urls ' ] . splitlines ( )
' notification_urls ' :
}
# Only use if present, if not set in n_object it should use the default system value
@ -549,7 +566,7 @@ def changedetection_app(config=None, datastore_o=None):
except Exception as e :
return make_response ( { ' error ' : str ( e ) } , 400 )
return ' OK '
return ' OK - Sent test notifications '
@app.route ( " /clear_history/<string:uuid> " , methods = [ ' GET ' ] )
@ -586,6 +603,12 @@ def changedetection_app(config=None, datastore_o=None):
output = render_template ( " clear_all_history.html " )
return output
def _watch_has_tag_options_set ( watch ) :
""" This should be fixed better so that Tag is some proper Model, a tag is just a Watch also """
for tag_uuid , tag in datastore . data [ ' settings ' ] [ ' application ' ] . get ( ' tags ' , { } ) . items ( ) :
if tag_uuid in watch . get ( ' tags ' , [ ] ) and ( tag . get ( ' include_filters ' ) or tag . get ( ' subtractive_selectors ' ) ) :
return True
@app.route ( " /edit/<string:uuid> " , methods = [ ' GET ' , ' POST ' ] )
@login_optionally_required
# https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
@ -596,7 +619,6 @@ def changedetection_app(config=None, datastore_o=None):
from . blueprint . browser_steps . browser_steps import browser_step_ui_config
from . import processors
using_default_check_time = True
# More for testing, possible to return the first/only
if not datastore . data [ ' watching ' ] . keys ( ) :
flash ( " No watches to edit " , " error " )
@ -621,10 +643,6 @@ def changedetection_app(config=None, datastore_o=None):
# be sure we update with a copy instead of accidently editing the live object by reference
default = deepcopy ( datastore . data [ ' watching ' ] [ uuid ] )
# Show system wide default if nothing configured
if all ( value == 0 or value == None for value in datastore . data [ ' watching ' ] [ uuid ] [ ' time_between_check ' ] . values ( ) ) :
default [ ' time_between_check ' ] = deepcopy ( datastore . data [ ' settings ' ] [ ' requests ' ] [ ' time_between_check ' ] )
# Defaults for proxy choice
if datastore . proxy_list is not None : # When enabled
# @todo
@ -662,18 +680,8 @@ def changedetection_app(config=None, datastore_o=None):
if request . args . get ( ' unpause_on_save ' ) :
extra_update_obj [ ' paused ' ] = False
# Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
# Assume we use the default value, unless something relevant is different, then use the form value
# values could be None, 0 etc.
# Set to None unless the next for: says that something is different
extra_update_obj [ ' time_between_check ' ] = dict . fromkeys ( form . time_between_check . data )
for k , v in form . time_between_check . data . items ( ) :
if v and v != datastore . data [ ' settings ' ] [ ' requests ' ] [ ' time_between_check ' ] [ k ] :
extra_update_obj [ ' time_between_check ' ] = form . time_between_check . data
using_default_check_time = False
break
extra_update_obj [ ' time_between_check ' ] = form . time_between_check . data
# Ignore text
form_ignore_text = form . ignore_text . data
@ -754,8 +762,8 @@ def changedetection_app(config=None, datastore_o=None):
extra_title = f " - Edit - { watch . label } " ,
form = form ,
has_default_notification_urls = True if len ( datastore . data [ ' settings ' ] [ ' application ' ] [ ' notification_urls ' ] ) else False ,
has_empty_checktime = using_default_check_time ,
has_extra_headers_file = len ( datastore . get_all_headers_in_textfile_for_watch ( uuid = uuid ) ) > 0 ,
has_special_tag_options = _watch_has_tag_options_set ( watch = watch ) ,
is_html_webdriver = is_html_webdriver ,
jq_support = jq_support ,
playwright_enabled = os . getenv ( ' PLAYWRIGHT_DRIVER_URL ' , False ) ,
@ -839,11 +847,13 @@ def changedetection_app(config=None, datastore_o=None):
flash ( " An error occurred, please see below. " , " error " )
output = render_template ( " settings.html " ,
form = form ,
hide_remove_pass = os . getenv ( " SALTED_PASS " , False ) ,
api_key = datastore . data [ ' settings ' ] [ ' application ' ] . get ( ' api_access_token ' ) ,
emailprefix = os . getenv ( ' NOTIFICATION_MAIL_BUTTON_PREFIX ' , False ) ,
settings_application = datastore . data [ ' settings ' ] [ ' application ' ] )
form = form ,
hide_remove_pass = os . getenv ( " SALTED_PASS " , False ) ,
min_system_recheck_seconds = int ( os . getenv ( ' MINIMUM_SECONDS_RECHECK_TIME ' , 3 ) ) ,
settings_application = datastore . data [ ' settings ' ] [ ' application ' ]
)
return output
@ -1279,9 +1289,8 @@ def changedetection_app(config=None, datastore_o=None):
url = request . form . get ( ' url ' ) . strip ( )
if datastore . url_exists ( url ) :
flash ( ' The URL {} already exists ' . format ( url ) , " error " )
return redirect ( url_for ( ' index ' ) )
flash ( f ' Warning, URL { url } already exists ' , " notice " )
add_paused = request . form . get ( ' edit_and_watch_submit_button ' ) != None
processor = request . form . get ( ' processor ' , ' text_json_diff ' )
new_uuid = datastore . add_watch ( url = url , tag = request . form . get ( ' tags ' ) . strip ( ) , extras = { ' paused ' : add_paused , ' processor ' : processor } )
@ -1645,14 +1654,14 @@ def notification_runner():
# Trim the log length
notification_debug_log = notification_debug_log [ - 100 : ]
# Thread runner to check every minute , look for new watches to feed into the Queue.
# Thread ed runner, look for new watches to feed into the Queue.
def ticker_thread_check_time_launch_checks ( ) :
import random
from changedetectionio import update_worker
proxy_last_called_time = { }
recheck_time_minimum_seconds = int ( os . getenv ( ' MINIMUM_SECONDS_RECHECK_TIME ' , 20 ) )
recheck_time_minimum_seconds = int ( os . getenv ( ' MINIMUM_SECONDS_RECHECK_TIME ' , 3 ) )
logger . debug ( f " System env MINIMUM_SECONDS_RECHECK_TIME { recheck_time_minimum_seconds } " )
# Spin up Workers that do the fetching
@ -1706,9 +1715,7 @@ def ticker_thread_check_time_launch_checks():
continue
# If they supplied an individual entry minutes to threshold.
watch_threshold_seconds = watch . threshold_seconds ( )
threshold = watch_threshold_seconds if watch_threshold_seconds > 0 else recheck_time_system_seconds
threshold = recheck_time_system_seconds if watch . get ( ' time_between_check_use_default ' ) else watch . threshold_seconds ( )
# #580 - Jitter plus/minus amount of time to make the check seem more random to the server
jitter = datastore . data [ ' settings ' ] [ ' requests ' ] . get ( ' jitter_seconds ' , 0 )
dgtlmoon
6 months ago
committed by
GitHub