From 212f15ad5fa5dde1cff59a5f4e615e71ff16ec19 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sun, 10 Sep 2023 13:44:24 +0200 Subject: [PATCH 01/21] Catch possible crash scenario for listing watches - `date_created` was missing on add (#1787) --- changedetectionio/store.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 388756d7..a8082cd5 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -327,6 +327,9 @@ class ChangeDetectionStore: if k in apply_extras: del apply_extras[k] + if not apply_extras.get('date_created'): + apply_extras['date_created'] = int(time.time()) + new_watch.update(apply_extras) new_watch.ensure_data_dir_exists() self.__data['watching'][new_uuid] = new_watch From 2180bb256d13ddf19cc72b5eabe627959a3f73fb Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 11 Sep 2023 10:22:35 +0200 Subject: [PATCH 02/21] UI - Make tgram:// and discord:// examples in notification settings link to how-to pages (#1785) --- changedetectionio/templates/_common_fields.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index 6b6f6e05..da2cebf2 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -13,9 +13,9 @@
  • Use AppRise URLs for notification to just about any service! Please read the notification services wiki here for important configuration notes.
    • -
  • discord:// only supports a maximum 2,000 characters of notification text, including the title.
    • -
  • tgram:// bots cant send messages to other bots, so you should specify chat ID of non-bot user.
    • -
  • tgram:// only supports very limited HTML and can fail when extra tags are sent, read more here (or use plaintext/markdown format)
    • +
  • discord:// only supports a maximum 2,000 characters of notification text, including the title.
    • +
  • tgram:// bots cant send messages to other bots, so you should specify chat ID of non-bot user.
    • +
  • tgram:// only supports very limited HTML and can fail when extra tags are sent, read more here (or use plaintext/markdown format)
  • gets://, posts://, puts://, deletes:// for direct API calls (or omit the "s" for non-SSL ie get://)
  • Accepts the {{ '{{token}}' }} placeholders listed below
From 8ac4757cd9a6efaa0587cdad992be2ca0319eceb Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 11 Sep 2023 13:18:05 +0200 Subject: [PATCH 03/21] UI - Fix spelling error --- changedetectionio/templates/_common_fields.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index da2cebf2..3de9a2a5 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -14,7 +14,7 @@
  • Use AppRise URLs for notification to just about any service! Please read the notification services wiki here for important configuration notes.
  • discord:// only supports a maximum 2,000 characters of notification text, including the title.
    • -
  • tgram:// bots cant send messages to other bots, so you should specify chat ID of non-bot user.
    • +
  • tgram:// bots can't send messages to other bots, so you should specify chat ID of non-bot user.
  • tgram:// only supports very limited HTML and can fail when extra tags are sent, read more here (or use plaintext/markdown format)
  • gets://, posts://, puts://, deletes:// for direct API calls (or omit the "s" for non-SSL ie get://)
  • Accepts the {{ '{{token}}' }} placeholders listed below
    • From 5dc3b00ec6311847a6728846bad6615ab8b5bc3e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 11 Sep 2023 23:27:10 +0200 Subject: [PATCH 04/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 700f4ba3..68e11329 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ The application also supports notifying you that it can follow this information ## Proxy Configuration -See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [BrightData proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) +See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) ## Raspberry Pi support? From a958e1fe201b5a172db73b2497529a794be3224a Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 12 Sep 2023 15:13:21 +0200 Subject: [PATCH 05/21] UI - "recheck all" button should ignore blank/empty "tag" setting when set --- changedetectionio/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index c9e12f07..10cc0505 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -1270,10 +1270,10 @@ def changedetection_app(config=None, datastore_o=None): update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False})) i = 1 - elif tag != None: + elif tag: # Items that have this current tag for watch_uuid, watch in datastore.data['watching'].items(): - if (tag != None and tag in watch.get('tags', {})): + if tag in watch.get('tags', {}): if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: update_q.put( queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}) From 2f2d0ea0f25aebe26bd9dff932169b93125b1a86 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 14 Sep 2023 13:07:01 +0200 Subject: [PATCH 06/21] RSS feeds - Fixing broken links from RSS index in some environments, refactor code (#152, #148, #1684, #1798) --- changedetectionio/__init__.py | 14 +++++------ changedetectionio/forms.py | 5 +++- changedetectionio/notification.py | 10 +++----- changedetectionio/store.py | 24 ++++++++++++++----- .../templates/_common_fields.jinja | 4 ---- changedetectionio/templates/settings.html | 15 ++++++------ 6 files changed, 38 insertions(+), 34 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 10cc0505..72838ac3 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -40,6 +40,8 @@ from changedetectionio.api import api_v1 __version__ = '0.45.1' +from changedetectionio.store import BASE_URL_NOT_SET_TEXT + datastore = None # Local @@ -356,12 +358,10 @@ def changedetection_app(config=None, datastore_o=None): # Include a link to the diff page, they will have to login here to see if password protection is enabled. # Description is the page you watch, link takes you to the diff JS UI page # Dict val base_url will get overriden with the env var if it is set. - ext_base_url = datastore.data['settings']['application'].get('base_url') - if ext_base_url: - # Go with overriden value - diff_link = {'href': "{}{}".format(ext_base_url, url_for('diff_history_page', uuid=watch['uuid'], _external=False))} - else: - diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)} + ext_base_url = datastore.data['settings']['application'].get('active_base_url') + + # Because we are called via whatever web server, flask should figure out the right path ( + diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)} fe.link(link=diff_link) @@ -714,7 +714,6 @@ def changedetection_app(config=None, datastore_o=None): output = render_template("edit.html", available_processors=processors.available_processors(), browser_steps_config=browser_step_ui_config, - current_base_url=datastore.data['settings']['application']['base_url'], emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), form=form, has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, @@ -804,7 +803,6 @@ def changedetection_app(config=None, datastore_o=None): output = render_template("settings.html", form=form, - current_base_url = datastore.data['settings']['application']['base_url'], hide_remove_pass=os.getenv("SALTED_PASS", False), api_key=datastore.data['settings']['application'].get('api_access_token'), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 4725b181..879500b0 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -502,7 +502,10 @@ class globalSettingsRequestForm(Form): class globalSettingsApplicationForm(commonSettingsForm): api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()]) - base_url = StringField('Base URL', validators=[validators.Optional()]) + base_url = StringField('Notification base URL override', + validators=[validators.Optional()], + render_kw={"placeholder": os.getenv('BASE_URL', 'Not set')} + ) empty_pages_are_a_change = BooleanField('Treat empty pages as a change?', default=False) fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()]) global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()]) diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 27c7afa7..6c0f53f6 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -208,15 +208,11 @@ def create_notification_parameters(n_object, datastore): watch_tag = '' # Create URLs to customise the notification with - base_url = datastore.data['settings']['application']['base_url'] + # active_base_url - set in store.py data property + base_url = datastore.data['settings']['application'].get('active_base_url') watch_url = n_object['watch_url'] - # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services - # like 'Join', so it's always best to atleast set something obvious so that they are not broken. - if base_url == '': - base_url = "" - diff_url = "{}/diff/{}".format(base_url, uuid) preview_url = "{}/preview/{}".format(base_url, uuid) @@ -226,7 +222,7 @@ def create_notification_parameters(n_object, datastore): # Valid_tokens also used as a field validator tokens.update( { - 'base_url': base_url if base_url is not None else '', + 'base_url': base_url, 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '', 'diff': n_object.get('diff', ''), # Null default in the case we use a test 'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test diff --git a/changedetectionio/store.py b/changedetectionio/store.py index a8082cd5..9ac2218b 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -18,6 +18,9 @@ import threading import time import uuid as uuid_builder +# Because the server will run as a daemon and wont know the URL for notification links when firing off a notification +BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)' + dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ]) # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods? @@ -175,12 +178,21 @@ class ChangeDetectionStore: @property def data(self): - # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url - env_base_url = os.getenv('BASE_URL','') - if not self.__data['settings']['application']['base_url']: - self.__data['settings']['application']['base_url'] = env_base_url.strip('" ') - - return self.__data + # Re #152, Return env base_url if not overriden + # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services + # like 'Join', so it's always best to atleast set something obvious so that they are not broken. + + active_base_url = BASE_URL_NOT_SET_TEXT + if self.__data['settings']['application'].get('base_url'): + active_base_url = self.__data['settings']['application'].get('base_url') + elif os.getenv('BASE_URL'): + active_base_url = os.getenv('BASE_URL') + + # I looked at various ways todo the following, but in the end just copying the dict seemed simplest/most reliable + # even given the memory tradeoff - if you know a better way.. maybe return d|self.__data.. or something + d = self.__data + d['settings']['application']['active_base_url'] = active_base_url.strip('" ') + return d # Delete a single watch by UUID def delete(self, uuid): diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index 3de9a2a5..99365d72 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -115,10 +115,6 @@
    -
    - URLs generated by changedetection.io (such as {{ '{{diff_url}}' }}) require the BASE_URL environment variable set.
    - Your BASE_URL var is currently "{{settings_application['current_base_url']}}" -
    Warning: Contents of {{ '{{diff}}' }}, {{ '{{diff_removed}}' }}, and {{ '{{diff_added}}' }} depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. More Here
diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 1d09faa5..7bca51fe 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -62,14 +62,6 @@ Allow access to view watch diff page when password is enabled (Good for sharing the diff page) -
- {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/", - class="m-d") }} - - Base URL used for the {{ '{{ base_url }}' }} token in notifications and RSS links.
Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"), - read more here. - -
{{ render_field(form.application.form.pager_size) }} Number of items per page in the watch overview list, 0 to disable. @@ -100,6 +92,13 @@ {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
+
+ {{ render_field(form.application.form.base_url, class="m-d") }} + + Base URL used for the {{ '{{ base_url }}' }} token in notification links.
+ Default value is the system environment variable 'BASE_URL' - read more here. + +
From 7355ac8d21a6ffff5d22964f7b61a0582055a4bc Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 14 Sep 2023 13:55:48 +0200 Subject: [PATCH 07/21] UI - Notifications - Tweak discord help text --- changedetectionio/templates/_common_fields.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index 99365d72..33b9bbe5 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -13,7 +13,7 @@
  • Use AppRise URLs for notification to just about any service! Please read the notification services wiki here for important configuration notes.
    • -
  • discord:// only supports a maximum 2,000 characters of notification text, including the title.
    • +
  • discord:// (or https://discord.com/api/webhooks...)) only supports a maximum 2,000 characters of notification text, including the title.
  • tgram:// bots can't send messages to other bots, so you should specify chat ID of non-bot user.
  • tgram:// only supports very limited HTML and can fail when extra tags are sent, read more here (or use plaintext/markdown format)
  • gets://, posts://, puts://, deletes:// for direct API calls (or omit the "s" for non-SSL ie get://)
    • From d497db639e709cb7ba17621d248ca4bea734649b Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 14 Sep 2023 14:16:08 +0200 Subject: [PATCH 08/21] UI - Notifications - Tidyup - Hide the notification tokens but show with a button/link --- changedetectionio/static/js/global-settings.js | 5 +++++ changedetectionio/static/js/watch-settings.js | 4 ++++ .../templates/_common_fields.jinja | 18 +++++++++++------- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/changedetectionio/static/js/global-settings.js b/changedetectionio/static/js/global-settings.js index dc7818bb..16ba33e1 100644 --- a/changedetectionio/static/js/global-settings.js +++ b/changedetectionio/static/js/global-settings.js @@ -32,5 +32,10 @@ $(document).ready(function () { window.getSelection().removeAllRanges(); }); + + $("#notification-token-toggle").click(function (e) { + e.preventDefault(); + $('#notification-tokens-info').toggle(); + }); }); diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index 353b23a3..0b419959 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -42,4 +42,8 @@ $(document).ready(function () { $('#notification_urls').val(''); e.preventDefault(); }); + $("#notification-token-toggle").click(function (e) { + e.preventDefault(); + $('#notification-tokens-info').toggle(); + }); }); diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index 33b9bbe5..8745ae48 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -35,18 +35,19 @@
{{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }} - Body for all notifications + Body for all notifications ‐ You can use Jinja2 templating in the notification title, body and URL, and tokens from below. + + +
+
+
Show token/placeholders
{{ render_field(form.notification_format , class="notification-format") }} Format for all notifications
-
-

- You can use Jinja2 templating in the notification title, body and URL. -

- +
From e3cf22fc27bef30b6b019cfd1f0e6825d7090cc0 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 14 Sep 2023 14:34:44 +0200 Subject: [PATCH 09/21] UI - Re-order notification field settings --- changedetectionio/templates/_common_fields.jinja | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index 8745ae48..2701d56c 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -42,11 +42,6 @@
Show token/placeholders
-
- - {{ render_field(form.notification_format , class="notification-format") }} - Format for all notifications -