From e27f66eb73da096665f046b74500d778d48839fd Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 16 May 2024 23:39:06 +0200 Subject: [PATCH 01/10] UI - Ability to preview/view single changes by timestamp using keyboard or select box(#1916) --- changedetectionio/__init__.py | 1 + changedetectionio/flask_app.py | 87 +++++------ changedetectionio/static/js/diff-overview.js | 7 + changedetectionio/static/js/diff-render.js | 7 +- changedetectionio/static/js/preview.js | 49 ++++++ changedetectionio/templates/preview.html | 151 +++++++++++-------- 6 files changed, 193 insertions(+), 109 deletions(-) create mode 100644 changedetectionio/static/js/preview.js diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 8ec6bb8d..07492851 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -175,6 +175,7 @@ def main(): # proxy_set_header Host "localhost"; # proxy_set_header X-Forwarded-Prefix /app; + if os.getenv('USE_X_SETTINGS'): logger.info("USE_X_SETTINGS is ENABLED") from werkzeug.middleware.proxy_fix import ProxyFix diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 49880b5d..fdd16294 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1063,6 +1063,8 @@ def changedetection_app(config=None, datastore_o=None): content = [] ignored_line_numbers = [] trigger_line_numbers = [] + versions = [] + timestamp = None # More for testing, possible to return the first/only if uuid == 'first': @@ -1082,57 +1084,53 @@ def changedetection_app(config=None, datastore_o=None): if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): is_html_webdriver = True - # Never requested successfully, but we detected a fetch error if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()): flash("Preview unavailable - No fetch/check completed or triggers not reached", "error") - output = render_template("preview.html", - content=content, - history_n=watch.history_n, - extra_stylesheets=extra_stylesheets, -# current_diff_url=watch['url'], - watch=watch, - uuid=uuid, - is_html_webdriver=is_html_webdriver, - last_error=watch['last_error'], - last_error_text=watch.get_error_text(), - last_error_screenshot=watch.get_error_snapshot()) - return output + else: + # So prepare the latest preview or not + preferred_version = request.args.get('version') + versions = list(watch.history.keys()) + timestamp = versions[-1] + if preferred_version and preferred_version in versions: + timestamp = preferred_version - timestamp = list(watch.history.keys())[-1] - try: - tmp = watch.get_history_snapshot(timestamp).splitlines() - - # Get what needs to be highlighted - ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] - - # .readlines will keep the \n, but we will parse it here again, in the future tidy this up - ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), - wordlist=ignore_rules, - mode='line numbers' - ) - - trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), - wordlist=watch['trigger_text'], - mode='line numbers' - ) - # Prepare the classes and lines used in the template - i=0 - for l in tmp: - classes=[] - i+=1 - if i in ignored_line_numbers: - classes.append('ignored') - if i in trigger_line_numbers: - classes.append('triggered') - content.append({'line': l, 'classes': ' '.join(classes)}) + try: + versions = list(watch.history.keys()) + tmp = watch.get_history_snapshot(timestamp).splitlines() + + # Get what needs to be highlighted + ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] + + # .readlines will keep the \n, but we will parse it here again, in the future tidy this up + ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), + wordlist=ignore_rules, + mode='line numbers' + ) + + trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), + wordlist=watch['trigger_text'], + mode='line numbers' + ) + # Prepare the classes and lines used in the template + i=0 + for l in tmp: + classes=[] + i+=1 + if i in ignored_line_numbers: + classes.append('ignored') + if i in trigger_line_numbers: + classes.append('triggered') + content.append({'line': l, 'classes': ' '.join(classes)}) - except Exception as e: - content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) + except Exception as e: + content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) output = render_template("preview.html", content=content, + current_version=timestamp, history_n=watch.history_n, extra_stylesheets=extra_stylesheets, + extra_title=f" - Diff - {watch.label} @ {timestamp}", ignored_line_numbers=ignored_line_numbers, triggered_line_numbers=trigger_line_numbers, current_diff_url=watch['url'], @@ -1142,7 +1140,10 @@ def changedetection_app(config=None, datastore_o=None): is_html_webdriver=is_html_webdriver, last_error=watch['last_error'], last_error_text=watch.get_error_text(), - last_error_screenshot=watch.get_error_snapshot()) + last_error_screenshot=watch.get_error_snapshot(), + versions=versions + ) + return output diff --git a/changedetectionio/static/js/diff-overview.js b/changedetectionio/static/js/diff-overview.js index 767cf6e1..95e6dd7a 100644 --- a/changedetectionio/static/js/diff-overview.js +++ b/changedetectionio/static/js/diff-overview.js @@ -8,6 +8,13 @@ $(document).ready(function () { } }) + $('.needs-localtime').each(function () { + for (var option of this.options) { + var dateObject = new Date(option.value * 1000); + option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"}); + } + }); + // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load window.addEventListener('hashchange', function (e) { toggle(location.hash); diff --git a/changedetectionio/static/js/diff-render.js b/changedetectionio/static/js/diff-render.js index 53f1d68f..ea69d364 100644 --- a/changedetectionio/static/js/diff-render.js +++ b/changedetectionio/static/js/diff-render.js @@ -79,12 +79,7 @@ $(document).ready(function () { $('#jump-next-diff').click(); } - $('.needs-localtime').each(function () { - for (var option of this.options) { - var dateObject = new Date(option.value * 1000); - option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"}); - } - }) + onDiffTypeChange( document.querySelector('#settings [name="diff_type"]:checked'), ); diff --git a/changedetectionio/static/js/preview.js b/changedetectionio/static/js/preview.js new file mode 100644 index 00000000..a9895cb2 --- /dev/null +++ b/changedetectionio/static/js/preview.js @@ -0,0 +1,49 @@ +function redirect_to_version(version) { + var currentUrl = window.location.href; + var baseUrl = currentUrl.split('?')[0]; // Base URL without query parameters + var anchor = ''; + + // Check if there is an anchor + if (baseUrl.indexOf('#') !== -1) { + anchor = baseUrl.substring(baseUrl.indexOf('#')); + baseUrl = baseUrl.substring(0, baseUrl.indexOf('#')); + } + window.location.href = baseUrl + '?version=' + version + anchor; +} + +document.addEventListener('keydown', function (event) { + var selectElement = document.getElementById('preview-version'); + if (selectElement) { + var selectedOption = selectElement.querySelector('option:checked'); + if (selectedOption) { + if (event.key === 'ArrowLeft') { + if (selectedOption.previousElementSibling) { + redirect_to_version(selectedOption.previousElementSibling.value); + } + } else if (event.key === 'ArrowRight') { + if (selectedOption.nextElementSibling) { + redirect_to_version(selectedOption.nextElementSibling.value); + } + } + } + } +}); + + +document.getElementById('preview-version').addEventListener('change', function () { + redirect_to_version(this.value); +}); + +var selectElement = document.getElementById('preview-version'); +if (selectElement) { + var selectedOption = selectElement.querySelector('option:checked'); + if (selectedOption) { + if (selectedOption.previousElementSibling) { + document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value; + } + if (selectedOption.nextElementSibling) { + document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value; + } + + } +} diff --git a/changedetectionio/templates/preview.html b/changedetectionio/templates/preview.html index 5cc61bed..8bc231e1 100644 --- a/changedetectionio/templates/preview.html +++ b/changedetectionio/templates/preview.html @@ -1,72 +1,103 @@ {% extends 'base.html' %} {% block content %} - + + + + {% if versions|length >= 2 %} +
+
+
+ + + +
+
+
+ Keyboard: + ← Previous   + → Next +
{% endif %} - const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}"; - - - -
- -
-
-
-
-
{{watch.error_text_ctime|format_seconds_ago}} seconds ago
- 
+    

+        
+    

+
+
+    

+        

+            
{{ watch.error_text_ctime|format_seconds_ago }} seconds ago

+            
             {{ last_error_text }}
         

-    

+        

 
-    

-        
{{watch.snapshot_error_screenshot_ctime|format_seconds_ago}} seconds ago

-        Current erroring screenshot from most recent request
-    

+        

+            
{{ watch.snapshot_error_screenshot_ctime|format_seconds_ago }} seconds ago
+            

+            Current erroring screenshot from most recent request
+        

 
-    

-        
{{watch.snapshot_text_ctime|format_timestamp_timeago}}

-        Grey lines are ignored Blue lines are triggers Pro-tip: Highlight text to add to ignore filters
+        

+            
{{ watch.snapshot_text_ctime|format_timestamp_timeago }}

+            Grey lines are ignored Blue lines are triggers
+            Pro-tip: Highlight text to add to ignore filters
 
-        
-            
-            
-                
-            
-            
-        

-                    {% for row in content %}
-                    
{{row.line}}

-                    {% endfor %}
-                

-    

+            
+                
+                
+                    
+                
+                
+            

+                        {% for row in content %}
+                            
{{ row.line }}

+                        {% endfor %}
+                    

+        

 
-     

-         

-             For now, Differences are performed on text, not graphically, only the latest screenshot is available.
-         

-         

-         {% if is_html_webdriver %}
-           {% if screenshot %}
-             
{{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}

-             Current screenshot from most recent request
-           {% else %}
-              No screenshot available just yet! Try rechecking the page.
-           {% endif %}
-         {% else %}
-           Screenshot requires Playwright/WebDriver enabled
-         {% endif %}
-     

-
+
+
+ For now, Differences are performed on text, not graphically, only the latest screenshot is available. +
+
+ {% if is_html_webdriver %} + {% if screenshot %} +
{{ watch.snapshot_screenshot_ctime|format_timestamp_timeago }}
+ Current screenshot from most recent request + {% else %} + No screenshot available just yet! Try rechecking the page. + {% endif %} + {% else %} + Screenshot requires Playwright/WebDriver enabled + {% endif %} +
+
{% endblock %} From add2c658b43ae8f63e487f8c0c3ccadc0250d5ff Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 17 May 2024 09:20:26 +0200 Subject: [PATCH 02/10] Notifications - Fixing truncated notifications when tgram:// or discord:// is used with other notification methods (#2372 #2299) --- changedetectionio/notification.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 4fa35738..41285ce4 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -122,10 +122,6 @@ def process_notification(n_object, datastore): # Insert variables into the notification content notification_parameters = create_notification_parameters(n_object, datastore) - # Get the notification body from datastore - n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) - n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) - n_format = valid_notification_formats.get( n_object.get('notification_format', default_notification_format), valid_notification_formats[default_notification_format], @@ -151,6 +147,11 @@ def process_notification(n_object, datastore): with apprise.LogCapture(level=apprise.logging.DEBUG) as logs: for url in n_object['notification_urls']: + + # Get the notification body from datastore + n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) + n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) + url = url.strip() if not url: logger.warning(f"Process Notification: skipping empty notification URL.") From f0ed4f64e8f39c4be1912bf537ceeac8ae3bea57 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 17 May 2024 17:47:35 +0200 Subject: [PATCH 03/10] RSS - Muted watches should not show in RSS feed (#2374 #2304) --- changedetectionio/flask_app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index fdd16294..13d3dfb6 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -338,8 +338,11 @@ def changedetection_app(config=None, datastore_o=None): # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away for uuid, watch in datastore.data['watching'].items(): + # @todo tag notification_muted skip also (improve Watch model) + if watch.get('notification_muted'): + continue if limit_tag and not limit_tag in watch['tags']: - continue + continue watch['uuid'] = uuid sorted_watches.append(watch) From 4293639f51695136bb57c462a256b4e305595845 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 20 May 2024 13:41:23 +0200 Subject: [PATCH 04/10] UI - CSS - Remove gradient border, it did not add much to the design #2377 --- .../static/images/gradient-border.png | Bin 21990 -> 0 bytes .../static/styles/scss/styles.scss | 1 - changedetectionio/static/styles/styles.css | 3 +-- changedetectionio/templates/base.html | 6 ------ 4 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 changedetectionio/static/images/gradient-border.png diff --git a/changedetectionio/static/images/gradient-border.png b/changedetectionio/static/images/gradient-border.png deleted file mode 100644 index 4c7705f8cb9753db40477fc2d3c4c7a4452cf241..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21990 zcmeFZcTkgC_&*p6VgkSjO_q`gB1Y4ewV|C0Dyf0KUGnCHxByTw7LlZRHBb<-xt`sKJ>uC)EM9f z7y)c3Q4p zYFm;{L2m_qJEanMZsqLd7=Sv1pmYRiwVuVcvT^iY6+D=D@A_eN;?xdjS-h=8_7}5T zgs$_8*=(|9OTtc>CQEDfY-}wvd2!I8O>S`OhG*Wk4sK%=LS5@l+hv-@Ls8)bLY~d; zFi6j@6Adx&4QI44n8DkNJv63`Q8NGwTdJ{xB+JU+re;2gm}cRru#8Wdzj)m6AqXwb z&THGS4@TI4_qivW-}TERm`HSUG%y8L(R{vVbfd``krht$u|En0tX{9h-c9Rxh2_O% zmrz8$ugot!+@WOg`7l;ICz5~u#0rAX^ouD!9;~7_i1>BR0Ay)EG1^&WtX3cEgz@?K z-9_p5;JWT#jKPY{!zbRNfBgS{id(IZ<~Q#+YeZ2uxj0(_`l=BQJ0$|%+f^oq%@?Pe zMwG8pyFxSxgFGZ*Z=-|z$Ea*cY7KSGG=4SOFL@>oFC8%Xn+PO2=5TyP7B8R-JRM{rx2QZY$0KS;2LmS2H^Og} zV1F4IouH4u5PX{oS$x-r;fC~g&L1?!8}`dr}Q zn8$V9Vcz=58cq|cXME)7K9ridJ}UGDJbzbmsvEeBUPl;j?GU1*$d@*l-N9C1urS}W zH8r?=#?-S=lFl36uIh;R-s-(8W*_9dut`M_X%Q_@PHr&gO|T?kCK4|>$o&0J#PVqB zMmA+?(t>=O*%*Cu*AoSYvEGGug#=5Qj0qXyrqFVAZ>urQ;`fb|r8UhBawSB-EDj=z z+{K8x?|Ne%c8gGB)ZSJI0lyo-W@f%(zf{V9oFK@FWDQ) zhJXJLn0o&8CtPWgqioNH2-wIkQc|-AEl%gft$NNdxvm5fwR?)nGul%z9mbaJ%$~ae z`bF|}WD;Zv1V|(wD;|vUZHpR;gN%ZwL{?@R=JdEtZ8Y5oF_|8;Y|jcdNNTs)h_9`Y zGUIpGG=9@iCLvCzpE?r<6Rh8W7)xmOh}b4H>=+iQ)S3*52WJsCn^I44G6*>*79#N= zCXgFhpdw%1#`h%P>1_M$?h~a$vDF#7UXW=~~mBPDLR zp+~dLu$-4s)G8goYpC5-?EJ+LleDmu9^d&H*k+S`vnAaS_a`(&6iklHBfnk`a*N`& z7rkp360gk>vCO|C&n1<{LfF!5t=hPvuy+JM?VR&{oj zDNmCvoLnjRjl*%?G2He347@fq_*H5$#H{%-KT*B?=oJUuw(a`j`1a}OU8d2Ll(~oe zL8)7>1oU4D9ea9Q^6a1Uy2{gPAujD+zbj(8f85`P{`h78|HudELA%^AzNeFeY%b-h zsqv6(k1Y->LjT0)Z{loaWD>-|d(m|$PFw`pREJ`=$__%h8?zwc^yVD$usB^W*EoJt zioLFm#1iAgZ5KpVBv%}nF4n4Sm(a0Ps01Uj+YZ;19F~CxjfDlkcm^`Rnn6rZ9Ycp` z>joS|Orw9**@ZJ~7mA88e> zQV5se*LrR17fF(hmk#6MlpDyUiaF~LP$1X`wb7-8?guW2fK82AV-=y4kJZ0~8Ei=k z_gR4SZ&{Nfhp|`i*M13kHss0O1va&S{CXhR7-7)ojuU#K92Npv$(H$WmQ^66V)}GI zrtmR4Ug&a|V&kpsIU?CAd!`np;iT_;)0{>La|ik18h*<;aJpM@)Ytc*3R`=(_Y z&5VSZpO_HR_IV?u-d zkS?~Wn_0~J>GkJe8&Tm>rl*LT@!LcA$a^ZX2>;#Ogt#)~Q`y0(a&mL(1%=NjwoklL zk+bQIk!UH+uKArUH>;u*14K)zdtO#!C!`TrVz7&|jo87rgjyyNu`o1|X^)JLtMP!_ z`56#{^8@JXAuhu92vbqUD;2kz;LwA$U83Q}3-7w-A1^&M=?=99Gk+^n<`J*d;V&!% z&otUY@R*PVX)-BkaDVwNV0|TBnOubyw`IUGPiQE7E3Q(~bIyayHJ^H^@!Z1O z5TrLOvgPRvg*_aiG*j#qiN7Cuh0K*lJv}~CA(fw>;H|3y*-em8@hH+yH0)*F_`P84 zhYgHu;GLQHmOT)hGxs%)1{EAmRl_23oNY;Q_?FjAUH_zpa;u!O8PS{R|x?irz96d@Q4> z-y}ViYL7^_Yn?x+jV!CK^FAQ&Kt;5X^|L2GPhq~MU%VEc4YpiX@Ga7SdIPQ~3 z@X%g{05vu)w@pA0!a6w-2GMv<);hw-Mp_exrSeSIsNC)D;BpNsGRBd&yv3k4E$NcZ zS!|i}wvZ_WRxsmagt3HLp%w8Ba~Z!J=Y|RJXIjx0$0(Gk`d#jk=#$ahRwoDMu!wQI zokg;a(WWhNvjQyL1`lu5C8Ry-spLzg+wL2)WP@BNt0B$Du7UBTpx~g~E1s)lPH|iF=j=Q6P6MkeS zO@S>9>Q#%^*4fwVwqXmGrFY!$ahL6U!a|k7234i@=?~^N7ex6SGQrTs%69zs3-fHf ztUKi1F|0LP#_D~^H8*272+gGROte1Mrit-b7DpPbG(Jnc0DGkdLdR)KFQ_x1h2Ojl zk>X5iV2K-VEfH$b`lC@<&3f5ySs!-88Pm_Sa#fWrDe)cidNW<bnh$K zGTkss6ucy-z?OXz-yNcK^Dy9K8uATAz@|pVY;O7ta{j ziMfpqYON>#@K!WC3Fp5^+#~6*;DR?QHccKrm;8VFA4J3NVVS? zg9b8aq^`NpQv3-yB~hXPYb%EO zYQn1Hua<*WTb$4zl7j1X+v-)qwuP3r_YxEittP=KLE*4+!0iDO1r9X7D&9LqPN&qD zrXGK=L~}A9iTAfoE|voAf1xn_CUJF%oT8AK9E4LS zarBMJ$zhNmM<;-NCTrOD$*q!^8etyb73^G56(xM4eb+KZ0J{pkDUA0D#(C0*I_JI=6 zjymamsyn%g-O)_F<$jkG^YyR}v`gO4E!sWmC~l$3O2xwfdR@EcF1xPqFUADDsS@8~ z;aX(z+E==UO2Ao^$sW+2G#dM&TM;rU+%2J8j|GRj^VXAn`33#Y?g@X2_@MuMTq5g0 z#F2`7^$Hgf2k){tvbo7F&v7Cyv&Wr<-DG}o)Ll_D1tx?5E|LcO%=%;Nzn$<-i$-qV_z2e4l&j&ubJ2-&_mM zCKpy&pQ1L7mkSXPg}zd6LsNg!uvGeJMwN}o{Jz7$iwRjN=Hwz`zN}7G>~yV_CFbgw zp@8AvF#>uVzzo-`?{F({#AftWGEmXNDHlg&Hv2m)p@`N7Rpjq z+`4P9ezZ2gHgspL`dEdt-r>}>2-eNp2G9?QLcJnE`%^2wiE1tcOZqX@_Q(2iHFQcU z4<1n4|Gu&}qfm_b`t;K>!DUz@!?>1&iH&RN3oh4|Y=yO-VXxoLz)^Jf|D$=;x5F~O zYGeMv(S@$iqAD9^%l={J*NlbyI`}N>Ew^e>>M*4Rsa77*>nyV%A_t9$6fm?n2|S7r zKpMo9_)7X1aN5h-z+|m{IN=bk^~AnE#4Y&#GKn$|!9E%|=+$Csv5s&6z2|S^UctpE zK;KO#EH3{5vghgT-VUP0_2K;$a2oG?jA9{s*}KP=P@;f7A~65q`9n(JH+MmccmxilLpf26MPfaCo|}}MEk)yQv|;(FOC+A z0@^#mDw3)kw+6khF;D+hHd=D}aczuW|I@w>N3+qgszCdrs|bT@-ir1oYb@U{)D&_P zls>O0&oHL&H>VykwSzU3BFeoP?4SPC6<$Q8J5Ey zf{8>j^WMXQ5kF{!i@a-G!_3%LeNWms`**2TF&6Ix+NHmIIZ!4ZJWB*sN>q#$V6|g= zW&7I;eR3W=^pXJywUb@)AV`Du@60zh4wVU7Z>`#A3ysW7XB9Pe9UBI^eD^9ak5_nH zCPI{}fB=h+%gPSJqI@sd1AiUwaic)`H&GV(nlgmKfVVet6a)eqdsW`U_RtP10PuXK zhNo%CvaVnr=XiuU0&hYqQh4g2dSJa{&HiPz_X&2qFq|f+>NUL@mEfiDMRf7N-gav) z%e-iIT^6?rJ8L~!aXMyhC1^%{New)h@bt)ji+=sOQ>Z)-cHJ|+?_?bFVO~C*@aQQq zlgg$K*gfc9plFk8W7b`m-s7Dsx>}V4SL^nN5tG()QTQ(J9MwuqTFrlsobe zqM5Bw1G7ej&Vm*ZfBG)Yrq%b#E3J%;y$kC6#`;Oe{hs?4;P1I~Rq0P#_|rqjq%L}v zEXp3jpQe7(H#vIomp)X=7vmy71o^k#?`1k(0IcL)dB8STC zhWn3n#hUzl52~_Ux$*^OMP`VlXKo3Rbf1?-o`ayaz)LwOi9gBTc1gisiB{+MLX@N) zZM2~OiW|1g*jRzA)7Tgy+nlQL1bLrXz;Wv0sE45bg0_ZC7JUl$zh6?~W)i9@a7C3j zH!;#^DDlyoXLncJcPF@QDOs%4d19yqSNx=rK&|rRglAbSd+qZ&n7?P)f^`<2p7nml zZEBBue(;TI{ib)H9X}iC!0*EpqB@|aUV49qlNEF|M2S`+)C=T^kJRCISVrXhW6=5y zp+AVc{K^*$CB`cEfIn=D7wU1$T-zR%c>_l`i3kEkmA?kfl!0La$>VU&g6L6;uc~RblPyi z&@_eZOAT+|XCu?Z`lLlKGls=al( zj0$@M(tob**-jiNT zB)Bni!Ry3;;vmK+JZ&A;dsC+Zy|=`_g=BDfkID;#4=#H}ayyI!6=9FOA8L1bCbJO> z-|pqh%rc(P4{Rpa>zE@{kp(vW$2m~Y1$aJu!?I`fA|*N89b02$Bod9RF^zDz(|p>A zJXbfZPQq5-@HsIA|G=*umv|9x{ta%Fb4+_mg|;x4R`rCfJ1Fm3Ajr*0IM*&WeORjg9DP8j6=kvJf_SId<7(j+C6#w(fLVc8@R^cHS&^&OH1bhB zhZqVL)0Uqta|eYgkYPU`csePQo)Bk7Rwe%_SE(u2o%^dwCBSV1IIJ)+ovFQJQpN%=wFkQz1^g^W8BxSImi zsdibh98V^3()O1oieZ1FvBYB$-3mJ zpnBnvTQLF`=%y-J(U{<^tA0bN2s6QUn(${8YpwGa|BqgP_Vdx*Y5u_T39%stbM056 zS+<7nC|75ayu?k{mVytkFp0JhqC(zp@5bJG&;lytNL4;mIkpPgt{&hM!T(0I!qhG3 z8$x!LdPQEKD$Usiq_e$4J0}3L4dNtO2xe<8hEq@Y?B}s6saH`L01<{{=O8I#Rsl9m zX;Q~^qmQ*8ehIrsQ}s-(!xwp~(@a{#u!kaF0FH$s&PmWM^x!#x*?3CMwG8z9JhVS@ zRPltHwO5VXJX_>Y!G?`pExVWXvTPyOE) zSdO?1si}lD4MpU0bJU-DP1TwjTc($v=UgE?C#fGgR;OqXI}n>SpEPn_?~I5l<}pRS z8cQyV%{QG+R0ZY`4U0<2UJ|LzKU1fdyvIqJ(~)J>C?AN*XtIdiu)^>3Yn=raF8rUT zm@^`S(<_Ph8y8j^dQOISz`6ntUJWo%I0-qm-f4Ff3iV`o*Yceha1L_F?}`u34hn6G zRcA5RfAETTz|X5i(&95ER_U!np>|v5VxvtfPXEm+ zuf1U#E!uWXWdDB^>3?Zkho?KC&IW~(;Qm}B4_XBQfjuLe9yd^DM%JuO4EGk6O}24+d^yc8#n95 z5h-n$=Kitk7=sj;YZOw$b1Z?)sUDYGaOn?D+3E{5r7ekiWm@20fdqWhy@$xQdB$c$ z3!ZlgF^v(xjMp8G7^1S2k=&L=Z6=~V;SS!kah+6~P_snXmF3=G?^t_#?j1X^?+_18 zW7rmtDhgvE`LZtD+L~Pk;7YEbvj|HEg!}MQw_^Qvd~D>9x@ya9F+LdOZAnIn*2O*v zU=Jaz>||F5i~fRaE7{@(!NlS`{c^*uvvpghPv8V_|A{hA->SQr{j{b^sX!jJ&A~bR zfJvj@lrVjQ>vosC?&!L3$?gZ|Q5%+*@9b?Ey@pWnnt_I|GT|*b^5tYhOSb1*m2w=D zsV*as57^F#Y6UQlzTqy{TBc(YISoR&tPi6J#VQw4s=#s&j$ST2Zs&2+mgp}TFhNC( zL1i%a4~iLNo*9S^^1d1XR8cxR5S=hwzrj4Ln44ly=vxHHPcay_B~EkHhfjZ?C_!DG zKR@k-0?2;7>lF@n3z{OF@8~-GVBxSXIVwv75zZT%^5v1c&;eJ;QN6_FXXG;@tp{ulLEmGS)o+^(=)MlP7E%hf8f zh{SV(Q}NcWUuw#dzbo3e7RBXxZ`;M_2snE$X4+9LFe>U&7*j^C`_Wu{Q_sFYLe|JS zE7Di>OrBshd#8_ALlIG?I9#Hizv-1;CmjPGP)+IZ6Xf}#Y@$_tz;i2i6trH48iS!X z4X*y)XPf0(5mmM*xWdi&S>mlb&A2EpT?Rf-L{m+%!dzu+UUWOD<^8%)*#*_Egf{9S z`nhN9uStsiz-)3(m@voJbL@Uz^$=5CYdSAhMbn4`k{!tOR^N2{R7KBUrHrS=YKN4| zn|(CwhiXyC1&j$<%krknZem@*9CY~Ve95nH@5=S{yMF!ORD5p(qRiSXV;t=Y72OVf zYCV_O)jFgQlO~KRkWL%>SvhNAspg|;o?Y!Q19eY-9xDoLe~d3rFH74Wm%8;`EX8n0 z$)_2#{~1t&s-bA0)lgT8N=XQIfj5QdK1iZbT^E8-_ywYyMuc1|*bdl#La~4v(zDY` zR8Vk9zgCI!{ytjeAnmQty%tvbMOLt~Sf%_KE2$t;^<%=q?*Ij#FRAMX+VM2!4$Iq7F;pha>KF}*@vrU*l{ zBYS@~bU|27UCV3@^k{yV%7!MYJI_?P1?&SWcihfU^=a$>fbZXDj*@Bh@KX{A&cC@C z@GH4EQ$5DFt@YsbnWIZvAh*yqRG0uvH7urZIK@Lapkv<7n|Q^VchApBQ@~X%cPhD6 zFIqYMHelyu)a&C;(h^c?>JsELAC;b(9f*Klxgw~;rWYH&c{G00851LGrSbKODu(U% zMl!Y6+2&lqt<4Zue10wJCMR6q*7Xn@ZIS5h@y27BqPHs7M5|YOa4;5Z|Kx7r2E_t3 z^J}7E4LDUxjcVOrvZ}irp4YzB)3U90vJEU|Db}sg?t7EI#dV&N2cEy?g>twS(esAx z`Q!v3QysyK;UXsZqx%=DD6bdKv`hE150st$jj{-mo$d)coL~V8zYg!0s9kXnN_cK? zELHm1kLQ?>C#|3yp+eS&NXYCrlKBsGL`&e$0ceVQa(QMh6VODYuf zG_@3*jBJk1?y{uI`u6Voob*l1xs0|?&(qosUCeKd{7bqG?eC=csO@y!K_#_oZC0ek zH)kDEPp|dH&K5Cl(jQRNqT7a2Rbf`}AL_{R>)}7bk!+rau$*@f6!#eE@%3;>SHYJ^ z=1pK%ZiI_ZaO)wW!1RlUkq=?$m#Einmx;(CtE znvBq`5>JBv%{^FuJ2Bo&IN{DS_haqQX-xr`{XOj|flWJ#FPG|nm$(zpz2MrpQZdAo zacfF;EsTL#nP0KxerZVs~JCYf@VYy9Ct6m5-k(+3N2N&qVxi@7d($ zori1rYz;=LEgJ!%kZp5eBK!YzCuz(me95+{;a%^WzyD?HBaxxX3S9MVFz(bz;kBV& z#>K7Nor@YAe<_lj)?GP!YV5px^E_@d!}>J-LL1NP8FyroJNH7pYZcZLt()#){iMja z0}dNr^XO#G>P~%1>CQ;jX66v0Lp_W#uDPxSDVnisOH~_%rmh^JiIR_m_uXv`xYhj7v%ZtGAZ7P;d!@frym~`u(#W>i?rSMq8W}{x`kz^0 zu450z`qO&sCWq_;wN?_mpq z%~kYYjSIQT@*@fzKcXWr+nFZ%y=F0>F2jnjabZn)Y@xU`gApWZXaBh}sOwI`u$&q+ zkZTu>)TGL)5=ZLPlDq;?CCOR1yx$GTeBc4IXZCrnVUPWtB*ImEQe9TMG?axEl!P_J`Lv@V?x;4tf z=kxSoxlLKsT& zOf9=qh3dH1;@&kkv3LJtd${(J;whGa}FAgH4}ogRM_9 zGY9tI;V>8DSrPHAv*mGnJ%;+Or>*OC+yoY;G!2`%b@4nXIedn6sBO5$qCYtE9rMD;CS|Vs z4OK0-e7RDW=7E26Bex8B&J2W9dKg3wpY0#NTxdK#lA9H0&T;m$SzQwps=Yj({KxOB zB>J#w@fMo5Y@f;c-kqgvJ(eiy%i^dx0*Q93Q7V;Jes%3<4EPeW6D^?4!`~WN;8w-1 z17E@F*e!N;y%HwE%0)Gg$~{nqoA0W=Y%(#|nsTa_ESQI`5`W#lrS+X&-CD<^Y=|KQ z+E1A6Nl}M!@rN3@}X;EFx=XgM4LhT z2Os9_yS!i6qB==jkMvMcPco2AEO#ru%i?F8lJK=J96QfjSCQ4VJM`GJxuw zPHqGf0zx89pj82vzc7rU8A0Yp@vKP8WLS;GWZ=+G;(AJBOK}b&oQ!Y>y01&rO08f+EN++2|7y`<=0osp7v8|&9xSTo|aJnz+V5Z>hc#)`b1FO6eH_=)Ye?#3$eBaP1pW5C$ zd;%gqt7+lm)FZt>FLzT2r;W=_GNE&9T+P$gRLbz*GYHZcAHQL3URiB=RG63fQi6AR zUL}o&*-IzuUp-<~Z*eQdj^t&m`$ps?G`W~2#*dxHvocrjTTD;}x@Q?h9E97kannYX z`tyYdEFtB}@|StwW55#|Q5S%|>4n#W)?W$-_OtRg^))i39(Z6FBa z3Tm41&r7z}b2Y_6N`kF9U0nkEaLv^yW3w zbj@EQhKg}r>QuoJ#zE_Bn|?3X4RyDN=0|IIsu09kTl^O%JbOnfqTIT%7?z1id^~?) zWHpyINvgM?Oh%bhJwdcb*c_c(QQP8&5NVlO+EY=R8I}?O>V=m_KA|C7Dn{?$hbVMl z)>p5>@I|3YZZsR}L2t81L&dFgL!@nljUs24L>lv>i(&iJ7L>h9aNY&BX8iPfUPjq; z@v$Ep7Opv8&dkhc$cuKD$7=@kS=hlx&+%*Z!Aha#zgo-i=YkiWas$$lDh4ZVAP|w? zb>|-A@&XicPjaRLVE`r(Azn2Iz3&^5d+zXGLp~f7v<#77O6*6XZy?R`V7M&;tv%_y zuu1}CU`D^OUpN+P{hPty7}vl)XDR!(C2zUjR260{{Ug?fpPEf|Qn$X1vDIhB73o72 zHLoKzQIT1n^Hsg)VZT3WEZ@ZLvRrQ03t!jDUUFTy z7~jD{T9ZnoXh%%GOfspmG%JUM7vv8_leOykS+ISY1VrvdBjasNpstimPLM_xF^gG- zQI~sa*SC>xYQ}L&o$so5H?he;)ehNW^)>A1>Ldve%+Il!^yEp#7fYV(7E3 zMGqI6!xySj#SOwWk+P3Ec>^ARC%699zrT0ia8t`3Gu%zZ|EZ3I^CLl8)inRh_h z^5gJ~ASsr{#Fy&X#5r3zR;_R3j~nVr*yMiUh<@P{B%-yp<$7mFR;SpR+A?|h#-8xE z-su!IbaHX{3{<9o`aSEf44Z!U{rAk~Qfg2jc`pZ#8Z#(VK0ai=z#~ZpFPtXL%w|8v zldfl)NEhVWOCcRFv!)jm9ALS!<dU3UQ35)u7 zbF+{9*z5qp%*RDOY$%9`TDci$_6ajP#_db;`RQlFEgX~Mtu33n&gBmzAMa&(`%w#` z-~UR!jZ*J33KaG)uFM)QvrSOT&OZaN*#l^f0RX#gv^%fE818aLV4vP6Ie3++m)giv zBf8+I_OM=gB=c}y)T*A8)ULX?oO}|Ow3S*J2UK%l$K@t>q)9DGQm=l{9QZ-Z?7n05 z<$CHtFv{{~FA*#17!K&st;s_lEHU_aJ^sq~EgX|luk}H0nH0%;AiWQcaxCX*qleP# zniFL3c~7_j&p5iTu~PSWV)*&n*burE|-kWyZYKc z@WvHZdio%(%qWC5lWZI8fVhFLo}o=%S?%%%-E!F=@^9hE$fz+N^)pXPu-bW-x>bGz zlmOE=SN}OQFjqWtK@%M4jpJ`Py-8xqvSL6zm<_=6B&vP?G-Z`_ZEI|md-yr?Bg-=$ z_!`=(m3p!=Yd2vm&fNR~=Y22LR+1d>_txd=1Y2?66k^j{^{Vd+N1|JP;*|hjb~$+K zG*C_sL~zB&A1}t|!6_e26HRy?L~&UpqNB?`w2(!}Db^X^w*4Hegz8FAz5rZO6M& z&D_o8xqJ^C%J$^er4f7eLRGf;^Kw$tYtEMO)-uF%D?g%leP^SFunp54Xgj*gRf(Ii z4CB764q11fD%+@+&M;)GaDQjX8vNNpz;NA_s4cBe-pSu}ysYt_@+{MIrOU|bi9Onp zTKSDU3FL$VaTptr(lfqlNY);yt#oAm!OGd_tly?()wlvzuG8+LirsR{(K+ew-;zHS zc{l^bf`?sq3p*-f60_zVuP2>j29$d3Cc{Q-Eo4$tfA<}!fC+@xIHmhtJkjqL-hYSe zL4=gwZ{n!aqKFrgOho7E-X|5QsoY;ACQe(yGfx#DGa=@bF6Itc^a)$*TK(-p^h4{- z*%xks?D7+{6Z&(3T-)a-m*N@iYsQ~zT7p04kklbVd-8dglB^>xEJPB1C?!uU=$DU`y`(OB9Zc0D zx4&n0Hh3_zP42G=o>j@KEJQhoKT9&HSo86zy-akVIK=4Fyc9P$ZB_a)y3oc4h8<|z zrTI|@Ao!h@+Bq3UfO*dK@<8*#~9cv5Ixx#`P%pCb&Z0v_JRfZ zUoHgz5Sg(@vDga|Hvcj&{(E^(1pf)_|Fn_>>@cHugZLBi(`_zZy}&LvNUvs>xL2=Q zmkXpbE z=oF0%piay^ol>33VqEJE>Lx@-gCkg_6aDxvrlCvQLYMA4EPW+Vfrh(Z z`0n(qX3Pxe5q3k(g?65AG-Jb-vJFNSOl>q$`MszV!9cGaE|cH15kbxA;O#8h_eaA# zgOHNlsx$uU^$eY?nOdW{g+55PBCD4f-fTmKZ91b$(>Lx*QloaGqR)Cur&6h}mbZK3 z3_ZO1W`GJ_zR`8Vq}r!aJr;Pw$(aXpeXpshkjEs_H~yZI3`NcU?4)b&;+T(7Q zYM1bQY3%??jSL8*njW44>0-JWu?}o61jCej)2W(LyEw!P(bERq-A|7R`JHT3??g4j zKwk(6G*iw9=R6F>Ayx(-65q&<<4tC06|L>03H4PBtI{kPexg~J zBKZjomU{TAs@j06iUrdL;uverh6jHck~Yh}L-d~O{VO+_c5c(Kd{L?6?2z57#NkUK z4~Ay+54;8y+>Mq({%VL#S$*ZG2&{9B*4KEG_JIJN^?Sb^@ws`OH>de=H$JiWCcbQ5 zUcZEB{^8xTJJwz##dogqv&|E>tk8e%qf3(V9z0IfD%Z>Z^vq2{Y5!}xfodfWP;-g% z#*lfQzrD@ZP;WaYUUFp>FaNT=h>XOUTf93w!o!sQs~uy^{W_xC)N#uBX8|+tm}2`k^EXU!yrz znZrT;8QErTKO#S_}(;!U52y>RnNCWXJ$wTAUCN_8Pg6`JV8_m=b^@i z9a|LxjWop8s0+PMA5C)i;^(I9;R?em@+({R)+6{MOho#PZbNg^0?VahSnmi0r0H^n zHt^E(wVDZbu}`${A!K2e>j8Dhz!*)O5l0&|EIIW8ar7woehG%rtr~xF+?!O9Q>&q# zybSd7gnEOxIX<=t?b1^35W1+I#7t{Dz8_(3$Gwun=MUxiZiZhW0u+C30 zexCKQSkdKSFNqE2@skvt1PznPUGek$CGVFr4MC=04tsd|T4^~~y;4fo`z}xhl3A@f z`u3uK(B0bpDAkXyEIcL{jOj3l?O1VB!SGsYB??ru`(SGYAA^yam&=Fkx)qzxt|mYE z^a{<-=<&jKj3?UQx?6j8q}e5+{v^AD!FpxA&Dx}+HKAD;Hg%mctx|Hz!qIDMvB3>_ zkaHBbXOPl2V>oJ+W&ZMVrnR>nLdsxu=@x@?YPP5XT@^qYa7$lN&8^ldeLKhE9K`z@ z^U++X_0BRD-P#D!{ZL=(P&0J+x8E1B)7~zX2U@Ft%q>u)lnrR&FqT*NlxlSC#`;3X z?TG*%$&SC)$-sH^b?96zweP#eRm73px%2D|J1fU4>u4jK{>A{u;EvwLH^ocFZ7(HW zHAlPuhQDxzqk7rn442+p)SLhYi+7u~?z8Jn9|8^Pe*E!Iw-ztN+s| zhpk0J|GhI+V}d6I(dNt8EIxDJ_Kf4QeaNgxn*L9@x`ya3o@$zxmP_gb{tIv zIq=?IokPy$l|U%7#fNe@Qj&aL!^`>))nRIT$(5k!J*1f>ZaJNT-(}H^n-|d`hs-yQ zjV%;Qjx4smGWo7SBk1ZSD-Ii)I$B#MxPSfbe%>**kT)K2Wctl=xDqw84HED2@>~FV zW9Zz*6Zr#g5>Nf6^p`zyTOs3L+$(+Mr+M#Fiejyd5)Qw{{#K#D!>bw)AA8p_e|OU^ zc}Sk?4tMGkS_<-aXCRwN!EKjH{kT=tqqOT|Cl@mp@as9XPVPIvo<`O$PS&(Y#R7H$y_diLEX%LT|TaS)9h-qWd@; zLin`dDx+jK0D7@=oZ}4XsW&5{1Kfg>gGzfgNIp_w_-mZR$O`(}HHqd!6Uql<^L5rS zqi4k7P8XKvEm$Vue8=w&N;fE*N{atH3OsCJa!57k)Re9jh}2depXWYohaG3U;CO_F zIKAx@m?|3IXKGz%x(3dHQkWwe%%(Q`fTt}!f@e(sYM63VEcKIgJ#lK`enk7-phcp3 zSyBZ(9HK*!U-wC|pNhR2o0NToLx*?6aKvfeme-Li2I^3$N z#07=6fG^xXCdZTgt0=@qm+3(ys<@F_7@9gqbN`;_WjJB!$&XdXfg!ukp@_53Rr^Nuwp3 z{f|(U4}>dQZMS!}=ZOeruz>TPqS;((=Vp4tzX6|(Q=UGwj(C8SzNq973h%T1>VPrj zC!Bmp2X>7(s*KjvLUZ+cE^`r7U@&UgEK->aiP%Xt+ZGe@;M@AWb^FI82`@coQ|(A2~-z!f5Lp z&Wu=seo6kEI9t0r!}a5Uk_9x}K_ah)?eDzi$sFOqYtin_k6xS=F*Kk>)HAiaGytEF zhFny03s5m_iPCw6%>pGfBfKfVrxH&<%`9o2F-I|XT(4BSaJhK+K&l9yRJRy>@dYsLC8CoF1O2pr0GrlU3gmF(dsq$ z2`kd=S3J>~ZRX*BmMd#aj^vp41EsM@pQ(rEDFl(4@m&K7Zll#;?=zb1SnC*0m@jN` zr!_$7N&-^^#A>51Byj6}yl)YwW1SzJ!<#~>b&b9GmX=(9PZ`fUzG#xAG}Rnkuhcy!tN)wj-bN_%uN$k9pK9zZ zb2N$nbU<&39>welt!*!gtOm`0+ag}RH(XKrGEwueo0?2+Mo@qFE0o;9$dC}p8a`;Q zty%Z?aa2#KOpNP5YB!?yuwR1OdDXPh3lI<1P9MTviZlTERA*u(VKHL2NBTWc>cwve z3a1RCIiE~WwyQ3>qo3QludK8Vs;wW47_}rsl}f339j~`=v%V7F5ZvF^%ofVOxf_ zM%PP*%A&a32c3=g3(t%tn#V_x@6PZt%2%nYCy-t!2%0xOt3L(apT4E;L_7ccBlwsX zBfQlM1)B%?ck)~MTk7emybT`rQFy?yXXa<_`-_9zs+@$M(9*ume61Fpdg}U|q1FX9 z<@tR(RfNjz9L?E4VA=m$09*j0|9LywIqEp5!BMP>JVMyF>sT$=-r>}J - - From 05bf3c9a5c1b7d4884149a0c54884aa377630d7d Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 20 May 2024 13:43:59 +0200 Subject: [PATCH 05/10] UI - Mobile - quick watch form element fixes --- changedetectionio/static/styles/scss/styles.scss | 3 +++ changedetectionio/static/styles/styles.css | 2 ++ 2 files changed, 5 insertions(+) diff --git a/changedetectionio/static/styles/scss/styles.scss b/changedetectionio/static/styles/scss/styles.scss index 0269bff0..1c1e8b5b 100644 --- a/changedetectionio/static/styles/scss/styles.scss +++ b/changedetectionio/static/styles/scss/styles.scss @@ -1082,6 +1082,9 @@ ul { li { list-style: none; font-size: 0.8rem; + > * { + display: inline-block; + } } } diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 073d8ac5..b09d5599 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -1172,6 +1172,8 @@ ul { #quick-watch-processor-type ul li { list-style: none; font-size: 0.8rem; } + #quick-watch-processor-type ul li > * { + display: inline-block; } .restock-label { padding: 3px; From a8959be348c94eb47282ad62e790eb562414515e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 20 May 2024 14:14:40 +0200 Subject: [PATCH 06/10] Testing - Fixing JSON test --- changedetectionio/tests/test_jsonpath_jq_selector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index 5dfdfef2..1202849f 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -479,8 +479,9 @@ def test_correct_header_detect(client, live_server): url_for("preview_page", uuid="first"), follow_redirects=True ) - assert b'"world":' in res.data - assert res.data.count(b'{') >= 2 + + assert b'"hello": 123,' in res.data + assert b'"world": 123' in res.data res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data From f49eb4567f21f779d917e562d3cb47bdbee0eb41 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 20 May 2024 15:11:15 +0200 Subject: [PATCH 07/10] Ability to set default User-Agent for either fetching types directly in the UI (#2375) --- .../content_fetchers/puppeteer.py | 1 - .../content_fetchers/requests.py | 5 -- changedetectionio/forms.py | 6 +++ changedetectionio/model/App.py | 5 ++ changedetectionio/processors/__init__.py | 4 ++ changedetectionio/store.py | 1 - changedetectionio/templates/settings.html | 16 ++++-- changedetectionio/tests/test_request.py | 52 +++++++++++++++---- 8 files changed, 70 insertions(+), 20 deletions(-) diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index cad1b6b8..a497cb16 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -9,7 +9,6 @@ from loguru import logger from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError - class fetcher(Fetcher): fetcher_description = "Puppeteer/direct {}/Javascript".format( os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize() diff --git a/changedetectionio/content_fetchers/requests.py b/changedetectionio/content_fetchers/requests.py index b743dbce..2c28cda7 100644 --- a/changedetectionio/content_fetchers/requests.py +++ b/changedetectionio/content_fetchers/requests.py @@ -30,11 +30,6 @@ class fetcher(Fetcher): if self.browser_steps_get_valid_steps(): raise BrowserStepsInUnsupportedFetcher(url=url) - # Make requests use a more modern looking user-agent - if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None): - request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36') - proxies = {} # Allows override the proxy on a per-request basis diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 2d64a227..673be9ca 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -526,6 +526,10 @@ class SingleExtraBrowser(Form): browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50}) # @todo do the validation here instead +class DefaultUAInputForm(Form): + html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": ""}) + if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"): + html_webdriver = StringField('Chrome requests', validators=[validators.Optional()], render_kw={"placeholder": ""}) # datastore.data['settings']['requests'].. class globalSettingsRequestForm(Form): @@ -537,6 +541,8 @@ class globalSettingsRequestForm(Form): extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5) extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5) + default_ua = FormField(DefaultUAInputForm, label="Default User-Agent overrides") + def validate_extra_proxies(self, extra_validators=None): for e in self.data['extra_proxies']: if e.get('proxy_name') or e.get('proxy_url'): diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 1202d5db..75384f17 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -6,6 +6,7 @@ from changedetectionio.notification import ( ) _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6 +DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36' class model(dict): base_config = { @@ -22,6 +23,10 @@ class model(dict): 'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None}, 'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")), # Default 45 seconds 'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")), # Number of threads, lower is better for slow connections + 'default_ua': { + 'html_requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT), + 'html_webdriver': None, + } }, 'application': { # Custom notification content diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index e2b54481..8702ee5d 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -97,6 +97,10 @@ class difference_detection_processor(): request_headers.update(self.datastore.get_all_base_headers()) request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid'))) + ua = self.datastore.data['settings']['requests'].get('default_ua') + if ua and ua.get(prefer_fetch_backend): + request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)}) + # https://github.com/psf/requests/issues/4525 # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot # do this by accident. diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 884c617a..afa6b2ae 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -554,7 +554,6 @@ class ChangeDetectionStore: return os.path.isfile(filepath) def get_all_base_headers(self): - from .model.App import parse_headers_from_text_file headers = {} # Global app settings headers.update(self.data['settings'].get('headers', {})) diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index e72c7818..0e3cea34 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -108,8 +108,6 @@

Use the Basic method (default) where your watched sites don't need Javascript to render.

The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

-
- Tip: Connect using Bright Data and Oxylabs Proxies, find out more here.
@@ -121,6 +119,18 @@ {{ render_field(form.application.form.webdriver_delay) }}
+
+ {{ render_field(form.requests.form.default_ua) }} + + Applied to all requests.

+ Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider all of the ways that the browser is detected. + +
+
+
+ Tip: Connect using Bright Data and Oxylabs Proxies, find out more here. + +
@@ -190,7 +200,7 @@ nav - + Chrome Chrome Webstore

diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 869ea349..cfbc7825 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -256,12 +256,40 @@ def test_method_in_request(client, live_server): def test_headers_textfile_in_request(client, live_server): #live_server_setup(live_server) # Add our URL to the import page + + webdriver_ua = "Hello fancy webdriver UA 1.0" + requests_ua = "Hello basic requests UA 1.1" + test_url = url_for('test_headers', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): # Because its no longer calling back to localhost but from the browser container, set in test-only.yml test_url = test_url.replace('localhost', 'cdio') - print ("TEST URL IS ",test_url) + form_data = { + "application-fetch_backend": "html_requests", + "application-minutes_between_check": 180, + "requests-default_ua-html_requests": requests_ua + } + + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + form_data["requests-default_ua-html_webdriver"] = webdriver_ua + + res = client.post( + url_for("settings_page"), + data=form_data, + follow_redirects=True + ) + assert b'Settings updated' in res.data + + res = client.get(url_for("settings_page")) + + # Only when some kind of real browser is setup + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + assert b'requests-default_ua-html_webdriver' in res.data + + # Field should always be there + assert b"requests-default_ua-html_requests" in res.data + # Add the test URL twice, we will check res = client.post( url_for("import_page"), @@ -272,15 +300,14 @@ def test_headers_textfile_in_request(client, live_server): wait_for_all_checks(client) - # Add some headers to a request res = client.post( url_for("edit_page", uuid="first"), data={ - "url": test_url, - "tags": "testtag", - "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests', - "headers": "xxx:ooo\ncool:yeah\r\n"}, + "url": test_url, + "tags": "testtag", + "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests', + "headers": "xxx:ooo\ncool:yeah\r\n"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -292,7 +319,7 @@ def test_headers_textfile_in_request(client, live_server): with open('test-datastore/headers.txt', 'w') as f: f.write("global-header: nice\r\nnext-global-header: nice") - with open('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt', 'w') as f: + with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f: f.write("watch-header: nice") client.get(url_for("form_watch_checknow"), follow_redirects=True) @@ -306,7 +333,7 @@ def test_headers_textfile_in_request(client, live_server): # Not needed anymore os.unlink('test-datastore/headers.txt') os.unlink('test-datastore/headers-testtag.txt') - os.unlink('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt') + os.unlink('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt') # The service should echo back the request verb res = client.get( url_for("preview_page", uuid="first"), @@ -319,7 +346,12 @@ def test_headers_textfile_in_request(client, live_server): assert b"Watch-Header:nice" in res.data assert b"Tag-Header:test" in res.data + # Check the custom UA from system settings page made it through + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + assert "User-Agent:".encode('utf-8') + webdriver_ua.encode('utf-8') in res.data + else: + assert "User-Agent:".encode('utf-8') + requests_ua.encode('utf-8') in res.data - #unlink headers.txt on start/stop + # unlink headers.txt on start/stop res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) - assert b'Deleted' in res.data \ No newline at end of file + assert b'Deleted' in res.data From 7b04b52e45a944f6772f6483bc1424af216c4941 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 20 May 2024 15:49:12 +0200 Subject: [PATCH 08/10] RSS and tags/groups - Fixes use active_tag_uuid, fixes broken RSS link in page html (#2379) --- changedetectionio/templates/base.html | 6 +++--- .../templates/watch-overview.html | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index 60ceb3bb..bbfe8634 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -6,7 +6,7 @@ Change Detection{{extra_title}} - + {% if extra_stylesheets %} @@ -83,8 +83,8 @@
  • - - + + diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 6a25208e..15f538fb 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -13,7 +13,7 @@
    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }} - {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag else '', placeholder="watch label / tag") }} + {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="watch label / tag") }} {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }} {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
    @@ -46,7 +46,7 @@ {% endif %} {% if search_q %}
    Searching "{{search_q}}"
    {% endif %}
    - All + All {% for uuid, tag in tags %} @@ -67,11 +67,11 @@ {% set link_order = "desc" if sort_order == 'asc' else "asc" %} {% set arrow_span = "" %} - # + # - Website - Last Checked - Last Changed + Website + Last Checked + Last Changed @@ -95,11 +95,11 @@ {{ loop.index+pagination.skip }} {% if not watch.paused %} - Pause checks + Pause checks {% else %} - UnPause checks + UnPause checks {% endif %} - Mute notifications + Mute notifications {{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}} @@ -204,7 +204,7 @@ all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}
  • - RSS Feed + RSS Feed
    • {{ pagination.links }} From cfc689e04603c165e0ebbe9d3151a454b40031ba Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 21 May 2024 12:13:05 +0200 Subject: [PATCH 09/10] Fix overflowing text --- changedetectionio/templates/edit.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index f8c0eba4..6d6d19fc 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -433,7 +433,8 @@ Unavailable") }}
    {% if visualselector_enabled %} - The Visual Selector tool lets you select the text elements that will be used for the change detection ‐ after the Browser Steps has completed, this tool is a helper to manage filters in the "CSS/JSONPath/JQ/XPath Filters" box of the Filters & Triggers tab. + The Visual Selector tool lets you select the text elements that will be used for the change detection ‐ after the Browser Steps has completed.
    + This tool is a helper to manage filters in the "CSS/JSONPath/JQ/XPath Filters" box of the Filters & Triggers tab.
    From 59cefe58e77a0ca0645444fecb93826d28451ccb Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 21 May 2024 17:03:50 +0200 Subject: [PATCH 10/10] Fetcher - Using pyppeteerstealth with puppeteer fetcher (#2203) --- .../content_fetchers/puppeteer.py | 30 +++++++++++++++++-- changedetectionio/flask_app.py | 2 +- requirements.txt | 2 +- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index a497cb16..725be3b3 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -92,15 +92,39 @@ class fetcher(Fetcher): ignoreHTTPSErrors=True ) except websockets.exceptions.InvalidStatusCode as e: - raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access)") + raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)") except websockets.exceptions.InvalidURI: raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://") except Exception as e: raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}") + + # Better is to launch chrome with the URL as arg + # non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab + # headless - ask a new page + self.page = (pages := await browser.pages) and len(pages) or await browser.newPage() + + try: + from pyppeteerstealth import inject_evasions_into_page + except ImportError: + logger.debug("pyppeteerstealth module not available, skipping") + pass else: - self.page = await browser.newPage() + # I tried hooking events via self.page.on(Events.Page.DOMContentLoaded, inject_evasions_requiring_obj_to_page) + # But I could never get it to fire reliably, so we just inject it straight after + await inject_evasions_into_page(self.page) - await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent'))) + # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..) + user_agent = None + if request_headers: + user_agent = next((value for key, value in request_headers.items() if key.lower().strip() == 'user-agent'), None) + if user_agent: + await self.page.setUserAgent(user_agent) + # Remove it so it's not sent again with headers after + [request_headers.pop(key) for key in list(request_headers) if key.lower().strip() == 'user-agent'.lower().strip()] + + if not user_agent: + # Attempt to strip 'HeadlessChrome' etc + await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent'))) await self.page.setBypassCSP(True) if request_headers: diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 13d3dfb6..41f80a77 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -771,7 +771,7 @@ def changedetection_app(config=None, datastore_o=None): jq_support=jq_support, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), settings_application=datastore.data['settings']['application'], - using_global_webdriver_wait=default['webdriver_delay'] is None, + using_global_webdriver_wait=not default['webdriver_delay'], uuid=uuid, visualselector_enabled=visualselector_enabled, watch=watch diff --git a/requirements.txt b/requirements.txt index 88f2c3c4..d6267bfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,8 +77,8 @@ jq~=1.3; python_version >= "3.8" and sys_platform == "linux" pillow # playwright is installed at Dockerfile build time because it's not available on all platforms -# experimental release pyppeteer-ng==2.0.0rc5 +pyppeteerstealth>=0.0.4 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup pytest ~=7.2