From eef56e52c68526e629282f8e72d994f7f6fa1eb5 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 23 May 2022 23:44:51 +0200 Subject: [PATCH] Adding new Visual Selector for choosing the area of the webpage to monitor - playwright/browserless only (#566) --- changedetectionio/__init__.py | 36 ++- changedetectionio/content_fetcher.py | 146 +++++++++++- changedetectionio/fetch_site_status.py | 6 +- changedetectionio/run_all_tests.sh | 23 ++ .../static/images/Playwright-icon.png | Bin 0 -> 6392 bytes changedetectionio/static/images/beta-logo.png | Bin 0 -> 12110 bytes changedetectionio/static/js/limit.js | 56 +++++ .../static/js/visual-selector.js | 219 ++++++++++++++++++ changedetectionio/static/js/watch-overview.js | 2 + changedetectionio/static/styles/styles.css | 30 ++- changedetectionio/static/styles/styles.scss | 37 +++ changedetectionio/store.py | 17 ++ changedetectionio/templates/diff.html | 15 -- changedetectionio/templates/edit.html | 47 ++++ changedetectionio/templates/preview.html | 14 -- .../templates/watch-overview.html | 1 + changedetectionio/tests/fetchers/__init__.py | 2 + changedetectionio/tests/fetchers/conftest.py | 3 + .../tests/fetchers/test_content.py | 48 ++++ changedetectionio/tests/test_trigger.py | 8 +- changedetectionio/update_worker.py | 7 +- 21 files changed, 670 insertions(+), 47 deletions(-) create mode 100644 changedetectionio/static/images/Playwright-icon.png create mode 100644 changedetectionio/static/images/beta-logo.png create mode 100644 changedetectionio/static/js/limit.js create mode 100644 changedetectionio/static/js/visual-selector.js create mode 100644 changedetectionio/tests/fetchers/__init__.py create mode 100644 changedetectionio/tests/fetchers/conftest.py create mode 100644 changedetectionio/tests/fetchers/test_content.py diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 425575f8..a5087e19 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -626,6 +626,12 @@ def changedetection_app(config=None, datastore_o=None): if request.method == 'POST' and not form.validate(): flash("An error occurred, please see below.", "error") + visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid) + + # Only works reliably with Playwright + visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and default['fetch_backend'] == 'html_webdriver' + + output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], @@ -633,7 +639,9 @@ def changedetection_app(config=None, datastore_o=None): has_empty_checktime=using_default_check_time, using_global_webdriver_wait=default['webdriver_delay'] is None, current_base_url=datastore.data['settings']['application']['base_url'], - emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False) + emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + visualselector_data_is_ready=visualselector_data_is_ready, + visualselector_enabled=visualselector_enabled ) return output @@ -976,10 +984,9 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/static//", methods=['GET']) def static_content(group, filename): - if group == 'screenshot': - - from flask import make_response + from flask import make_response + if group == 'screenshot': # Could be sensitive, follow password requirements if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated: abort(403) @@ -998,6 +1005,26 @@ def changedetection_app(config=None, datastore_o=None): except FileNotFoundError: abort(404) + + if group == 'visual_selector_data': + # Could be sensitive, follow password requirements + if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated: + abort(403) + + # These files should be in our subdirectory + try: + # set nocache, set content-type + watch_dir = datastore_o.datastore_path + "/" + filename + response = make_response(send_from_directory(filename="elements.json", directory=watch_dir, path=watch_dir + "/elements.json")) + response.headers['Content-type'] = 'application/json' + response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' + response.headers['Pragma'] = 'no-cache' + response.headers['Expires'] = 0 + return response + + except FileNotFoundError: + abort(404) + # These files should be in our subdirectory try: return send_from_directory("static/{}".format(group), path=filename) @@ -1150,7 +1177,6 @@ def changedetection_app(config=None, datastore_o=None): # paste in etc return redirect(url_for('index')) - # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index 0deb8966..5ac95927 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -27,6 +27,117 @@ class Fetcher(): status_code = None content = None headers = None + + fetcher_description = "No description" + xpath_element_js = """ + // Include the getXpath script directly, easier than fetching + !function(e,n){"object"==typeof exports&&"undefined"!=typeof module?module.exports=n():"function"==typeof define&&define.amd?define(n):(e=e||self).getXPath=n()}(this,function(){return function(e){var n=e;if(n&&n.id)return'//*[@id="'+n.id+'"]';for(var o=[];n&&Node.ELEMENT_NODE===n.nodeType;){for(var i=0,r=!1,d=n.previousSibling;d;)d.nodeType!==Node.DOCUMENT_TYPE_NODE&&d.nodeName===n.nodeName&&i++,d=d.previousSibling;for(d=n.nextSibling;d;){if(d.nodeName===n.nodeName){r=!0;break}d=d.nextSibling}o.push((n.prefix?n.prefix+":":"")+n.localName+(i||r?"["+(i+1)+"]":"")),n=n.parentNode}return o.length?"/"+o.reverse().join("/"):""}}); + + + const findUpTag = (el) => { + let r = el + chained_css = []; + depth=0; + + // Strategy 1: Keep going up until we hit an ID tag, imagine it's like #list-widget div h4 + while (r.parentNode) { + if(depth==5) { + break; + } + if('' !==r.id) { + chained_css.unshift("#"+r.id); + final_selector= chained_css.join('>'); + // Be sure theres only one, some sites have multiples of the same ID tag :-( + if (window.document.querySelectorAll(final_selector).length ==1 ) { + return final_selector; + } + return null; + } else { + chained_css.unshift(r.tagName.toLowerCase()); + } + r=r.parentNode; + depth+=1; + } + return null; + } + + + // @todo - if it's SVG or IMG, go into image diff mode + var elements = window.document.querySelectorAll("div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary"); + var size_pos=[]; + // after page fetch, inject this JS + // build a map of all elements and their positions (maybe that only include text?) + var bbox; + for (var i = 0; i < elements.length; i++) { + bbox = elements[i].getBoundingClientRect(); + + // forget really small ones + if (bbox['width'] <20 && bbox['height'] < 20 ) { + continue; + } + + // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes + // it should not traverse when we know we can anchor off just an ID one level up etc.. + // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match + + // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us. + xpath_result=false; + + try { + var d= findUpTag(elements[i]); + if (d) { + xpath_result =d; + } + } catch (e) { + var x=1; + } + +// You could swap it and default to getXpath and then try the smarter one + // default back to the less intelligent one + if (!xpath_result) { + xpath_result = getXPath(elements[i]); + } + if(window.getComputedStyle(elements[i]).visibility === "hidden") { + continue; + } + + size_pos.push({ + xpath: xpath_result, + width: Math.round(bbox['width']), + height: Math.round(bbox['height']), + left: Math.floor(bbox['left']), + top: Math.floor(bbox['top']), + childCount: elements[i].childElementCount + }); + } + + + // inject the current one set in the css_filter, which may be a CSS rule + // used for displaying the current one in VisualSelector, where its not one we generated. + if (css_filter.length) { + // is it xpath? + if (css_filter.startsWith('/') ) { + q=document.evaluate(css_filter, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + } else { + q=document.querySelector(css_filter); + } + bbox = q.getBoundingClientRect(); + if (bbox && bbox['width'] >0 && bbox['height']>0) { + size_pos.push({ + xpath: css_filter, + width: bbox['width'], + height: bbox['height'], + left: bbox['left'], + top: bbox['top'], + childCount: q.childElementCount + }); + } + } +// https://stackoverflow.com/questions/1145850/how-to-get-height-of-entire-document-with-javascript + return {'size_pos':size_pos, 'browser_width': window.innerWidth, 'browser_height':document.body.scrollHeight}; + """ + xpath_data = None + # Will be needed in the future by the VisualSelector, always get this where possible. screenshot = False fetcher_description = "No description" @@ -47,7 +158,8 @@ class Fetcher(): request_headers, request_body, request_method, - ignore_status_codes=False): + ignore_status_codes=False, + current_css_filter=None): # Should set self.error, self.status_code and self.content pass @@ -128,7 +240,8 @@ class base_html_playwright(Fetcher): request_headers, request_body, request_method, - ignore_status_codes=False): + ignore_status_codes=False, + current_css_filter=None): from playwright.sync_api import sync_playwright import playwright._impl._api_types @@ -148,8 +261,8 @@ class base_html_playwright(Fetcher): proxy=self.proxy ) page = context.new_page() - page.set_viewport_size({"width": 1280, "height": 1024}) try: + # Bug - never set viewport size BEFORE page.goto response = page.goto(url, timeout=timeout * 1000, wait_until='commit') # Wait_until = commit # - `'commit'` - consider operation to be finished when network response is received and the document started loading. @@ -166,14 +279,27 @@ class base_html_playwright(Fetcher): if len(page.content().strip()) == 0: raise EmptyReply(url=url, status_code=None) + # Bug 2(?) Set the viewport size AFTER loading the page + page.set_viewport_size({"width": 1280, "height": 1024}) + # Bugish - Let the page redraw/reflow + page.set_viewport_size({"width": 1280, "height": 1024}) + self.status_code = response.status self.content = page.content() self.headers = response.all_headers() + if current_css_filter is not None: + page.evaluate("var css_filter='{}'".format(current_css_filter)) + else: + page.evaluate("var css_filter=''") + + self.xpath_data = page.evaluate("async () => {" + self.xpath_element_js + "}") + # Bug 3 in Playwright screenshot handling # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it # JPEG is better here because the screenshots can be very very large page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}) - self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=90) + self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=92) + context.close() browser.close() @@ -225,7 +351,8 @@ class base_html_webdriver(Fetcher): request_headers, request_body, request_method, - ignore_status_codes=False): + ignore_status_codes=False, + current_css_filter=None): from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities @@ -245,6 +372,10 @@ class base_html_webdriver(Fetcher): self.quit() raise + self.driver.set_window_size(1280, 1024) + self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))) + self.screenshot = self.driver.get_screenshot_as_png() + # @todo - how to check this? is it possible? self.status_code = 200 # @todo somehow we should try to get this working for WebDriver @@ -254,8 +385,6 @@ class base_html_webdriver(Fetcher): time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) self.content = self.driver.page_source self.headers = {} - self.screenshot = self.driver.get_screenshot_as_png() - self.quit() # Does the connection to the webdriver work? run a test connection. def is_ready(self): @@ -292,7 +421,8 @@ class html_requests(Fetcher): request_headers, request_body, request_method, - ignore_status_codes=False): + ignore_status_codes=False, + current_css_filter=None): proxies={} diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index c8b95321..8629f454 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -94,6 +94,7 @@ class perform_site_check(): # If the klass doesnt exist, just use a default klass = getattr(content_fetcher, "html_requests") + proxy_args = self.set_proxy_from_list(watch) fetcher = klass(proxy_override=proxy_args) @@ -104,7 +105,8 @@ class perform_site_check(): elif system_webdriver_delay is not None: fetcher.render_extract_delay = system_webdriver_delay - fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code) + fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code, watch['css_filter']) + fetcher.quit() # Fetching complete, now filters # @todo move to class / maybe inside of fetcher abstract base? @@ -236,4 +238,4 @@ class perform_site_check(): if not watch['title'] or not len(watch['title']): update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content) - return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot + return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot, fetcher.xpath_data diff --git a/changedetectionio/run_all_tests.sh b/changedetectionio/run_all_tests.sh index 82b603f3..c2bbf9aa 100755 --- a/changedetectionio/run_all_tests.sh +++ b/changedetectionio/run_all_tests.sh @@ -22,3 +22,26 @@ echo "RUNNING WITH BASE_URL SET" export BASE_URL="https://really-unique-domain.io" pytest tests/test_notification.py + +# Now for the selenium and playwright/browserless fetchers +# Note - this is not UI functional tests - just checking that each one can fetch the content + +echo "TESTING WEBDRIVER FETCH > SELENIUM/WEBDRIVER..." +docker run -d --name $$-test_selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome-debug:3.141.59 +# takes a while to spin up +sleep 5 +export WEBDRIVER_URL=http://localhost:4444/wd/hub +pytest tests/fetchers/test_content.py +unset WEBDRIVER_URL +docker kill $$-test_selenium + +echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..." +# Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt +pip3 install playwright~=1.22 +docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable +# takes a while to spin up +sleep 5 +export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 +pytest tests/fetchers/test_content.py +unset PLAYWRIGHT_DRIVER_URL +docker kill $$-test_browserless \ No newline at end of file diff --git a/changedetectionio/static/images/Playwright-icon.png b/changedetectionio/static/images/Playwright-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..75db893b6a8d93e7225759f894e2a3ae3de090f8 GIT binary patch literal 6392 zcmb7J^;6W3(|_M_^Z}BBASFn5mz;E`ba#i+eYAvtAX0LKl%#Zc0v;jV(kb0YBXM6p z|H1R@%HI_Ze~g3mPj;OMMKW-;?7cUztPdjdpHx4;Rl1~7DGFn;g{=hofp)&nhxdStqJf1c;OsVl7L5MO+~di9ld z%YQKa-P=DDmYn(Z@1H7+WjZ|h$RkM^>Ny=%H;EsLY;i?y4k%QMW3q;{{U&^7X-<8g zL(Z2F5fZ6xC(VhT_3uxwG8Mx6dS3kvnf|L$vk!hTrS!9yt#Qn9qi2GS>n=@g@Z<15 z3(Kemma4n4`xxRZF=MKxOw|34pWAnq5A|6exN=(AKXt~ZH>NXP-)mj0*)Z5yHsy+! zo4Cjq_}c6r3+K|h+|*@Nbb(p#>~K8gh}4=*dZ1MGb`IBabCa7cKXn>hx8s|E1dkey z_whbcl{?~myj4|#S;F2m4@oBMzivBP!swSy?IaA4m|tI=odM^4l}C5B3vVHegV(uT81(SWD+*L^MN!Jad0>Pjt8>(hi>6j@O1K#UAy=QT{FMyxxO5 zX&ghNiG4xIi^0K2c?lw1nIDN@UT3IDpBbh23`-iX7Xsj9%g#(2(mXMkXSU!Ji4|+ogOCavOiQ@rxT0SBrEUyxWhcz>M1$w*;+^MV>B+y+oe;zms zSs{e2e#-j^fTvTyNMW{bjLq2+TZf@5a>XJA$p z+p>}(R+Hhk9xVZ_W?A(LAYJso!_gTDl<`u#vO7e0jm45oWzS<;gjw>REv z0s#O%4&T9f9k}fNbZ{({9uOG!TE}YBC;Gx-c)sC_6cs3WYFR9qyfvm6DNn*>(|exZ zAFlurO4oN%CVmWMi4UzSZt{ZluuO_Wd9^w@%jGu&brAsAMGgToa>Lt-^R z3F~!G;yK^9f^iZ4mJ<5zASEmO}xD%j$&%yJkxwgcD_S z?OmFo?cj+Aj(Dchu4wt7B1q90=%4h02>&0`FP3SCv9%eNE&b8CgM77kicc zWcu+SG9o%sU}~%26}WdU%}QAa_ju&7Mh8g*Z3Nog_*=g@-o4( zke<_im5}%2dTL8~li#VNg~5n&*ygwsr`VBg*4D`LGor6@A&i^r=BtH!*`(1j8vgtu zAI8`66}9u_dN9Nz7$BVZI+(N%jt5k*f_SA76Z{9U-_5krNFNup<0gCg^hZ)FDcC-@ zls)&6BUR=E9}nV_SlXL{pn3&du)KWx-HdzCa8m}&iZ9>ZsL~oH!|PEFOzuN0#y2}2 z9TE?4IUGC%eOk70TE`sX3e!oB93sJX)_w81NDtH z3|}%BH0uMeoJXomPt;pmNZ~$|gQYO6c>@zO#WGX!d^Jew4|!*z;mun3$ukTyjCTSZ zy5geSahM+IFZ3RwjnJ(qDV>`bw;Tk7ZB(-LRRZqG={0vU;ZIpgrw?`0Vt|y z3Gf`#Q4lJQgF!zsjnLR!ocPIBlkSfMR>uBaLij2_j=tO&i(&&h-|ED;Yg+RGPD(Yz zZx_ifV>@_8KS4(x%#9Rx(>6y(0=MR=3kQYV7tXOV(W{I`K~ki8ddE~%hX+p{Kumv( z3w?gi*QCtMe!lW9JjEPSBhn2+}Di)*vvh^15a8)8~?4k_2pykAE$LG^#K~7O4eaf8ykq_{CZ$euhaNCHx^KQlilCa=oOWq#`@s#1~!*Kv~)jJ6T#(J z*q0*7_B(pC5=bG${uYXE>z)ywZv!(1=4NGJ(zi*bq$%=Xua9C$KCDwHgTG^(_%>~D zs7oNE`Hi10Z3dnsAb#vVlV9Wf2z1Loz%8ghmo?b(%Wl0Q-I_Y7i`06>TxndbQB_4R z7vE>6YCdIF&P^^jP-#a75>Ks9SYdA?1l2-Yc&P#(*acl44gOt$6^FeDPf&F2!GeTJ zWd$~o!DQMs@}(D?SRc~ocknu)hdHfl^@=E!9y1=~Bfoavo_KQ1Xg^lKv(9yyw|J#jPZbpu_L1&DFEim_N*nT&G~a@E=xIrMP#;&snfh22G#F*= z%MctK`2?9~N}_ zeFnKBh9^|zSw_AHD93fr>I{@yp4Ixc-45YiL+u_=FufebfTgju9ZuIWjq(@WdFjTx z<#*OTIzwl3qUVpguz`LO{V9A>X;F#GGFqQ=1I0eBQ+&EmhlYPOpTI;U*Euj?(jtq3 z9mjvaFfvsz)P2tuxYR``xz?Ip>byRrNHjc^2ey(U{+7MyH0*3)b@MDb5?${;e^|Qn zvC2M7r_N|MXx(kA39((j5Sh8JbMJmX8U8dU343Z)1>R)tZ3)XlRW19u#gkIlKCc3B zziX!J2}xvnE`)7j2E96YYqMUPFF_-g$9kG>GCy9B?D057M2Ke`x(eboxB2}TXI_<{ z@f)|XqA2?4TS%Z*2z4jwOnR%2J8;Jq1Q%U??%`CnO%kE0}Ty;&^Z{2M-kK!1gZJ-9f}zn{&!ncU%59EWJZrfg=} ziQ?lV%MMCtT#VEL=n<+%Dltlqj1py7VTAN2X(__7{7&yaw*h&4?Ow-DIA{|1KJcJw zkDOcBKQ=|a-sS1p*5oVz+jPk@n|%Ls2Xr`Lij*>{$&@b!U3>cLINbzeeOxT-pO&l> zd7_jK5BPywnfFBIRx}B`7*M?<1mEq%uw)27dKujjQN1rMuPghGS6<|yA4g?;9id^r`EDgg)#H5Z1vg3ke zTQfhSm~@w`nAWK0P6kT+fP+2TjoX4*b9f3K6PY_bQM>ELXEv5vgkl-0fw-=5fr5lld;}&pSLMH*T$V*8~6ptdTHo4VtX(dQv+$N z_3qNm0!O6Sp)S$x39W%3^Da5_T;nyyin^{uoUZM;E3noF(j~*DPc^20f)B2&iZMo8 zc5K%e-%iN`*|EuSaCd2DN;n%T&9ZjvQl0(mq0P1AI9i@|1yM(<`Sql%Wbb>?tS0cQ z&fx(KC8BhiI}v9K@;#X46LUIh#DFYLT92>d4L^C!&%*Zh4o&GPed#e+eQUZssd*sg zQ$ELuFfqrsqy;?NU-zEe97^OtNz#V&H*FtotSTs{kb8e3WbVi0lJ}Kri+_X|lJ%KS z7v&Vw_`J_+?!zZpr;BGg@uQ~oM;CiHq&KL0U8wmB>rCCx443eKdpr=wXvlYVd_WH$ z$%{cY-v#gYQn-dZpg@I9Yo7TNYZune)`u~8xAuF^ak#pszTcUY^-S;vr#{NGfIMGr zsBO$w)7F=Dbbffx_>mKU^&`$bWZ~#%X788}#~Yew27garE&;!gq}snE%u3?P6d)12 zV{SVF(if<+3?H_h`WiDF8FGNWD2TDRlr)fp)9C%#t=}G<&-&#i!Y9G^Iqdpr;bYYevNV|ZdvfSL?b>-b8tg0)w7bEp0fm>`~ z-&>BTHW1`?YpUc0?XHa{@t7LbtKlFyfW>Mm(F+r|2fyHucDd2dtSU?fUg^RiwY#?G z#|h=m+pv7RM0Va9tRLnJm|Mz2$I=R35$xMp}#hOS&tA=!g3Y!QYq=aNt7{5t6FD4 zZh?Fjn%1xDI3S`GUoe0FCbC>rO0VcysT(+yGxHRhC zbXt8NEb9{(e5vG=hxuG`*-#g~`D&gruHl^%d%1IHEY82Btv;ZD`{|p)+`zWy4*s0Y ze(*$|)D0nuNg6DRt(#FGd{to2rw7;C5d29Q?6S6m{K4qggBBXikEW4VGwmxGa!aCx#qS69=2&;w zm%ZoNF?#Azzo3iZWyRYWx%(dxB3gW&9H}kaYIoT0go4`h5)=O@Re(QK8?B#aB1Ax( zCM8C07Jn2HJ?^Jly9kphNM7wKmJMnNw@y zDJy_Q2rSa%a$^3HVk1YS+r?6j@|`?s@C%N9;x0_iXP4Vq97^FFvDM5Z|h;Y-(-E161MYi3flCA01xu&Kfp zuR{-ymc9%KJh35}sYHIh#Ij!|q~9i`{k-Wu{PmEm7v+(;eT55wPscrf@YOaE&wfEf zoZr$wsMDix$%f9{@x$amSO}XLPRjxzPdXPr*n+}rEDjMhB@+sFUWvr%zpkom`_Q0O z<9+_*Qh;ZL(ZfIdKkls%oM}LgeM!;oBw9e4R&{7^QRd9YH^#S)`>TvbQE*a z9>;KyiVotc5<4QP*Y>PnFC+~%1cwE8>i02N^)iE^{#3+dORbDd?eg>2|N1q*DAaoX z*8#8r!;8@ltZBX`Dx(DD2d{})&U-XbV=~0g%BflLSRJz*3w1SDxP_T;5V(j}OCRN& z{zOd)xg{)+s01oUC#t(4UQAee(@LFKzuvv``_a;#+>`j2(fQ$p?rxiT+IRQgrn3!7 z<%}cASU6EBTCc@_^AVC?6(9b)a$y;0AiobzsyyZ_Uq0(RO589a6yVv`orj~xwy!fu@ci$n-GbWDUXL$ zngTtCFFt*K1BGF>aH_wx9ru%X&7neLO?yGBGXmi#v4)T$kekbh!TPmJ=kWC?57Y2-eT@+ap z+O*$GtVj7U)z~BHUU-tT!b0wKV;BOSb`Fns=o$|92h>VxvUs4r$l6c=zu|S0^S;BS zm(rJeCVy0ur3``}^QzsHXS3~%Da6@j&NdzX58@vMpu%3Z$3Cd+WK#QxoR1iU!W6$`*TWGpxB7M=|c7&fy_EisI zf?OB7@)_@63Sl=f$3 zVo0&5TLAStOv~Jo!^7?F%$#)o=*I0y#6&u~K`t2jU!!t(d45c}sVibivxCDWMxMl4 zusfauqfi^gX8Y2rKvNKTZoX#Ze$HjR>auw=AR_Gg^@?v>&FdFC0K8g!J?i~qubI+R zz?h1{)d)Y(_?bYEnT|+&un% z(+wep^LDLslEL~Ru!|!nPd-%uk+mTr1B&@lR{foJRS~y^F2j8+lM#yK$ZQ|?94s6* z?08|5H>aAu!iq_0-=__&#QG!kXQFl86B@N`^@oSL*<(Zrtk`z*?iF11OUwOoD^oWC z=e4o5w!-Qcs2cD{ie^Xh>X;rwJ>mn}WwxonXM^G?&Ht77`_LF~Gz(W$sp6RbeJucG Mc@4R08Ow literal 0 HcmV?d00001 diff --git a/changedetectionio/static/images/beta-logo.png b/changedetectionio/static/images/beta-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..e5533ee23315e360d9a29e36c3e8db28b01ef601 GIT binary patch literal 12110 zcmaKS1yEegv+v>#%ib|O1uTIUJIXyi+Q{8iZ-DmoQDSVQ~LMKHB003ApC`9S?-v3WQMSiWzH7YUy05S%5 z6%8jP0~abg2U}B%FD6t@ZgwVACaxBy0D$Y_QL46@@rP>23wQV%|COlq-3GO>HSD%J zCU_92D6r(dSs@vhfP6G&X?b`AoWJmLvugC*r9L2>nj|yAW;Ue$rRr`p)sph_(eu%l zW}nr{yYu%q&yE-O2xHQIbEmhflbLfa!t!n2>sP{?Pdf#|J{t!YQpZ=lH!QmY?rkgB zv13o0jzpQfmJet2by_RfUj=QsGKp5Q1z);9Pc?nM_qjL@5RX^e5bJoL(D`|Da!)Ka zPD#mM8P?{dukgrqA&g|^YQJtGgSfSPsi#-&MV93v=Iv(Y9rb+BwCaEIaB*>}&3f@( zs`>5`h5hN;&DATy=V9%XpXTKmg{|?oWb!qw;omelK$(8 zTP#PNJ)q zXn%5b1}_yvP=J$|sNZ*l9h{iXwnDTMYD6cN9`gWvqC#M&Ya`B!qBgb-uhbjcb_s8q zyWjdX*Jpu-i}bZu{9ozZj5q!09(m=*{nneDHJ6w0`4OzSiU$q zlUHC-YNeYUP2ZeaB{sS^M+gM|Xr`>aoN!-A=bLRvvFfOOzYTSq=9jQ7I2nGMP?a^} zNmV`Yr#p0gNH*ov2pcg;g|cotLU@*2&<~2~b+x2sRhYCR%G8{U*0gzsutlqekLhZ^nvTGO z>=_sCR|O@9RHWfk{M0dW_XxZz9+4m($z@Y3L9-~6sg*w9oLDJys>WL+nsTk}=OHSu zYs(B<9kesnUQssutb~BK0;`5U!AXXvYi}oeL$id66|&KqN7CpMW&_X)YXnK>%S;lv zh?s;;TuV$agqx34ehUnLcRNnDsto^1T<*`y>k!Vu6X#5$`t6*#4)jjp(xdL4xTMuq za{j^H@+SW(ejoYHQQp2!cmg|4H+XD{%31Fu;DPmAk@vVq2QxL0k$^l zX7p8!X{FUf)EEcSt^HBs-0^ zuvUmrKBz-~?1#0ij(U>S;V!w;{dxm^&xS<_^wWI!Ti2;fDI~>0$6>dWoli~J{B~ez ze?LkuN|b%i)gUxNeWN7XPvhI4?5BSrCcyUd)C)Nt3!k;yN7?0Co|}aSp1#ag;O|jH_6D!vw6*|kI;Ph}^6swm738Ss4l`qZ-lYWjNOf*o5k5mZ!=9~ZJs}Spa(cqCzduTpPDL;8 zBXh^?52@nEr#F@y?-rxp&j&3tC*&wNNId&1e zDc&t2jAk3^({b?sE`_pgOAX`H$zvO@p`e2xxoJVvHF53?@IO{goCHcB>7~KVFtw#1 zzpVw1xWX*eaSGohi9d|NknValuENF?t8RcjS#vC@PQcX+|%S= zMl@c1R8p>nUCFBX%h_3rBI?wMIU6=`y^Cp`7wxvq!c>prU0#)-=bntE#D0)k)2U^W zXS*KmfpO(zVVyx*;YcUN?pRHh=!GA_A)6!@Wf7bz5JdWI?@CQ+&0((JQ)c`i>!two zG5=BehQrn$D%3izb_w3%hOrz`?ECF@;OLrjuor#ZxLnlSZ4~TvWyD5v4*j(MrlwWT zD+M(Su*C2GQ%3&eLnyZ5ic<6)1Nr6kO!H>e#o{Ofc$%1KDucl2eljay1ZsFswOQ9z z(>b~wR5KZYi3kqVkkRnC1}Y%2Gpux45Nd``SNmzL;C&gx%FcEnl@G&8r;v(S6F#MQ zl!U0L7iq2pVU5s<3pCu!+`EA!<5%*otCsgLUEVEb%hArI6^P)IZbbprOcB7z`kyR4 z0@i!u;0a=)w}UKm4OzJ*EnIt-CaBCvib+MCMxom2F`EAFbpS0(>TRjj#Xln?g?AJu_u_KkaB*kwkbQjTu`UijD5$TP!}+^zqh5_l(K7M_PsLF%NTTOM z_W(gggRN^sMW#8Ut_r#a6$%%bs#1{{^P4N#YKxtK<}pB%+BnYj?)kc%di8YPr$y&L zVPgT70=SWy0_D{e@1ACAf9P}3JE@2V@f(0h0rIA`=Ql8B1P2^VW26|e>$iEz`P!Ti z_%m_B2ia^S)lTa~OrHHvVPj;xTsh+j{C;U@ElG;hPB;={aU@BDI7~`Kl!K#nXpNnf zc>*L@wd0^ofR3#Hyu03|*sQXq)@hSsL{ zl>o%x7LEcUq7*n0t&zfA;p+*+d(3Y?x$Ga6;i+X((~P4Hy4= z?gmG1c>GtzO4YG+_sglcQ;fG>6kOdY9D3Tj9%e*a%I$FP?|hQD(fSOA65HPae`Jr| zAv}5VNCQX6%$&E(iQm+oz)!hwDW*|~=HL-6a-Rak!-RuKpYpN#3VMerd)6@)p2a9q zZ2P<9r?YF@9Dm($e(=f084+>XtGplgqz9gi(*d)$s8h17hJ@s~JLD{6U&@px(JIRf zD$JGU%37O=dxnMfp1+~jTuMFviM7f?7DCDK`a6MwpRxQ_sH+OuP1I=GR(?GZzU2A* zy)HE?nZDYknobt-t(nRc>pa=L(Dfr?5p{nAK^KZDeK>h1wZ@s+(GR6A;X217 zvu<(TCTA&{_TT2dMErvE;d4goBc6(#(Cr=ZpVY@Z4TI+eo_zv5w$M!{XIJ%(mq02536o~4q z2?hM9kO6X~hfJEfwxeioM;7oI{SzXRezG@%)b8(Ddlv+)9v6jTpz|C*KT_W>`%TRu zE5UN5^!btafTQq|}lrP^~5HVUIm|&?yh%X-+U7^XRtqCkSh%XJSZj2U_`G2uYvhNz6x5bfd z-{YMz$G|U-P@iz1nr)vt1-8bwT{ufKe2{4{%soXhB7PK7#rA{*GVgpK#p)}UX(u{Q z;9-TU{!4lfA(380mvQMo;p6_6Esb~h$+T#^o{^Oi(+FE<^zY)p;`sFO^cu{M9T%WAvYU$kB#lMZ4G*m!B z*fr_dD(BCnQOl#FNi6eTwn??U+j`ZWeZFJu)Y1&(DCSy&iwyySP0nF>br?PH6I#0yCa#X_wiM-F?_`NF|E8CV}0jomeYL&zn^= z^*ukzW*>V#sS5g%1yTyWbE4)AwcwpB-UY)x8YA;qbEWjI%%eib4c`xb47Eqj;zl{V z@jbQY9H)M7qhw^q&re3n$B>@vX0q2pV@OGM0q2zLf>?V#8S8C~?8AA^ZR}6ajpv8! zKM%yo1XLv<5y4oD`yCWjknZ2Zj_{S6iC5TWSUXGNyN|}3k<0MI&ZbbsB%l^EnPB*BsU`GXV-UyO%s96= zHRJ#f7PwGK>JGQNn6C)f(|Cr&rCj==+I1EP4OJUMjOrFA=CR~G$>HH)k~|c#@hJhL z#S}Q29?hAvRt8WvM8;eVc{%kvM+XiqhvkV@@ejMiK{u0t_flULZKh5k%6XGmvt}Q51q7$_U z_V55XJ3Oipq(rTqnhjb_xZI4?kybI)xfBO-*jc|dCjP# zIke0hT+7$`p*H`Jqxz0;zee#Z)x3RakaPE{hiBUlz1l%R#U9jM)Nm5iTME3d-0C07 zlO4Z5>LM?)SkO^_lj1~>+(7~USxhPvQ5=0Re~N;Vdl+Gf5Ln}`Efm^o;X#ls=CGgn zG!MAdWWUNi{;HXe{50jb1KdqKJ`zE$iVGE5 zAav6h7viXFEZuJ9;sU~uVb+%qsd`)%7^B>kFox}V*mCqDpC_*WGN(ZgtVR{sqOk} zgJlWB3}LWCG|n7UJYS%V&rnUHJNK@tA_MV*XkGz1Rq{$ruOI%zJJoMD8C~YK8T6}L zy!r1);=B!*o?Rz@5yA(y=MG7=kdK8Xp|*`~N#NtZeGGl~qqn8*G#-V~+P>aN{k|4o{=h78Z~hg?6njz1BE z$geY60{q#BAO>g8v&)*{ms&AoMFi@?uv@ z_Y|2T-dEKXyBv|^nIcTnBQ%R*5l|Ea^0Si*Z_f5wlSta1zQ;HsQ8)_t0ie}lU{ozV)N=g7jETQa0xFFL z$pt=t!q72i4=CD@#b~FVg+EBAs7-;I#VrJHz#$FrKe$E$2obwWwG4Ddm&YaU=hd2+Um4=;&7|4-_5|$xNa*hRXDaqp|eD zqH8PuIea)L8qX3BA!-A5Y>r2h8;{!oM5E*zXAA{Kp#PD9v%(sP^`P)R%oijS#Zv?P z;%oRr<%S#ApNXqI&5ZKW^@};*s75ZpvWrRK1x|h+b&f^OPFex2! z$U9QT+*@grOIPj}9TK6ym^+j~=uTNlChC|3(-!x zXfXx5!a*59B@t!r$Ihy16gn@m@!T2>8=DxO9ioknKu#zz2ZKPc5vL0B*Iv6Y)JEhs zUs8oG%*94_!l|&a97!G(%5=*m3bHitp7i(JqDaEFtQcBi-lGx*($Z=hwD-|UN52Kr z_4HMdY0-;rul=jw2hcrdo|*l(h%wCSTNt?-x#f|cNie)nk>rTJxt=a~sgyBQV6as> z7Vlyo^8R2mvQbU3A!C$I*<*JSzX;;m-6GdT{U(LB!;&(7ym)z&7pw-`s#3$2lsE&u zgP+BUCFqZFs3p}&^F+I!nAH6m45ulBaJhdSZ@5s*=`rd1oUf)}!GF3rfyl4c0%e}# zJ)|e(j1>gi72@~L&7%3fsojGs0oeVZag66QnFu#IB+NvC^y80eHF5q#{tIvA5Ht$f z<-!SzGt7-XTkzsd~~9>PlE21q9;t_yf|XOJpEc z0vG{$W=^5+O@8E9&Pmi(CmSeCD(`&kmhG@?;_YwCwodBu_pRIygC%P{UH^-dRTNn3LG710@6fC{ zQ+%VHvljkb7|NDM+&DkuOiu7bmI|ABd3>NEe{A!DP)|V~{(GzO(f0upzc-}JAS{AL z&|2+sNl{OTWC^VR$NTC!UxZEZkL2Brh(1y^gUfptG+Ku^dM2VDPZ5=`cF7@ABv$ic&bBQeddW@wW&PQd6Vf3d}(Bjvg21UtG z619iSwrvXWUj_3LDtC$6wS-CqB~_!ND(Vp%#AE2jWQ244S)sefcd27r6Kd!&-i|_g z-(KO&II@s1IzGtG%Ih$2dW~dz`n!6t% zCNAYJ5bNG=2VJV?lxXLIN9->vI+3IUf7=%ucHnh<)hh{0R zLjz9c&EU6)X^}1A`JhZZJV}N7v_)xKtm?9xBZAJjG3Kq5vTbWq>ce+#-0HKvR~aRk zJT@lC4Or973fudF+8rW1L8^_^FzwAuIScrHoslg{5|K?CpXX`0O3>(WTm9Wc`ZHE~ zvL^pal7EkJ=#5MF$H|smSVx{);xtYqo*K^c^FeL_f||Cb0c$#^#Y2<7%!fz+H$mmK zmCi1b4qf-KnH8b?k$QXV1N-N*@oX6a^RDGpsSg%J$u<->W%^5VEH8bLG@FuD#8Bld zhf4veZy_?=W*GERlWoDiC)Hj*2_!!MH4zzEzJP9UY`t@>-?8+mro8!kV9VS1@O85Y zat>({Zi)O+^CmFXVe;%qgVO5lTPtqrwdz{63)3?b7i+8+`yINq&xOzd!+3pK-qyH)%+hz_0}w2HpBeevIt zSof@we~7}^yCHiz5`VY;nHe1Sq5wV8{nA-iczx2ld-5eRHF5cj(k3!90Lb-%(D7r| zx+{OC9sqy~w2+WcfJsRFPs;2yQI_TvBLwXdCGYvHEGwx+UdFf)oz1TaB(R@m#?KZn zBGCHE(Qt{8!UbkxkMy_d=!ohm3HbJ_0=vSW>K1NecPnp~hVHqNsV4Mnt)BAaSm<~K zPP}E9tJI2o-09Cpx(!46CT8T%xb1+>-4LqxLrHOYV8OmQAy3biZ)dD0jy{n;Q+^Sz z;f0wc8!UE{bx)=K?r_WG>kO}!@%$ucI*AlU*-%mc*xN8}>}Mu5>|~k2C(cU9ud%1E z#>KtCv{6*D7ys5+N9}{H{p_IXY$Jn@0b^k$t7G~$2S}qE1P)bDkVG??51lB9Go+S$ zu*Z2(n0gx{ORbI;L5@)*8nAR3@X~Q&Q~tOWn2H~c;78PVM-1oRB$~U;x7rsu#RvB| z)}F9(t-jwr%&=Gcn@|Yez0U93!S8}YxI}^WwK(cj2j%lA{ByUpr-)W2jLT(sAuAi( za<4RDfW+Z#TQsHAE>?glae&eKan}jCSikmnLU;ma*Yu}+ z8pedM_s%lCmbq+xf#C|_654K@KvW~vXo2a@3j;IN>vpk1g^LBzoy$^YLq<^~QE2rUMP8UsYbd)kUh zLMAYy%?JVPu#ia2;+ zjrjZp5_>Ni3ML_k7&22qVllC4I0=9>Fl3aNBy`jma6ka`BU1MgG`2?!TmWzQ3G84G z3@i3c!U$913++}B9rgpGkuR&N?#@>sXquS#QUSxl{PK8W!S-pIjrc2#{E+*H7;X=q zm?UzjB~6SU?ZyoP;Qsm~F`|3JuD!`Ds=4kXxDhE<3yR-vPZt|Aoa7fe7Sf7<5(<{| z%R>es*@MHuzsfNElncjwwoqg87=cE#ig402P{eOYw8t$(D=DObt+)PACHzp%7_eXN zj;)Wx=0=abJ*Fg0uY_S8t<|xP)0b6j#f=M4*qJXe1}yf1oG=mNbtLsPN;NdMyKRZKRo(H)f;OFqY`1==C zuO9e>lMv%~DETe^(R7RZm>(%e9*oXT2}2$bwSCL}bnL=O87Jh^Iyr(uGrth7dD@`Atj`?n`G3T#yD z5K;=oR{_!Mk~BscwBz%kc)v_x64fqmCvDq=Y^0Yjc603)4G@4R$kU|d)Sz^Dk~d)V zkMO}MsdUs;-^s(mxVigTS-w5%sZ~I>(_MN!GZY-(cPl_!B*b5>mBR)?GV9N$soLJk z+UmmLbbfN57tDBhc?res_gP1Zco?zWIBwl7FI8?U0u9vmNhF%jq%tU4+=U8>Q1Z^q zbTDNr3$I<#%+I58;`=*xB5F7~fIYn+3NJ^txhLb)a?hw~3=?mGSJ6fFznK-mVFf2^ zVrfa761BppF~k_5%+TuX;g0V!Gr&XVo>xOVTrgy|b9y&BSj%VPTul{Zzx*1@dof5K z?Rb^}i-VaaQ50L`C!1_3*?ms^^7^hRg~H$C2qys}S-fI{-7bJOw(8s3g5*+Dc=F)D zt_|nhyhc)_B~?A=GJYSQaCSTVE(0C!6QTNok7A2?uqqb6z5uvNBCm%VhDZMONbJj} zO-kacr<0+Sgs98wjiubE9LK)MK)a>e9TD1#iYC{)3N&9#N|2c7?BPP_-<0$$CHcO!x2eVb18U6&_KNGqY zHC&ff>-|T9?K}kBU!GmA?0?24A&0I<8!vnDzUsc=xnpb?^6-Ogf0GKN49AHH>M-o6 z4@aNxi)?vD_`;)ee`HG?-1XnRB+g*MLcEp5#UzILev~(&$0e#0b=M@sqJK&&#W4|U zI{rbHcV*C}_hy}%DuEd(DF+olN797IcvaNI?f7$Vo*wSUwG?B<(RCO8#T*r}NnKT> zhxsaKCO`Lfug#uvA&uDtO`x_mZmScT|1!3;xA1wU&znsiYyY0V;dffu7dsP;7IBt* z-F|SW{_QQke?6n}fX8_XZed zI@WJChtL1^H|}KIX8K9AGwF({6oW${g{%fr1}I{M(Fsu;TTFpfLB)IJr4k22Z~KHu;(8TT6NsH?IFW!~ zqYqcGz76FQ$NC-Hq}{*%vU1$cNG6>#6Ak?c2XO=Epq?Brhu(U=zL#;eSCP9nx%LB~ z%cqOX#WgAr`~)JJ?<;Bo`M#pl9yc%)AGshmHciwUqp_>2TYU?FqC!}}PtuWV=MN{e zMvGkY7w2H!sphX=`M$1t*;g{jn`Y!5q`QotAAFV-hzSc6ot6UGSV{Dy+S1sagVXRB zI(ZznzNYHpRuZi&YNcs6fY=V^XqRU4`LF?Y%MP-JR^6fCm>cVqqH5eg{n}1W8}iAv zE4DnjBtXZyXi+)Vvr*qGeur*pQ}<_3%NxR4w;JByjm0G$ks9f)xQkW&rG<3T|5!=U zZtOT~qgU*dJ;(cEgBIQ9mmZU%ZHkb?0v?D^b352yV`YzYqbUsnFUcv;e)vCh>^mQ z{V*{=lJs!g5YX{{csOl&9%62$D|}0)XKb}ZyD3Tz6G+mmpIg*$WVAPErZjgWNkcv4 z@a{s{JtI(*OFU=r9Je;c^K#;=z?Qn*{KbW`_q!NTF43IDMdsQV9R~Gv;`62A@K;vR z*g6{uL6OJob3QI92VHWe$bX0%0^!3(1P7&zQgv31TzYkFPuT1}Evq*O3t^}9f# zHu5wkAwu8nxUitH2*HKxcP!V|eJ8LN!=u?@~uo&}6(NCQ@@ zBV!mG+}b3T0uyE_$o@;C@VQ0kOk`*BfGClp`6Z!-?W*tTH2UXd=>uUAWBOdLO!VNA zH@LVJu(+HR*8UkM@L%<63hu@wdeu1rnmtrpoXQjuTf&DZw#a>k_G<25)G#q} z{}JSy2olD{W|E3YC07C+q>ogl;<1K&tTwz-?3(^(wr$^=GanI0V?p(1!wOEtg0BXI zU3Uf%zuX@L_Zf^0NvtT#8$4bg5!ZTtIBaNmvDISo6#B=^u+u}$dKuZx)jgRgK18@z z@ev+>3GiF5NvpR$R{}-CoMnu(!+&) z>RTwJ89ug)%H#L}VH;4Xr!^FFpG~Y0dkr1-h}%(@=$p$ayJctX4LQaz3$MQfI~%WynE-{U42FhDPn4hu@@dEBqsx-i6Yr$ zIViU6nIJBTB48&*T7P6v=KjuKPLA)hUSuvE9cqREimbc)@Tl96?Aw7Bd&aKqALmu3 zrZi?}h7p=A?sj~dqcq}*ke9Pf^M#*q%RcvbG~Obt>_i3Ab6tuW#8-2VeUa<8)5X>I zYCIXAnN(pvFbfxwn2Kc3t$-{`|W=cpw6yIi(N`-OhX=e+BK(?D0gAA-$he z^whvbgH@@w@ zJYE!eq1>dqbaI=>4DamEx9aj>s{z$wx9*8Pd^j!B@S7-A>(JC;eW1yQ+P+<^vyrrt zlzhZx5Nnt7xastgzKjAR!}IaDy@UV$9kJ6q5CVKff2rdrp9saF9~~)ix$~)4KF=RM z4W}_#b*gbzVN54(!Vt92xn{$PXBl z$GgPji!Gj-69|1jV;R+8G%YRWK2KlCGv6T<60!N8h`iiYi;U~42KDC4#+86`DSZTw zK?$REt7`cF&SaPXURnsdX44-3ZGk7w#>VOMc0vY(hOD?a4)D;^yDUiamuPxL58$)wv-3M6=D7z z^-))|vovf2-EVZ|u;KkU?!^qNs7?w+zI%LcT9jx~SLMn; zi0FQ+G&5%dYgo}KSAE^z5y!6vthL%4^)cuCM-#`C7*O?X9*s3zK6 zwO3Vk?qb85y`15Wf}KNxt*VlYb8Z*QGuj^MwwGPH($meg8kB1Pm0H09Yi0}LXJO$c zTQos3aMFb3EazxpBHKIPIjE{1Rr_-UmxEVYJh)SD^pOlARADx)Bs;I)U) z?SCZ3ZbkoR6wKj2Qz!j5iSyr%|6kqJE&3`F4= limit) { + lastEventTimestamp = now; + baseFunction.apply(self, args); + } + }; +}; \ No newline at end of file diff --git a/changedetectionio/static/js/visual-selector.js b/changedetectionio/static/js/visual-selector.js new file mode 100644 index 00000000..e6fa9091 --- /dev/null +++ b/changedetectionio/static/js/visual-selector.js @@ -0,0 +1,219 @@ +// Horrible proof of concept code :) +// yes - this is really a hack, if you are a front-ender and want to help, please get in touch! + +$(document).ready(function() { + + $('#visualselector-tab').click(function () { + $("img#selector-background").off('load'); + bootstrap_visualselector(); + }); + + $(document).on('keydown', function(event) { + if ($("img#selector-background").is(":visible")) { + if (event.key == "Escape") { + state_clicked=false; + ctx.clearRect(0, 0, c.width, c.height); + } + } + }); + + // For when the page loads + if(!window.location.hash || window.location.hash != '#visualselector') { + $("img#selector-background").attr('src',''); + return; + } + + // Handle clearing button/link + $('#clear-selector').on('click', function(event) { + if(!state_clicked) { + alert('Oops, Nothing selected!'); + } + state_clicked=false; + ctx.clearRect(0, 0, c.width, c.height); + }); + + + bootstrap_visualselector(); + + var current_selected_i; + var state_clicked=false; + + var c; + + // greyed out fill context + var xctx; + // redline highlight context + var ctx; + + var current_default_xpath; + var x_scale=1; + var y_scale=1; + var selector_image; + var selector_image_rect; + var vh; + var selector_data; + + + function bootstrap_visualselector() { + if ( 1 ) { + // bootstrap it, this will trigger everything else + $("img#selector-background").bind('load', function () { + console.log("Loaded background..."); + c = document.getElementById("selector-canvas"); + // greyed out fill context + xctx = c.getContext("2d"); + // redline highlight context + ctx = c.getContext("2d"); + current_default_xpath =$("#css_filter").val(); + fetch_data(); + $('#selector-canvas').off("mousemove"); + // screenshot_url defined in the edit.html template + }).attr("src", screenshot_url); + } + } + + function fetch_data() { + // Image is ready + $('.fetching-update-notice').html("Fetching element data.."); + + $.ajax({ + url: watch_visual_selector_data_url, + context: document.body + }).done(function (data) { + $('.fetching-update-notice').html("Rendering.."); + selector_data = data; + console.log("Reported browser width from backend: "+data['browser_width']); + state_clicked=false; + set_scale(); + reflow_selector(); + $('.fetching-update-notice').fadeOut(); + }); + }; + + + + function set_scale() { + + // some things to check if the scaling doesnt work + // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq + selector_image = $("img#selector-background")[0]; + selector_image_rect = selector_image.getBoundingClientRect(); + + // make the canvas the same size as the image + $('#selector-canvas').attr('height', selector_image_rect.height); + $('#selector-canvas').attr('width', selector_image_rect.width); + $('#selector-wrapper').attr('width', selector_image_rect.width); + x_scale = selector_image_rect.width / selector_data['browser_width']; + y_scale = selector_image_rect.height / selector_image.naturalHeight; + ctx.strokeStyle = 'rgba(255,0,0, 0.9)'; + ctx.fillStyle = 'rgba(255,0,0, 0.1)'; + ctx.lineWidth = 3; + console.log("scaling set x: "+x_scale+" by y:"+y_scale); + $("#selector-current-xpath").css('max-width', selector_image_rect.width); + } + + function reflow_selector() { + $(window).resize(function() { + set_scale(); + highlight_current_selected_i(); + }); + var selector_currnt_xpath_text=$("#selector-current-xpath span"); + + set_scale(); + + console.log(selector_data['size_pos'].length + " selectors found"); + + // highlight the default one if we can find it in the xPath list + // or the xpath matches the default one + found = false; + if(current_default_xpath.length) { + for (var i = selector_data['size_pos'].length; i!==0; i--) { + var sel = selector_data['size_pos'][i-1]; + if(selector_data['size_pos'][i - 1].xpath == current_default_xpath) { + console.log("highlighting "+current_default_xpath); + current_selected_i = i-1; + highlight_current_selected_i(); + found = true; + break; + } + } + if(!found) { + alert("unfortunately your existing CSS/xPath Filter was no longer found!"); + } + } + + + $('#selector-canvas').bind('mousemove', function (e) { + if(state_clicked) { + return; + } + ctx.clearRect(0, 0, c.width, c.height); + current_selected_i=null; + + // Reverse order - the most specific one should be deeper/"laster" + // Basically, find the most 'deepest' + var found=0; + ctx.fillStyle = 'rgba(205,0,0,0.35)'; + for (var i = selector_data['size_pos'].length; i!==0; i--) { + // draw all of them? let them choose somehow? + var sel = selector_data['size_pos'][i-1]; + // If we are in a bounding-box + if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale + && + e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale + + ) { + + // FOUND ONE + set_current_selected_text(sel.xpath); + ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); + ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); + + // no need to keep digging + // @todo or, O to go out/up, I to go in + // or double click to go up/out the selector? + current_selected_i=i-1; + found+=1; + break; + } + } + + }.debounce(5)); + + function set_current_selected_text(s) { + selector_currnt_xpath_text[0].innerHTML=s; + } + + function highlight_current_selected_i() { + if(state_clicked) { + state_clicked=false; + xctx.clearRect(0,0,c.width, c.height); + return; + } + + var sel = selector_data['size_pos'][current_selected_i]; + if (sel[0] == '/') { + // @todo - not sure just checking / is right + $("#css_filter").val('xpath:'+sel.xpath); + } else { + $("#css_filter").val(sel.xpath); + } + xctx.fillStyle = 'rgba(205,205,205,0.95)'; + xctx.strokeStyle = 'rgba(225,0,0,0.9)'; + xctx.lineWidth = 3; + xctx.fillRect(0,0,c.width, c.height); + // Clear out what only should be seen (make a clear/clean spot) + xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); + xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale); + state_clicked=true; + set_current_selected_text(sel.xpath); + + } + + + $('#selector-canvas').bind('mousedown', function (e) { + highlight_current_selected_i(); + }); + } + +}); diff --git a/changedetectionio/static/js/watch-overview.js b/changedetectionio/static/js/watch-overview.js index 1431b1b9..a06034b1 100644 --- a/changedetectionio/static/js/watch-overview.js +++ b/changedetectionio/static/js/watch-overview.js @@ -4,6 +4,7 @@ $(function () { $(this).closest('.unviewed').removeClass('unviewed'); }); + $('.with-share-link > *').click(function () { $("#copied-clipboard").remove(); @@ -20,5 +21,6 @@ $(function () { $(this).remove(); }); }); + }); diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 26300bea..724be932 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -338,7 +338,8 @@ footer { padding-top: 110px; } div.tabs.collapsable ul li { display: block; - border-radius: 0px; } + border-radius: 0px; + margin-right: 0px; } input[type='text'] { width: 100%; } /* @@ -429,6 +430,15 @@ and also iPads specifically. .tab-pane-inner:target { display: block; } +#beta-logo { + height: 50px; + right: -3px; + top: -3px; + position: absolute; } + +#selector-header { + padding-bottom: 1em; } + .edit-form { min-width: 70%; /* so it cant overflow */ @@ -454,6 +464,24 @@ ul { .time-check-widget tr input[type="number"] { width: 5em; } +#selector-wrapper { + height: 600px; + overflow-y: scroll; + position: relative; } + #selector-wrapper > img { + position: absolute; + z-index: 4; + max-width: 100%; } + #selector-wrapper > canvas { + position: relative; + z-index: 5; + max-width: 100%; } + #selector-wrapper > canvas:hover { + cursor: pointer; } + +#selector-current-xpath { + font-size: 80%; } + #webdriver-override-options input[type="number"] { width: 5em; } diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss index 6066bcde..ca97be4a 100644 --- a/changedetectionio/static/styles/styles.scss +++ b/changedetectionio/static/styles/styles.scss @@ -469,6 +469,7 @@ footer { div.tabs.collapsable ul li { display: block; border-radius: 0px; + margin-right: 0px; } input[type='text'] { @@ -613,6 +614,18 @@ $form-edge-padding: 20px; padding: 0px; } +#beta-logo { + height: 50px; + // looks better when it's hanging off a little + right: -3px; + top: -3px; + position: absolute; +} + +#selector-header { + padding-bottom: 1em; +} + .edit-form { min-width: 70%; /* so it cant overflow */ @@ -649,6 +662,30 @@ ul { } } +#selector-wrapper { + height: 600px; + overflow-y: scroll; + position: relative; + //width: 100%; + > img { + position: absolute; + z-index: 4; + max-width: 100%; + } + >canvas { + position: relative; + z-index: 5; + max-width: 100%; + &:hover { + cursor: pointer; + } + } +} + +#selector-current-xpath { + font-size: 80%; +} + #webdriver-override-options { input[type="number"] { width: 5em; diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 2ef09c54..45d9a0a5 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -372,6 +372,15 @@ class ChangeDetectionStore: return False + def visualselector_data_is_ready(self, watch_uuid): + output_path = "{}/{}".format(self.datastore_path, watch_uuid) + screenshot_filename = "{}/last-screenshot.png".format(output_path) + elements_index_filename = "{}/elements.json".format(output_path) + if path.isfile(screenshot_filename) and path.isfile(elements_index_filename) : + return True + + return False + # Save as PNG, PNG is larger but better for doing visual diff in the future def save_screenshot(self, watch_uuid, screenshot: bytes): output_path = "{}/{}".format(self.datastore_path, watch_uuid) @@ -380,6 +389,14 @@ class ChangeDetectionStore: f.write(screenshot) f.close() + def save_xpath_data(self, watch_uuid, data): + output_path = "{}/{}".format(self.datastore_path, watch_uuid) + fname = "{}/elements.json".format(output_path) + with open(fname, 'w') as f: + f.write(json.dumps(data)) + f.close() + + def sync_to_json(self): logging.info("Saving JSON..") print("Saving JSON..") diff --git a/changedetectionio/templates/diff.html b/changedetectionio/templates/diff.html index c32da120..db8b7f73 100644 --- a/changedetectionio/templates/diff.html +++ b/changedetectionio/templates/diff.html @@ -39,9 +39,6 @@
@@ -63,18 +60,6 @@ Diff algorithm from the amazing github.com/kpdecker/jsdiff - -{% if screenshot %} -
-

- For now, only the most recent screenshot is saved and displayed.
- Note: No changedetection is performed on the image yet, but we are working on that in an upcoming release. -

- - -
-{% endif %} - diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 17ad6eb7..bf5e7aa3 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -5,12 +5,18 @@ + +
@@ -18,6 +24,7 @@ @@ -194,6 +201,46 @@ nav
+
+ + +
+
+ {% if visualselector_enabled %} + {% if visualselector_data_is_ready %} +
+ Clear selection + One moment, fetching screenshot and element information.. +
+
+ + + + + + +
+
Currently: Loading...
+ + +

Beta! The Visual Selector is new and there may be minor bugs, please report pages that dont work, help us to improve this software!

+ + + {% else %} + Screenshot and element data is not available or not yet ready. + {% endif %} + {% else %} + +

Sorry, this functionality only works with Playwright/Chrome enabled watches.

+

Enable the Playwright Chrome fetcher, or alternatively try our very affordable subscription based service.

+

This is because Selenium/WebDriver can not extract full page screenshots reliably.

+ + + {% endif %} +
+
+
+
diff --git a/changedetectionio/templates/preview.html b/changedetectionio/templates/preview.html index 25ff4986..7015d795 100644 --- a/changedetectionio/templates/preview.html +++ b/changedetectionio/templates/preview.html @@ -10,9 +10,6 @@
@@ -31,16 +28,5 @@
- -{% if screenshot %} -
-

- For now, only the most recent screenshot is saved and displayed.
- Note: No changedetection is performed on the image yet, but we are working on that in an upcoming release. -

- - -
-{% endif %}
{% endblock %} \ No newline at end of file diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 1f37d5bc..f2d0c857 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -3,6 +3,7 @@ {% from '_helpers.jinja' import render_simple_field %} +
diff --git a/changedetectionio/tests/fetchers/__init__.py b/changedetectionio/tests/fetchers/__init__.py new file mode 100644 index 00000000..085b3d78 --- /dev/null +++ b/changedetectionio/tests/fetchers/__init__.py @@ -0,0 +1,2 @@ +"""Tests for the app.""" + diff --git a/changedetectionio/tests/fetchers/conftest.py b/changedetectionio/tests/fetchers/conftest.py new file mode 100644 index 00000000..430513d4 --- /dev/null +++ b/changedetectionio/tests/fetchers/conftest.py @@ -0,0 +1,3 @@ +#!/usr/bin/python3 + +from .. import conftest diff --git a/changedetectionio/tests/fetchers/test_content.py b/changedetectionio/tests/fetchers/test_content.py new file mode 100644 index 00000000..02c2c026 --- /dev/null +++ b/changedetectionio/tests/fetchers/test_content.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +import time +from flask import url_for +from ..util import live_server_setup +import logging + + +def test_fetch_webdriver_content(client, live_server): + live_server_setup(live_server) + + ##################### + res = client.post( + url_for("settings_page"), + data={"application-empty_pages_are_a_change": "", + "requests-time_between_check-minutes": 180, + 'application-fetch_backend': "html_webdriver"}, + follow_redirects=True + ) + + assert b"Settings updated." in res.data + + # Add our URL to the import page + res = client.post( + url_for("import_page"), + data={"urls": "https://changedetection.io/ci-test.html"}, + follow_redirects=True + ) + + assert b"1 Imported" in res.data + time.sleep(3) + attempt = 0 + while attempt < 20: + res = client.get(url_for("index")) + if not b'Checking now' in res.data: + break + logging.getLogger().info("Waiting for check to not say 'Checking now'..") + time.sleep(3) + attempt += 1 + + + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + logging.getLogger().info("Looking for correct fetched HTML (text) from server") + + assert b'cool it works' in res.data \ No newline at end of file diff --git a/changedetectionio/tests/test_trigger.py b/changedetectionio/tests/test_trigger.py index 7eacaff5..66b8121e 100644 --- a/changedetectionio/tests/test_trigger.py +++ b/changedetectionio/tests/test_trigger.py @@ -121,7 +121,7 @@ def test_trigger_functionality(client, live_server): res = client.get(url_for("index")) assert b'unviewed' not in res.data - # Just to be sure.. set a regular modified change.. + # Now set the content which contains the trigger text time.sleep(sleep_time_for_fetch_thread) set_modified_with_trigger_text_response() @@ -130,6 +130,12 @@ def test_trigger_functionality(client, live_server): res = client.get(url_for("index")) assert b'unviewed' in res.data + # https://github.com/dgtlmoon/changedetection.io/issues/616 + # Apparently the actual snapshot that contains the trigger never shows + res = client.get(url_for("diff_history_page", uuid="first")) + assert b'foobar123' in res.data + + # Check the preview/highlighter, we should be able to see what we triggered on, but it should be highlighted res = client.get(url_for("preview_page", uuid="first")) # We should be able to see what we ignored diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index c23ae82a..0e2b344f 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -40,10 +40,11 @@ class update_worker(threading.Thread): contents = "" screenshot = False update_obj= {} + xpath_data = False now = time.time() try: - changed_detected, update_obj, contents, screenshot = update_handler.run(uuid) + changed_detected, update_obj, contents, screenshot, xpath_data = update_handler.run(uuid) # Re #342 # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes. @@ -55,6 +56,7 @@ class update_worker(threading.Thread): except content_fetcher.ReplyWithContentButNoText as e: # Totally fine, it's by choice - just continue on, nothing more to care about # Page had elements/content but no renderable text + self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found."}) pass except content_fetcher.EmptyReply as e: # Some kind of custom to-str handler in the exception handler that does this? @@ -148,6 +150,9 @@ class update_worker(threading.Thread): # Always save the screenshot if it's available if screenshot: self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot) + if xpath_data: + self.datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data) + self.current_uuid = None # Done self.q.task_done()