When fetching a snapshot via Chrome, make the most recent screenshot available on the Diff and Preview pages (#516)

changedetectionio/__init__.py

@@ -625,6 +625,7 @@ def changedetection_app(config=None, datastore_o=None):
             form.notification_body.data = datastore.data['settings']['application']['notification_body']
             form.notification_format.data = datastore.data['settings']['application']['notification_format']
             form.base_url.data = datastore.data['settings']['application']['base_url']
+            form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot']
 
         if request.method == 'POST' and form.data.get('removepassword_button') == True:
             # Password unset is a GET, but we can lock the session to a salted env password to always need the password
@@ -647,7 +648,8 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data
             datastore.data['settings']['application']['global_ignore_text'] =  form.global_ignore_text.data
             datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data
-
+            datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data
+            
             if form.trigger_check.data:
                 if len(form.notification_urls.data):
                     n_object = {'watch_url': "Test from changedetection.io!",
@@ -776,6 +778,9 @@ def changedetection_app(config=None, datastore_o=None):
         except Exception as e:
             previous_version_file_contents = "Unable to read {}.\n".format(previous_file)
 
+
+        screenshot_url = datastore.get_screenshot(uuid)
+
         output = render_template("diff.html", watch_a=watch,
                                  newest=newest_version_file_contents,
                                  previous=previous_version_file_contents,
@@ -786,7 +791,8 @@ def changedetection_app(config=None, datastore_o=None):
                                  current_previous_version=str(previous_version),
                                  current_diff_url=watch['url'],
                                  extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
-                                 left_sticky=True)
+                                 left_sticky=True,
+                                 screenshot=screenshot_url)
 
         return output
 
@@ -846,15 +852,17 @@ def changedetection_app(config=None, datastore_o=None):
         else:
             content.append({'line': "No history found", 'classes': ''})
 
-
+        screenshot_url = datastore.get_screenshot(uuid)
         output = render_template("preview.html",
                                  content=content,
                                  extra_stylesheets=extra_stylesheets,
                                  ignored_line_numbers=ignored_line_numbers,
                                  triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
+                                 screenshot=screenshot_url,
                                  watch=watch,
                                  uuid=uuid)
+        
         return output
 
     @app.route("/settings/notification-logs", methods=['GET'])
@@ -967,6 +975,28 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
+        if group == 'screenshot':
+
+            from flask import make_response
+
+            # Could be sensitive, follow password requirements
+            if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+                abort(403)
+
+            # These files should be in our subdirectory
+            try:
+                # set nocache, set content-type
+                watch_dir = datastore_o.datastore_path + "/" + filename
+                response = make_response(send_from_directory(filename="last-screenshot.png", directory=watch_dir, path=watch_dir + "/last-screenshot.png"))
+                response.headers['Content-type'] = 'image/png'
+                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response.headers['Pragma'] = 'no-cache'
+                response.headers['Expires'] = 0
+                return response
+
+            except FileNotFoundError:
+                abort(404)
+
         # These files should be in our subdirectory
         try:
             return send_from_directory("static/{}".format(group), path=filename)

changedetectionio/content_fetcher.py

@@ -42,6 +42,14 @@ class Fetcher():
         # Should set self.error, self.status_code and self.content
         pass
 
+    @abstractmethod
+    def quit(self):
+        return
+
+    @abstractmethod
+    def screenshot(self):
+        return
+
     @abstractmethod
     def get_last_status_code(self):
         return self.status_code
@@ -116,16 +124,16 @@ class html_webdriver(Fetcher):
         # request_body, request_method unused for now, until some magic in the future happens.
 
         # check env for WEBDRIVER_URL
-        driver = webdriver.Remote(
+        self.driver = webdriver.Remote(
             command_executor=self.command_executor,
             desired_capabilities=DesiredCapabilities.CHROME,
             proxy=self.proxy)
 
         try:
-            driver.get(url)
+            self.driver.get(url)
         except WebDriverException as e:
             # Be sure we close the session window
-            driver.quit()
+            self.quit()
             raise
 
         # @todo - how to check this? is it possible?
@@ -135,26 +143,33 @@ class html_webdriver(Fetcher):
 
         # @todo - dom wait loaded?
         time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-        self.content = driver.page_source
+        self.content = self.driver.page_source
         self.headers = {}
 
-        driver.quit()
+    def screenshot(self):
-
+        return self.driver.get_screenshot_as_png()
 
+    # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
         from selenium.common.exceptions import WebDriverException
 
-        driver = webdriver.Remote(
+        self.driver = webdriver.Remote(
             command_executor=self.command_executor,
             desired_capabilities=DesiredCapabilities.CHROME)
 
         # driver.quit() seems to cause better exceptions
-        driver.quit()
+        self.quit()
-
         return True
 
+    def quit(self):
+        if self.driver:
+            try:
+                self.driver.quit()
+            except Exception as e:
+                print("Exception in chrome shutdown/quit" + str(e))
+
 # "html_requests" is listed as the default fetcher in store.py!
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"

changedetectionio/fetch_site_status.py

@@ -21,6 +21,7 @@ class perform_site_check():
         timestamp = int(time.time())  # used for storage etc too
 
         changed_detected = False
+        screenshot = False # as bytes
         stripped_text_from_html = ""
 
         watch = self.datastore.data['watching'][uuid]
@@ -171,5 +172,9 @@ class perform_site_check():
                     if not watch['title'] or not len(watch['title']):
                         update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
 
+            if self.datastore.data['settings']['application'].get('real_browser_save_screenshot', True):
+                screenshot = fetcher.screenshot()
 
-        return changed_detected, update_obj, text_content_before_ignored_filter
+            fetcher.quit()
+
+        return changed_detected, update_obj, text_content_before_ignored_filter, screenshot

changedetectionio/forms.py

@@ -345,7 +345,6 @@ class watchForm(commonSettingsForm):
         return result
 
 class globalSettingsForm(commonSettingsForm):
-
     password = SaltyPasswordField()
     minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                                [validators.NumberRange(min=1)])
@@ -355,4 +354,5 @@ class globalSettingsForm(commonSettingsForm):
     global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
     ignore_whitespace = BooleanField('Ignore whitespace')
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/static/js/tabs.js

@@ -1,6 +1,11 @@
 // Rewrite this is a plugin.. is all this JS really 'worth it?'
 
 
+if(!window.location.hash) {
+  var tab=document.querySelectorAll("#default-tab a");
+  tab[0].click();
+}
+
 window.addEventListener('hashchange', function() {
   var tabs = document.getElementsByClassName('active');
   while (tabs[0]) {
@@ -21,7 +26,6 @@ if (!has_errors.length) {
   focus_error_tab();
 }
 
-
 function set_active_tab() {
   var tab=document.querySelectorAll("a[href='"+location.hash+"']");
   if (tab.length) {

changedetectionio/static/styles/diff.css

@@ -1,7 +1,8 @@
 #diff-ui {
   background: #fff;
   padding: 2em;
-  margin: 1em;
+  margin-left: 1em;
+  margin-right: 1em;
   border-radius: 5px;
   font-size: 11px; }
   #diff-ui table {
@@ -70,3 +71,8 @@ td#diff-col div {
 /* ignored and triggered? make it obvious error */
 .ignored.triggered {
   background-color: #ff0000; }
+
+.tab-pane-inner#screenshot {
+  text-align: center; }
+  .tab-pane-inner#screenshot img {
+    max-width: 99%; }

changedetectionio/static/styles/diff.scss

@@ -2,7 +2,8 @@
 
     background: #fff;
     padding: 2em;
-    margin: 1em;
+    margin-left: 1em;
+    margin-right: 1em;
     border-radius: 5px;
     font-size: 11px;
 
@@ -85,4 +86,11 @@ td#diff-col div {
 /* ignored and triggered? make it obvious error */
 .ignored.triggered {
   background-color: #ff0000;
+}
+
+.tab-pane-inner#screenshot {
+  text-align: center;
+  img {
+    max-width: 99%;
+  }
 }

changedetectionio/static/styles/styles.css

@@ -317,7 +317,7 @@ footer {
     right: auto; }
   section.content {
     padding-top: 110px; }
-  div.tabs ul li {
+  div.tabs.collapsable ul li {
     display: block;
     border-radius: 0px; }
   input[type='text'] {
@@ -403,14 +403,15 @@ and also iPads specifically.
   padding: 20px;
   border-radius: 5px; }
 
+.tab-pane-inner {
+  padding: 0px; }
+  .tab-pane-inner:not(:target) {
+    display: none; }
+  .tab-pane-inner:target {
+    display: block; }
+
 .edit-form {
   min-width: 70%; }
-  .edit-form .tab-pane-inner {
-    padding: 0px; }
-    .edit-form .tab-pane-inner:not(:target) {
-      display: none; }
-    .edit-form .tab-pane-inner:target {
-      display: block; }
   .edit-form .box-wrap {
     position: relative; }
   .edit-form .inner {

changedetectionio/static/styles/styles.scss

@@ -35,7 +35,7 @@ a.github-link {
 
 section.content {
   padding-top: 5em;
-  padding-bottom: 5em;
+  padding-bottom: 1em;
   flex-direction: column;
   display: flex;
   align-items: center;
@@ -437,7 +437,7 @@ footer {
   }
 
   // Make the tabs easier to hit, they will be all nice and horizontal
-  div.tabs ul li {
+  div.tabs.collapsable ul li {
     display: block;
     border-radius: 0px;
   }
@@ -573,10 +573,7 @@ $form-edge-padding: 20px;
   }
 }
 
-
+.tab-pane-inner {
-.edit-form {
-  min-width: 70%;
-  .tab-pane-inner {
     &:not(:target) {
         display: none;
     }
@@ -585,7 +582,11 @@ $form-edge-padding: 20px;
     }
     // doesnt need padding because theres another row of buttons/activity
     padding: 0px;
-  }
+}
+
+.edit-form {
+  min-width: 70%;
+
   .box-wrap {
     position: relative;
   }

changedetectionio/store.py

@@ -57,6 +57,7 @@ class ChangeDetectionStore:
                     'notification_title': default_notification_title,
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
+                    'real_browser_save_screenshot': True,
                 }
             }
         }
@@ -381,6 +382,22 @@ class ChangeDetectionStore:
 
         return fname
 
+    def get_screenshot(self, watch_uuid):
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/last-screenshot.png".format(output_path)
+        if path.isfile(fname):
+            return fname
+
+        return False
+
+    # Save as PNG, PNG is larger but better for doing visual diff in the future
+    def save_screenshot(self, watch_uuid, screenshot: bytes):
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/last-screenshot.png".format(output_path)
+        with open(fname, 'wb') as f:
+            f.write(screenshot)
+            f.close()
+
     def sync_to_json(self):
         logging.info("Saving JSON..")
 

changedetectionio/templates/diff.html

@@ -1,7 +1,6 @@
 {% extends 'base.html' %}
 
 {% block content %}
-
 <div id="settings">
     <h1>Differences</h1>
     <form class="pure-form " action="" method="GET">
@@ -35,21 +34,45 @@
 <div id="diff-jump">
     <a onclick="next_diff();">Jump</a>
 </div>
+
+{% if screenshot %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+        <li class="tab"><a href="#screenshot">Screenshot</a></li>
+    </ul>
+</div>
+{% endif %}
+
 <div id="diff-ui">
-    <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
+     <div class="tab-pane-inner" id="text">
-    <table>
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.
-        <tbody>
+         </div>
-        <tr>
+         <table>
-            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+             <tbody>
-            <td id="a" style="display: none;">{{previous}}</td>
+             <tr>
-            <td id="b" style="display: none;">{{newest}}</td>
+                 <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-            <td id="diff-col">
+                 <td id="a" style="display: none;">{{previous}}</td>
-                <span id="result"></span>
+                 <td id="b" style="display: none;">{{newest}}</td>
-            </td>
+                 <td id="diff-col">
-        </tr>
+                     <span id="result"></span>
-        </tbody>
+                 </td>
-    </table>
+             </tr>
-    Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+             </tbody>
+         </table>
+         Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+     </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
 
 </div>
 

changedetectionio/templates/edit.html

@@ -7,7 +7,7 @@
 
 <div class="edit-form monospaced-textarea">
 
-    <div class="tabs">
+    <div class="tabs collapsable">
         <ul>
             <li class="tab" id="default-tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>

changedetectionio/templates/preview.html

@@ -6,18 +6,40 @@
     <h1>Current - {{watch.last_checked|format_timestamp_timeago}}</h1>
 </div>
 
+{% if screenshot %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+        <li class="tab"><a href="#screenshot">Screenshot</a></li>
+    </ul>
+</div>
+{% endif %}
+
 <div id="diff-ui">
-    <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
+    <div class="tab-pane-inner" id="text">
-    <table>
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
-        <tbody>
+        <table>
-        <tr>
+            <tbody>
-            <td id="diff-col">
+            <tr>
+                <td id="diff-col">
                     {% for row in content %}
                     <div class="{{row.classes}}">{{row.line}}</div>
                     {% endfor %}
-            </td>
+                </td>
-        </tr>
+            </tr>
-        </tbody>
+            </tbody>
-    </table>
+        </table>
+    </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
 </div>
 {% endblock %}

changedetectionio/templates/settings.html

@@ -8,7 +8,7 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 
 <div class="edit-form">
-    <div class="tabs">
+    <div class="tabs collapsable">
         <ul>
             <li class="tab" id="default-tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#notifications">Notifications</a></li>
@@ -50,6 +50,12 @@
                         {{ render_field(form.extract_title_as_title) }}
                         <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
                     </div>
+
+                    <div class="pure-control-group">
+                        {{ render_field(form.real_browser_save_screenshot) }}
+                        <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
+                    </div>
+
                 </fieldset>
             </div>
 

changedetectionio/update_worker.py

@@ -38,11 +38,12 @@ class update_worker(threading.Thread):
 
                     changed_detected = False
                     contents = ""
+                    screenshot = False
                     update_obj= {}
                     now = time.time()
 
                     try:
-                        changed_detected, update_obj, contents = update_handler.run(uuid)
+                        changed_detected, update_obj, contents, screenshot = update_handler.run(uuid)
 
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@@ -140,6 +141,9 @@ class update_worker(threading.Thread):
                         # Always record that we atleast tried
                         self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
                                                                            'last_checked': round(time.time())})
+                        # Always save the screenshot if it's available
+                        if screenshot:
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
 
                 self.current_uuid = None  # Done
                 self.q.task_done()