Self-Hosted
/
changedetection.io
mirror of https://github.com/dgtlmoon/changedetection.io.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

When fetching a snapshot via Chrome, make the most recent screenshot available on the Diff and Preview pages (#516)

Browse Source
pull/522/head
dgtlmoon 3 years ago committed by GitHub
parent ffd2a89d60
commit 9fe4f95990
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 201 additions and 59 deletions
Show Stats Download Patch File Download Diff File

36
changedetectionio/__init__.py
Unescape View File

@ -625,6 +625,7 @@ def changedetection_app(config=None, datastore_o=None):
form.notification_body.data = datastore.data['settings']['application']['notification_body']
form.notification_format.data = datastore.data['settings']['application']['notification_format']
form.base_url.data = datastore.data['settings']['application']['base_url']
form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot']
if request.method == 'POST' and form.data.get('removepassword_button') == True:
# Password unset is a GET, but we can lock the session to a salted env password to always need the password
@ -647,7 +648,8 @@ def changedetection_app(config=None, datastore_o=None):
datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data
datastore.data['settings']['application']['global_ignore_text'] = form.global_ignore_text.data
datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data
datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data
if form.trigger_check.data:
if len(form.notification_urls.data):
n_object = {'watch_url': "Test from changedetection.io!",
@ -776,6 +778,9 @@ def changedetection_app(config=None, datastore_o=None):
except Exception as e:
previous_version_file_contents = "Unable to read {}.\n".format(previous_file)
screenshot_url = datastore.get_screenshot(uuid)
output = render_template("diff.html", watch_a=watch,
newest=newest_version_file_contents,
previous=previous_version_file_contents,
@ -786,7 +791,8 @@ def changedetection_app(config=None, datastore_o=None):
current_previous_version=str(previous_version),
current_diff_url=watch['url'],
extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
left_sticky=True)
left_sticky=True,
screenshot=screenshot_url)
return output
@ -846,15 +852,17 @@ def changedetection_app(config=None, datastore_o=None):
else:
content.append({'line': "No history found", 'classes': ''})
screenshot_url = datastore.get_screenshot(uuid)
output = render_template("preview.html",
content=content,
extra_stylesheets=extra_stylesheets,
ignored_line_numbers=ignored_line_numbers,
triggered_line_numbers=trigger_line_numbers,
current_diff_url=watch['url'],
screenshot=screenshot_url,
watch=watch,
uuid=uuid)
return output
@app.route("/settings/notification-logs", methods=['GET'])
@ -967,6 +975,28 @@ def changedetection_app(config=None, datastore_o=None):
@app.route("/static/<string:group>/<string:filename>", methods=['GET'])
def static_content(group, filename):
if group == 'screenshot':
from flask import make_response
# Could be sensitive, follow password requirements
if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
abort(403)
# These files should be in our subdirectory
try:
# set nocache, set content-type
watch_dir = datastore_o.datastore_path + "/" + filename
response = make_response(send_from_directory(filename="last-screenshot.png", directory=watch_dir, path=watch_dir + "/last-screenshot.png"))
response.headers['Content-type'] = 'image/png'
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
response.headers['Expires'] = 0
return response
except FileNotFoundError:
abort(404)
# These files should be in our subdirectory
try:
return send_from_directory("static/{}".format(group), path=filename)

33
changedetectionio/content_fetcher.py
Unescape View File

@ -42,6 +42,14 @@ class Fetcher():
# Should set self.error, self.status_code and self.content
pass
@abstractmethod
def quit(self):
return
@abstractmethod
def screenshot(self):
return
@abstractmethod
def get_last_status_code(self):
return self.status_code
@ -116,16 +124,16 @@ class html_webdriver(Fetcher):
# request_body, request_method unused for now, until some magic in the future happens.
# check env for WEBDRIVER_URL
driver = webdriver.Remote(
self.driver = webdriver.Remote(
command_executor=self.command_executor,
desired_capabilities=DesiredCapabilities.CHROME,
proxy=self.proxy)
try:
driver.get(url)
self.driver.get(url)
except WebDriverException as e:
# Be sure we close the session window
driver.quit()
self.quit()
raise
# @todo - how to check this? is it possible?
@ -135,26 +143,33 @@ class html_webdriver(Fetcher):
# @todo - dom wait loaded?
time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
self.content = driver.page_source
self.content = self.driver.page_source
self.headers = {}
driver.quit()
def screenshot(self):
return self.driver.get_screenshot_as_png()
# Does the connection to the webdriver work? run a test connection.
def is_ready(self):
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.common.exceptions import WebDriverException
driver = webdriver.Remote(
self.driver = webdriver.Remote(
command_executor=self.command_executor,
desired_capabilities=DesiredCapabilities.CHROME)
# driver.quit() seems to cause better exceptions
driver.quit()
self.quit()
return True
def quit(self):
if self.driver:
try:
self.driver.quit()
except Exception as e:
print("Exception in chrome shutdown/quit" + str(e))
# "html_requests" is listed as the default fetcher in store.py!
class html_requests(Fetcher):
fetcher_description = "Basic fast Plaintext/HTTP Client"

7
changedetectionio/fetch_site_status.py
Unescape View File

@ -21,6 +21,7 @@ class perform_site_check():
timestamp = int(time.time()) # used for storage etc too
changed_detected = False
screenshot = False # as bytes
stripped_text_from_html = ""
watch = self.datastore.data['watching'][uuid]
@ -171,5 +172,9 @@ class perform_site_check():
if not watch['title'] or not len(watch['title']):
update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
if self.datastore.data['settings']['application'].get('real_browser_save_screenshot', True):
screenshot = fetcher.screenshot()
return changed_detected, update_obj, text_content_before_ignored_filter
fetcher.quit()
return changed_detected, update_obj, text_content_before_ignored_filter, screenshot

4
changedetectionio/forms.py
Unescape View File

@ -345,7 +345,6 @@ class watchForm(commonSettingsForm):
return result
class globalSettingsForm(commonSettingsForm):
password = SaltyPasswordField()
minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
[validators.NumberRange(min=1)])
@ -355,4 +354,5 @@ class globalSettingsForm(commonSettingsForm):
global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
ignore_whitespace = BooleanField('Ignore whitespace')
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})

6
changedetectionio/static/js/tabs.js
Unescape View File

@ -1,6 +1,11 @@
// Rewrite this is a plugin.. is all this JS really 'worth it?'
if(!window.location.hash) {
var tab=document.querySelectorAll("#default-tab a");
tab[0].click();
}
window.addEventListener('hashchange', function() {
var tabs = document.getElementsByClassName('active');
while (tabs[0]) {
@ -21,7 +26,6 @@ if (!has_errors.length) {
focus_error_tab();
}
function set_active_tab() {
var tab=document.querySelectorAll("a[href='"+location.hash+"']");
if (tab.length) {

8
changedetectionio/static/styles/diff.css
Unescape View File

@ -1,7 +1,8 @@
#diff-ui {
background: #fff;
padding: 2em;
margin: 1em;
margin-left: 1em;
margin-right: 1em;
border-radius: 5px;
font-size: 11px; }
#diff-ui table {
@ -70,3 +71,8 @@ td#diff-col div {
/* ignored and triggered? make it obvious error */
.ignored.triggered {
background-color: #ff0000; }
.tab-pane-inner#screenshot {
text-align: center; }
.tab-pane-inner#screenshot img {
max-width: 99%; }

10
changedetectionio/static/styles/diff.scss
Unescape View File

@ -2,7 +2,8 @@
background: #fff;
padding: 2em;
margin: 1em;
margin-left: 1em;
margin-right: 1em;
border-radius: 5px;
font-size: 11px;
@ -85,4 +86,11 @@ td#diff-col div {
/* ignored and triggered? make it obvious error */
.ignored.triggered {
background-color: #ff0000;
}
.tab-pane-inner#screenshot {
text-align: center;
img {
max-width: 99%;
}
}

15
changedetectionio/static/styles/styles.css
Unescape View File

@ -317,7 +317,7 @@ footer {
right: auto; }
section.content {
padding-top: 110px; }
div.tabs ul li {
div.tabs.collapsable ul li {
display: block;
border-radius: 0px; }
input[type='text'] {
@ -403,14 +403,15 @@ and also iPads specifically.
padding: 20px;
border-radius: 5px; }
.tab-pane-inner {
padding: 0px; }
.tab-pane-inner:not(:target) {
display: none; }
.tab-pane-inner:target {
display: block; }
.edit-form {
min-width: 70%; }
.edit-form .tab-pane-inner {
padding: 0px; }
.edit-form .tab-pane-inner:not(:target) {
display: none; }
.edit-form .tab-pane-inner:target {
display: block; }
.edit-form .box-wrap {
position: relative; }
.edit-form .inner {

15
changedetectionio/static/styles/styles.scss
Unescape View File

@ -35,7 +35,7 @@ a.github-link {
section.content {
padding-top: 5em;
padding-bottom: 5em;
padding-bottom: 1em;
flex-direction: column;
display: flex;
align-items: center;
@ -437,7 +437,7 @@ footer {
}
// Make the tabs easier to hit, they will be all nice and horizontal
div.tabs ul li {
div.tabs.collapsable ul li {
display: block;
border-radius: 0px;
}
@ -573,10 +573,7 @@ $form-edge-padding: 20px;
}
}
.edit-form {
min-width: 70%;
.tab-pane-inner {
.tab-pane-inner {
&:not(:target) {
display: none;
}
@ -585,7 +582,11 @@ $form-edge-padding: 20px;
}
// doesnt need padding because theres another row of buttons/activity
padding: 0px;
}
}
.edit-form {
min-width: 70%;
.box-wrap {
position: relative;
}

17
changedetectionio/store.py
Unescape View File

@ -57,6 +57,7 @@ class ChangeDetectionStore:
'notification_title': default_notification_title,
'notification_body': default_notification_body,
'notification_format': default_notification_format,
'real_browser_save_screenshot': True,
}
}
}
@ -381,6 +382,22 @@ class ChangeDetectionStore:
return fname
def get_screenshot(self, watch_uuid):
output_path = "{}/{}".format(self.datastore_path, watch_uuid)
fname = "{}/last-screenshot.png".format(output_path)
if path.isfile(fname):
return fname
return False
# Save as PNG, PNG is larger but better for doing visual diff in the future
def save_screenshot(self, watch_uuid, screenshot: bytes):
output_path = "{}/{}".format(self.datastore_path, watch_uuid)
fname = "{}/last-screenshot.png".format(output_path)
with open(fname, 'wb') as f:
f.write(screenshot)
f.close()
def sync_to_json(self):
logging.info("Saving JSON..")

53
changedetectionio/templates/diff.html
Unescape View File

@ -1,7 +1,6 @@
{% extends 'base.html' %}
{% block content %}
<div id="settings">
<h1>Differences</h1>
<form class="pure-form " action="" method="GET">
@ -35,21 +34,45 @@
<div id="diff-jump">
<a onclick="next_diff();">Jump</a>
</div>
{% if screenshot %}
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
<div class="tabs">
<ul>
<li class="tab" id="default-tab"><a href="#text">Text</a></li>
<li class="tab"><a href="#screenshot">Screenshot</a></li>
</ul>
</div>
{% endif %}
<div id="diff-ui">
<div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
<table>
<tbody>
<tr>
<!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
<td id="a" style="display: none;">{{previous}}</td>
<td id="b" style="display: none;">{{newest}}</td>
<td id="diff-col">
<span id="result"></span>
</td>
</tr>
</tbody>
</table>
Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
<div class="tab-pane-inner" id="text">
<div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.
</div>
<table>
<tbody>
<tr>
<!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
<td id="a" style="display: none;">{{previous}}</td>
<td id="b" style="display: none;">{{newest}}</td>
<td id="diff-col">
<span id="result"></span>
</td>
</tr>
</tbody>
</table>
Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
</div>
{% if screenshot %}
<div class="tab-pane-inner" id="screenshot">
<p>
<i>For now, only the most recent screenshot is saved and displayed.</i>
</p>
<img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
</div>
{% endif %}
</div>

2
changedetectionio/templates/edit.html
Unescape View File

@ -7,7 +7,7 @@
<div class="edit-form monospaced-textarea">
<div class="tabs">
<div class="tabs collapsable">
<ul>
<li class="tab" id="default-tab"><a href="#general">General</a></li>
<li class="tab"><a href="#request">Request</a></li>

40
changedetectionio/templates/preview.html
Unescape View File

@ -6,18 +6,40 @@
<h1>Current - {{watch.last_checked|format_timestamp_timeago}}</h1>
</div>
{% if screenshot %}
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
<div class="tabs">
<ul>
<li class="tab" id="default-tab"><a href="#text">Text</a></li>
<li class="tab"><a href="#screenshot">Screenshot</a></li>
</ul>
</div>
{% endif %}
<div id="diff-ui">
<span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
<table>
<tbody>
<tr>
<td id="diff-col">
<div class="tab-pane-inner" id="text">
<span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
<table>
<tbody>
<tr>
<td id="diff-col">
{% for row in content %}
<div class="{{row.classes}}">{{row.line}}</div>
{% endfor %}
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</div>
{% if screenshot %}
<div class="tab-pane-inner" id="screenshot">
<p>
<i>For now, only the most recent screenshot is saved and displayed.</i>
</p>
<img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
</div>
{% endif %}
</div>
{% endblock %}

8
changedetectionio/templates/settings.html
Unescape View File

@ -8,7 +8,7 @@
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
<div class="edit-form">
<div class="tabs">
<div class="tabs collapsable">
<ul>
<li class="tab" id="default-tab"><a href="#general">General</a></li>
<li class="tab"><a href="#notifications">Notifications</a></li>
@ -50,6 +50,12 @@
{{ render_field(form.extract_title_as_title) }}
<span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
</div>
<div class="pure-control-group">
{{ render_field(form.real_browser_save_screenshot) }}
<span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
</div>
</fieldset>
</div>

6
changedetectionio/update_worker.py
Unescape View File

@ -38,11 +38,12 @@ class update_worker(threading.Thread):
changed_detected = False
contents = ""
screenshot = False
update_obj= {}
now = time.time()
try:
changed_detected, update_obj, contents = update_handler.run(uuid)
changed_detected, update_obj, contents, screenshot = update_handler.run(uuid)
# Re #342
# In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@ -140,6 +141,9 @@ class update_worker(threading.Thread):
# Always record that we atleast tried
self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
'last_checked': round(time.time())})
# Always save the screenshot if it's available
if screenshot:
self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
self.current_uuid = None # Done
self.q.task_done()

Write Preview
Loading…
Cancel
Save