Chrome/Webdriver support for Javascript websites (#114)
JS Support via fetching the page over WebDriver/Selenium network Refactor forms (Split into logical tabs)pull/187/head
dgtlmoon
3 years ago
committed by
GitHub
parent
1f821d6e8b
commit
9e08f326be
@ -0,0 +1,87 @@
|
|||||||
|
name: Javascript/Webdriver support - Test, build and push to Docker Hub :javascript tag
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ javascript-browser ]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- name: Set up Python 3.9
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: 3.9
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
pip install flake8 pytest
|
||||||
|
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||||
|
|
||||||
|
- name: Lint with flake8
|
||||||
|
run: |
|
||||||
|
# stop the build if there are Python syntax errors or undefined names
|
||||||
|
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||||
|
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||||
|
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||||
|
|
||||||
|
- name: Create release metadata
|
||||||
|
run: |
|
||||||
|
# COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
|
||||||
|
echo ${{ github.sha }} > backend/source.txt
|
||||||
|
echo ${{ github.ref }} > backend/tag.txt
|
||||||
|
|
||||||
|
- name: Test with pytest
|
||||||
|
run: |
|
||||||
|
# Each test is totally isolated and performs its own cleanup/reset
|
||||||
|
cd backend; ./run_all_tests.sh
|
||||||
|
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v1
|
||||||
|
with:
|
||||||
|
image: tonistiigi/binfmt:latest
|
||||||
|
platforms: all
|
||||||
|
|
||||||
|
- name: Login to Docker Hub
|
||||||
|
uses: docker/login-action@v1
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
id: buildx
|
||||||
|
uses: docker/setup-buildx-action@v1
|
||||||
|
with:
|
||||||
|
install: true
|
||||||
|
version: latest
|
||||||
|
driver-opts: image=moby/buildkit:master
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
id: docker_build
|
||||||
|
uses: docker/build-push-action@v2
|
||||||
|
with:
|
||||||
|
context: ./
|
||||||
|
file: ./Dockerfile
|
||||||
|
push: true
|
||||||
|
tags: |
|
||||||
|
${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:javascript-dev
|
||||||
|
platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
|
||||||
|
cache-from: type=local,src=/tmp/.buildx-cache
|
||||||
|
cache-to: type=local,dest=/tmp/.buildx-cache
|
||||||
|
|
||||||
|
- name: Image digest
|
||||||
|
run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
|
||||||
|
|
||||||
|
# failed: Cache service responded with 503
|
||||||
|
# - name: Cache Docker layers
|
||||||
|
# uses: actions/cache@v2
|
||||||
|
# with:
|
||||||
|
# path: /tmp/.buildx-cache
|
||||||
|
# key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||||
|
# restore-keys: |
|
||||||
|
# ${{ runner.os }}-buildx-
|
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,137 @@
|
|||||||
|
import os
|
||||||
|
import time
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
||||||
|
from selenium.common.exceptions import WebDriverException
|
||||||
|
import urllib3.exceptions
|
||||||
|
|
||||||
|
|
||||||
|
class EmptyReply(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class Fetcher():
|
||||||
|
error = None
|
||||||
|
status_code = None
|
||||||
|
content = None # Should be bytes?
|
||||||
|
|
||||||
|
fetcher_description ="No description"
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_error(self):
|
||||||
|
return self.error
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def run(self, url, timeout, request_headers):
|
||||||
|
# Should set self.error, self.status_code and self.content
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_last_status_code(self):
|
||||||
|
return self.status_code
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
# Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
|
||||||
|
def is_ready(self):
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Maybe for the future, each fetcher provides its own diff output, could be used for text, image
|
||||||
|
# the current one would return javascript output (as we use JS to generate the diff)
|
||||||
|
#
|
||||||
|
# Returns tuple(mime_type, stream)
|
||||||
|
# @abstractmethod
|
||||||
|
# def return_diff(self, stream_a, stream_b):
|
||||||
|
# return
|
||||||
|
|
||||||
|
def available_fetchers():
|
||||||
|
import inspect
|
||||||
|
from backend import content_fetcher
|
||||||
|
p=[]
|
||||||
|
for name, obj in inspect.getmembers(content_fetcher):
|
||||||
|
if inspect.isclass(obj):
|
||||||
|
# @todo html_ is maybe better as fetcher_ or something
|
||||||
|
# In this case, make sure to edit the default one in store.py and fetch_site_status.py
|
||||||
|
if "html_" in name:
|
||||||
|
t=tuple([name,obj.fetcher_description])
|
||||||
|
p.append(t)
|
||||||
|
|
||||||
|
return p
|
||||||
|
|
||||||
|
class html_webdriver(Fetcher):
|
||||||
|
fetcher_description = "WebDriver Chrome/Javascript"
|
||||||
|
command_executor = ''
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.command_executor = os.getenv("WEBDRIVER_URL",'http://browser-chrome:4444/wd/hub')
|
||||||
|
|
||||||
|
def run(self, url, timeout, request_headers):
|
||||||
|
|
||||||
|
# check env for WEBDRIVER_URL
|
||||||
|
driver = webdriver.Remote(
|
||||||
|
command_executor=self.command_executor,
|
||||||
|
desired_capabilities=DesiredCapabilities.CHROME)
|
||||||
|
|
||||||
|
try:
|
||||||
|
driver.get(url)
|
||||||
|
except WebDriverException as e:
|
||||||
|
# Be sure we close the session window
|
||||||
|
driver.quit()
|
||||||
|
raise
|
||||||
|
|
||||||
|
# @todo - how to check this? is it possible?
|
||||||
|
self.status_code = 200
|
||||||
|
|
||||||
|
# @todo - dom wait loaded?
|
||||||
|
time.sleep(5)
|
||||||
|
self.content = driver.page_source
|
||||||
|
|
||||||
|
driver.quit()
|
||||||
|
|
||||||
|
|
||||||
|
def is_ready(self):
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
||||||
|
from selenium.common.exceptions import WebDriverException
|
||||||
|
|
||||||
|
driver = webdriver.Remote(
|
||||||
|
command_executor='http://browser-chrome:4444/wd/hub',
|
||||||
|
desired_capabilities=DesiredCapabilities.CHROME)
|
||||||
|
|
||||||
|
# driver.quit() seems to cause better exceptions
|
||||||
|
driver.quit()
|
||||||
|
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
# "html_requests" is listed as the default fetcher in store.py!
|
||||||
|
class html_requests(Fetcher):
|
||||||
|
fetcher_description = "Basic fast Plaintext/HTTP Client"
|
||||||
|
|
||||||
|
def run(self, url, timeout, request_headers):
|
||||||
|
import requests
|
||||||
|
try:
|
||||||
|
r = requests.get(url,
|
||||||
|
headers=request_headers,
|
||||||
|
timeout=timeout,
|
||||||
|
verify=False)
|
||||||
|
|
||||||
|
html = r.text
|
||||||
|
|
||||||
|
# Usually from networkIO/requests level
|
||||||
|
except (
|
||||||
|
requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout,
|
||||||
|
requests.exceptions.MissingSchema) as e:
|
||||||
|
self.error = str(e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
self.error = "Other exception" + str(e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# @todo test this
|
||||||
|
if not r or not html or not len(html):
|
||||||
|
raise EmptyReply(url)
|
||||||
|
|
||||||
|
self.status_code = r.status_code
|
||||||
|
self.content = html
|
||||||
|
|
||||||
|
After Width: | Height: | Size: 14 KiB |
@ -0,0 +1,51 @@
|
|||||||
|
// Rewrite this is a plugin.. is all this JS really 'worth it?'
|
||||||
|
|
||||||
|
|
||||||
|
window.addEventListener('hashchange', function() {
|
||||||
|
var tabs = document.getElementsByClassName('active');
|
||||||
|
while (tabs[0]) {
|
||||||
|
tabs[0].classList.remove('active')
|
||||||
|
}
|
||||||
|
set_active_tab();
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
var has_errors=document.querySelectorAll(".messages .error");
|
||||||
|
if (!has_errors.length) {
|
||||||
|
if (document.location.hash == "" ) {
|
||||||
|
document.location.hash = "#general";
|
||||||
|
document.getElementById("default-tab").className = "active";
|
||||||
|
} else {
|
||||||
|
set_active_tab();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
focus_error_tab();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function set_active_tab() {
|
||||||
|
var tab=document.querySelectorAll("a[href='"+location.hash+"']");
|
||||||
|
if (tab.length) {
|
||||||
|
tab[0].parentElement.className="active";
|
||||||
|
}
|
||||||
|
// hash could move the page down
|
||||||
|
window.scrollTo(0, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function focus_error_tab() {
|
||||||
|
// time to use jquery or vuejs really,
|
||||||
|
// activate the tab with the error
|
||||||
|
var tabs = document.querySelectorAll('.tabs li a'),i;
|
||||||
|
for (i = 0; i < tabs.length; ++i) {
|
||||||
|
var tab_name=tabs[i].hash.replace('#','');
|
||||||
|
var pane_errors=document.querySelectorAll('#'+tab_name+' .error')
|
||||||
|
if (pane_errors.length) {
|
||||||
|
document.location.hash = '#'+tab_name;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -1,80 +1,112 @@
|
|||||||
{% extends 'base.html' %}
|
{% extends 'base.html' %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
{% from '_helpers.jinja' import render_field %}
|
{% from '_helpers.jinja' import render_field %}
|
||||||
|
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
|
||||||
|
|
||||||
<div class="edit-form monospaced-textarea">
|
<div class="edit-form monospaced-textarea">
|
||||||
<form class="pure-form pure-form-stacked" action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
|
|
||||||
<fieldset>
|
<div class="tabs">
|
||||||
<div class="pure-control-group">
|
<ul>
|
||||||
{{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
|
<li class="tab" id="default-tab"><a href="#general">General</a></li>
|
||||||
</div>
|
<li class="tab"><a href="#notifications">Notifications</a></li>
|
||||||
<div class="pure-control-group">
|
<li class="tab"><a href="#filters">Filters</a></li>
|
||||||
{{ render_field(form.title, class="m-d") }}
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
<div class="pure-control-group">
|
|
||||||
{{ render_field(form.tag) }}
|
<div class="box-wrap inner">
|
||||||
|
<form class="pure-form pure-form-stacked"
|
||||||
|
action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
|
||||||
|
|
||||||
|
<div class="tab-pane-inner" id="general">
|
||||||
|
<fieldset>
|
||||||
|
<div class="pure-control-group">
|
||||||
|
{{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
|
||||||
|
</div>
|
||||||
|
<div class="pure-control-group">
|
||||||
|
{{ render_field(form.title, class="m-d") }}
|
||||||
|
</div>
|
||||||
|
<div class="pure-control-group">
|
||||||
|
{{ render_field(form.tag) }}
|
||||||
|
</div>
|
||||||
|
<div class="pure-control-group">
|
||||||
|
{{ render_field(form.minutes_between_check) }}
|
||||||
|
{% if using_default_minutes %}
|
||||||
|
<span class="pure-form-message-inline">Currently using the <a
|
||||||
|
href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
|
||||||
|
{% else %}
|
||||||
|
<span class="pure-form-message-inline">Set to blank to use the <a
|
||||||
|
href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
<fieldset class="pure-group">
|
||||||
|
{{ render_field(form.headers, rows=5, placeholder="Example
|
||||||
|
Cookie: foobar
|
||||||
|
User-Agent: wonderbra 1.0") }}
|
||||||
|
<span class="pure-form-message-inline">
|
||||||
|
Note: ONLY used by Basic fast Plaintext/HTTP Client
|
||||||
|
</span>
|
||||||
|
</fieldset>
|
||||||
|
</fieldset>
|
||||||
</div>
|
</div>
|
||||||
<div class="pure-control-group">
|
<div class="tab-pane-inner" id="notifications">
|
||||||
{{ render_field(form.minutes_between_check) }}
|
<fieldset>
|
||||||
{% if using_default_minutes %}
|
<div class="pure-control-group">
|
||||||
<span class="pure-form-message-inline">Currently using the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
|
{{ render_field(form.notification_urls, rows=5, placeholder="Examples:
|
||||||
{% else %}
|
Gitter - gitter://token/room
|
||||||
<span class="pure-form-message-inline">Set to blank to use the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
|
Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
|
||||||
{% endif %}
|
AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
|
||||||
|
SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
|
||||||
|
") }}
|
||||||
|
<span class="pure-form-message-inline">Use <a target=_new
|
||||||
|
href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
|
||||||
|
<span class="pure-form-message-inline">Note: This overrides any global settings notification URLs</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pure-controls">
|
||||||
|
{{ render_field(form.trigger_check, rows=5) }}
|
||||||
|
</div>
|
||||||
|
</fieldset>
|
||||||
</div>
|
</div>
|
||||||
<div class="pure-control-group">
|
<div class="tab-pane-inner" id="filters">
|
||||||
{{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.", class="m-d") }}
|
<fieldset>
|
||||||
<span class="pure-form-message-inline">
|
<div class="pure-control-group">
|
||||||
|
{{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
|
||||||
|
class="m-d") }}
|
||||||
|
<span class="pure-form-message-inline">
|
||||||
<ul>
|
<ul>
|
||||||
<li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
|
<li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
|
||||||
<li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
|
<li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a
|
||||||
|
href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
|
||||||
</ul>
|
</ul>
|
||||||
Please be sure that you thoroughly understand how to write CSS or JSONPath selector rules before filing an issue on GitHub! <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
|
Please be sure that you thoroughly understand how to write CSS or JSONPath selector rules before filing an issue on GitHub! <a
|
||||||
|
href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<!-- @todo: move to tabs --->
|
|
||||||
<fieldset class="pure-group">
|
</fieldset>
|
||||||
{{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
|
<fieldset class="pure-group">
|
||||||
/some.regex\d{2}/ for case-INsensitive regex
|
{{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
|
||||||
") }}
|
/some.regex\d{2}/ for case-INsensitive regex
|
||||||
<span class="pure-form-message-inline">
|
") }}
|
||||||
|
<span class="pure-form-message-inline">
|
||||||
Each line processed separately, any line matching will be ignored.<br/>
|
Each line processed separately, any line matching will be ignored.<br/>
|
||||||
Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
|
Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
|
||||||
</span>
|
</span>
|
||||||
|
|
||||||
</fieldset>
|
</fieldset>
|
||||||
|
|
||||||
<fieldset class="pure-group">
|
|
||||||
{{ render_field(form.headers, rows=5, placeholder="Example
|
|
||||||
Cookie: foobar
|
|
||||||
User-Agent: wonderbra 1.0") }}
|
|
||||||
</fieldset>
|
|
||||||
|
|
||||||
<div class="pure-control-group">
|
|
||||||
{{ render_field(form.notification_urls, rows=5, placeholder="Examples:
|
|
||||||
Gitter - gitter://token/room
|
|
||||||
Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
|
|
||||||
AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
|
|
||||||
SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
|
|
||||||
") }}
|
|
||||||
<span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
|
|
||||||
<span class="pure-form-message-inline">Note: This overrides any global settings notification URLs</span>
|
|
||||||
</div>
|
</div>
|
||||||
|
<div id="actions">
|
||||||
|
<div class="pure-control-group">
|
||||||
|
|
||||||
<div class="pure-controls">
|
<button type="submit" class="pure-button pure-button-primary">Save</button>
|
||||||
{{ render_field(form.trigger_check, rows=5) }}
|
<a href="{{url_for('api_delete', uuid=uuid)}}"
|
||||||
|
class="pure-button button-small button-error ">Delete</a>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="pure-control-group">
|
</form>
|
||||||
<button type="submit" class="pure-button pure-button-primary">Save</button>
|
</div>
|
||||||
</div>
|
|
||||||
<br/>
|
|
||||||
<div class="pure-control-group">
|
|
||||||
<a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
|
|
||||||
<a href="{{url_for('api_delete', uuid=uuid)}}"
|
|
||||||
class="pure-button button-small button-error ">Delete</a>
|
|
||||||
</div>
|
|
||||||
</fieldset>
|
|
||||||
</form>
|
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
Loading…
Reference in new issue