Chrome/Webdriver support for Javascript websites (#114)
JS Support via fetching the page over WebDriver/Selenium network Refactor forms (Split into logical tabs)pull/187/head
dgtlmoon
3 years ago
committed by
GitHub
parent
1f821d6e8b
commit
9e08f326be
@ -0,0 +1,87 @@
|
|||||||
|
name: Javascript/Webdriver support - Test, build and push to Docker Hub :javascript tag
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ javascript-browser ]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- name: Set up Python 3.9
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: 3.9
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
pip install flake8 pytest
|
||||||
|
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||||
|
|
||||||
|
- name: Lint with flake8
|
||||||
|
run: |
|
||||||
|
# stop the build if there are Python syntax errors or undefined names
|
||||||
|
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||||
|
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||||
|
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||||
|
|
||||||
|
- name: Create release metadata
|
||||||
|
run: |
|
||||||
|
# COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
|
||||||
|
echo ${{ github.sha }} > backend/source.txt
|
||||||
|
echo ${{ github.ref }} > backend/tag.txt
|
||||||
|
|
||||||
|
- name: Test with pytest
|
||||||
|
run: |
|
||||||
|
# Each test is totally isolated and performs its own cleanup/reset
|
||||||
|
cd backend; ./run_all_tests.sh
|
||||||
|
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v1
|
||||||
|
with:
|
||||||
|
image: tonistiigi/binfmt:latest
|
||||||
|
platforms: all
|
||||||
|
|
||||||
|
- name: Login to Docker Hub
|
||||||
|
uses: docker/login-action@v1
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
id: buildx
|
||||||
|
uses: docker/setup-buildx-action@v1
|
||||||
|
with:
|
||||||
|
install: true
|
||||||
|
version: latest
|
||||||
|
driver-opts: image=moby/buildkit:master
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
id: docker_build
|
||||||
|
uses: docker/build-push-action@v2
|
||||||
|
with:
|
||||||
|
context: ./
|
||||||
|
file: ./Dockerfile
|
||||||
|
push: true
|
||||||
|
tags: |
|
||||||
|
${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:javascript-dev
|
||||||
|
platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
|
||||||
|
cache-from: type=local,src=/tmp/.buildx-cache
|
||||||
|
cache-to: type=local,dest=/tmp/.buildx-cache
|
||||||
|
|
||||||
|
- name: Image digest
|
||||||
|
run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
|
||||||
|
|
||||||
|
# failed: Cache service responded with 503
|
||||||
|
# - name: Cache Docker layers
|
||||||
|
# uses: actions/cache@v2
|
||||||
|
# with:
|
||||||
|
# path: /tmp/.buildx-cache
|
||||||
|
# key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||||
|
# restore-keys: |
|
||||||
|
# ${{ runner.os }}-buildx-
|
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,137 @@
|
|||||||
|
import os
|
||||||
|
import time
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
||||||
|
from selenium.common.exceptions import WebDriverException
|
||||||
|
import urllib3.exceptions
|
||||||
|
|
||||||
|
|
||||||
|
class EmptyReply(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class Fetcher():
|
||||||
|
error = None
|
||||||
|
status_code = None
|
||||||
|
content = None # Should be bytes?
|
||||||
|
|
||||||
|
fetcher_description ="No description"
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_error(self):
|
||||||
|
return self.error
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def run(self, url, timeout, request_headers):
|
||||||
|
# Should set self.error, self.status_code and self.content
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_last_status_code(self):
|
||||||
|
return self.status_code
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
# Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
|
||||||
|
def is_ready(self):
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Maybe for the future, each fetcher provides its own diff output, could be used for text, image
|
||||||
|
# the current one would return javascript output (as we use JS to generate the diff)
|
||||||
|
#
|
||||||
|
# Returns tuple(mime_type, stream)
|
||||||
|
# @abstractmethod
|
||||||
|
# def return_diff(self, stream_a, stream_b):
|
||||||
|
# return
|
||||||
|
|
||||||
|
def available_fetchers():
|
||||||
|
import inspect
|
||||||
|
from backend import content_fetcher
|
||||||
|
p=[]
|
||||||
|
for name, obj in inspect.getmembers(content_fetcher):
|
||||||
|
if inspect.isclass(obj):
|
||||||
|
# @todo html_ is maybe better as fetcher_ or something
|
||||||
|
# In this case, make sure to edit the default one in store.py and fetch_site_status.py
|
||||||
|
if "html_" in name:
|
||||||
|
t=tuple([name,obj.fetcher_description])
|
||||||
|
p.append(t)
|
||||||
|
|
||||||
|
return p
|
||||||
|
|
||||||
|
class html_webdriver(Fetcher):
|
||||||
|
fetcher_description = "WebDriver Chrome/Javascript"
|
||||||
|
command_executor = ''
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.command_executor = os.getenv("WEBDRIVER_URL",'http://browser-chrome:4444/wd/hub')
|
||||||
|
|
||||||
|
def run(self, url, timeout, request_headers):
|
||||||
|
|
||||||
|
# check env for WEBDRIVER_URL
|
||||||
|
driver = webdriver.Remote(
|
||||||
|
command_executor=self.command_executor,
|
||||||
|
desired_capabilities=DesiredCapabilities.CHROME)
|
||||||
|
|
||||||
|
try:
|
||||||
|
driver.get(url)
|
||||||
|
except WebDriverException as e:
|
||||||
|
# Be sure we close the session window
|
||||||
|
driver.quit()
|
||||||
|
raise
|
||||||
|
|
||||||
|
# @todo - how to check this? is it possible?
|
||||||
|
self.status_code = 200
|
||||||
|
|
||||||
|
# @todo - dom wait loaded?
|
||||||
|
time.sleep(5)
|
||||||
|
self.content = driver.page_source
|
||||||
|
|
||||||
|
driver.quit()
|
||||||
|
|
||||||
|
|
||||||
|
def is_ready(self):
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
||||||
|
from selenium.common.exceptions import WebDriverException
|
||||||
|
|
||||||
|
driver = webdriver.Remote(
|
||||||
|
command_executor='http://browser-chrome:4444/wd/hub',
|
||||||
|
desired_capabilities=DesiredCapabilities.CHROME)
|
||||||
|
|
||||||
|
# driver.quit() seems to cause better exceptions
|
||||||
|
driver.quit()
|
||||||
|
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
# "html_requests" is listed as the default fetcher in store.py!
|
||||||
|
class html_requests(Fetcher):
|
||||||
|
fetcher_description = "Basic fast Plaintext/HTTP Client"
|
||||||
|
|
||||||
|
def run(self, url, timeout, request_headers):
|
||||||
|
import requests
|
||||||
|
try:
|
||||||
|
r = requests.get(url,
|
||||||
|
headers=request_headers,
|
||||||
|
timeout=timeout,
|
||||||
|
verify=False)
|
||||||
|
|
||||||
|
html = r.text
|
||||||
|
|
||||||
|
# Usually from networkIO/requests level
|
||||||
|
except (
|
||||||
|
requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout,
|
||||||
|
requests.exceptions.MissingSchema) as e:
|
||||||
|
self.error = str(e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
self.error = "Other exception" + str(e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# @todo test this
|
||||||
|
if not r or not html or not len(html):
|
||||||
|
raise EmptyReply(url)
|
||||||
|
|
||||||
|
self.status_code = r.status_code
|
||||||
|
self.content = html
|
||||||
|
|
||||||
|
After Width: | Height: | Size: 14 KiB |
@ -0,0 +1,51 @@
|
|||||||
|
// Rewrite this is a plugin.. is all this JS really 'worth it?'
|
||||||
|
|
||||||
|
|
||||||
|
window.addEventListener('hashchange', function() {
|
||||||
|
var tabs = document.getElementsByClassName('active');
|
||||||
|
while (tabs[0]) {
|
||||||
|
tabs[0].classList.remove('active')
|
||||||
|
}
|
||||||
|
set_active_tab();
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
var has_errors=document.querySelectorAll(".messages .error");
|
||||||
|
if (!has_errors.length) {
|
||||||
|
if (document.location.hash == "" ) {
|
||||||
|
document.location.hash = "#general";
|
||||||
|
document.getElementById("default-tab").className = "active";
|
||||||
|
} else {
|
||||||
|
set_active_tab();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
focus_error_tab();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function set_active_tab() {
|
||||||
|
var tab=document.querySelectorAll("a[href='"+location.hash+"']");
|
||||||
|
if (tab.length) {
|
||||||
|
tab[0].parentElement.className="active";
|
||||||
|
}
|
||||||
|
// hash could move the page down
|
||||||
|
window.scrollTo(0, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function focus_error_tab() {
|
||||||
|
// time to use jquery or vuejs really,
|
||||||
|
// activate the tab with the error
|
||||||
|
var tabs = document.querySelectorAll('.tabs li a'),i;
|
||||||
|
for (i = 0; i < tabs.length; ++i) {
|
||||||
|
var tab_name=tabs[i].hash.replace('#','');
|
||||||
|
var pane_errors=document.querySelectorAll('#'+tab_name+' .error')
|
||||||
|
if (pane_errors.length) {
|
||||||
|
document.location.hash = '#'+tab_name;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Loading…
Reference in new issue