Configurable "Browser Steps" when Playwright/Chrome is configured (enter text, scroll, wait for text, click button etc) (#478)

export-regex-csv
dgtlmoon 2 years ago committed by GitHub
parent c98536ace4
commit 5b530ff61c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

1
.gitignore vendored

@ -8,6 +8,7 @@ __pycache__
build build
dist dist
venv venv
test-datastore/*
test-datastore test-datastore
*.egg-info* *.egg-info*
.vscode/settings.json .vscode/settings.json

@ -25,7 +25,7 @@ RUN pip install --target=/dependencies -r /requirements.txt
# Playwright is an alternative to Selenium # Playwright is an alternative to Selenium
# Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
# https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported) # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
RUN pip install --target=/dependencies playwright~=1.26 \ RUN pip install --target=/dependencies playwright~=1.27.1 \
|| echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled." || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
# Final image stage # Final image stage

@ -4,6 +4,9 @@ recursive-include changedetectionio/static *
recursive-include changedetectionio/model * recursive-include changedetectionio/model *
recursive-include changedetectionio/tests * recursive-include changedetectionio/tests *
recursive-include changedetectionio/res * recursive-include changedetectionio/res *
prune changedetectionio/static/package-lock.json
prune changedetectionio/static/styles/node_modules
prune changedetectionio/static/styles/package-lock.json
include changedetection.py include changedetection.py
global-exclude *.pyc global-exclude *.pyc
global-exclude node_modules global-exclude node_modules

@ -69,6 +69,17 @@ Available when connected to a <a href="https://github.com/dgtlmoon/changedetecti
<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference " title="Self-hosted web page change monitoring context difference " /> <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference " title="Self-hosted web page change monitoring context difference " />
### Perform interactive browser steps
Fill in text boxes, click buttons and more, setup your changedetection scenario.
Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, refining searches.
<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference " title="Website change detection with interactive browser steps, login, cookies etc" />
After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
Requires Playwright to be enabled.
## Installation ## Installation
### Docker ### Docker

@ -1,18 +1,20 @@
#!/usr/bin/python3 #!/usr/bin/python3
import datetime import datetime
import flask_login
import logging
import os import os
import pytz
import queue import queue
import threading import threading
import time import time
import timeago
from copy import deepcopy from copy import deepcopy
from distutils.util import strtobool
from feedgen.feed import FeedGenerator
from threading import Event from threading import Event
import flask_login
import logging
import pytz
import timeago
from feedgen.feed import FeedGenerator
from flask import ( from flask import (
Flask, Flask,
abort, abort,
@ -27,7 +29,6 @@ from flask import (
) )
from flask_login import login_required from flask_login import login_required
from flask_restful import abort, Api from flask_restful import abort, Api
from flask_wtf import CSRFProtect from flask_wtf import CSRFProtect
from changedetectionio import html_tools from changedetectionio import html_tools
@ -44,7 +45,6 @@ ticker_thread = None
extra_stylesheets = [] extra_stylesheets = []
update_q = queue.PriorityQueue() update_q = queue.PriorityQueue()
notification_q = queue.Queue() notification_q = queue.Queue()
app = Flask(__name__, app = Flask(__name__,
@ -97,7 +97,7 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
# Worker thread tells us which UUID it is currently processing. # Worker thread tells us which UUID it is currently processing.
for t in running_update_threads: for t in running_update_threads:
if t.current_uuid == watch_obj['uuid']: if t.current_uuid == watch_obj['uuid']:
return '<span class="loader"></span><span> Checking now</span>' return '<span class="spinner"></span><span> Checking now</span>'
if watch_obj['last_checked'] == 0: if watch_obj['last_checked'] == 0:
return 'Not yet' return 'Not yet'
@ -525,6 +525,7 @@ def changedetection_app(config=None, datastore_o=None):
def edit_page(uuid): def edit_page(uuid):
from changedetectionio import forms from changedetectionio import forms
from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
using_default_check_time = True using_default_check_time = True
# More for testing, possible to return the first/only # More for testing, possible to return the first/only
@ -558,6 +559,8 @@ def changedetection_app(config=None, datastore_o=None):
data=default, data=default,
) )
# form.browser_steps[0] can be assumed that we 'goto url' first
if datastore.proxy_list is None: if datastore.proxy_list is None:
# @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
del form.proxy del form.proxy
@ -650,6 +653,7 @@ def changedetection_app(config=None, datastore_o=None):
watch.get('fetch_backend', None) is None and system_uses_webdriver) else False watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
output = render_template("edit.html", output = render_template("edit.html",
browser_steps_config=browser_step_ui_config,
current_base_url=datastore.data['settings']['application']['base_url'], current_base_url=datastore.data['settings']['application']['base_url'],
emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
form=form, form=form,
@ -661,7 +665,6 @@ def changedetection_app(config=None, datastore_o=None):
settings_application=datastore.data['settings']['application'], settings_application=datastore.data['settings']['application'],
using_global_webdriver_wait=default['webdriver_delay'] is None, using_global_webdriver_wait=default['webdriver_delay'] is None,
uuid=uuid, uuid=uuid,
visualselector_data_is_ready=visualselector_data_is_ready,
visualselector_enabled=visualselector_enabled, visualselector_enabled=visualselector_enabled,
watch=watch watch=watch
) )
@ -1190,7 +1193,6 @@ def changedetection_app(config=None, datastore_o=None):
else: else:
# No tag, no uuid, add everything. # No tag, no uuid, add everything.
for watch_uuid, watch in datastore.data['watching'].items(): for watch_uuid, watch in datastore.data['watching'].items():
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
update_q.put((1, watch_uuid)) update_q.put((1, watch_uuid))
i += 1 i += 1
@ -1308,9 +1310,11 @@ def changedetection_app(config=None, datastore_o=None):
# paste in etc # paste in etc
return redirect(url_for('index')) return redirect(url_for('index'))
import changedetectionio.blueprint.browser_steps as browser_steps
app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
# @todo handle ctrl break # @todo handle ctrl break
ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
threading.Thread(target=notification_runner).start() threading.Thread(target=notification_runner).start()
# Check for new release version, but not when running in test/build or pytest # Check for new release version, but not when running in test/build or pytest

@ -0,0 +1,226 @@
# HORRIBLE HACK BUT WORKS :-) PR anyone?
#
# Why?
# `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()
# - this flask app is not async()
# - browserless has a single timeout/keepalive which applies to the session made at .connect_over_cdp()
#
# So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run
# and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user
# that their time is up, insert another coin. (reload)
#
# Bigger picture
# - It's horrible that we have this click+wait deal, some nice socket.io solution using something similar
# to what the browserless debug UI already gives us would be smarter..
#
# OR
# - Some API call that should be hacked into browserless or playwright that we can "/api/bump-keepalive/{session_id}/60"
# So we can tell it that we need more time (run this on each action)
#
# OR
# - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes)
from distutils.util import strtobool
from flask import Blueprint, request, make_response
from flask_login import login_required
import os
import logging
from changedetectionio.store import ChangeDetectionStore
browsersteps_live_ui_o = {}
browsersteps_playwright_browser_interface = None
browsersteps_playwright_browser_interface_start_time = None
browsersteps_playwright_browser_interface_browser = None
browsersteps_playwright_browser_interface_end_time = None
def cleanup_playwright_session():
print("Cleaning up old playwright session because time was up")
global browsersteps_playwright_browser_interface
global browsersteps_live_ui_o
global browsersteps_playwright_browser_interface_browser
global browsersteps_playwright_browser_interface
global browsersteps_playwright_browser_interface_start_time
global browsersteps_playwright_browser_interface_end_time
import psutil
current_process = psutil.Process()
children = current_process.children(recursive=True)
for child in children:
print (child)
print('Child pid is {}'.format(child.pid))
# .stop() hangs sometimes if its called when there are no children to process
# but how do we know this is our child? dunno
if children:
browsersteps_playwright_browser_interface.stop()
browsersteps_live_ui_o = {}
browsersteps_playwright_browser_interface = None
browsersteps_playwright_browser_interface_start_time = None
browsersteps_playwright_browser_interface_browser = None
browsersteps_playwright_browser_interface_end_time = None
print ("Cleaning up old playwright session because time was up - done")
def construct_blueprint(datastore: ChangeDetectionStore):
browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
@login_required
@browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
def browsersteps_ui_update():
import base64
import playwright._impl._api_types
import time
from changedetectionio.blueprint.browser_steps import browser_steps
global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
global browsersteps_playwright_browser_interface_browser
global browsersteps_playwright_browser_interface
global browsersteps_playwright_browser_interface_start_time
step_n = None
remaining =0
uuid = request.args.get('uuid')
browsersteps_session_id = request.args.get('browsersteps_session_id')
if not browsersteps_session_id:
return make_response('No browsersteps_session_id specified', 500)
# Because we don't "really" run in a context manager ( we make the playwright interface global/long-living )
# We need to manage the shutdown when the time is up
if browsersteps_playwright_browser_interface_end_time:
remaining = browsersteps_playwright_browser_interface_end_time-time.time()
if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
cleanup_playwright_session()
return make_response('Browser session expired, please reload the Browser Steps interface', 500)
# Actions - step/apply/etc, do the thing and return state
if request.method == 'POST':
# @todo - should always be an existing session
step_operation = request.form.get('operation')
step_selector = request.form.get('selector')
step_optional_value = request.form.get('optional_value')
step_n = int(request.form.get('step_n'))
is_last_step = strtobool(request.form.get('is_last_step'))
if step_operation == 'Goto site':
step_operation = 'goto_url'
step_optional_value = None
step_selector = datastore.data['watching'][uuid].get('url')
# @todo try.. accept.. nice errors not popups..
try:
this_session = browsersteps_live_ui_o.get(browsersteps_session_id)
if not this_session:
print("Browser exited")
return make_response('Browser session ran out of time :( Please reload this page.', 401)
this_session.call_action(action_name=step_operation,
selector=step_selector,
optional_value=step_optional_value)
except playwright._impl._api_types.TimeoutError as e:
print("Element wasnt found :-(", step_operation)
return make_response("Element was not found on page", 401)
except playwright._impl._api_types.Error as e:
# Browser/playwright level error
print("Browser error - got playwright._impl._api_types.Error, try reloading the session/browser")
print (str(e))
# Try to find something of value to give back to the user
for l in str(e).splitlines():
if 'DOMException' in l:
return make_response(l, 401)
return make_response('Browser session ran out of time :( Please reload this page.', 401)
# Get visual selector ready/update its data (also use the current filter info from the page?)
# When the last 'apply' button was pressed
# @todo this adds overhead because the xpath selection is happening twice
u = this_session.page.url
if is_last_step and u:
(screenshot, xpath_data) = this_session.request_visualselector_data()
datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
# Setup interface
if request.method == 'GET':
if not browsersteps_playwright_browser_interface:
print("Starting connection with playwright")
logging.debug("browser_steps.py connecting")
from playwright.sync_api import sync_playwright
browsersteps_playwright_browser_interface = sync_playwright().start()
time.sleep(1)
# At 20 minutes, some other variable is closing it
# @todo find out what it is and set it
seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
# keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
try:
browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
except Exception as e:
if 'ECONNREFUSED' in str(e):
return make_response('Unable to start the Playwright session properly, is it running?', 401)
browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
print("Starting connection with playwright - done")
if not browsersteps_live_ui_o.get(browsersteps_session_id):
# Boot up a new session
proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
proxy = None
if proxy_id:
proxy_url = datastore.proxy_list.get(proxy_id).get('url')
if proxy_url:
proxy = {'server': proxy_url}
print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
# Begin the new "Playwright Context" that re-uses the playwright interface
# Each session is a "Playwright Context" as a list, that uses the playwright interface
browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
playwright_browser=browsersteps_playwright_browser_interface_browser,
proxy=proxy)
this_session = browsersteps_live_ui_o[browsersteps_session_id]
if not this_session.page:
cleanup_playwright_session()
return make_response('Browser session ran out of time :( Please reload this page.', 401)
try:
state = this_session.get_current_state()
except playwright._impl._api_types.Error as e:
return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
p = {'screenshot': "data:image/png;base64,{}".format(
base64.b64encode(state[0]).decode('ascii')),
'xpath_data': state[1],
'session_age_start': this_session.age_start,
'browser_time_remaining': round(remaining)
}
# @todo BSON/binary JSON, faster xfer, OR pick it off the disk
return p
return browser_steps_blueprint

@ -0,0 +1,268 @@
#!/usr/bin/python3
import os
import time
import re
from random import randint
# Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
# 0- off, 1- on
browser_step_ui_config = {'Choose one': '0 0',
# 'Check checkbox': '1 0',
# 'Click button containing text': '0 1',
# 'Scroll to bottom': '0 0',
# 'Scroll to element': '1 0',
# 'Scroll to top': '0 0',
# 'Switch to iFrame by index number': '0 1'
# 'Uncheck checkbox': '1 0',
# @todo
'Check checkbox': '1 0',
'Click X,Y': '0 1',
'Click element if exists': '1 0',
'Click element': '1 0',
'Click element containing text': '0 1',
'Enter text in field': '1 1',
# 'Extract text and use as filter': '1 0',
'Goto site': '0 0',
'Press Enter': '0 0',
'Select by label': '1 1',
'Scroll down': '0 0',
'Uncheck checkbox': '1 0',
'Wait for seconds': '0 1',
'Wait for text': '0 1',
# 'Press Page Down': '0 0',
# 'Press Page Up': '0 0',
# weird bug, come back to it later
}
# Good reference - https://playwright.dev/python/docs/input
# https://pythonmana.com/2021/12/202112162236307035.html
#
# ONLY Works in Playwright because we need the fullscreen screenshot
class steppable_browser_interface():
page = None
# Convert and perform "Click Button" for example
def call_action(self, action_name, selector=None, optional_value=None):
now = time.time()
call_action_name = re.sub('[^0-9a-zA-Z]+', '_', action_name.lower())
if call_action_name == 'choose_one':
return
print("> action calling", call_action_name)
# https://playwright.dev/python/docs/selectors#xpath-selectors
if selector.startswith('/') and not selector.startswith('//'):
selector = "xpath=" + selector
action_handler = getattr(self, "action_" + call_action_name)
# Support for Jinja2 variables in the value and selector
from jinja2 import Environment
jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
if selector and ('{%' in selector or '{{' in selector):
selector = str(jinja2_env.from_string(selector).render())
if optional_value and ('{%' in optional_value or '{{' in optional_value):
optional_value = str(jinja2_env.from_string(optional_value).render())
action_handler(selector, optional_value)
self.page.wait_for_timeout(3 * 1000)
print("Call action done in", time.time() - now)
def action_goto_url(self, url, optional_value):
# self.page.set_viewport_size({"width": 1280, "height": 5000})
now = time.time()
response = self.page.goto(url, timeout=0, wait_until='domcontentloaded')
print("Time to goto URL", time.time() - now)
# Wait_until = commit
# - `'commit'` - consider operation to be finished when network response is received and the document started loading.
# Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
# This seemed to solve nearly all 'TimeoutErrors'
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
self.page.wait_for_timeout(extra_wait * 1000)
def action_click_element_containing_text(self, selector=None, value=''):
if not len(value.strip()):
return
elem = self.page.get_by_text(value)
if elem.count():
elem.first.click(delay=randint(200, 500))
def action_enter_text_in_field(self, selector, value):
if not len(selector.strip()):
return
self.page.fill(selector, value, timeout=10 * 1000)
def action_click_element(self, selector, value):
print("Clicking element")
if not len(selector.strip()):
return
self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
def action_click_element_if_exists(self, selector, value):
import playwright._impl._api_types as _api_types
print("Clicking element if exists")
if not len(selector.strip()):
return
try:
self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
except _api_types.TimeoutError as e:
return
except _api_types.Error as e:
# Element was there, but page redrew and now its long long gone
return
def action_click_x_y(self, selector, value):
x, y = value.strip().split(',')
x = int(float(x.strip()))
y = int(float(y.strip()))
self.page.mouse.click(x=x, y=y, delay=randint(200, 500))
def action_scroll_down(self, selector, value):
# Some sites this doesnt work on for some reason
self.page.mouse.wheel(0, 600)
self.page.wait_for_timeout(1000)
def action_wait_for_seconds(self, selector, value):
self.page.wait_for_timeout(int(value) * 1000)
# @todo - in the future make some popout interface to capture what needs to be set
# https://playwright.dev/python/docs/api/class-keyboard
def action_press_enter(self, selector, value):
self.page.keyboard.press("Enter", delay=randint(200, 500))
def action_press_page_up(self, selector, value):
self.page.keyboard.press("PageUp", delay=randint(200, 500))
def action_press_page_down(self, selector, value):
self.page.keyboard.press("PageDown", delay=randint(200, 500))
def action_check_checkbox(self, selector, value):
self.page.locator(selector).check()
def action_uncheck_checkbox(self, selector, value):
self.page.locator(selector).uncheck()
# Responsible for maintaining a live 'context' with browserless
# @todo - how long do contexts live for anyway?
class browsersteps_live_ui(steppable_browser_interface):
context = None
page = None
render_extra_delay = 1
stale = False
# bump and kill this if idle after X sec
age_start = 0
# use a special driver, maybe locally etc
command_executor = os.getenv(
"PLAYWRIGHT_BROWSERSTEPS_DRIVER_URL"
)
# if not..
if not command_executor:
command_executor = os.getenv(
"PLAYWRIGHT_DRIVER_URL",
'ws://playwright-chrome:3000'
).strip('"')
browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
def __init__(self, playwright_browser, proxy=None):
self.age_start = time.time()
self.playwright_browser = playwright_browser
if self.context is None:
self.connect(proxy=proxy)
# Connect and setup a new context
def connect(self, proxy=None):
# Should only get called once - test that
keep_open = 1000 * 60 * 5
now = time.time()
# @todo handle multiple contexts, bind a unique id from the browser on each req?
self.context = self.playwright_browser.new_context(
# @todo
# user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
# proxy=self.proxy,
# This is needed to enable JavaScript execution on GitHub and others
bypass_csp=True,
# Should never be needed
accept_downloads=False,
proxy=proxy
)
self.page = self.context.new_page()
# self.page.set_default_navigation_timeout(keep_open)
self.page.set_default_timeout(keep_open)
# @todo probably this doesnt work
self.page.on(
"close",
self.mark_as_closed,
)
# Listen for all console events and handle errors
self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
print("time to browser setup", time.time() - now)
self.page.wait_for_timeout(1 * 1000)
def mark_as_closed(self):
print("Page closed, cleaning up..")
@property
def has_expired(self):
if not self.page:
return True
def get_current_state(self):
"""Return the screenshot and interactive elements mapping, generally always called after action_()"""
from pkg_resources import resource_string
xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
now = time.time()
self.page.wait_for_timeout(1 * 1000)
# The actual screenshot
screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
self.page.evaluate("var include_filters=''")
# Go find the interactive elements
# @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4'
xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
# So the JS will find the smallest one first
xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
print("Time to complete get_current_state of browser", time.time() - now)
# except
# playwright._impl._api_types.Error: Browser closed.
# @todo show some countdown timer?
return (screenshot, xpath_data)
def request_visualselector_data(self):
"""
Does the same that the playwright operation in content_fetcher does
This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
@todo refactor and remove duplicate code, add include_filters
:param xpath_data:
:param screenshot:
:param current_include_filters:
:return:
"""
self.page.evaluate("var include_filters=''")
from pkg_resources import resource_string
# The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
# @todo dont duplicate these selectors, or just let them both use the same data?
xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
xpath_element_js = xpath_element_js.replace('%ELEMENTS%',
'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section')
xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
return (screenshot, xpath_data)

@ -1,7 +1,7 @@
from abc import abstractmethod from abc import abstractmethod
from pkg_resources import resource_string
import chardet import chardet
import json import json
import logging
import os import os
import requests import requests
import sys import sys
@ -30,6 +30,12 @@ class JSActionExceptions(Exception):
self.message = message self.message = message
return return
class BrowserStepsStepTimout(Exception):
def __init__(self, step_n):
self.step_n = step_n
return
class PageUnloadable(Exception): class PageUnloadable(Exception):
def __init__(self, status_code, url, screenshot=False, message=False): def __init__(self, status_code, url, screenshot=False, message=False):
# Set this so we can use it in other parts of the app # Set this so we can use it in other parts of the app
@ -70,6 +76,8 @@ class Fetcher():
status_code = None status_code = None
content = None content = None
headers = None headers = None
browser_steps = None
browser_steps_screenshot_path = None
fetcher_description = "No description" fetcher_description = "No description"
webdriver_js_execute_code = None webdriver_js_execute_code = None
@ -86,8 +94,10 @@ class Fetcher():
render_extract_delay = 0 render_extract_delay = 0
def __init__(self): def __init__(self):
from pkg_resources import resource_string
# The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8') self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
self.xpath_element_js = self.xpath_element_js.replace('%ELEMENTS%', 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section')
@abstractmethod @abstractmethod
def get_error(self): def get_error(self):
@ -113,11 +123,62 @@ class Fetcher():
def get_last_status_code(self): def get_last_status_code(self):
return self.status_code return self.status_code
@abstractmethod
def screenshot_step(self, step_n):
return None
@abstractmethod @abstractmethod
# Return true/false if this checker is ready to run, in the case it needs todo some special config check etc # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
def is_ready(self): def is_ready(self):
return True return True
def iterate_browser_steps(self):
from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
from playwright._impl._api_types import TimeoutError
from jinja2 import Environment
jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
step_n = 0
if self.browser_steps is not None and len(self.browser_steps):
interface = steppable_browser_interface()
interface.page = self.page
valid_steps = filter(lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), self.browser_steps)
for step in valid_steps:
step_n += 1
print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
self.screenshot_step("before-"+str(step_n))
self.save_step_html("before-"+str(step_n))
try:
optional_value = step['optional_value']
selector = step['selector']
# Support for jinja2 template in step values, with date module added
if '{%' in step['optional_value'] or '{{' in step['optional_value']:
optional_value = str(jinja2_env.from_string(step['optional_value']).render())
if '{%' in step['selector'] or '{{' in step['selector']:
selector = str(jinja2_env.from_string(step['selector']).render())
getattr(interface, "call_action")(action_name=step['operation'],
selector=selector,
optional_value=optional_value)
self.screenshot_step(step_n)
self.save_step_html(step_n)
except TimeoutError:
# Stop processing here
raise BrowserStepsStepTimout(step_n=step_n)
# It's always good to reset these
def delete_browser_steps_screenshots(self):
import glob
if self.browser_steps_screenshot_path is not None:
dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
files = glob.glob(dest)
for f in files:
os.unlink(f)
# Maybe for the future, each fetcher provides its own diff output, could be used for text, image # Maybe for the future, each fetcher provides its own diff output, could be used for text, image
# the current one would return javascript output (as we use JS to generate the diff) # the current one would return javascript output (as we use JS to generate the diff)
@ -136,7 +197,6 @@ def available_fetchers():
return p return p
class base_html_playwright(Fetcher): class base_html_playwright(Fetcher):
fetcher_description = "Playwright {}/Javascript".format( fetcher_description = "Playwright {}/Javascript".format(
os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize() os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@ -174,15 +234,26 @@ class base_html_playwright(Fetcher):
# allow per-watch proxy selection override # allow per-watch proxy selection override
if proxy_override: if proxy_override:
# https://playwright.dev/docs/network#http-proxy self.proxy = {'server': proxy_override}
from urllib.parse import urlparse
parsed = urlparse(proxy_override) def screenshot_step(self, step_n=''):
proxy_url = "{}://{}:{}".format(parsed.scheme, parsed.hostname, parsed.port)
self.proxy = {'server': proxy_url} # There's a bug where we need to do it twice or it doesnt take the whole page, dont know why.
if parsed.username: self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
self.proxy['username'] = parsed.username screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
if parsed.password:
self.proxy['password'] = parsed.password if self.browser_steps_screenshot_path is not None:
destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
logging.debug("Saving step screenshot to {}".format(destination))
with open(destination, 'wb') as f:
f.write(screenshot)
def save_step_html(self, step_n):
content = self.page.content()
destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
logging.debug("Saving step HTML to {}".format(destination))
with open(destination, 'w') as f:
f.write(content)
def run(self, def run(self,
url, url,
@ -195,9 +266,9 @@ class base_html_playwright(Fetcher):
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
import playwright._impl._api_types import playwright._impl._api_types
from playwright._impl._api_types import Error, TimeoutError
response = None
self.delete_browser_steps_screenshots()
response = None
with sync_playwright() as p: with sync_playwright() as p:
browser_type = getattr(p, self.browser_type) browser_type = getattr(p, self.browser_type)
@ -217,89 +288,86 @@ class base_html_playwright(Fetcher):
accept_downloads=False accept_downloads=False
) )
self.page = context.new_page()
if len(request_headers): if len(request_headers):
context.set_extra_http_headers(request_headers) context.set_extra_http_headers(request_headers)
page = context.new_page()
try: try:
page.set_default_navigation_timeout(90000) self.page.set_default_navigation_timeout(90000)
page.set_default_timeout(90000) self.page.set_default_timeout(90000)
# Listen for all console events and handle errors # Listen for all console events and handle errors
page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}")) self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
# Bug - never set viewport size BEFORE page.goto # Bug - never set viewport size BEFORE page.goto
# Waits for the next navigation. Using Python context manager # Waits for the next navigation. Using Python context manager
# prevents a race condition between clicking and waiting for a navigation. # prevents a race condition between clicking and waiting for a navigation.
with page.expect_navigation(): with self.page.expect_navigation():
response = page.goto(url, wait_until='load') response = self.page.goto(url, wait_until='load')
# Wait_until = commit
# - `'commit'` - consider operation to be finished when network response is received and the document started loading.
# Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
# This seemed to solve nearly all 'TimeoutErrors'
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
self.page.wait_for_timeout(extra_wait * 1000)
if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
self.page.evaluate(self.webdriver_js_execute_code)
except playwright._impl._api_types.TimeoutError as e: except playwright._impl._api_types.TimeoutError as e:
context.close() context.close()
browser.close() browser.close()
# This can be ok, we will try to grab what we could retrieve # This can be ok, we will try to grab what we could retrieve
pass pass
except Exception as e: except Exception as e:
print("other exception when page.goto") print ("other exception when page.goto")
print(str(e)) print (str(e))
context.close() context.close()
browser.close() browser.close()
raise PageUnloadable(url=url, status_code=None, message=e.message) raise PageUnloadable(url=url, status_code=None)
if response is None: if response is None:
context.close() context.close()
browser.close() browser.close()
print("response object was none") print ("response object was none")
raise EmptyReply(url=url, status_code=None) raise EmptyReply(url=url, status_code=None)
# Bug 2(?) Set the viewport size AFTER loading the page
self.page.set_viewport_size({"width": 1280, "height": 1024})
# Removed browser-set-size, seemed to be needed to make screenshots work reliably in older playwright versions # Run Browser Steps here
# Was causing exceptions like 'waiting for page but content is changing' etc self.iterate_browser_steps()
# https://www.browserstack.com/docs/automate/playwright/change-browser-window-size 1280x720 should be the default
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
time.sleep(extra_wait) time.sleep(extra_wait)
if self.webdriver_js_execute_code is not None:
try: self.content = self.page.content()
page.evaluate(self.webdriver_js_execute_code)
except Exception as e:
# Is it possible to get a screenshot?
error_screenshot = False
try:
page.screenshot(type='jpeg',
clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024},
quality=1)
# The actual screenshot
error_screenshot = page.screenshot(type='jpeg',
full_page=True,
quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
except Exception as s:
pass
raise JSActionExceptions(status_code=response.status, screenshot=error_screenshot, message=str(e), url=url)
else:
# JS eval was run, now we also wait some time if possible to let the page settle
if self.render_extract_delay:
page.wait_for_timeout(self.render_extract_delay * 1000)
page.wait_for_timeout(500)
self.content = page.content()
self.status_code = response.status self.status_code = response.status
if len(self.page.content().strip()) == 0:
context.close()
browser.close()
print ("Content was empty")
raise EmptyReply(url=url, status_code=None)
# Bug 2(?) Set the viewport size AFTER loading the page
self.page.set_viewport_size({"width": 1280, "height": 1024})
self.status_code = response.status
self.content = self.page.content()
self.headers = response.all_headers() self.headers = response.all_headers()
# So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
if current_include_filters is not None: if current_include_filters is not None:
page.evaluate("var include_filters={}".format(json.dumps(current_include_filters))) self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
else: else:
page.evaluate("var include_filters=''") self.page.evaluate("var include_filters=''")
self.xpath_data = page.evaluate("async () => {" + self.xpath_element_js + "}") self.xpath_data = self.page.evaluate("async () => {" + self.xpath_element_js.replace('%ELEMENTS%', 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary') + "}")
# Bug 3 in Playwright screenshot handling # Bug 3 in Playwright screenshot handling
# Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
@ -310,26 +378,17 @@ class base_html_playwright(Fetcher):
# acceptable screenshot quality here # acceptable screenshot quality here
try: try:
# Quality set to 1 because it's not used, just used as a work-around for a bug, no need to change this. # Quality set to 1 because it's not used, just used as a work-around for a bug, no need to change this.
page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1) self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
# The actual screenshot # The actual screenshot
self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72))) self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
except Exception as e: except Exception as e:
context.close() context.close()
browser.close() browser.close()
raise ScreenshotUnavailable(url=url, status_code=None) raise ScreenshotUnavailable(url=url, status_code=None)
if len(self.content.strip()) == 0:
context.close()
browser.close()
print("Content was empty")
raise EmptyReply(url=url, status_code=None, screenshot=self.screenshot)
context.close() context.close()
browser.close() browser.close()
if not ignore_status_codes and self.status_code!=200:
raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, page_html=self.content, screenshot=self.screenshot)
class base_html_webdriver(Fetcher): class base_html_webdriver(Fetcher):
if os.getenv("WEBDRIVER_URL"): if os.getenv("WEBDRIVER_URL"):
fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL")) fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
@ -423,7 +482,6 @@ class base_html_webdriver(Fetcher):
def is_ready(self): def is_ready(self):
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.common.exceptions import WebDriverException
self.driver = webdriver.Remote( self.driver = webdriver.Remote(
command_executor=self.command_executor, command_executor=self.command_executor,

@ -108,6 +108,11 @@ class perform_site_check():
elif system_webdriver_delay is not None: elif system_webdriver_delay is not None:
fetcher.render_extract_delay = system_webdriver_delay fetcher.render_extract_delay = system_webdriver_delay
# Possible conflict
if prefer_backend == 'html_webdriver':
fetcher.browser_steps = watch.get('browser_steps', None)
fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, uuid)
if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip(): if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code') fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')

@ -1,11 +1,10 @@
import os
import re import re
from wtforms import ( from wtforms import (
BooleanField, BooleanField,
Field,
Form, Form,
IntegerField, IntegerField,
PasswordField,
RadioField, RadioField,
SelectField, SelectField,
StringField, StringField,
@ -13,15 +12,17 @@ from wtforms import (
TextAreaField, TextAreaField,
fields, fields,
validators, validators,
widgets, widgets
) )
from wtforms.fields import FieldList
from wtforms.validators import ValidationError from wtforms.validators import ValidationError
# default
# each select <option data-enabled="enabled-0-0"
from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
from changedetectionio import content_fetcher from changedetectionio import content_fetcher
from changedetectionio.notification import ( from changedetectionio.notification import (
default_notification_body,
default_notification_format,
default_notification_title,
valid_notification_formats, valid_notification_formats,
) )
@ -323,7 +324,6 @@ class ValidateCSSJSONXPATHInput(object):
except: except:
raise ValidationError("A system-error occurred when validating your jq expression") raise ValidationError("A system-error occurred when validating your jq expression")
class quickWatchForm(Form): class quickWatchForm(Form):
url = fields.URLField('URL', validators=[validateURL()]) url = fields.URLField('URL', validators=[validateURL()])
tag = StringField('Group tag', [validators.Optional()]) tag = StringField('Group tag', [validators.Optional()])
@ -342,6 +342,17 @@ class commonSettingsForm(Form):
webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
message="Should contain one or more seconds")]) message="Should contain one or more seconds")])
class SingleBrowserStep(Form):
operation = SelectField('Operation', [validators.Optional()], choices=browser_step_ui_config.keys())
# maybe better to set some <script>var..
selector = StringField('Selector', [validators.Optional()], render_kw={"placeholder": "CSS or xPath selector"})
optional_value = StringField('value', [validators.Optional()], render_kw={"placeholder": "Value"})
# @todo move to JS? ajax fetch new field?
# remove_button = SubmitField('-', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Remove'})
# add_button = SubmitField('+', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Add new step after'})
class watchForm(commonSettingsForm): class watchForm(commonSettingsForm):
url = fields.URLField('URL', validators=[validateURL()]) url = fields.URLField('URL', validators=[validateURL()])
@ -364,8 +375,9 @@ class watchForm(commonSettingsForm):
ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False) ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
check_unique_lines = BooleanField('Only trigger when new lines appear', default=False) check_unique_lines = BooleanField('Only trigger when new lines appear', default=False)
trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()]) trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
if os.getenv("PLAYWRIGHT_DRIVER_URL"):
browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
text_should_not_be_present = StringListField('Block change-detection if text matches', [validators.Optional(), ValidateListRegex()]) text_should_not_be_present = StringListField('Block change-detection if text matches', [validators.Optional(), ValidateListRegex()])
webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()]) webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"}) save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})

@ -1,8 +1,5 @@
// Include the getXpath script directly, easier than fetching // Include the getXpath script directly, easier than fetching
!function (e, n) { function getxpath(e) {
"object" == typeof exports && "undefined" != typeof module ? module.exports = n() : "function" == typeof define && define.amd ? define(n) : (e = e || self).getXPath = n()
}(this, function () {
return function (e) {
var n = e; var n = e;
if (n && n.id) return '//*[@id="' + n.id + '"]'; if (n && n.id) return '//*[@id="' + n.id + '"]';
for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) { for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
@ -18,15 +15,30 @@
} }
return o.length ? "/" + o.reverse().join("/") : "" return o.length ? "/" + o.reverse().join("/") : ""
} }
});
const findUpTag = (el) => { const findUpTag = (el) => {
let r = el let r = el
chained_css = []; chained_css = [];
depth = 0; depth = 0;
// Strategy 1: Keep going up until we hit an ID tag, imagine it's like #list-widget div h4 // Strategy 1: If it's an input, with name, and there's only one, prefer that
if (el.name !== undefined && el.name.length) {
var proposed = el.tagName + "[name=" + el.name + "]";
var proposed_element = window.document.querySelectorAll(proposed);
if(proposed_element.length) {
if (proposed_element.length === 1) {
return proposed;
} else {
// Some sites change ID but name= stays the same, we can hit it if we know the index
// Find all the elements that match and work out the input[n]
var n=Array.from(proposed_element).indexOf(el);
// Return a Playwright selector for nthinput[name=zipcode]
return proposed+" >> nth="+n;
}
}
}
// Strategy 2: Keep going up until we hit an ID tag, imagine it's like #list-widget div h4
while (r.parentNode) { while (r.parentNode) {
if (depth == 5) { if (depth == 5) {
break; break;
@ -50,7 +62,8 @@ const findUpTag = (el) => {
// @todo - if it's SVG or IMG, go into image diff mode // @todo - if it's SVG or IMG, go into image diff mode
var elements = window.document.querySelectorAll("div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary"); // %ELEMENTS% replaced at injection time because different interfaces use it with different settings
var elements = window.document.querySelectorAll("%ELEMENTS%");
var size_pos = []; var size_pos = [];
// after page fetch, inject this JS // after page fetch, inject this JS
// build a map of all elements and their positions (maybe that only include text?) // build a map of all elements and their positions (maybe that only include text?)
@ -85,7 +98,7 @@ for (var i = 0; i < elements.length; i++) {
try { try {
// I've seen on FB and eBay that this doesnt work // I've seen on FB and eBay that this doesnt work
// ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44) // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
xpath_result = getXPath(elements[i]); xpath_result = getxpath(elements[i]);
} catch (e) { } catch (e) {
console.log(e); console.log(e);
continue; continue;
@ -101,8 +114,11 @@ for (var i = 0; i < elements.length; i++) {
width: Math.round(bbox['width']), width: Math.round(bbox['width']),
height: Math.round(bbox['height']), height: Math.round(bbox['height']),
left: Math.floor(bbox['left']), left: Math.floor(bbox['left']),
top: Math.floor(bbox['top']) top: Math.floor(bbox['top']),
tagName: (elements[i].tagName) ? elements[i].tagName.toLowerCase() : '',
tagtype: (elements[i].type) ? elements[i].type.toLowerCase() : ''
}); });
} }

@ -0,0 +1,425 @@
$(document).ready(function () {
// duplicate
var csrftoken = $('input[name=csrf_token]').val();
$.ajaxSetup({
beforeSend: function (xhr, settings) {
if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
xhr.setRequestHeader("X-CSRFToken", csrftoken)
}
}
})
var browsersteps_session_id;
var browserless_seconds_remaining=0;
var apply_buttons_disabled = false;
var include_text_elements = $("#include_text_elements");
var xpath_data;
var current_selected_i;
var state_clicked = false;
var c;
// redline highlight context
var ctx;
var last_click_xy = {'x': -1, 'y': -1}
$(window).resize(function () {
set_scale();
});
$('a#browsersteps-tab').click(function () {
start();
});
// Show seconds remaining until playwright/browserless needs to restart the session
// (See comment at the top of changedetectionio/blueprint/browser_steps/__init__.py )
setInterval(() => {
if (browserless_seconds_remaining >= 1) {
document.getElementById('browserless-seconds-remaining').innerText = browserless_seconds_remaining + " seconds remaining in session";
browserless_seconds_remaining -= 1;
}
}, "1000")
if (window.location.hash == '#browser-steps') {
start();
}
window.addEventListener('hashchange', function () {
if (window.location.hash == '#browser-steps') {
start();
}
// For when the page loads
if (!window.location.hash || window.location.hash != '#browser-steps') {
$("img#browsersteps-img").attr('src', '');
return;
}
});
function set_scale() {
// some things to check if the scaling doesnt work
// - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
selector_image = $("img#browsersteps-img")[0];
selector_image_rect = selector_image.getBoundingClientRect();
// make the canvas and input steps the same size as the image
$('#browsersteps-selector-canvas').attr('height', selector_image_rect.height).attr('width', selector_image_rect.width);
//$('#browsersteps-selector-wrapper').attr('width', selector_image_rect.width);
$('#browser-steps-ui').attr('width', selector_image_rect.width);
x_scale = selector_image_rect.width / xpath_data['browser_width'];
y_scale = selector_image_rect.height / selector_image.naturalHeight;
ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
ctx.fillStyle = 'rgba(255,0,0, 0.1)';
ctx.lineWidth = 3;
console.log("scaling set x: " + x_scale + " by y:" + y_scale);
}
// bootstrap it, this will trigger everything else
$('#browsersteps-img').bind('load', function () {
$('body').addClass('full-width');
console.log("Loaded background...");
document.getElementById("browsersteps-selector-canvas");
c = document.getElementById("browsersteps-selector-canvas");
// redline highlight context
ctx = c.getContext("2d");
// @todo is click better?
$('#browsersteps-selector-canvas').off("mousemove mousedown click");
// Undo disable_browsersteps_ui
$("#browser_steps select,input").removeAttr('disabled').css('opacity', '1.0');
$("#browser-steps-ui").css('opacity', '1.0');
// init
set_scale();
// @todo click ? some better library?
$('#browsersteps-selector-canvas').bind('click', function (e) {
// https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
e.preventDefault()
});
$('#browsersteps-selector-canvas').bind('mousedown', function (e) {
// https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
e.preventDefault()
console.log(e);
console.log("current xpath in index is "+current_selected_i);
last_click_xy = {'x': parseInt((1 / x_scale) * e.offsetX), 'y': parseInt((1 / y_scale) * e.offsetY)}
process_selected(current_selected_i);
current_selected_i = false;
// if process selected returned false, then best we can do is offer a x,y click :(
if (!found_something) {
var first_available = $("ul#browser_steps li.empty").first();
$('select', first_available).val('Click X,Y').change();
$('input[type=text]', first_available).first().val(last_click_xy['x'] + ',' + last_click_xy['y']);
draw_circle_on_canvas(e.offsetX, e.offsetY);
}
});
$('#browsersteps-selector-canvas').bind('mousemove', function (e) {
// checkbox if find elements is enabled
ctx.clearRect(0, 0, c.width, c.height);
ctx.fillStyle = 'rgba(255,0,0, 0.1)';
ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
// Add in offset
if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
var targetOffset = $(e.target).offset();
e.offsetX = e.pageX - targetOffset.left;
e.offsetY = e.pageY - targetOffset.top;
}
current_selected_i = false;
// Reverse order - the most specific one should be deeper/"laster"
// Basically, find the most 'deepest'
//$('#browsersteps-selector-canvas').css('cursor', 'pointer');
for (var i = xpath_data['size_pos'].length; i !== 0; i--) {
// draw all of them? let them choose somehow?
var sel = xpath_data['size_pos'][i - 1];
// If we are in a bounding-box
if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
&&
e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
) {
// Only highlight these interesting types
if (1) {
ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
current_selected_i = i - 1;
break;
// find the smallest one at this x,y
// does it mean sort the xpath list by size (w*h) i think so!
} else {
if ( include_text_elements[0].checked === true) {
// blue one with background instead?
ctx.fillStyle = 'rgba(0,0,255, 0.1)';
ctx.strokeStyle = 'rgba(0,0,200, 0.7)';
$('#browsersteps-selector-canvas').css('cursor', 'grab');
ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
current_selected_i = i - 1;
break;
}
}
}
}
}.debounce(10));
});
// $("#browser-steps-fieldlist").bind('mouseover', function(e) {
// console.log(e.xpath_data_index);
// });
// callback for clicking on an xpath on the canvas
function process_selected(xpath_data_index) {
found_something = false;
var first_available = $("ul#browser_steps li.empty").first();
if (xpath_data_index !== false) {
// Nothing focused, so fill in a new one
// if inpt type button or <button>
// from the top, find the next not used one and use it
var x = xpath_data['size_pos'][xpath_data_index];
console.log(x);
if (x && first_available.length) {
// @todo will it let you click shit that has a layer ontop? probably not.
if (x['tagtype'] === 'text' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search' ) {
$('select', first_available).val('Enter text in field').change();
$('input[type=text]', first_available).first().val(x['xpath']);
$('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
found_something = true;
} else {
// Assume it's just for clicking on
// what are we clicking on?
if (x['tagName'].startsWith('h')|| x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit'|| x['tagtype'] === 'checkbox'|| x['tagtype'] === 'radio'|| x['tagtype'] === 'li') {
$('select', first_available).val('Click element').change();
$('input[type=text]', first_available).first().val(x['xpath']);
found_something = true;
}
}
first_available.xpath_data_index=xpath_data_index;
if (!found_something) {
if ( include_text_elements[0].checked === true) {
// Suggest that we use as filter?
// @todo filters should always be in the last steps, nothing non-filter after it
found_something = true;
ctx.strokeStyle = 'rgba(0,0,255, 0.9)';
ctx.fillStyle = 'rgba(0,0,255, 0.1)';
$('select', first_available).val('Extract text and use as filter').change();
$('input[type=text]', first_available).first().val(x['xpath']);
include_text_elements[0].checked = false;
}
}
}
} else {
}
}
function draw_circle_on_canvas(x, y) {
ctx.beginPath();
ctx.arc(x, y, 8, 0, 2 * Math.PI, false);
ctx.fillStyle = 'rgba(255,0,0, 0.6)';
ctx.fill();
}
function start() {
console.log("Starting browser-steps UI");
browsersteps_session_id=Date.now();
// @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
$('#browser_steps >li:first-child').removeClass('empty');
$('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
$('#browser-steps-ui .loader').show();
$('.clear,.remove', $('#browser_steps >li:first-child')).hide();
$.ajax({
type: "GET",
url: browser_steps_sync_url+"&browsersteps_session_id="+browsersteps_session_id,
statusCode: {
400: function () {
// More than likely the CSRF token was lost when the server restarted
alert("There was a problem processing the request, please reload the page.");
}
}
}).done(function (data) {
xpath_data = data.xpath_data;
$('#browsersteps-img').attr('src', data.screenshot);
// This should trigger 'Goto site'
$('#browser_steps >li:first-child .apply').click();
browserless_seconds_remaining = data.browser_time_remaining;
}).fail(function (data) {
console.log(data);
alert('There was an error communicating with the server.');
});
}
function disable_browsersteps_ui() {
$("#browser_steps select,input").attr('disabled', 'disabled').css('opacity', '0.5');
$("#browser-steps-ui").css('opacity', '0.3');
$('#browsersteps-selector-canvas').off("mousemove mousedown click");
}
////////////////////////// STEPS UI ////////////////////
$('ul#browser_steps [type="text"]').keydown(function (e) {
if (e.keyCode === 13) {
// hitting [enter] in a browser-step input should trigger the 'Apply'
e.preventDefault();
$(".apply", $(this).closest('li')).click();
return false;
}
});
// Look up which step was selected, and enable or disable the related extra fields
// So that people using it dont' get confused
$('ul#browser_steps select').on("change", function () {
var config = browser_steps_config[$(this).val()].split(' ');
var elem_selector = $('tr:nth-child(2) input', $(this).closest('tbody'));
var elem_value = $('tr:nth-child(3) input', $(this).closest('tbody'));
if (config[0] == 0) {
$(elem_selector).fadeOut();
} else {
$(elem_selector).fadeIn();
}
if (config[1] == 0) {
$(elem_value).fadeOut();
} else {
$(elem_value).fadeIn();
}
if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
// @todo handle scale
$(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
}
}).change();
function set_greyed_state() {
$('ul#browser_steps select').not('option:selected[value="Choose one"]').closest('li').removeClass('empty');
$('ul#browser_steps select option:selected[value="Choose one"]').closest('li').addClass('empty');
}
// Add the extra buttons to the steps
$('ul#browser_steps li').each(function (i) {
$(this).append('<div class="control">' +
'<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;' +
'<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
'<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>' +
'</div>')
}
);
$('ul#browser_steps li .control .clear').click(function (element) {
$("select", $(this).closest('li')).val("Choose one").change();
$(":text", $(this).closest('li')).val('');
});
$('ul#browser_steps li .control .remove').click(function (element) {
// so you wanna remove the 2nd (3rd spot 0,1,2,...)
var p = $("#browser_steps li").index($(this).closest('li'));
var elem_to_remove = $("#browser_steps li")[p];
$('.clear', elem_to_remove).click();
$("#browser_steps li").slice(p, 10).each(function (index) {
// get the next one's value from where we clicked
var next = $("#browser_steps li")[p + index + 1];
if (next) {
// and set THIS ones value from the next one
var n = $('input', next);
$("select", $(this)).val($('select', next).val());
$('input', this)[0].value = $(n)[0].value;
$('input', this)[1].value = $(n)[1].value;
// Triggers reconfiguring the field based on the system config
$("select", $(this)).change();
}
});
// Reset their hidden/empty states
set_greyed_state();
});
$('ul#browser_steps li .control .apply').click(function (event) {
// sequential requests @todo refactor
if(apply_buttons_disabled) {
return;
}
var current_data = $(event.currentTarget).closest('li');
$('#browser-steps-ui .loader').fadeIn();
apply_buttons_disabled=true;
$('ul#browser_steps li .control .apply').css('opacity',0.5);
$("#browsersteps-img").css('opacity',0.65);
var is_last_step = 0;
var step_n = $(event.currentTarget).data('step-index');
// On the last step, we should also be getting data ready for the visual selector
$('ul#browser_steps li select').each(function (i) {
if ($(this).val() !== 'Choose one') {
is_last_step += 1;
}
});
if (is_last_step == (step_n+1)) {
is_last_step = true;
} else {
is_last_step = false;
}
// POST the currently clicked step form widget back and await response, redraw
$.ajax({
method: "POST",
url: browser_steps_sync_url+"&browsersteps_session_id="+browsersteps_session_id,
data: {
'operation': $("select[id$='operation']", current_data).first().val(),
'selector': $("input[id$='selector']", current_data).first().val(),
'optional_value': $("input[id$='optional_value']", current_data).first().val(),
'step_n': step_n,
'is_last_step': is_last_step
},
statusCode: {
400: function () {
// More than likely the CSRF token was lost when the server restarted
alert("There was a problem processing the request, please reload the page.");
}
}
}).done(function (data) {
// it should return the new state (selectors available and screenshot)
xpath_data = data.xpath_data;
$('#browsersteps-img').attr('src', data.screenshot);
$('#browser-steps-ui .loader').fadeOut();
apply_buttons_disabled=false;
$("#browsersteps-img").css('opacity',1);
$('ul#browser_steps li .control .apply').css('opacity',1);
browserless_seconds_remaining = data.browser_time_remaining;
}).fail(function (data) {
console.log(data);
if (data.responseText.includes("Browser session expired")) {
disable_browsersteps_ui();
}
apply_buttons_disabled=false;
$('ul#browser_steps li .control .apply').css('opacity',1);
$("#browsersteps-img").css('opacity',1);
//$('#browsersteps-selector-wrapper .loader').fadeOut(2500);
});
});
$("ul#browser_steps select").change(function () {
set_greyed_state();
}).change();
});

@ -0,0 +1,34 @@
$(document).ready(function(){
checkUserVal();
$('#fetch_backend input').on('change', checkUserVal);
});
var checkUserVal = function(){
if($('#fetch_backend input:checked').val()=='html_requests') {
$('#request-override').show();
$('#webdriver-stepper').hide();
} else {
$('#request-override').hide();
$('#webdriver-stepper').show();
}
};
$('a.row-options').on('click', function(){
var row=$(this.closest('tr'));
switch($(this).data("action")) {
case 'remove':
$(row).remove();
break;
case 'add':
var new_row=$(row).clone(true).insertAfter($(row));
$('input', new_new).val("");
break;
case 'add':
var new_row=$(row).clone(true).insertAfter($(row));
$('input', new_new).val("");
break;
case 'resend-step':
break;
}
});

@ -3,7 +3,8 @@
window.addEventListener('hashchange', function () { window.addEventListener('hashchange', function () {
var tabs = document.getElementsByClassName('active'); var tabs = document.getElementsByClassName('active');
while (tabs[0]) { while (tabs[0]) {
tabs[0].classList.remove('active') tabs[0].classList.remove('active');
document.body.classList.remove('full-width');
} }
set_active_tab(); set_active_tab();
}, false); }, false);
@ -20,6 +21,7 @@ if (!has_errors.length) {
} }
function set_active_tab() { function set_active_tab() {
document.body.classList.remove('full-width');
var tab = document.querySelectorAll("a[href='" + location.hash + "']"); var tab = document.querySelectorAll("a[href='" + location.hash + "']");
if (tab.length) { if (tab.length) {
tab[0].parentElement.className = "active"; tab[0].parentElement.className = "active";
@ -45,4 +47,3 @@ function focus_error_tab() {

@ -1,10 +1,10 @@
// Horrible proof of concept code :) // Horrible proof of concept code :)
// yes - this is really a hack, if you are a front-ender and want to help, please get in touch! // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
$(document).ready(function() { $(document).ready(function () {
var current_selected_i; var current_selected_i;
var state_clicked=false; var state_clicked = false;
var c; var c;
@ -13,9 +13,9 @@ $(document).ready(function() {
// redline highlight context // redline highlight context
var ctx; var ctx;
var current_default_xpath=[]; var current_default_xpath = [];
var x_scale=1; var x_scale = 1;
var y_scale=1; var y_scale = 1;
var selector_image; var selector_image;
var selector_image_rect; var selector_image_rect;
var selector_data; var selector_data;
@ -27,27 +27,27 @@ $(document).ready(function() {
bootstrap_visualselector(); bootstrap_visualselector();
}); });
$(document).on('keydown', function(event) { $(document).on('keydown', function (event) {
if ($("img#selector-background").is(":visible")) { if ($("img#selector-background").is(":visible")) {
if (event.key == "Escape") { if (event.key == "Escape") {
state_clicked=false; state_clicked = false;
ctx.clearRect(0, 0, c.width, c.height); ctx.clearRect(0, 0, c.width, c.height);
} }
} }
}); });
// For when the page loads // For when the page loads
if(!window.location.hash || window.location.hash != '#visualselector') { if (!window.location.hash || window.location.hash != '#visualselector') {
$("img#selector-background").attr('src',''); $("img#selector-background").attr('src', '');
return; return;
} }
// Handle clearing button/link // Handle clearing button/link
$('#clear-selector').on('click', function(event) { $('#clear-selector').on('click', function (event) {
if(!state_clicked) { if (!state_clicked) {
alert('Oops, Nothing selected!'); alert('Oops, Nothing selected!');
} }
state_clicked=false; state_clicked = false;
ctx.clearRect(0, 0, c.width, c.height); ctx.clearRect(0, 0, c.width, c.height);
xctx.clearRect(0, 0, c.width, c.height); xctx.clearRect(0, 0, c.width, c.height);
$("#include_filters").val(''); $("#include_filters").val('');
@ -77,58 +77,61 @@ $(document).ready(function() {
// screenshot_url defined in the edit.html template // screenshot_url defined in the edit.html template
}).attr("src", screenshot_url); }).attr("src", screenshot_url);
} }
// Tell visualSelector that the image should update
var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
$("img#selector-background").attr('src',s)
} }
function fetch_data() { function fetch_data() {
// Image is ready // Image is ready
$('.fetching-update-notice').html("Fetching element data.."); $('.fetching-update-notice').html("Fetching element data..");
$.ajax({ $.ajax({
url: watch_visual_selector_data_url, url: watch_visual_selector_data_url,
context: document.body context: document.body
}).done(function (data) { }).done(function (data) {
$('.fetching-update-notice').html("Rendering.."); $('.fetching-update-notice').html("Rendering..");
selector_data = data; selector_data = data;
console.log("Reported browser width from backend: "+data['browser_width']); console.log("Reported browser width from backend: " + data['browser_width']);
state_clicked=false; state_clicked = false;
set_scale(); set_scale();
reflow_selector(); reflow_selector();
$('.fetching-update-notice').fadeOut(); $('.fetching-update-notice').fadeOut();
}); });
}; };
function set_scale() { function set_scale() {
// some things to check if the scaling doesnt work // some things to check if the scaling doesnt work
// - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
selector_image = $("img#selector-background")[0]; $("#selector-wrapper").show();
selector_image_rect = selector_image.getBoundingClientRect(); selector_image = $("img#selector-background")[0];
selector_image_rect = selector_image.getBoundingClientRect();
// make the canvas the same size as the image
$('#selector-canvas').attr('height', selector_image_rect.height); // make the canvas the same size as the image
$('#selector-canvas').attr('width', selector_image_rect.width); $('#selector-canvas').attr('height', selector_image_rect.height);
$('#selector-wrapper').attr('width', selector_image_rect.width); $('#selector-canvas').attr('width', selector_image_rect.width);
x_scale = selector_image_rect.width / selector_data['browser_width']; $('#selector-wrapper').attr('width', selector_image_rect.width);
y_scale = selector_image_rect.height / selector_image.naturalHeight; x_scale = selector_image_rect.width / selector_data['browser_width'];
ctx.strokeStyle = 'rgba(255,0,0, 0.9)'; y_scale = selector_image_rect.height / selector_image.naturalHeight;
ctx.fillStyle = 'rgba(255,0,0, 0.1)'; ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
ctx.lineWidth = 3; ctx.fillStyle = 'rgba(255,0,0, 0.1)';
console.log("scaling set x: "+x_scale+" by y:"+y_scale); ctx.lineWidth = 3;
$("#selector-current-xpath").css('max-width', selector_image_rect.width); console.log("scaling set x: " + x_scale + " by y:" + y_scale);
$("#selector-current-xpath").css('max-width', selector_image_rect.width);
} }
function reflow_selector() { function reflow_selector() {
$(window).resize(function() { $(window).resize(function () {
set_scale(); set_scale();
highlight_current_selected_i(); highlight_current_selected_i();
}); });
var selector_currnt_xpath_text=$("#selector-current-xpath span"); var selector_currnt_xpath_text = $("#selector-current-xpath span");
set_scale(); set_scale();
console.log(selector_data['size_pos'].length + " selectors found"); console.log(selector_data['size_pos'].length + " selectors found");
// highlight the default one if we can find it in the xPath list // highlight the default one if we can find it in the xPath list
// or the xpath matches the default one // or the xpath matches the default one
@ -156,84 +159,84 @@ $(document).ready(function() {
} }
$('#selector-canvas').bind('mousemove', function (e) { $('#selector-canvas').bind('mousemove', function (e) {
if(state_clicked) { if (state_clicked) {
return; return;
} }
ctx.clearRect(0, 0, c.width, c.height); ctx.clearRect(0, 0, c.width, c.height);
current_selected_i=null; current_selected_i = null;
// Add in offset
if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
var targetOffset = $(e.target).offset();
e.offsetX = e.pageX - targetOffset.left;
e.offsetY = e.pageY - targetOffset.top;
}
// Add in offset // Reverse order - the most specific one should be deeper/"laster"
if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) { // Basically, find the most 'deepest'
var targetOffset = $(e.target).offset(); var found = 0;
e.offsetX = e.pageX - targetOffset.left; ctx.fillStyle = 'rgba(205,0,0,0.35)';
e.offsetY = e.pageY - targetOffset.top; for (var i = selector_data['size_pos'].length; i !== 0; i--) {
} // draw all of them? let them choose somehow?
var sel = selector_data['size_pos'][i - 1];
// If we are in a bounding-box
if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
&&
e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
) {
// FOUND ONE
set_current_selected_text(sel.xpath);
ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
// no need to keep digging
// @todo or, O to go out/up, I to go in
// or double click to go up/out the selector?
current_selected_i = i - 1;
found += 1;
break;
}
}
// Reverse order - the most specific one should be deeper/"laster" }.debounce(5));
// Basically, find the most 'deepest'
var found=0; function set_current_selected_text(s) {
ctx.fillStyle = 'rgba(205,0,0,0.35)'; selector_currnt_xpath_text[0].innerHTML = s;
for (var i = selector_data['size_pos'].length; i!==0; i--) {
// draw all of them? let them choose somehow?
var sel = selector_data['size_pos'][i-1];
// If we are in a bounding-box
if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
&&
e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
) {
// FOUND ONE
set_current_selected_text(sel.xpath);
ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
// no need to keep digging
// @todo or, O to go out/up, I to go in
// or double click to go up/out the selector?
current_selected_i=i-1;
found+=1;
break;
}
} }
}.debounce(5)); function highlight_current_selected_i() {
if (state_clicked) {
state_clicked = false;
xctx.clearRect(0, 0, c.width, c.height);
return;
}
function set_current_selected_text(s) { var sel = selector_data['size_pos'][current_selected_i];
selector_currnt_xpath_text[0].innerHTML=s; if (sel[0] == '/') {
} // @todo - not sure just checking / is right
$("#include_filters").val('xpath:' + sel.xpath);
} else {
$("#include_filters").val(sel.xpath);
}
xctx.fillStyle = 'rgba(205,205,205,0.95)';
xctx.strokeStyle = 'rgba(225,0,0,0.9)';
xctx.lineWidth = 3;
xctx.fillRect(0, 0, c.width, c.height);
// Clear out what only should be seen (make a clear/clean spot)
xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
state_clicked = true;
set_current_selected_text(sel.xpath);
function highlight_current_selected_i() {
if(state_clicked) {
state_clicked=false;
xctx.clearRect(0,0,c.width, c.height);
return;
} }
var sel = selector_data['size_pos'][current_selected_i];
if (sel[0] == '/') { $('#selector-canvas').bind('mousedown', function (e) {
// @todo - not sure just checking / is right highlight_current_selected_i();
$("#include_filters").val('xpath:'+sel.xpath); });
} else {
$("#include_filters").val(sel.xpath);
}
xctx.fillStyle = 'rgba(205,205,205,0.95)';
xctx.strokeStyle = 'rgba(225,0,0,0.9)';
xctx.lineWidth = 3;
xctx.fillRect(0,0,c.width, c.height);
// Clear out what only should be seen (make a clear/clean spot)
xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
state_clicked=true;
set_current_selected_text(sel.xpath);
}
$('#selector-canvas').bind('mousedown', function (e) {
highlight_current_selected_i();
});
} }
}); });

@ -0,0 +1,81 @@
#browser_steps {
/* convert rows to horizontal cells */
th {
display: none;
}
li {
list-style: decimal;
padding: 5px;
.control {
padding-left: 5px;
padding-right: 5px;
a {
font-size: 70%;
}
}
&.empty {
padding: 0px;
opacity: 0.35;
.control {
display: none;
}
}
&:hover {
background: #eee;
}
> label {
display: none;
}
}
}
#browser-steps-fieldlist {
height: 100%;
overflow-y: scroll;
}
#browser-steps .flex-wrapper {
display: flex;
flex-flow: row;
height: 600px; /*@todo make this dynamic */
}
/* this is duplicate :( */
#browsersteps-selector-wrapper {
height: 100%;
width: 100%;
overflow-y: scroll;
position: relative;
//width: 100%;
> img {
position: absolute;
max-width: 100%;
}
> canvas {
position: relative;
max-width: 100%;
&:hover {
cursor: pointer;
}
}
.loader {
position: absolute;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
margin-left: -40px;
z-index: 100;
}
/* nice tall skinny one */
.spinner, .spinner:after {
width: 80px;
height: 80px;
font-size: 3px;
}
}

@ -0,0 +1,44 @@
/* spinner */
.spinner,
.spinner:after {
border-radius: 50%;
width: 10px;
height: 10px;
}
.spinner {
margin: 0px auto;
font-size: 3px;
vertical-align: middle;
display: inline-block;
text-indent: -9999em;
border-top: 1.1em solid rgba(38,104,237, 0.2);
border-right: 1.1em solid rgba(38,104,237, 0.2);
border-bottom: 1.1em solid rgba(38,104,237, 0.2);
border-left: 1.1em solid #2668ed;
-webkit-transform: translateZ(0);
-ms-transform: translateZ(0);
transform: translateZ(0);
-webkit-animation: load8 1.1s infinite linear;
animation: load8 1.1s infinite linear;
}
@-webkit-keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg);
}
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg);
}
}
@keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg);
}
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg);
}
}

@ -1,9 +1,107 @@
/* /*
* -- BASE STYLES -- * -- BASE STYLES --
* Most of these are inherited from Base, but I want to change a few. * Most of these are inherited from Base, but I want to change a few.
* nvm use v14.18.1 && npm install && npm run build nvm use v14.18.1 && npm install && npm run build
* or npm run watch * or npm run watch
*/ */
/* spinner */
.spinner,
.spinner:after {
border-radius: 50%;
width: 10px;
height: 10px; }
.spinner {
margin: 0px auto;
font-size: 3px;
vertical-align: middle;
display: inline-block;
text-indent: -9999em;
border-top: 1.1em solid rgba(38, 104, 237, 0.2);
border-right: 1.1em solid rgba(38, 104, 237, 0.2);
border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
border-left: 1.1em solid #2668ed;
-webkit-transform: translateZ(0);
-ms-transform: translateZ(0);
transform: translateZ(0);
-webkit-animation: load8 1.1s infinite linear;
animation: load8 1.1s infinite linear; }
@-webkit-keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg); }
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg); } }
@keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg); }
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg); } }
#browser_steps {
/* convert rows to horizontal cells */ }
#browser_steps th {
display: none; }
#browser_steps li {
list-style: decimal;
padding: 5px; }
#browser_steps li .control {
padding-left: 5px;
padding-right: 5px; }
#browser_steps li .control a {
font-size: 70%; }
#browser_steps li.empty {
padding: 0px;
opacity: 0.35; }
#browser_steps li.empty .control {
display: none; }
#browser_steps li:hover {
background: #eee; }
#browser_steps li > label {
display: none; }
#browser-steps-fieldlist {
height: 100%;
overflow-y: scroll; }
#browser-steps .flex-wrapper {
display: flex;
flex-flow: row;
height: 600px;
/*@todo make this dynamic */ }
/* this is duplicate :( */
#browsersteps-selector-wrapper {
height: 100%;
width: 100%;
overflow-y: scroll;
position: relative;
/* nice tall skinny one */ }
#browsersteps-selector-wrapper > img {
position: absolute;
max-width: 100%; }
#browsersteps-selector-wrapper > canvas {
position: relative;
max-width: 100%; }
#browsersteps-selector-wrapper > canvas:hover {
cursor: pointer; }
#browsersteps-selector-wrapper .loader {
position: absolute;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
margin-left: -40px;
z-index: 100; }
#browsersteps-selector-wrapper .spinner, #browsersteps-selector-wrapper .spinner:after {
width: 80px;
height: 80px;
font-size: 3px; }
.arrow { .arrow {
border: solid #1b98f8; border: solid #1b98f8;
border-width: 0 2px 2px 0; border-width: 0 2px 2px 0;
@ -132,7 +230,7 @@ body:after, body:before {
.fetch-error { .fetch-error {
padding-top: 1em; padding-top: 1em;
font-size: 60%; font-size: 80%;
max-width: 400px; max-width: 400px;
display: block; } display: block; }
@ -252,12 +350,12 @@ footer {
#top-right-menu { #top-right-menu {
/* /*
position: absolute; position: absolute;
right: 0px; right: 0px;
background: linear-gradient(to right, #fff0, #fff 10%); background: linear-gradient(to right, #fff0, #fff 10%);
padding-left: 20px; padding-left: 20px;
padding-right: 10px; padding-right: 10px;
*/ } */ }
.sticky-tab { .sticky-tab {
position: absolute; position: absolute;
@ -356,10 +454,10 @@ footer {
input[type='text'] { input[type='text'] {
width: 100%; } width: 100%; }
/* /*
Max width before this PARTICULAR table gets nasty Max width before this PARTICULAR table gets nasty
This query will take effect for any screen smaller than 760px This query will take effect for any screen smaller than 760px
and also iPads specifically. and also iPads specifically.
*/ */
.watch-table { .watch-table {
/* Force table to not be like tables anymore */ /* Force table to not be like tables anymore */
/* Force table to not be like tables anymore */ /* Force table to not be like tables anymore */
@ -446,7 +544,7 @@ and also iPads specifically.
.tab-pane-inner:target { .tab-pane-inner:target {
display: block; } display: block; }
#beta-logo { .beta-logo {
height: 50px; height: 50px;
right: -3px; right: -3px;
top: -3px; top: -3px;
@ -455,6 +553,9 @@ and also iPads specifically.
#selector-header { #selector-header {
padding-bottom: 1em; } padding-bottom: 1em; }
body.full-width .edit-form {
width: 95%; }
.edit-form { .edit-form {
min-width: 70%; min-width: 70%;
/* so it cant overflow */ /* so it cant overflow */
@ -481,7 +582,7 @@ ul {
width: 5em; } width: 5em; }
#selector-wrapper { #selector-wrapper {
height: 600px; height: 100%;
overflow-y: scroll; overflow-y: scroll;
position: relative; } position: relative; }
#selector-wrapper > img { #selector-wrapper > img {
@ -507,44 +608,24 @@ ul {
#api-key-copy { #api-key-copy {
color: #0078e7; } color: #0078e7; }
/* spinner */ .button-green {
.loader, background-color: #42dd53; }
.loader:after {
border-radius: 50%; .button-red {
width: 10px; background-color: #dd4242; }
height: 10px; }
.noselect {
.loader { -webkit-touch-callout: none;
margin: 0px auto; /* iOS Safari */
font-size: 3px; -webkit-user-select: none;
vertical-align: middle; /* Safari */
display: inline-block; -moz-user-select: none;
text-indent: -9999em; /* Old versions of Firefox */
border-top: 1.1em solid rgba(38, 104, 237, 0.2); -ms-user-select: none;
border-right: 1.1em solid rgba(38, 104, 237, 0.2); /* Internet Explorer/Edge */
border-bottom: 1.1em solid rgba(38, 104, 237, 0.2); user-select: none;
border-left: 1.1em solid #2668ed; /* Non-prefixed version, currently
-webkit-transform: translateZ(0); supported by Chrome, Edge, Opera and Firefox */ }
-ms-transform: translateZ(0);
transform: translateZ(0);
-webkit-animation: load8 1.1s infinite linear;
animation: load8 1.1s infinite linear; }
@-webkit-keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg); }
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg); } }
@keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg); }
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg); } }
.snapshot-age { .snapshot-age {
padding: 4px; padding: 4px;

@ -1,15 +1,19 @@
/* /*
* -- BASE STYLES -- * -- BASE STYLES --
* Most of these are inherited from Base, but I want to change a few. * Most of these are inherited from Base, but I want to change a few.
* nvm use v14.18.1 && npm install && npm run build nvm use v14.18.1 && npm install && npm run build
* or npm run watch * or npm run watch
*/ */
@import "parts/spinners";
@import "parts/browser-steps";
@import "parts/_arrows.scss"; @import "parts/_arrows.scss";
body { body {
color: #333; color: #333;
background: #262626; background: #262626;
} }
.pure-table-even { .pure-table-even {
background: #fff; background: #fff;
} }
@ -209,46 +213,50 @@ body:after, body:before {
} }
.messages { .messages {
li { li {
list-style: none; list-style: none;
padding: 1em; padding: 1em;
border-radius: 10px; border-radius: 10px;
color: #fff; color: #fff;
font-weight: bold; font-weight: bold;
&.message {
background: rgba(255, 255, 255, .2); &.message {
} background: rgba(255, 255, 255, .2);
&.error { }
background: rgba(255, 1, 1, .5);
} &.error {
&.notice { background: rgba(255, 1, 1, .5);
background: rgba(255, 255, 255, .5);
}
} }
&.with-share-link {
> *:hover { &.notice {
cursor:pointer; background: rgba(255, 255, 255, .5);
}
} }
}
&.with-share-link {
> *:hover {
cursor: pointer;
}
}
} }
#notification-customisation { #notification-customisation {
border: 1px solid #ccc; border: 1px solid #ccc;
padding: 0.5rem; padding: 0.5rem;
border-radius: 5px; border-radius: 5px;
} }
#notification-error-log { #notification-error-log {
border: 1px solid #ccc; border: 1px solid #ccc;
padding: 1rem; padding: 1rem;
border-radius: 5px; border-radius: 5px;
overflow-wrap: break-word; overflow-wrap: break-word;
} }
#token-table { #token-table {
&.pure-table td, &.pure-table th { &.pure-table td, &.pure-table th {
font-size: 80%; font-size: 80%;
} }
} }
#new-watch-form { #new-watch-form {
@ -256,13 +264,16 @@ body:after, body:before {
padding: 1em; padding: 1em;
border-radius: 10px; border-radius: 10px;
margin-bottom: 1em; margin-bottom: 1em;
input { input {
display: inline-block; display: inline-block;
margin-bottom: 5px; margin-bottom: 5px;
} }
.label { .label {
display: none; display: none;
} }
legend { legend {
color: #fff; color: #fff;
font-weight: bold; font-weight: bold;
@ -281,8 +292,6 @@ body:after, body:before {
} }
#diff-col { #diff-col {
padding-left: 40px; padding-left: 40px;
} }
@ -296,15 +305,16 @@ body:after, body:before {
border-top-right-radius: 5px; border-top-right-radius: 5px;
border-bottom-right-radius: 5px; border-bottom-right-radius: 5px;
box-shadow: 5px 0 5px -2px #888; box-shadow: 5px 0 5px -2px #888;
a {
color: #1b98f8; a {
cursor: grabbing; color: #1b98f8;
-moz-user-select: none; cursor: grabbing;
-webkit-user-select: none; -moz-user-select: none;
-ms-user-select: none; -webkit-user-select: none;
user-select: none; -ms-user-select: none;
-o-user-select: none; user-select: none;
} -o-user-select: none;
}
} }
footer { footer {
@ -319,14 +329,14 @@ footer {
} }
#top-right-menu { #top-right-menu {
// Just let flex overflow the x axis for now // Just let flex overflow the x axis for now
/* /*
position: absolute; position: absolute;
right: 0px; right: 0px;
background: linear-gradient(to right, #fff0, #fff 10%); background: linear-gradient(to right, #fff0, #fff 10%);
padding-left: 20px; padding-left: 20px;
padding-right: 10px; padding-right: 10px;
*/ */
} }
.sticky-tab { .sticky-tab {
@ -335,12 +345,15 @@ footer {
font-size: 65%; font-size: 65%;
background: #fff; background: #fff;
padding: 10px; padding: 10px;
&#left-sticky { &#left-sticky {
left: 0px; left: 0px;
} }
&#right-sticky { &#right-sticky {
right: 0px; right: 0px;
} }
&#hosted-sticky { &#hosted-sticky {
right: 0px; right: 0px;
top: 100px; top: 100px;
@ -374,43 +387,49 @@ footer {
} }
.monospaced-textarea { .monospaced-textarea {
textarea { textarea {
width: 100%; width: 100%;
font-family: monospace; font-family: monospace;
white-space: pre; white-space: pre;
overflow-wrap: normal; overflow-wrap: normal;
overflow-x: scroll; overflow-x: scroll;
} }
} }
.pure-form { .pure-form {
fieldset { fieldset {
padding-top: 0px; padding-top: 0px;
ul {
padding-bottom: 0px; ul {
margin-bottom: 0px; padding-bottom: 0px;
} margin-bottom: 0px;
} }
.pure-control-group, .pure-group, .pure-controls { }
padding-bottom: 1em;
div { .pure-control-group, .pure-group, .pure-controls {
margin: 0px; padding-bottom: 1em;
}
.checkbox { div {
> * { margin: 0px;
display: inline; }
vertical-align: middle;
} .checkbox {
> label { > * {
padding-left: 5px; display: inline;
} vertical-align: middle;
} }
> label {
padding-left: 5px;
}
} }
}
/* The input fields with errors */ /* The input fields with errors */
.error { .error {
input { input {
background-color: #ffebeb; background-color: #ffebeb;
} }
} }
@ -422,9 +441,10 @@ footer {
vertical-align: middle; vertical-align: middle;
-webkit-box-sizing: border-box; -webkit-box-sizing: border-box;
box-sizing: border-box; box-sizing: border-box;
li { li {
margin-left: 1em; margin-left: 1em;
color: #dd0000; color: #dd0000;
} }
} }
@ -435,19 +455,22 @@ footer {
textarea { textarea {
width: 100%; width: 100%;
} }
.inline-radio { .inline-radio {
ul { ul {
margin: 0px; margin: 0px;
list-style: none; list-style: none;
li {
> * { li {
display: inline-block; > * {
} display: inline-block;
} }
} }
}
} }
} }
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) { @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
.box { .box {
max-width: 95% max-width: 95%
@ -462,7 +485,6 @@ footer {
} }
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) { @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
div.sticky-tab#hosted-sticky { div.sticky-tab#hosted-sticky {
@ -486,11 +508,11 @@ footer {
width: 100%; width: 100%;
} }
/* /*
Max width before this PARTICULAR table gets nasty Max width before this PARTICULAR table gets nasty
This query will take effect for any screen smaller than 760px This query will take effect for any screen smaller than 760px
and also iPads specifically. and also iPads specifically.
*/ */
.watch-table { .watch-table {
/* Force table to not be like tables anymore */ /* Force table to not be like tables anymore */
thead, tbody, th, td, tr { thead, tbody, th, td, tr {
@ -567,19 +589,19 @@ and also iPads specifically.
- Rely always on width in CSS - Rely always on width in CSS
*/ */
@media only screen and (min-width: 761px) { @media only screen and (min-width: 761px) {
/* m-d is medium-desktop */ /* m-d is medium-desktop */
.m-d { .m-d {
min-width: 80%; min-width: 80%;
} }
} }
.tabs { .tabs {
ul { ul {
margin: 0px; margin: 0px;
padding: 0px; padding: 0px;
display:block; display: block;
li { li {
margin-right: 3px; margin-right: 3px;
display: inline-block; display: inline-block;
@ -588,13 +610,15 @@ and also iPads specifically.
border-top-right-radius: 5px; border-top-right-radius: 5px;
background-color: rgba(255, 255, 255, 0.2); background-color: rgba(255, 255, 255, 0.2);
&.active,:target { &.active, :target {
background-color: #fff; background-color: #fff;
a { a {
color: #222; color: #222;
font-weight: bold; font-weight: bold;
} }
} }
a { a {
display: block; display: block;
padding: 0.8em; padding: 0.8em;
@ -606,7 +630,7 @@ and also iPads specifically.
$form-edge-padding: 20px; $form-edge-padding: 20px;
.pure-form-stacked { .pure-form-stacked {
>div:first-child { > div:first-child {
display: block; display: block;
} }
} }
@ -620,39 +644,48 @@ $form-edge-padding: 20px;
} }
.tab-pane-inner { .tab-pane-inner {
&:not(:target) { &:not(:target) {
display: none; display: none;
} }
&:target {
display: block; &:target {
} display: block;
// doesnt need padding because theres another row of buttons/activity }
padding: 0px;
// doesnt need padding because theres another row of buttons/activity
padding: 0px;
} }
#beta-logo { .beta-logo {
height: 50px; height: 50px;
// looks better when it's hanging off a little // looks better when it's hanging off a little
right: -3px; right: -3px;
top: -3px; top: -3px;
position: absolute; position: absolute;
} }
#selector-header { #selector-header {
padding-bottom: 1em; padding-bottom: 1em;
}
body.full-width {
.edit-form {
width: 95%;
}
} }
.edit-form { .edit-form {
min-width: 70%; min-width: 70%;
/* so it cant overflow */ /* so it cant overflow */
max-width: 95%; max-width: 95%;
.box-wrap { .box-wrap {
position: relative; position: relative;
} }
.inner { .inner {
background: #fff;; background: #fff;;
padding: $form-edge-padding; padding: $form-edge-padding;
} }
#actions { #actions {
display: block; display: block;
background: #fff; background: #fff;
@ -664,38 +697,41 @@ $form-edge-padding: 20px;
} }
ul { ul {
padding-left: 1em; padding-left: 1em;
padding-top: 0px; padding-top: 0px;
margin-top: 4px; margin-top: 4px;
} }
.time-check-widget { .time-check-widget {
tr { tr {
display: inline; display: inline;
input[type="number"] {
width: 5em; input[type="number"] {
} width: 5em;
} }
}
} }
#selector-wrapper { #selector-wrapper {
height: 600px; height: 100%;
overflow-y: scroll; overflow-y: scroll;
position: relative; position: relative;
//width: 100%; //width: 100%;
> img { > img {
position: absolute; position: absolute;
z-index: 4; z-index: 4;
max-width: 100%; max-width: 100%;
} }
>canvas {
> canvas {
position: relative; position: relative;
z-index: 5; z-index: 5;
max-width: 100%; max-width: 100%;
&:hover {
cursor: pointer; &:hover {
} cursor: pointer;
} }
}
} }
#selector-current-xpath { #selector-current-xpath {
@ -703,9 +739,9 @@ ul {
} }
#webdriver-override-options { #webdriver-override-options {
input[type="number"] { input[type="number"] {
width: 5em; width: 5em;
} }
} }
#api-key { #api-key {
@ -718,48 +754,22 @@ ul {
color: #0078e7; color: #0078e7;
} }
/* spinner */ .button-green {
.loader, background-color: #42dd53;
.loader:after {
border-radius: 50%;
width: 10px;
height: 10px;
} }
.loader {
margin: 0px auto; .button-red {
font-size: 3px; background-color: #dd4242;
vertical-align: middle; }
display: inline-block;
text-indent: -9999em; .noselect {
border-top: 1.1em solid rgba(38,104,237, 0.2); -webkit-touch-callout: none; /* iOS Safari */
border-right: 1.1em solid rgba(38,104,237, 0.2); -webkit-user-select: none; /* Safari */
border-bottom: 1.1em solid rgba(38,104,237, 0.2); -moz-user-select: none; /* Old versions of Firefox */
border-left: 1.1em solid #2668ed; -ms-user-select: none; /* Internet Explorer/Edge */
-webkit-transform: translateZ(0); user-select: none;
-ms-transform: translateZ(0); /* Non-prefixed version, currently
transform: translateZ(0); supported by Chrome, Edge, Opera and Firefox */
-webkit-animation: load8 1.1s infinite linear;
animation: load8 1.1s infinite linear;
}
@-webkit-keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg);
}
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg);
}
}
@keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg);
}
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg);
}
} }
.snapshot-age { .snapshot-age {

@ -303,8 +303,8 @@ class ChangeDetectionStore:
'text_should_not_be_present', 'text_should_not_be_present',
'title', 'title',
'trigger_text', 'trigger_text',
'webdriver_js_execute_code',
'url', 'url',
'webdriver_js_execute_code',
]: ]:
if res.get(k): if res.get(k):
if k != 'css_filter': if k != 'css_filter':

@ -13,11 +13,17 @@
const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}'); const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
{% endif %} {% endif %}
const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
</script> </script>
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script> <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script> <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script> <script type="text/javascript" src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script> {% if playwright_enabled %}
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
{% endif %}
<div class="edit-form monospaced-textarea"> <div class="edit-form monospaced-textarea">
@ -25,6 +31,9 @@
<ul> <ul>
<li class="tab" id=""><a href="#general">General</a></li> <li class="tab" id=""><a href="#general">General</a></li>
<li class="tab"><a href="#request">Request</a></li> <li class="tab"><a href="#request">Request</a></li>
{% if playwright_enabled %}
<li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
{% endif %}
<li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li> <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
<li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li> <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
<li class="tab"><a href="#notifications">Notifications</a></li> <li class="tab"><a href="#notifications">Notifications</a></li>
@ -135,6 +144,44 @@ User-Agent: wonderbra 1.0") }}
</div> </div>
</fieldset> </fieldset>
</div> </div>
{% if playwright_enabled %}
<div class="tab-pane-inner" id="browser-steps">
<img class="beta-logo" src="/static/images/beta-logo.png">
<fieldset>
<div class="pure-control-group">
<!--
Too hard right now, better to just send the events to the fetcher for now and leave it in the final screenshot
and/or report an error
<a id="play-steps" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Play steps ▶</a>
-->
<!--- Do this later -->
<div class="checkbox" style="display: none;">
<input type=checkbox id="include_text_elements" > <label for="include_text_elements">Turn on text finder</label>
</div>
<div class="flex-wrapper" >
<div id="browser-steps-ui" class="noselect" style="width: 100%; background-color: #eee; border-radius: 5px;">
<div class="noselect" id="browsersteps-selector-wrapper" style="width: 100%">
<span class="loader">
<div class="spinner"></div>
</span>
<img class="noselect" id="browsersteps-img" src="" style="max-width: 100%; width: 100%;" />
<canvas class="noselect" id="browsersteps-selector-canvas" style="max-width: 100%; width: 100%;"></canvas>
</div>
</div>
<div id="browser-steps-fieldlist" style="padding-left: 1em; width: 350px; font-size: 80%;" >
<span id="browserless-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
{{ render_field(form.browser_steps) }}
</div>
</div>
</div>
</fieldset>
</div>
{% endif %}
<div class="tab-pane-inner" id="notifications"> <div class="tab-pane-inner" id="notifications">
<fieldset> <fieldset>
@ -295,39 +342,32 @@ Unavailable") }}
</div> </div>
<div class="tab-pane-inner visual-selector-ui" id="visualselector"> <div class="tab-pane-inner visual-selector-ui" id="visualselector">
<img id="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}"> <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
<strong>Pro-tip:</strong> This tool is only for limiting which elements will be included on a change-detection, not for interacting with browser directly.
<fieldset> <fieldset>
<div class="pure-control-group"> <div class="pure-control-group">
{% if visualselector_enabled %} {% if visualselector_enabled %}
{% if visualselector_data_is_ready %}
<div id="selector-header">
<a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
<i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
</div>
<div id="selector-wrapper">
<!-- request the screenshot and get the element offset info ready -->
<!-- use img src ready load to know everything is ready to map out -->
<!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
<img id="selector-background" />
<canvas id="selector-canvas"></canvas>
</div>
<div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
<span class="pure-form-message-inline"> <span class="pure-form-message-inline">
<p><span style="font-weight: bold">Beta!</span> The Visual Selector is new and there may be minor bugs, please report pages that dont work, help us to improve this software!</p> The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br/><br/>
</span> </span>
{% else %} <div id="selector-header">
<span class="pure-form-message-inline">Screenshot and element data is not available or not yet ready.</span> <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
{% endif %} <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
</div>
<div id="selector-wrapper" style="display: none">
<!-- request the screenshot and get the element offset info ready -->
<!-- use img src ready load to know everything is ready to map out -->
<!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
<img id="selector-background" />
<canvas id="selector-canvas"></canvas>
</div>
<div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
{% else %} {% else %}
<span class="pure-form-message-inline"> <span class="pure-form-message-inline">
<p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p> <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
<p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p> <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
<p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p> <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
</span> </span>
{% endif %} {% endif %}
</div> </div>

@ -5,7 +5,7 @@ from flask import url_for
from ..util import live_server_setup, wait_for_all_checks from ..util import live_server_setup, wait_for_all_checks
import logging import logging
# Requires playwright to be installed
def test_fetch_webdriver_content(client, live_server): def test_fetch_webdriver_content(client, live_server):
live_server_setup(live_server) live_server_setup(live_server)

@ -10,8 +10,9 @@ def test_visual_selector_content_ready(client, live_server):
import json import json
assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test" assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
live_server_setup(live_server)
time.sleep(1) time.sleep(1)
live_server_setup(live_server)
# Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
test_url = "https://changedetection.io/ci-test/test-runjs.html" test_url = "https://changedetection.io/ci-test/test-runjs.html"
@ -35,7 +36,6 @@ def test_visual_selector_content_ready(client, live_server):
follow_redirects=True follow_redirects=True
) )
assert b"unpaused" in res.data assert b"unpaused" in res.data
time.sleep(1)
wait_for_all_checks(client) wait_for_all_checks(client)
uuid = extract_UUID_from_client(client) uuid = extract_UUID_from_client(client)

@ -113,6 +113,34 @@ class update_worker(threading.Thread):
self.notification_q.put(n_object) self.notification_q.put(n_object)
print("Sent filter not found notification for {}".format(watch_uuid)) print("Sent filter not found notification for {}".format(watch_uuid))
def send_step_failure_notification(self, watch_uuid, step_n):
watch = self.datastore.data['watching'].get(watch_uuid, False)
if not watch:
return
threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
'notification_body': "Your configured browser step at position {} for {{watch['url']}} "
"did not appear on the page after {} attempts, did the page change layout? "
"Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n"
"Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
'notification_format': 'text'}
if len(watch['notification_urls']):
n_object['notification_urls'] = watch['notification_urls']
elif len(self.datastore.data['settings']['application']['notification_urls']):
n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
# Only prepare to notify if the rules above matched
if 'notification_urls' in n_object:
n_object.update({
'watch_url': watch['url'],
'uuid': watch_uuid
})
self.notification_q.put(n_object)
print("Sent step not found notification for {}".format(watch_uuid))
def cleanup_error_artifacts(self, uuid): def cleanup_error_artifacts(self, uuid):
# All went fine, remove error artifacts # All went fine, remove error artifacts
cleanup_files = ["last-error-screenshot.png", "last-error.txt"] cleanup_files = ["last-error-screenshot.png", "last-error.txt"]
@ -213,6 +241,32 @@ class update_worker(threading.Thread):
process_changedetection_results = True process_changedetection_results = True
except content_fetcher.BrowserStepsStepTimout as e:
if not self.datastore.data['watching'].get(uuid):
continue
err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
# So that we get a trigger when the content is added again
'previous_md5': ''})
if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
c += 1
# Send notification if we reached the threshold?
threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
0)
print("Step for {} not found, consecutive_filter_failures: {}".format(uuid, c))
if threshold > 0 and c >= threshold:
if not self.datastore.data['watching'][uuid].get('notification_muted'):
self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
c = 0
self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
process_changedetection_results = False
except content_fetcher.EmptyReply as e: except content_fetcher.EmptyReply as e:
# Some kind of custom to-str handler in the exception handler that does this? # Some kind of custom to-str handler in the exception handler that does this?
err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code) err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)

Binary file not shown.

After

Width:  |  Height:  |  Size: 301 KiB

@ -57,4 +57,3 @@ jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux"
# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future # Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
pillow pillow
# playwright is installed at Dockerfile build time because it's not available on all platforms # playwright is installed at Dockerfile build time because it's not available on all platforms

Loading…
Cancel
Save