Self-Hosted
/
changedetection.io
mirror of https://github.com/dgtlmoon/changedetection.io.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

BrowserSteps - Refactored to re-use playwright context which should solve some errors

Browse Source
search-list
dgtlmoon 2 years ago
parent 5f338d7824
commit e4f6d54ae2
5 changed files with 138 additions and 163 deletions
Show Stats Download Patch File Download Diff File

219
changedetectionio/blueprint/browser_steps/__init__.py
Unescape View File

@ -27,58 +27,103 @@ import os
import logging import logging
from changedetectionio.store import ChangeDetectionStore from changedetectionio.store import ChangeDetectionStore
from changedetectionio import login_optionally_required from changedetectionio import login_optionally_required
browsersteps_live_ui_o = {}
browsersteps_playwright_browser_interface = None browsersteps_sessions = {}
browsersteps_playwright_browser_interface_browser = None io_interface_context = None
browsersteps_playwright_browser_interface_context = None
browsersteps_playwright_browser_interface_end_time = None
browsersteps_playwright_browser_interface_start_time = None def construct_blueprint(datastore: ChangeDetectionStore):
browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
def cleanup_playwright_session():
def start_browsersteps_session(watch_uuid):
global browsersteps_live_ui_o from . import nonContext
global browsersteps_playwright_browser_interface from . import browser_steps
global browsersteps_playwright_browser_interface_browser import time
global browsersteps_playwright_browser_interface_context global browsersteps_sessions
global browsersteps_playwright_browser_interface_end_time global io_interface_context
global browsersteps_playwright_browser_interface_start_time
browsersteps_live_ui_o = {} # We keep the playwright session open for many minutes
browsersteps_playwright_browser_interface = None seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
browsersteps_playwright_browser_interface_browser = None
browsersteps_playwright_browser_interface_end_time = None browsersteps_start_session = {'start_time': time.time()}
browsersteps_playwright_browser_interface_start_time = None
# You can only have one of these running
print("Cleaning up old playwright session because time was up, calling .goodbye()") # This should be very fine to leave running for the life of the application
# @idea - Make it global so the pool of watch fetchers can use it also
if not io_interface_context:
io_interface_context = nonContext.c_sync_playwright()
# Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
io_interface_context = io_interface_context.start()
# keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
try: try:
browsersteps_playwright_browser_interface_context.goodbye() browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
except Exception as e: except Exception as e:
print ("Got exception in shutdown, probably OK") if 'ECONNREFUSED' in str(e):
print (str(e)) return make_response('Unable to start the Playwright Browser session, is it running?', 401)
else:
return make_response(str(e), 401)
browsersteps_playwright_browser_interface_context = None proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
proxy = None
if proxy_id:
proxy_url = datastore.proxy_list.get(proxy_id).get('url')
if proxy_url:
print ("Cleaning up old playwright session because time was up - done") # Playwright needs separate username and password values
from urllib.parse import urlparse
parsed = urlparse(proxy_url)
proxy = {'server': proxy_url}
def construct_blueprint(datastore: ChangeDetectionStore): if parsed.username:
proxy['username'] = parsed.username
browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates") if parsed.password:
proxy['password'] = parsed.password
print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
# Tell Playwright to connect to Chrome and setup a new session via our stepper interface
browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
playwright_browser=browsersteps_start_session['browser'],
proxy=proxy)
# For test
#browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
return browsersteps_start_session
@login_optionally_required
@browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
def browsersteps_start_session():
# A new session was requested, return sessionID
import uuid
browsersteps_session_id = str(uuid.uuid4())
watch_uuid = request.args.get('uuid')
global browsersteps_sessions
print("Starting connection with playwright")
logging.debug("browser_steps.py connecting")
browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
print("Starting connection with playwright - done")
return {'browsersteps_session_id': browsersteps_session_id}
# A request for an action was received
@login_optionally_required @login_optionally_required
@browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST']) @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
def browsersteps_ui_update(): def browsersteps_ui_update():
import base64 import base64
import playwright._impl._api_types import playwright._impl._api_types
import time global browsersteps_sessions
from changedetectionio.blueprint.browser_steps import browser_steps from changedetectionio.blueprint.browser_steps import browser_steps
global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
global browsersteps_playwright_browser_interface_browser
global browsersteps_playwright_browser_interface
global browsersteps_playwright_browser_interface_start_time
step_n = None
remaining =0 remaining =0
uuid = request.args.get('uuid') uuid = request.args.get('uuid')
@ -87,13 +132,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
if not browsersteps_session_id: if not browsersteps_session_id:
return make_response('No browsersteps_session_id specified', 500) return make_response('No browsersteps_session_id specified', 500)
# Because we don't "really" run in a context manager ( we make the playwright interface global/long-living ) if not browsersteps_sessions.get(browsersteps_session_id):
# We need to manage the shutdown when the time is up return make_response('No session exists under that ID', 500)
if browsersteps_playwright_browser_interface_end_time:
remaining = browsersteps_playwright_browser_interface_end_time-time.time()
if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
cleanup_playwright_session()
return make_response('Browser session expired, please reload the Browser Steps interface', 401)
# Actions - step/apply/etc, do the thing and return state # Actions - step/apply/etc, do the thing and return state
if request.method == 'POST': if request.method == 'POST':
@ -112,12 +153,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# @todo try.. accept.. nice errors not popups.. # @todo try.. accept.. nice errors not popups..
try: try:
this_session = browsersteps_live_ui_o.get(browsersteps_session_id) browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(action_name=step_operation,
if not this_session:
print("Browser exited")
return make_response('Browser session ran out of time :( Please reload this page.', 401)
this_session.call_action(action_name=step_operation,
selector=step_selector, selector=step_selector,
optional_value=step_optional_value) optional_value=step_optional_value)
@ -129,77 +165,19 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# Get visual selector ready/update its data (also use the current filter info from the page?) # Get visual selector ready/update its data (also use the current filter info from the page?)
# When the last 'apply' button was pressed # When the last 'apply' button was pressed
# @todo this adds overhead because the xpath selection is happening twice # @todo this adds overhead because the xpath selection is happening twice
u = this_session.page.url u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
if is_last_step and u: if is_last_step and u:
(screenshot, xpath_data) = this_session.request_visualselector_data() (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot) datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data) datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
# Setup interface # if not this_session.page:
if request.method == 'GET': # cleanup_playwright_session()
# return make_response('Browser session ran out of time :( Please reload this page.', 401)
if not browsersteps_playwright_browser_interface:
print("Starting connection with playwright")
logging.debug("browser_steps.py connecting")
global browsersteps_playwright_browser_interface_context
from . import nonContext
browsersteps_playwright_browser_interface_context = nonContext.c_sync_playwright()
browsersteps_playwright_browser_interface = browsersteps_playwright_browser_interface_context.start()
# At 20 minutes, some other variable is closing it
# @todo find out what it is and set it
seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
# keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
try:
browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
except Exception as e:
if 'ECONNREFUSED' in str(e):
return make_response('Unable to start the Playwright session properly, is it running?', 401)
browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
print("Starting connection with playwright - done")
if not browsersteps_live_ui_o.get(browsersteps_session_id):
# Boot up a new session
proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
proxy = None
if proxy_id:
proxy_url = datastore.proxy_list.get(proxy_id).get('url')
if proxy_url:
# Playwright needs separate username and password values
from urllib.parse import urlparse
parsed = urlparse(proxy_url)
proxy = {'server': proxy_url}
if parsed.username:
proxy['username'] = parsed.username
if parsed.password:
proxy['password'] = parsed.password
print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
# Begin the new "Playwright Context" that re-uses the playwright interface
# Each session is a "Playwright Context" as a list, that uses the playwright interface
browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
playwright_browser=browsersteps_playwright_browser_interface_browser,
proxy=proxy)
this_session = browsersteps_live_ui_o[browsersteps_session_id]
if not this_session.page:
cleanup_playwright_session()
return make_response('Browser session ran out of time :( Please reload this page.', 401)
response = None
if request.method == 'POST':
# Screenshots and other info only needed on requesting a step (POST) # Screenshots and other info only needed on requesting a step (POST)
try: try:
state = this_session.get_current_state() state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
except playwright._impl._api_types.Error as e: except playwright._impl._api_types.Error as e:
return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401) return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
@ -212,7 +190,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format( output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
base64.b64encode(state[0]).decode('ascii')), base64.b64encode(state[0]).decode('ascii')),
'xpath_data': state[1], 'xpath_data': state[1],
'session_age_start': this_session.age_start, 'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
'browser_time_remaining': round(remaining) 'browser_time_remaining': round(remaining)
}) })
@ -225,13 +203,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# No longer needed # No longer needed
os.unlink(tmp_file) os.unlink(tmp_file)
elif request.method == 'GET':
# Just enough to get the session rolling, it will call for goto-site via POST next
response = make_response({
'session_age_start': this_session.age_start,
'browser_time_remaining': round(remaining)
})
return response return response
return browser_steps_blueprint return browser_steps_blueprint

11
changedetectionio/blueprint/browser_steps/browser_steps.py
Unescape View File

@ -71,10 +71,10 @@ class steppable_browser_interface():
optional_value = str(jinja2_env.from_string(optional_value).render()) optional_value = str(jinja2_env.from_string(optional_value).render())
action_handler(selector, optional_value) action_handler(selector, optional_value)
self.page.wait_for_timeout(3 * 1000) self.page.wait_for_timeout(1.5 * 1000)
print("Call action done in", time.time() - now) print("Call action done in", time.time() - now)
def action_goto_url(self, selector, value): def action_goto_url(self, selector=None, value=None):
# self.page.set_viewport_size({"width": 1280, "height": 5000}) # self.page.set_viewport_size({"width": 1280, "height": 5000})
now = time.time() now = time.time()
response = self.page.goto(value, timeout=0, wait_until='commit') response = self.page.goto(value, timeout=0, wait_until='commit')
@ -105,7 +105,8 @@ class steppable_browser_interface():
print("Clicking element") print("Clicking element")
if not len(selector.strip()): if not len(selector.strip()):
return return
self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
def action_click_element_if_exists(self, selector, value): def action_click_element_if_exists(self, selector, value):
import playwright._impl._api_types as _api_types import playwright._impl._api_types as _api_types
@ -137,13 +138,13 @@ class steppable_browser_interface():
def action_wait_for_text(self, selector, value): def action_wait_for_text(self, selector, value):
import json import json
v = json.dumps(value) v = json.dumps(value)
self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000) self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
def action_wait_for_text_in_element(self, selector, value): def action_wait_for_text_in_element(self, selector, value):
import json import json
s = json.dumps(selector) s = json.dumps(selector)
v = json.dumps(value) v = json.dumps(value)
self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000) self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
# @todo - in the future make some popout interface to capture what needs to be set # @todo - in the future make some popout interface to capture what needs to be set
# https://playwright.dev/python/docs/api/class-keyboard # https://playwright.dev/python/docs/api/class-keyboard

3
changedetectionio/content_fetcher.py
Unescape View File

@ -182,7 +182,8 @@ class Fetcher():
optional_value=optional_value) optional_value=optional_value)
self.screenshot_step(step_n) self.screenshot_step(step_n)
self.save_step_html(step_n) self.save_step_html(step_n)
except TimeoutError: except TimeoutError as e:
print(str(e))
# Stop processing here # Stop processing here
raise BrowserStepsStepTimout(step_n=step_n) raise BrowserStepsStepTimout(step_n=step_n)

8
changedetectionio/static/js/browser-steps.js
Unescape View File

@ -238,7 +238,7 @@ $(document).ready(function () {
function start() { function start() {
console.log("Starting browser-steps UI"); console.log("Starting browser-steps UI");
browsersteps_session_id = Date.now(); browsersteps_session_id = false;
// @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
$('#browser_steps >li:first-child').removeClass('empty'); $('#browser_steps >li:first-child').removeClass('empty');
set_first_gotosite_disabled(); set_first_gotosite_disabled();
@ -246,7 +246,7 @@ $(document).ready(function () {
$('.clear,.remove', $('#browser_steps >li:first-child')).hide(); $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
$.ajax({ $.ajax({
type: "GET", type: "GET",
url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id, url: browser_steps_start_url,
statusCode: { statusCode: {
400: function () { 400: function () {
// More than likely the CSRF token was lost when the server restarted // More than likely the CSRF token was lost when the server restarted
@ -254,12 +254,12 @@ $(document).ready(function () {
} }
} }
}).done(function (data) { }).done(function (data) {
xpath_data = data.xpath_data;
$("#loading-status-text").fadeIn(); $("#loading-status-text").fadeIn();
browsersteps_session_id = data.browsersteps_session_id;
// This should trigger 'Goto site' // This should trigger 'Goto site'
console.log("Got startup response, requesting Goto-Site (first) step fake click"); console.log("Got startup response, requesting Goto-Site (first) step fake click");
$('#browser_steps >li:first-child .apply').click(); $('#browser_steps >li:first-child .apply').click();
browserless_seconds_remaining = data.browser_time_remaining; browserless_seconds_remaining = 500;
set_first_gotosite_disabled(); set_first_gotosite_disabled();
}).fail(function (data) { }).fail(function (data) {
console.log(data); console.log(data);

2
changedetectionio/templates/edit.html
Unescape View File

@ -14,7 +14,9 @@
{% endif %} {% endif %}
const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}'); const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}"; const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
</script> </script>
<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script> <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>

Write Preview
Loading…
Cancel
Save