changedetection.io/backend/__init__.py

#!/usr/bin/python3


# @todo logging
# @todo extra options for url like , verify=False etc.
# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
# @todo option for interval day/6 hour/etc
# @todo on change detected, config for calling some API
# @todo make tables responsive!
# @todo fetch title into json
# https://distill.io/features
# proxy per check
#  - flask_cors, itsdangerous,MarkupSafe

import time
import os
import timeago

import threading
import queue

from flask import Flask, render_template, request, send_file, send_from_directory,  abort, redirect, url_for

from feedgen.feed import FeedGenerator
from flask import make_response
import datetime
import pytz

datastore = None

# Local
running_update_threads = []
ticker_thread = None

messages = []
extra_stylesheets = []

update_q = queue.Queue()

app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")

# Stop browser caching of assets
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0

app.config['STOP_THREADS'] = False

# Disables caching of the templates
app.config['TEMPLATES_AUTO_RELOAD'] = True


# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
# running or something similar.
@app.template_filter('format_last_checked_time')
def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
    # Worker thread tells us which UUID it is currently processing.
    for t in running_update_threads:
        if t.current_uuid == watch_obj['uuid']:
            return "Checking now.."

    if watch_obj['last_checked'] == 0:
        return 'Not yet'

    return timeago.format(int(watch_obj['last_checked']), time.time())


# @app.context_processor
# def timeago():
#    def _timeago(lower_time, now):
#        return timeago.format(lower_time, now)
#    return dict(timeago=_timeago)

@app.template_filter('format_timestamp_timeago')
def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
    return timeago.format(timestamp, time.time())
    # return timeago.format(timestamp, time.time())
    # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)


def changedetection_app(config=None, datastore_o=None):
    global datastore
    datastore = datastore_o
    # Hmm
    app.config.update(dict(DEBUG=True))
    app.config.update(config or {})

    # Setup cors headers to allow all domains
    # https://flask-cors.readthedocs.io/en/latest/
    #    CORS(app)

    # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
    # You can divide up the stuff like this

    @app.route("/", methods=['GET'])
    def index():
        global messages

        limit_tag = request.args.get('tag')

        # Sort by last_changed and add the uuid which is usually the key..
        sorted_watches = []
        for uuid, watch in datastore.data['watching'].items():

            if limit_tag != None:
                # Support for comma separated list of tags.
                for tag_in_watch in watch['tag'].split(','):
                    tag_in_watch = tag_in_watch.strip()
                    if tag_in_watch == limit_tag:
                        watch['uuid'] = uuid
                        sorted_watches.append(watch)

            else:
                watch['uuid'] = uuid
                sorted_watches.append(watch)

        sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)

        existing_tags = datastore.get_all_tags()
        rss = request.args.get('rss')

        if rss:
            fg = FeedGenerator()
            fg.title('changedetection.io')
            fg.description('Feed description')
            fg.link(href='https://changedetection.io')

            for watch in sorted_watches:
                if not watch['viewed']:
                    fe = fg.add_entry()
                    fe.title(watch['url'])
                    fe.link(href=watch['url'])
                    fe.description(watch['url'])
                    fe.guid(watch['uuid'], permalink=False)
                    dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
                    dt = dt.replace(tzinfo=pytz.UTC)
                    fe.pubDate(dt)

            response = make_response(fg.rss_str())
            response.headers.set('Content-Type', 'application/rss+xml')
            return response

        else:
            output = render_template("watch-overview.html",
                                     watches=sorted_watches,
                                     messages=messages,
                                     tags=existing_tags,
                                     active_tag=limit_tag,
                                     has_unviewed=datastore.data['has_unviewed'])

            # Show messages but once.
            messages = []


        return output

    @app.route("/scrub", methods=['GET', 'POST'])
    def scrub_page():
        from pathlib import Path

        global messages

        if request.method == 'POST':
            confirmtext = request.form.get('confirmtext')

            if confirmtext == 'scrub':

                for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
                    os.unlink(txt_file_path)

                for uuid, watch in datastore.data['watching'].items():
                    watch['last_checked'] = 0
                    watch['last_changed'] = 0
                    watch['previous_md5'] = None
                    watch['history'] = {}

                datastore.needs_write = True
                messages.append({'class': 'ok', 'message': 'Cleaned all version history.'})
            else:
                messages.append({'class': 'error', 'message': 'Wrong confirm text.'})

            return redirect(url_for('index'))

        return render_template("scrub.html")

    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
    # the excluded text.
    def get_current_checksum_include_ignore_text(uuid):

        import hashlib
        from backend import fetch_site_status

        # Get the most recent one
        newest_history_key = datastore.get_val(uuid, 'newest_history_key')

        # 0 means that theres only one, so that there should be no 'unviewed' history availabe
        if newest_history_key == 0:
            newest_history_key = list(datastore.data['watching'][uuid]['history'].keys())[0]

        if newest_history_key:
            with open(datastore.data['watching'][uuid]['history'][newest_history_key],
                      encoding='utf-8') as file:
                raw_content = file.read()

                handler = fetch_site_status.perform_site_check(datastore=datastore)
                stripped_content = handler.strip_ignore_text(raw_content,
                                                          datastore.data['watching'][uuid]['ignore_text'])

                checksum = hashlib.md5(stripped_content).hexdigest()
                return checksum

        return datastore.data['watching'][uuid]['previous_md5']

    @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
    def edit_page(uuid):
        global messages
        import validators

        # More for testing, possible to return the first/only
        if uuid == 'first':
            uuid = list(datastore.data['watching'].keys()).pop()

        if request.method == 'POST':

            url = request.form.get('url').strip()
            tag = request.form.get('tag').strip()

            # Extra headers
            form_headers = request.form.get('headers').strip().split("\n")
            extra_headers = {}
            if form_headers:
                for header in form_headers:
                    if len(header):
                        parts = header.split(':', 1)
                        if len(parts) == 2:
                            extra_headers.update({parts[0].strip(): parts[1].strip()})

            update_obj = {'url': url,
                          'tag': tag,
                          'headers': extra_headers
                          }

            # Ignore text
            form_ignore_text = request.form.get('ignore-text').strip()
            ignore_text = []
            if len(form_ignore_text):
                for text in form_ignore_text.split("\n"):
                    text = text.strip()
                    if len(text):
                        ignore_text.append(text)

                datastore.data['watching'][uuid]['ignore_text'] = ignore_text

                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
                if len( datastore.data['watching'][uuid]['history']):
                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)



            validators.url(url)  # @todo switch to prop/attr/observer
            datastore.data['watching'][uuid].update(update_obj)
            datastore.needs_write = True

            messages.append({'class': 'ok', 'message': 'Updated watch.'})

            return redirect(url_for('index'))

        else:
            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)

        return output

    @app.route("/settings", methods=['GET', "POST"])
    def settings_page():
        global messages
        if request.method == 'POST':
            try:
                minutes = int(request.values.get('minutes').strip())
            except ValueError:
                messages.append({'class': 'error', 'message': "Invalid value given, use an integer."})

            else:
                if minutes >= 5:
                    datastore.data['settings']['requests']['minutes_between_check'] = minutes
                    datastore.needs_write = True

                    messages.append({'class': 'ok', 'message': "Updated"})
                else:
                    messages.append(
                        {'class': 'error', 'message': "Must be atleast 5 minutes."})

        output = render_template("settings.html", messages=messages,
                                 minutes=datastore.data['settings']['requests']['minutes_between_check'])
        messages = []

        return output

    @app.route("/import", methods=['GET', "POST"])
    def import_page():
        import validators
        global messages
        remaining_urls = []

        good = 0

        if request.method == 'POST':
            urls = request.values.get('urls').split("\n")
            for url in urls:
                url = url.strip()
                if len(url) and validators.url(url):
                    new_uuid = datastore.add_watch(url=url.strip(), tag="")
                    # Straight into the queue.
                    update_q.put(new_uuid)
                    good += 1
                else:
                    if len(url):
                        remaining_urls.append(url)

            messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})

            if len(remaining_urls) == 0:
                return redirect(url_for('index'))
        else:
            output = render_template("import.html",
                                     messages=messages,
                                     remaining="\n".join(remaining_urls)
                                     )
            messages = []
        return output

    # Clear all statuses, so we do not see the 'unviewed' class
    @app.route("/api/mark-all-viewed", methods=['GET'])
    def mark_all_viewed():

        # Save the current newest history as the most recently viewed
        for watch_uuid, watch in datastore.data['watching'].items():
            datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])

        messages.append({'class': 'ok', 'message': "Cleared all statuses."})
        return redirect(url_for('index'))

    @app.route("/diff/<string:uuid>", methods=['GET'])
    def diff_history_page(uuid):
        global messages

        # More for testing, possible to return the first/only
        if uuid == 'first':
            uuid= list(datastore.data['watching'].keys()).pop()

        extra_stylesheets = ['/static/css/diff.css']
        try:
            watch = datastore.data['watching'][uuid]
        except KeyError:
            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
            return redirect(url_for('index'))

        dates = list(watch['history'].keys())
        # Convert to int, sort and back to str again
        dates = [int(i) for i in dates]
        dates.sort(reverse=True)
        dates = [str(i) for i in dates]


        if len(dates) < 2:
            messages.append({'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
            return redirect(url_for('index'))

        # Save the current newest history as the most recently viewed
        datastore.set_last_viewed(uuid, dates[0])

        newest_file = watch['history'][dates[0]]
        with open(newest_file, 'r') as f:
            newest_version_file_contents = f.read()

        previous_version = request.args.get('previous_version')

        try:
            previous_file = watch['history'][previous_version]
        except KeyError:
            # Not present, use a default value, the second one in the sorted list.
            previous_file = watch['history'][dates[1]]

        with open(previous_file, 'r') as f:
            previous_version_file_contents = f.read()

        output = render_template("diff.html", watch_a=watch,
                                 messages=messages,
                                 newest=newest_version_file_contents,
                                 previous=previous_version_file_contents,
                                 extra_stylesheets=extra_stylesheets,
                                 versions=dates[1:],
                                 newest_version_timestamp=dates[0],
                                 current_previous_version=str(previous_version),
                                 current_diff_url=watch['url'])

        return output

    @app.route("/favicon.ico", methods=['GET'])
    def favicon():
        return send_from_directory("/app/static/images", filename="favicon.ico")

    # We're good but backups are even better!
    @app.route("/backup", methods=['GET'])
    def get_backup():
        import zipfile
        from pathlib import Path

        # create a ZipFile object
        backupname = "changedetection-backup-{}.zip".format(int(time.time()))

        # We only care about UUIDS from the current index file
        uuids = list(datastore.data['watching'].keys())

        with zipfile.ZipFile(os.path.join(app.config['datastore_path'], backupname), 'w',
                             compression=zipfile.ZIP_DEFLATED,
                             compresslevel=6) as zipObj:

            # Be sure we're written fresh
            datastore.sync_to_json()

            # Add the index
            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"))
            # Add any snapshot data we find
            for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
                parent_p = txt_file_path.parent
                if parent_p.name in uuids:
                    zipObj.write(txt_file_path)

        return send_file(os.path.join(app.config['datastore_path'], backupname),
                         as_attachment=True,
                         mimetype="application/zip",
                         attachment_filename=backupname)

    @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
    def static_content(group, filename):
        # These files should be in our subdirectory
        full_path = os.path.realpath(__file__)
        p = os.path.dirname(full_path)

        try:
            return send_from_directory("{}/static/{}".format(p, group), filename=filename)
        except FileNotFoundError:
            abort(404)

    @app.route("/api/add", methods=['POST'])
    def api_watch_add():
        global messages

        # @todo add_watch should throw a custom Exception for validation etc
        new_uuid = datastore.add_watch(url=request.form.get('url').strip(), tag=request.form.get('tag').strip())
        # Straight into the queue.
        update_q.put(new_uuid)

        messages.append({'class': 'ok', 'message': 'Watch added.'})
        return redirect(url_for('index'))

    @app.route("/api/delete", methods=['GET'])
    def api_delete():
        global messages
        uuid = request.args.get('uuid')
        datastore.delete(uuid)
        messages.append({'class': 'ok', 'message': 'Deleted.'})

        return redirect(url_for('index'))

    @app.route("/api/checknow", methods=['GET'])
    def api_watch_checknow():

        global messages

        tag = request.args.get('tag')
        uuid = request.args.get('uuid')
        i = 0

        running_uuids = []
        for t in running_update_threads:
            running_uuids.append(t.current_uuid)

        # @todo check thread is running and skip

        if uuid:
            if uuid not in running_uuids:
                update_q.put(uuid)
            i = 1

        elif tag != None:
            # Items that have this current tag
            for watch_uuid, watch in datastore.data['watching'].items():
                if (tag != None and tag in watch['tag']):
                    i += 1
                    if watch_uuid not in running_uuids:
                        update_q.put(watch_uuid)
        else:
            # No tag, no uuid, add everything.
            for watch_uuid, watch in datastore.data['watching'].items():
                i += 1
                if watch_uuid not in running_uuids:
                    update_q.put(watch_uuid)

        messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
        return redirect(url_for('index', tag=tag))

    # @todo handle ctrl break
    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()

    return app


# Requests for checking on the site use a pool of thread Workers managed by a Queue.
class Worker(threading.Thread):
    current_uuid = None

    def __init__(self, q, *args, **kwargs):
        self.q = q
        super().__init__(*args, **kwargs)

    def run(self):
        from backend import fetch_site_status

        update_handler = fetch_site_status.perform_site_check(datastore=datastore)

        while True:

            try:
                uuid = self.q.get(block=True, timeout=1)
            except queue.Empty:
                # We have a chance to kill this thread that needs to monitor for new jobs..
                # Delays here would be caused by a current response object pending
                # @todo switch to threaded response handler
                if app.config['STOP_THREADS']:
                    return
            else:
                self.current_uuid = uuid

                if uuid in list(datastore.data['watching'].keys()):

                    try:
                        changed_detected, result, contents = update_handler.run(uuid)

                    except PermissionError as s:
                        app.logger.error("File permission error updating", uuid, str(s))
                    else:
                        if result:

                            datastore.update_watch(uuid=uuid, update_obj=result)
                            if changed_detected:
                                # A change was detected
                                datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)


                self.current_uuid = None  # Done
                self.q.task_done()


# Thread runner to check every minute, look for new watches to feed into the Queue.
def ticker_thread_check_time_launch_checks():
    # Spin up Workers.
    for _ in range(datastore.data['settings']['requests']['workers']):
        new_worker = Worker(update_q)
        running_update_threads.append(new_worker)
        new_worker.start()

    # Every minute check for new UUIDs to follow up on
    while True:

        if app.config['STOP_THREADS']:
            return

        running_uuids = []
        for t in running_update_threads:
            running_uuids.append(t.current_uuid)

        # Look at the dataset, find a stale watch to process
        minutes = datastore.data['settings']['requests']['minutes_between_check']
        for uuid, watch in datastore.data['watching'].items():
            if watch['last_checked'] <= time.time() - (minutes * 60):

                # @todo maybe update_q.queue is enough?
                if not uuid in running_uuids and uuid not in update_q.queue:
                    update_q.put(uuid)

        # Should be low so we can break this out in testing
        time.sleep(1)