diff --git a/.gitignore b/.gitignore index 4c36359..156795a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,55 @@ -.venv -pip-selfcheck.json -.errors -2015*.txt -2016*.txt -2017*.txt -*.cfg -*.pyc -shreddit.conf -shreddit.yml -praw.ini +# Docs +docs/_build/ + +# Byte-compiled / optimized / DLL files __pycache__/ -.*.swp +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# PyBuilder +target/ + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..376dc78 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +help: + @echo "build - Build package" + @echo "install - Install package to local system" + @echo "clean - Clean built artifacts" + @echo "test - Run test suite with coverage" + +build: + python setup.py build + python setup.py bdist_wheel + +install: + pip install dist/*.whl --upgrade --force-reinstall --no-deps + python setup.py clean + +clean: + find . -type f -name "*.pyc" -delete + rm -rf ./build ./dist ./*.egg-info diff --git a/install.sh b/install.sh deleted file mode 100755 index b00c7b1..0000000 --- a/install.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env sh - -virtualenv . -source ./bin/activate -pip install -r requirements.txt - -if [ ! -f "shreddit.yml" ]; then - cp "shreddit.yml.example" "shreddit.yml" - $EDITOR shreddit.yml -fi - diff --git a/lambda_handler.py b/lambda_handler.py new file mode 100644 index 0000000..e7e7d97 --- /dev/null +++ b/lambda_handler.py @@ -0,0 +1,12 @@ +"""This module contains the handler function called by AWS. +""" +from shreddit.shredder import shred +import yaml + + +def lambda_handler(event, context): + with open("shreddit.yml") as fh: + config = yaml.safe_load(fh) + if not config: + raise Exception("No config options passed!") + shred(config) diff --git a/oauth_test.py b/oauth_test.py deleted file mode 100644 index 563d0c6..0000000 --- a/oauth_test.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -''' -Simple script to check if your oauth is working. -''' -import praw -import sys - -r = praw.Reddit('Shreddit oauth test') -try: - r.refresh_access_information() - if r.is_oauth_session(): - sys.exit(0) - else: - sys.exit(2) -except: - sys.exit(1) - diff --git a/run.sh b/run.sh deleted file mode 100755 index b7b4f25..0000000 --- a/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env sh -source ./bin/activate -pip install --upgrade praw -python ./shreddit.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..724d001 --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +"""Setup script for shreddit. +""" +from setuptools import setup +from codecs import open +from os import path + +VERSION = "2.0.0" +DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." + +here = path.abspath(path.dirname(__file__)) + +with open(path.join(here, "README.md"), encoding='utf-8') as filein: + long_description = filein.read() + +with open(path.join(here, "requirements.txt"), encoding="utf-8") as filein: + requirements = [line.strip() for line in filein.readlines()] + +setup( + name="shreddit", + version=VERSION, + description=DESCRIPTION, + long_description=long_description, + url="https://github.com/scott-hand/Shreddit", + author="Scott Hand", + author_email="scott@vkgfx.com", + classifiers=["Development Status :: 3 - Alpha", + "Intended Audience :: End Users/Desktop", + "Programming Language :: Python :: 2"], + packages=["shreddit"], + install_requires=requirements, + entry_points={ + "console_scripts": [ + "shreddit=shreddit.app:main" + ] + } +) diff --git a/shreddit.py b/shreddit.py deleted file mode 100755 index 1e744c9..0000000 --- a/shreddit.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import logging -import argparse -import json -import yaml -import praw - -from re import sub -from random import shuffle, randint -from datetime import datetime, timedelta -from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, - HTTPException, OAuthAppRequired) -from praw.objects import Comment, Submission - -logging.basicConfig(stream=sys.stdout) -log = logging.getLogger(__name__) -log.setLevel(level=logging.WARNING) - -try: - from loremipsum import get_sentence # This only works on Python 2 -except ImportError: - def get_sentence(): - return '''I have been Shreddited for privacy!''' - - os_wordlist = '/usr/share/dict/words' - if os.name == 'posix' and os.path.isfile(os_wordlist): - # Generate a random string of words from our system's dictionary - fh = open(os_wordlist) - words = fh.read().splitlines() - fh.close() - shuffle(words) - - def get_sentence(): - return ' '.join(words[:randint(50, 150)]) - -assert get_sentence - -parser = argparse.ArgumentParser() -parser.add_argument( - '-c', - '--config', - help="config file to use instead of the default shreddit.cfg" -) -args = parser.parse_args() - -if args.config: - config_file = args.config -else: - config_file = 'shreddit.yml' - -with open(config_file, 'r') as fh: - config = yaml.safe_load(fh) -if config is None: - raise Exception("No config options passed!") - -save_directory = config.get('save_directory', '.') - -r = praw.Reddit(user_agent="shreddit/4.2") -if save_directory: - r.config.store_json_result = True - -if config.get('verbose', True): - log.setLevel(level=logging.DEBUG) - -try: - # Try to login with OAuth2 - r.refresh_access_information() - log.debug("Logged in with OAuth.") -except (HTTPException, OAuthAppRequired) as e: - log.warning('''You should migrate to OAuth2 using get_secret.py before - Reddit disables this login method.''') - try: - try: - r.login(config['username'], config['password']) - except InvalidUserPass: - r.login() # Supply details on the command line - except InvalidUser as e: - raise InvalidUser("User does not exist.", e) - except InvalidUserPass as e: - raise InvalidUserPass("Specified an incorrect password.", e) - except RateLimitExceeded as e: - raise RateLimitExceeded("You're doing that too much.", e) - -log.info("Logged in as {user}.".format(user=r.user)) -log.debug("Deleting messages before {time}.".format( - time=datetime.now() - timedelta(hours=config['hours']))) - -whitelist = config.get('whitelist', []) -whitelist_ids = config.get('whitelist_ids', []) - -if config.get('whitelist'): - log.debug("Keeping messages from subreddits {subs}".format( - subs=', '.join(whitelist)) - ) - - -def get_things(after=None): - limit = None - item = config.get('item', 'comments') - sort = config.get('sort', 'new') - log.debug("Deleting items: {item}".format(item=item)) - if item == "comments": - return r.user.get_comments(limit=limit, sort=sort) - elif item == "submitted": - return r.user.get_submitted(limit=limit, sort=sort) - elif item == "overview": - return r.user.get_overview(limit=limit, sort=sort) - else: - raise Exception("Your deletion section is wrong") - - -def remove_things(things): - for thing in things: - log.debug('Starting remove function on: {thing}'.format(thing=thing)) - # Seems to be in users's timezone. Unclear. - thing_time = datetime.fromtimestamp(thing.created_utc) - # Exclude items from being deleted unless past X hours. - after_time = datetime.now() - timedelta(hours=config.get('hours', 24)) - if thing_time > after_time: - if thing_time + timedelta(hours=config.get('nuke_hours', 4320)) < datetime.utcnow(): - pass - continue - # For edit_only we're assuming that the hours aren't altered. - # This saves time when deleting (you don't edit already edited posts). - if config.get('edit_only'): - end_time = after_time - timedelta(hours=config.get('hours', 24)) - if thing_time < end_time: - continue - - if str(thing.subreddit).lower() in config.get('whitelist', []) \ - or thing.id in config.get('whitelist_ids', []): - continue - - if config.get('whitelist_distinguished') and thing.distinguished: - continue - if config.get('whitelist_gilded') and thing.gilded: - continue - if 'max_score' in config and thing.score > config['max_score']: - continue - - if config.get('save_directory'): - save_directory = config['save_directory'] - if not os.path.exists(save_directory): - os.makedirs(save_directory) - with open("%s/%s.json" % (save_directory, thing.id), "w") as fh: - json.dump(thing.json_dict, fh) - - if config.get('trial_run'): # Don't do anything, trial mode! - log.debug("Would have deleted {thing}: '{content}'".format( - thing=thing.id, content=thing)) - continue - - if config.get('clear_vote'): - thing.clear_vote() - - if isinstance(thing, Submission): - log.info('Deleting submission: #{id} {url}'.format( - id=thing.id, - url=thing.url.encode('utf-8')) - ) - elif isinstance(thing, Comment): - rep_format = config.get('replacement_format') - if rep_format == 'random': - replacement_text = get_sentence() - elif rep_format == 'dot': - replacement_text = '.' - else: - replacement_text = rep_format - - msg = '/r/{3}/ #{0} with:\n\t"{1}" to\n\t"{2}"'.format( - thing.id, - sub(b'\n\r\t', ' ', thing.body[:78].encode('utf-8')), - replacement_text[:78], - thing.subreddit - ) - - if config.get('edit_only'): - log.info('Editing (not removing) {msg}'.format(msg=msg)) - else: - log.info('Editing and deleting {msg}'.format(msg=msg)) - - thing.edit(replacement_text) - if not config.get('edit_only'): - thing.delete() - -remove_things(get_things()) diff --git a/shreddit/__init__.py b/shreddit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shreddit/app.py b/shreddit/app.py new file mode 100644 index 0000000..e00e17c --- /dev/null +++ b/shreddit/app.py @@ -0,0 +1,29 @@ +"""This module contains script entrypoints for shreddit. +""" +import argparse +import yaml +from shreddit.oauth import oauth_test +from shreddit.shredder import shred + + +def main(): + parser = argparse.ArgumentParser(description="Command-line frontend to the shreddit library.") + parser.add_argument("-c", "--config", help="Config file to use instead of the default shreddit.cfg") + parser.add_argument("-p", "--praw", help="PRAW config (if not ./praw.ini)") + parser.add_argument("-t", "--test-oauth", help="Perform OAuth test and exit", action="store_true") + args = parser.parse_args() + + if args.test_oauth: + oauth_test(args.praw) + return + + with open(args.config or "shreddit.yml") as fh: + config = yaml.safe_load(fh) + if not config: + raise Exception("No config options passed!") + + shred(config, args.praw) + + +if __name__ == "__main__": + main() diff --git a/shreddit/oauth.py b/shreddit/oauth.py new file mode 100644 index 0000000..8276c43 --- /dev/null +++ b/shreddit/oauth.py @@ -0,0 +1,22 @@ +"""This module contains a function that tests OAuth session validity. +""" +import os +import praw + + +def oauth_test(praw_ini): + if praw_ini: + # PRAW won't panic if the file is invalid, so check first + if not os.path.exists(praw_ini): + print("PRAW configuration file \"{}\" not found.".format(praw_ini)) + return + praw.settings.CONFIG.read(praw_ini) + r = praw.Reddit("Shreddit oauth test") + try: + r.refresh_access_information() + if r.is_oauth_session(): + print("Session is valid.") + else: + print("Session is not a valid OAuth session.") + except Exception as e: + print("Error encountered while checking credentials:\n{}".format(e)) diff --git a/shreddit/shredder.py b/shreddit/shredder.py new file mode 100644 index 0000000..5f53010 --- /dev/null +++ b/shreddit/shredder.py @@ -0,0 +1,172 @@ +import os +import sys +import logging +import argparse +import json +import yaml +import praw +from re import sub +from random import shuffle, randint +from datetime import datetime, timedelta +from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, HTTPException, OAuthAppRequired) +from praw.objects import Comment, Submission +try: + from loremipsum import get_sentence # This only works on Python 2 +except ImportError: + def get_sentence(): + return "I have been Shreddited for privacy!" + + os_wordlist = "/usr/share/dict/words" + if os.name == "posix" and os.path.isfile(os_wordlist): + # Generate a random string of words from our system's dictionary + fh = open(os_wordlist) + words = fh.read().splitlines() + fh.close() + shuffle(words) + + def get_sentence(): + return " ".join(words[:randint(50, 150)]) +assert get_sentence + + +def shred(config, praw_ini=None): + logging.basicConfig(stream=sys.stdout) + log = logging.getLogger("shreddit") + log.setLevel(level=logging.WARNING) + + if praw_ini: + # PRAW won't panic if the file is invalid, so check first + if not os.path.exists(praw_ini): + print("PRAW configuration file \"{}\" not found.".format(praw_ini)) + return + praw.settings.CONFIG.read(praw_ini) + + save_directory = config.get("save_directory", ".") + + r = praw.Reddit(user_agent="shreddit/4.2") + if save_directory: + r.config.store_json_result = True + + if config.get("verbose", True): + log.setLevel(level=logging.DEBUG) + + try: + # Try to login with OAuth2 + r.refresh_access_information() + log.debug("Logged in with OAuth.") + except (HTTPException, OAuthAppRequired) as e: + log.warning("You should migrate to OAuth2 using get_secret.py before Reddit disables this login method.") + try: + try: + r.login(config["username"], config["password"]) + except InvalidUserPass: + r.login() # Supply details on the command line + except InvalidUser as e: + raise InvalidUser("User does not exist.", e) + except InvalidUserPass as e: + raise InvalidUserPass("Specified an incorrect password.", e) + except RateLimitExceeded as e: + raise RateLimitExceeded("You're doing that too much.", e) + + log.info("Logged in as {user}.".format(user=r.user)) + log.debug("Deleting messages before {time}.".format( + time=datetime.now() - timedelta(hours=config["hours"]))) + + whitelist = config.get("whitelist", []) + whitelist_ids = config.get("whitelist_ids", []) + + if config.get("whitelist"): + log.debug("Keeping messages from subreddits {subs}".format(subs=", ".join(whitelist))) + + remove_things(r, config, log, get_things(r, config, log)) + + +def get_things(r, config, log, after=None): + limit = None + item = config.get("item", "comments") + sort = config.get("sort", "new") + log.debug("Deleting items: {item}".format(item=item)) + if item == "comments": + return r.user.get_comments(limit=limit, sort=sort) + elif item == "submitted": + return r.user.get_submitted(limit=limit, sort=sort) + elif item == "overview": + return r.user.get_overview(limit=limit, sort=sort) + else: + raise Exception("Your deletion section is wrong") + + +def remove_things(r, config, log, things): + for thing in things: + log.debug("Starting remove function on: {thing}".format(thing=thing)) + # Seems to be in users's timezone. Unclear. + thing_time = datetime.fromtimestamp(thing.created_utc) + # Exclude items from being deleted unless past X hours. + after_time = datetime.now() - timedelta(hours=config.get("hours", 24)) + if thing_time > after_time: + if thing_time + timedelta(hours=config.get("nuke_hours", 4320)) < datetime.utcnow(): + pass + continue + # For edit_only we're assuming that the hours aren't altered. + # This saves time when deleting (you don't edit already edited posts). + if config.get("edit_only"): + end_time = after_time - timedelta(hours=config.get("hours", 24)) + if thing_time < end_time: + continue + + if str(thing.subreddit).lower() in config.get("whitelist", []) \ + or thing.id in config.get("whitelist_ids", []): + continue + + if config.get("whitelist_distinguished") and thing.distinguished: + continue + if config.get("whitelist_gilded") and thing.gilded: + continue + if "max_score" in config and thing.score > config["max_score"]: + continue + + if config.get("save_directory"): + save_directory = config["save_directory"] + if not os.path.exists(save_directory): + os.makedirs(save_directory) + with open("%s/%s.json" % (save_directory, thing.id), "w") as fh: + json.dump(thing.json_dict, fh) + + if config.get("trial_run"): # Don't do anything, trial mode! + log.debug("Would have deleted {thing}: '{content}'".format( + thing=thing.id, content=thing)) + continue + + if config.get("clear_vote"): + thing.clear_vote() + + if isinstance(thing, Submission): + log.info("Deleting submission: #{id} {url}".format( + id=thing.id, + url=thing.url.encode("utf-8")) + ) + elif isinstance(thing, Comment): + rep_format = config.get("replacement_format") + if rep_format == "random": + replacement_text = get_sentence() + elif rep_format == "dot": + replacement_text = "." + else: + replacement_text = rep_format + + msg = '/r/{3}/ #{0} with:\n\t"{1}" to\n\t"{2}"'.format( + thing.id, + sub(b"\n\r\t", " ", thing.body[:78].encode("utf-8")), + replacement_text[:78], + thing.subreddit + ) + + if config.get("edit_only"): + log.info("Editing (not removing) {msg}".format(msg=msg)) + else: + log.info("Editing and deleting {msg}".format(msg=msg)) + + thing.edit(replacement_text) + if not config.get("edit_only"): + thing.delete() +