From 65fbf900050694128707320e257ebdf53fb40491 Mon Sep 17 00:00:00 2001 From: David Trail Date: Fri, 30 Sep 2016 13:38:51 +0100 Subject: [PATCH 01/62] Debug level added to config --- shreddit.py | 9 +++++---- shreddit.yml.example | 8 ++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/shreddit.py b/shreddit.py index 1e744c9..0a0d144 100755 --- a/shreddit.py +++ b/shreddit.py @@ -17,7 +17,7 @@ from praw.objects import Comment, Submission logging.basicConfig(stream=sys.stdout) log = logging.getLogger(__name__) -log.setLevel(level=logging.WARNING) +log.setLevel(level=logging.DEBUG) try: from loremipsum import get_sentence # This only works on Python 2 @@ -58,12 +58,13 @@ if config is None: save_directory = config.get('save_directory', '.') -r = praw.Reddit(user_agent="shreddit/4.2") +r = praw.Reddit(user_agent="shreddit/4.3") if save_directory: r.config.store_json_result = True if config.get('verbose', True): - log.setLevel(level=logging.DEBUG) + log_level = config.get('debug', 'WARNING') # Default to WARNING only + log.setLevel(level=getattr(logging, log_level)) try: # Try to login with OAuth2 @@ -176,7 +177,7 @@ def remove_things(things): replacement_text[:78], thing.subreddit ) - + if config.get('edit_only'): log.info('Editing (not removing) {msg}'.format(msg=msg)) else: diff --git a/shreddit.yml.example b/shreddit.yml.example index f23bf0d..81de823 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -2,8 +2,8 @@ # Login details for Reddit. Fill out if you don't wish # to be prompted for a login every time you run Shreddit. ### NOTE: This may be deprecated as you can specify in praw.ini instead -username: -password: +username: +password: # How many hours of comments you want to keep # 24 hours in a day, @@ -68,4 +68,8 @@ save_directory: /tmp # options: [random, dot, "user entered string"] replacement_format: random +# Debug level, how much output you want +# See: https://docs.python.org/3/library/logging.html#logging-levels +debug: DEBUG + # vim: syntax=yaml ts=2 From 92028f21d78f9baad172fa31da28a3d06a334fff Mon Sep 17 00:00:00 2001 From: David Trail Date: Fri, 30 Sep 2016 14:11:36 +0100 Subject: [PATCH 02/62] Optimise and minor bug fix random string generation --- shreddit.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/shreddit.py b/shreddit.py index 0a0d144..fee463c 100755 --- a/shreddit.py +++ b/shreddit.py @@ -7,9 +7,9 @@ import argparse import json import yaml import praw +import random from re import sub -from random import shuffle, randint from datetime import datetime, timedelta from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, HTTPException, OAuthAppRequired) @@ -19,25 +19,6 @@ logging.basicConfig(stream=sys.stdout) log = logging.getLogger(__name__) log.setLevel(level=logging.DEBUG) -try: - from loremipsum import get_sentence # This only works on Python 2 -except ImportError: - def get_sentence(): - return '''I have been Shreddited for privacy!''' - - os_wordlist = '/usr/share/dict/words' - if os.name == 'posix' and os.path.isfile(os_wordlist): - # Generate a random string of words from our system's dictionary - fh = open(os_wordlist) - words = fh.read().splitlines() - fh.close() - shuffle(words) - - def get_sentence(): - return ' '.join(words[:randint(50, 150)]) - -assert get_sentence - parser = argparse.ArgumentParser() parser.add_argument( '-c', @@ -63,7 +44,7 @@ if save_directory: r.config.store_json_result = True if config.get('verbose', True): - log_level = config.get('debug', 'WARNING') # Default to WARNING only + log_level = config.get('debug', 'DEBUG') log.setLevel(level=getattr(logging, log_level)) try: @@ -98,6 +79,24 @@ if config.get('whitelist'): ) +def get_sentence(): + return '''I have been Shreddited for privacy!''' +try: + # Provide a method that works on windows + from loremipsum import get_sentence +except ImportError: + # Module unavailable, use the default phrase + pass +os_wordlist = '/usr/share/dict/words' +if os.name == 'posix' and os.path.isfile(os_wordlist): + # Generate a random string of words from our system's dictionary + fh = open(os_wordlist) + words = fh.read().splitlines() + fh.close() + def get_sentence(): + return ' '.join(random.sample(words, random.randint(50,75))) + + def get_things(after=None): limit = None item = config.get('item', 'comments') From 4e785dca72e19b47697b3bf56b97f89677902f84 Mon Sep 17 00:00:00 2001 From: David Trail Date: Fri, 30 Sep 2016 14:13:35 +0100 Subject: [PATCH 03/62] Allow use of own wordlist --- shreddit.py | 2 +- shreddit.yml.example | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/shreddit.py b/shreddit.py index fee463c..373acf1 100755 --- a/shreddit.py +++ b/shreddit.py @@ -87,7 +87,7 @@ try: except ImportError: # Module unavailable, use the default phrase pass -os_wordlist = '/usr/share/dict/words' +os_wordlist = config.get('wordlist', '/usr/share/dict/words') if os.name == 'posix' and os.path.isfile(os_wordlist): # Generate a random string of words from our system's dictionary fh = open(os_wordlist) diff --git a/shreddit.yml.example b/shreddit.yml.example index 81de823..49e5f7a 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -72,4 +72,7 @@ replacement_format: random # See: https://docs.python.org/3/library/logging.html#logging-levels debug: DEBUG +# Define your own wordlist to use as substitution text +wordlist: False + # vim: syntax=yaml ts=2 From cf2e670ad19c69c906abef021f488204f3abbfcf Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 8 Oct 2016 10:05:52 -0700 Subject: [PATCH 04/62] Fixed wordlist. Fixed whitelist. --- shreddit.py | 27 +++++++++++++++++---------- shreddit.yml.example | 5 +++-- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/shreddit.py b/shreddit.py index 373acf1..4252070 100755 --- a/shreddit.py +++ b/shreddit.py @@ -8,6 +8,7 @@ import json import yaml import praw import random +import string from re import sub from datetime import datetime, timedelta @@ -73,10 +74,11 @@ log.debug("Deleting messages before {time}.".format( whitelist = config.get('whitelist', []) whitelist_ids = config.get('whitelist_ids', []) -if config.get('whitelist'): +if whitelist: log.debug("Keeping messages from subreddits {subs}".format( subs=', '.join(whitelist)) ) + whitelist = set([string.lower(subr) for subr in whitelist]) def get_sentence(): @@ -87,14 +89,19 @@ try: except ImportError: # Module unavailable, use the default phrase pass -os_wordlist = config.get('wordlist', '/usr/share/dict/words') -if os.name == 'posix' and os.path.isfile(os_wordlist): - # Generate a random string of words from our system's dictionary - fh = open(os_wordlist) - words = fh.read().splitlines() - fh.close() - def get_sentence(): - return ' '.join(random.sample(words, random.randint(50,75))) + +if config.get('replacement_format') == 'random': + wordlist = config.get('wordlist') + if not wordlist: + os_wordlist = '/usr/share/dict/words' + if os.name == 'posix' and os.path.isfile(os_wordlist): + # Generate a random string of words from our system's dictionary + with open(os_wordlist) as fh: + wordlist = fh.read().splitlines() + if wordlist: + def get_sentence(): + return ' '.join(random.sample(wordlist, min(len(wordlist), + random.randint(50,75)))) def get_things(after=None): @@ -130,7 +137,7 @@ def remove_things(things): if thing_time < end_time: continue - if str(thing.subreddit).lower() in config.get('whitelist', []) \ + if str(thing.subreddit).lower() in whitelist \ or thing.id in config.get('whitelist_ids', []): continue diff --git a/shreddit.yml.example b/shreddit.yml.example index 49e5f7a..216eb5e 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -72,7 +72,8 @@ replacement_format: random # See: https://docs.python.org/3/library/logging.html#logging-levels debug: DEBUG -# Define your own wordlist to use as substitution text -wordlist: False +# Define your own wordlist to use as substitution text when +# replacement_format == random +wordlist: [] # vim: syntax=yaml ts=2 From c1dcdea5fdcb4bf30af109599463299b0a344c54 Mon Sep 17 00:00:00 2001 From: David Trail Date: Sun, 9 Oct 2016 21:51:33 +0100 Subject: [PATCH 05/62] Fix string module error --- shreddit.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shreddit.py b/shreddit.py index 4252070..bf9fc96 100755 --- a/shreddit.py +++ b/shreddit.py @@ -8,7 +8,6 @@ import json import yaml import praw import random -import string from re import sub from datetime import datetime, timedelta @@ -75,10 +74,10 @@ whitelist = config.get('whitelist', []) whitelist_ids = config.get('whitelist_ids', []) if whitelist: + whitelist = set([subr.lower() for subr in whitelist]) log.debug("Keeping messages from subreddits {subs}".format( subs=', '.join(whitelist)) ) - whitelist = set([string.lower(subr) for subr in whitelist]) def get_sentence(): From 30bdd82abca355c8b2dea078db8dfb07bcc745ce Mon Sep 17 00:00:00 2001 From: David Trail Date: Fri, 14 Oct 2016 01:57:51 +0100 Subject: [PATCH 06/62] minor bash enhancements --- oauth_test.py => oauth_check.py | 0 run.sh | 6 +++++- 2 files changed, 5 insertions(+), 1 deletion(-) rename oauth_test.py => oauth_check.py (100%) diff --git a/oauth_test.py b/oauth_check.py similarity index 100% rename from oauth_test.py rename to oauth_check.py diff --git a/run.sh b/run.sh index b7b4f25..b2b49da 100755 --- a/run.sh +++ b/run.sh @@ -1,4 +1,8 @@ #!/usr/bin/env sh -source ./bin/activate +if [[ -f './bin/activate' ]]; then + source ./bin/activate +elif [[ -f '.venv/bin/activate' ]]; then + source '.venv/bin/activate' +fi pip install --upgrade praw python ./shreddit.py From 09b9adf6f9fe7252c50f77f3d9cc559df1ad4100 Mon Sep 17 00:00:00 2001 From: noflag Date: Fri, 28 Oct 2016 21:53:59 -0700 Subject: [PATCH 07/62] Cleaned it up Removed useless old YAML message, fixed a bit of grammar --- README.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a8fd19a..4d2bac8 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,6 @@ Shreddit ======= -YAML Upgrade ------------ - -**Note! Important! The latest version uses YAML format over the old simpleconfigparser! -This means you'll have to migrate your config file to yaml!** - User Login deprecation -------------------- @@ -15,9 +9,9 @@ Reddit intends to disable username-password based authentication to access its A Description ----------- -Shreddit is a Python command line program which will take a user's post history on the website [Reddit](http://reddit.com) and after having the user edit a config file will systematically go through the user's history deleting one post/submission at a time until only those whitelisted remain. +Shreddit is a Python command line program which will take a user's post history on the website [Reddit](http://reddit.com) and, after having the user edit a config file, will systematically go through the user's history deleting one post/submission at a time until only those whitelisted remain. -**Note:** When it became known that post edits were *not* saved but post deletions *were* saved, code was added to edit your post prior to deletion. In fact you can actually turn off deletion all together and just have lorem ipsum (or a message about Shreddit) but this will increase how long it takes the script to run as it will be going over all of your messages every run! +**Note:** When it became known that post edits were *not* saved but post deletions *were* saved, code was added to edit your post prior to deletion. In fact you can actually turn off deletion all together and just have lorem ipsum (or a message about Shreddit) but this will increase how long it takes the script to run as it will be going over all of your messages every run! This should be considered the most secure option with current information. Basically it lets you maintain your normal reddit account while having your history scrubbed after a certain amount of time. @@ -89,6 +83,6 @@ OAuth2 Instructions Caveats ----------- -- Only your previous 1,000 comments are accessible on Reddit. So good luck deleting the others. There may be ways to hack around this via iterating using sorting by top/best/controversial/new but for now I am unsure. I believe it best to set the script settings and run it as a cron job and then it won't be a problem unless you post *a lot*. I do, however, think that it may be a caching issue and perhaps after a certain time period your post history would, once again, become available as a block of 1,000. So you needn't despair yet! +- Only your previous 1,000 comments are accessible on Reddit, so good luck deleting the others. There may be ways to hack around this via iterating using sorting by top/best/controversial/new but for now I am unsure. I believe it best to set the script settings and run it as a cron job and then it won't be a problem unless you post *a lot*. I do, however, think that it may be a caching issue and perhaps after a certain time period your post history would, once again, become available as a block of 1,000. So you needn't despair yet! - We are relying on Reddit admin words that they do not store edits, deleted posts are still stored in the database they are merely inaccessible to the public. From 6225a9966b7f7ebce38b4a1fa75d381ba0cd57b4 Mon Sep 17 00:00:00 2001 From: David Trail Date: Tue, 22 Nov 2016 09:44:15 +0000 Subject: [PATCH 08/62] Up the user agent string --- shreddit/shredder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 912e278..b20caa6 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -61,7 +61,7 @@ class Shredder(object): self.shred() def _connect(self, praw_ini, username, password): - self._r = praw.Reddit(user_agent="shreddit/4.2") + self._r = praw.Reddit(user_agent="shreddit/4.4") if praw_ini: # PRAW won't panic if the file is invalid, so check first if not os.path.exists(praw_ini): @@ -118,7 +118,7 @@ class Shredder(object): short_text = sub(b"\n\r\t", " ", comment.body[:35].encode("utf-8")) msg = "/r/{}/ #{} ({}) with: {}".format(comment.subreddit, comment.id, short_text, replacement_text) - + if self._edit_only: self._logger.debug("Editing (not removing) {msg}".format(msg=msg)) else: From 7d19968f5038f6bf3b2f72d84e6757dd062dfe7e Mon Sep 17 00:00:00 2001 From: David Trail Date: Tue, 22 Nov 2016 09:49:24 +0000 Subject: [PATCH 09/62] Description at the top of the file --- README.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1c55d10..f1eea9e 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,30 @@ Shreddit ======== +Description +----------- +Shreddit is a Python command line program which will take a user's post history +on the website [Reddit](http://reddit.com), and will systematically go through +the user's history deleting one post/submission at a time until only those +whitelisted remain. + +**Note:** When it became known that post edits were *not* saved but post +deletions *were* saved, code was added to edit your post prior to deletion. In +fact you can actually turn off deletion all together and just have lorem ipsum +(or a message about Shreddit) but this will increase how long it takes the +script to run as it will be going over all of your messages every run! + +It allows you to maintain your normal reddit account while having your history +scrubbed after a certain amount of time. + User Login deprecation ---------------------- - Reddit intends to disable username-password based authentication to access its APIs in the near future. You can specify your username and password in the `shreddit.yml` or the `praw.ini` to make it work **FOR NOW**. But consider looking at the [OAuth2 instructions](#oauth2-instructions) if you intend to use this program in the future. -Description ------------ - Shreddit is a Python command line program which will take a user's post history on the website [Reddit](http://reddit.com), and will systematically go through the user's history deleting one post/submission at a time until only those From 68c2ca06fcbda3ac0b96537ac0dce1cf8d01ed2d Mon Sep 17 00:00:00 2001 From: David Trail Date: Tue, 22 Nov 2016 09:50:03 +0000 Subject: [PATCH 10/62] =?UTF-8?q?Don't=20duplicate=20the=20description?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index f1eea9e..afdfc08 100644 --- a/README.md +++ b/README.md @@ -25,20 +25,6 @@ APIs in the near future. You can specify your username and password in the looking at the [OAuth2 instructions](#oauth2-instructions) if you intend to use this program in the future. -Shreddit is a Python command line program which will take a user's post history -on the website [Reddit](http://reddit.com), and will systematically go through -the user's history deleting one post/submission at a time until only those -whitelisted remain. - -**Note:** When it became known that post edits were *not* saved but post -deletions *were* saved, code was added to edit your post prior to deletion. In -fact you can actually turn off deletion all together and just have lorem ipsum -(or a message about Shreddit) but this will increase how long it takes the -script to run as it will be going over all of your messages every run! - -It allows you to maintain your normal reddit account while having your history -scrubbed after a certain amount of time. - Installation ([Click here for Windows instructions](#for-windows-users)) ------------------------------------------------------------------------ 1. Clone the repository From 94734b3db2b1f3ec79d4479f9a7de5362c0cec6d Mon Sep 17 00:00:00 2001 From: David Trail Date: Fri, 25 Nov 2016 10:32:48 +0000 Subject: [PATCH 11/62] This should skip clearing votes on subs > 6m old --- shreddit/shredder.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index b20caa6..a59b001 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -9,7 +9,8 @@ import praw import time from re import sub from datetime import datetime, timedelta -from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, HTTPException, OAuthAppRequired) +from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, + HTTPException, OAuthAppRequired) from praw.objects import Comment, Submission from shreddit.util import get_sentence @@ -131,8 +132,11 @@ class Shredder(object): if self._keep_a_copy and self._save_directory: self._save_item(item) if self._clear_vote: - item.clear_vote() - self._api_calls.append(int(time.time())) + try: + item.clear_vote() + self._api_calls.append(int(time.time())) + except HTTPException: + self._logger.debug("Couldn't clear vote on {item}".format(item=item)) if isinstance(item, Submission): self._remove_submission(item) elif isinstance(item, Comment): From 2a1e2fb0a551ba2c1ea9f1ca2010649332d28ac9 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 28 Nov 2016 16:17:15 -0600 Subject: [PATCH 12/62] Makes it easier for users to get the default config --- .gitignore | 1 + setup.py | 6 +-- shreddit/app.py | 11 ++++- shreddit/shreddit.yml.example | 83 +++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 shreddit/shreddit.yml.example diff --git a/.gitignore b/.gitignore index e336702..84bf6ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ shreddit.yml praw.ini +Makefile # Docs docs/_build/ diff --git a/setup.py b/setup.py index c6b8ee3..fc369f2 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "2.0.0" +VERSION = "2.1.0" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) @@ -27,8 +27,8 @@ setup( "Intended Audience :: End Users/Desktop", "Programming Language :: Python"], packages=["shreddit"], - install_requires=["arrow", "backports-abc", "decorator", "praw", "PyYAML", - "requests", "six", "tornado", "update-checker", "wheel"], + install_requires=["arrow", "backports-abc", "praw", "PyYAML", "requests", "six", "tornado"], + package_data={"shreddit": ["shreddit.yml.example"]}, entry_points={ "console_scripts": [ "shreddit=shreddit.app:main" diff --git a/shreddit/app.py b/shreddit/app.py index 0eeaca8..b25c2d5 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -3,6 +3,8 @@ import argparse import yaml import logging +import os +import pkg_resources from shreddit import default_config from shreddit.oauth import oauth_test from shreddit.shredder import Shredder @@ -19,7 +21,14 @@ def main(): oauth_test(args.praw) return - with open(args.config or "shreddit.yml") as fh: + config_filename = args.config or "shreddit.yml" + if not os.path.isfile(config_filename): + print("No configuration file could be found. Paste the following into a file called \"shreddit.yml\" and " \ + "try running shreddit again:\n\n") + print(pkg_resources.resource_string("shreddit", "shreddit.yml.example")) + return + + with open(config_filename) as fh: # Not doing a simple update() here because it's preferable to only set attributes that are "whitelisted" as # configuration options in the form of default values. user_config = yaml.safe_load(fh) diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example new file mode 100644 index 0000000..38088a8 --- /dev/null +++ b/shreddit/shreddit.yml.example @@ -0,0 +1,83 @@ +--- +# Login details for Reddit. Fill out if you don't wish +# to be prompted for a login every time you run Shreddit. +### NOTE: This may be deprecated as you can specify in praw.ini instead +username: +password: + +# How many hours of comments you want to keep +# 24 hours in a day, +# 168 hours in a week, +# 672 hours in two fortnights, +# 720 hours in a month (30 days), +# 8766 hours in a year (365.25 days) +hours: 24 + +# Max score, higher than this and it'll be kept. +max_score: 100 + +# Options: new, top, controversial, more? +sort: new + +# Enables print statements to notify you of what's going on +verbose: True + +# Removes your vote before deleting the item +clear_vote: False + +# Options: comments, submitted, overview +# See: https://github.com/mellort/reddit_api/blob/master/reddit/objects.py#L359 +# Overview: both submissions and comments. Comments / Submitted are as expected +item: overview + +# Anything in this list won't be deleted, coma delimited +# spaces should work as .strip() is called after splitting +# on comma. +whitelist: [AskScience, TheCulture, redditdev, programming, charity, netsec] + +# If you want any specific posts to be whitelisted stick 'em in here +whitelist_ids: [] + +# If you set this then no editing or deleting will be done +# but the output from the program will be shown as an example +trial_run: False + +# Don't delete but *do* edit, could prove... interesting to see a comment +# with 5000 upvotes and it's just a lorem ipsum! +edit_only: False + +# Ignore distinguished comments. +whitelist_distinguished: True + +# Ignore gilded (gold) comments +whitelist_gilded: True + +# Delete everything older that this date, **This ignores whitelists**. +# Can be used as a second deletion, as in "delete items older than 24 hours +# except on whitelisted subreddits but after 3 months delete everything. +nuke_hours: 720 + +# Save a copy to disk of comments and posts before deleting them. +keep_a_copy: False +save_directory: /tmp + +# Replacement text format +# Defines what you want to edit deleted content with pre-deletion (to ensure +# it's not saved in their database). +# Default: Random string. But this can be detected as spam in some cases. +# options: [random, dot, "user entered string"] +replacement_format: random + +# Debug level, how much output you want +# See: https://docs.python.org/3/library/logging.html#logging-levels +debug: DEBUG + +# Define your own wordlist to use as substitution text when +# replacement_format == random +wordlist: [] + +# Batch cooldown +# This controls how long (in seconds) to wait between each set of 1000 deletions. +batch_cooldown: 10 + +# vim: syntax=yaml ts=2 From f93d5e0abac19ce671dd2554417c8c43f3b95126 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 28 Nov 2016 16:17:36 -0600 Subject: [PATCH 13/62] Freezes up-to-date requirements --- requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index b45e408..92b2c37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -arrow -decorator +arrow==0.9.0 +decorator==4.0.10 praw==3.6.0 -PyYAML -requests -six +PyYAML==3.12 +requests==2.12.1 +six==1.10.0 backports-abc==0.4 tornado==4.3 -update-checker==0.11 +update-checker==0.15 wheel==0.24.0 From 20e8271e11c7353b69f746cbd1b28c1b8d182d72 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 28 Nov 2016 17:22:33 -0600 Subject: [PATCH 14/62] Updates the README with pip installation info --- README.md | 169 ++++++++++++++++++++++++++---------------------------- 1 file changed, 82 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index afdfc08..74dce83 100644 --- a/README.md +++ b/README.md @@ -1,112 +1,107 @@ -Shreddit -======== - -Description ------------ -Shreddit is a Python command line program which will take a user's post history -on the website [Reddit](http://reddit.com), and will systematically go through -the user's history deleting one post/submission at a time until only those -whitelisted remain. - -**Note:** When it became known that post edits were *not* saved but post -deletions *were* saved, code was added to edit your post prior to deletion. In -fact you can actually turn off deletion all together and just have lorem ipsum -(or a message about Shreddit) but this will increase how long it takes the -script to run as it will be going over all of your messages every run! - -It allows you to maintain your normal reddit account while having your history +# Shreddit + +Shreddit is a Python command line program which will take a user's post history on the website +[Reddit](http://reddit.com), and will systematically go through the user's history deleting one post/submission at a +time until only those whitelisted remain. It allows you to maintain your normal reddit account while having your history scrubbed after a certain amount of time. -User Login deprecation ----------------------- -Reddit intends to disable username-password based authentication to access its -APIs in the near future. You can specify your username and password in the -`shreddit.yml` or the `praw.ini` to make it work **FOR NOW**. But consider -looking at the [OAuth2 instructions](#oauth2-instructions) if you intend to use -this program in the future. - -Installation ([Click here for Windows instructions](#for-windows-users)) ------------------------------------------------------------------------- -1. Clone the repository -2. Run `python setup.py install`. Usually this is run in the context of a - virtualenv or with administrative permissions for system-wide installation. - virtual environment) -3. Copy `shreddit.yml.example` to `shreddit.yml` and edit it to your liking. - - Make sure you specify your credentials in the file. - - See the [OAuth2 instructions](#oauth2-instructions) if you don't want to - use username-password based authentication. - -- It's useful to have it run as an event, you can set this up as you like but I - suggest `cron` via `crontab -e` and adding a line such as - `@hourly cd $HOME/Shreddit && source bin/activate && shreddit` See below for - more. -- Adding your password to the praw.ini and adding the additional output line - can provide extra debugging help. - -Cron examples -------------- - -- Run `crontab -e` to edit your cron file. If you have access to something like - vixie-cron then each user can have their own personal cron job! +When it became known that post edits were *not* saved but post deletions *were* saved, code was added to edit your post +prior to deletion. In fact you can actually turn off deletion all together and just have lorem ipsum (or a message +about Shreddit) but this will increase how long it takes the script to run as it will be going over all of your messages +every run. + +## User Login deprecation + +Reddit intends to disable username-password based authentication to access its APIs in the near future. You can specify +your username and password in the `shreddit.yml` or the `praw.ini` to make it work **FOR NOW**. But consider looking at +the [OAuth2 instructions](#oauth2-instructions) if you intend to use this program in the future. + +## Pip Installation + +`pip install -U shreddit` will install the package and its dependencies, and it will add a `shreddit` command line +utility to your PATH. This is typically either run in a virtualenv or using administrative privileges for global +installation. + +## Manual Installation + +1. Clone the `shreddit` repository to a directory. +2. From the directory, run `pip install -r requirements.txt` +3. Run `python setup.py install` to install the package and the `shreddit` command line utility. This is typically + either run in a virtualenv or using administrative privileges for global installation. + +## Usage + +After installing the `shreddit` command line utility, the first step is setting up the tool's configuration file. Simply +typing `shreddit` will print a message with an example config. Copy the message from `---` onwards and save it as +`shreddit.yml`. Now, the tool may be used by simply typing `shreddit` from this directory. Alternatively, if you named +the configuration file something different such as `config.yml`, you may use it with `shreddit -c config.yml`. + +### Automating + +The easiest way to automate this tool after the first run is by using the cron utility. Run `crontab -e` to edit your +user's crontab settings. + +**Examples:** - Run every hour on the hour - `0 * * * * shreddit -c ` + `0 * * * * shreddit -c ` - Run at 3am every morning - `0 3 * * * shreddit -c ` + `0 3 * * * shreddit -c ` - Run once a month on the 1st of the month - `0 0 1 * * shreddit -c ` + `0 0 1 * * shreddit -c ` + +If virtualenv was used, be sure to add `source /full/path/to/venv/bin/activate &&` before the command. For example: -If virtualenv was used, be sure to add -`source /full/path/to/venv/bin/activate &&` -before the command. For example: +`0 * * * * source /full/path/to/venv/bin/activate && shreddit -c ` -`0 * * * * source /full/path/to/venv/bin/activate && -shreddit -c ` +### Command Line Options -For Windows users ------------------ +``` +$ shreddit --help +usage: shreddit [-h] [-c CONFIG] [-p PRAW] [-t] + +Command-line frontend to the shreddit library. + +optional arguments: + -h, --help show this help message and exit + -c CONFIG, --config CONFIG + Config file to use instead of the default shreddit.yml + -p PRAW, --praw PRAW PRAW config (if not ./praw.ini) + -t, --test-oauth Perform OAuth test and exit +``` + +## For Windows users 1. Make sure you have Python installed. [Click here for the Python download page](https://www.python.org/downloads/). - - **Note:** Install either `python 2.x` or `python 3.x`, not both. -2. Clone the repository (or download and extract the zip file) -3. Open command prompt to the folder with the zip file (Shreddit-master.zip), - and type `pip install -U Shreddit-master.zip` -4. Open `shreddit.yml.example` in the zip edit it to your liking, and rename the - file to `shreddit.yml`. - - Make sure you specify credentials in the file. - - See the [OAuth2 instructions](#oauth2-instructions) if you don't want to - use username-password based authentication. -5. Type `shreddit` in the open command prompt window to run the program. - -OAuth2 Instructions -------------------- + - **Note:** Install either `python 2.x` or `python 3.x`, not both. +2. Follow the [pip installation](#pip-installation) instructions. +3. Open a new command prompt and verify that the `shreddit` command works before moving on to the [usage](#usage) + section. + +## OAuth2 Instructions 1. Visit: https://www.reddit.com/prefs/apps 2. Click on 'Create app'. - - Fill in the name and select the 'script' option - - Under "redirect uri" put http://127.0.0.1:65010 -3. Copy from or rename `praw.ini.example` to `praw.ini` and open it. Enter the - values from the Reddit page. - - oauth\_client\_id = { The ID displayed next to the icon thingy (under + - Fill in the name and select the 'script' option + - Under "redirect uri" put http://127.0.0.1:65010 +3. Copy from or rename `praw.ini.example` to `praw.ini` and open it. Enter the values from the Reddit page. + - oauth\_client\_id = { The ID displayed next to the icon thingy (under "personal use script") } - - oauth\_client\_secret = { The secret } - - oauth\_redirect\_uri = http://127.0.0.1:65010 - - Save the file. + - oauth\_client\_secret = { The secret } + - oauth\_redirect\_uri = http://127.0.0.1:65010 + - Save the file. 4. Run `python get_secret.py` in the command prompt. 5. Your browser will open to a page on Reddit listing requested permissions. 6. Click 'Allow'. -Caveats -------- +## Caveats -- Certain limitations in the Reddit API and the PRAW library make it difficult - to delete more than 1,000 comments. While deleting >1000 comments is planned, - it is necessary right now to rerun the program until they are all deleted. +- Certain limitations in the Reddit API and the PRAW library make it difficult to delete more than 1,000 comments. + While deleting >1000 comments is planned, it is necessary right now to rerun the program until they are all deleted. -- We are relying on Reddit admin words that they do not store edits, deleted - posts are still stored in the database they are merely inaccessible to the - public. +- We are relying on Reddit admin words that they do not store edits, deleted posts are still stored in the database + they are merely inaccessible to the public. From ebdd8221d9617de8295e5fb20182b659d89b0c86 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 28 Nov 2016 17:22:58 -0600 Subject: [PATCH 15/62] Updates setup.py with origin repo author info --- setup.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index fc369f2..8831329 100644 --- a/setup.py +++ b/setup.py @@ -12,17 +12,14 @@ here = path.abspath(path.dirname(__file__)) with open(path.join(here, "README.md"), encoding='utf-8') as filein: long_description = filein.read() -with open(path.join(here, "requirements.txt"), encoding="utf-8") as filein: - requirements = [line.strip() for line in filein.readlines()] - setup( name="shreddit", version=VERSION, description=DESCRIPTION, long_description=long_description, - url="https://github.com/scott-hand/Shreddit", - author="Scott Hand", - author_email="scott@vkgfx.com", + url="https://github.com/x89/Shreddit", + author="David John", + author_email="david@vaunt.eu", classifiers=["Development Status :: 4 - Beta", "Intended Audience :: End Users/Desktop", "Programming Language :: Python"], From 3c7f57a0507ff3fef73304faeec5a2ac313d63e0 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 28 Nov 2016 17:30:52 -0600 Subject: [PATCH 16/62] Gets package ready for PyPI --- setup.cfg | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b88034e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py index 8831329..3b5497c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "2.1.0" +VERSION = "3.0.0" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) From 4c78f404e2a81d68f91452e51465c1ab5edf238b Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 28 Nov 2016 20:51:56 -0600 Subject: [PATCH 17/62] Allows users to whitelist and blacklist multireddits --- shreddit.yml.example | 4 ++++ shreddit/__init__.py | 2 ++ shreddit/shredder.py | 30 +++++++++++++++++++++++++----- shreddit/shreddit.yml.example | 4 ++++ 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/shreddit.yml.example b/shreddit.yml.example index 38088a8..498e98f 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -38,6 +38,10 @@ whitelist: [AskScience, TheCulture, redditdev, programming, charity, netsec] # If you want any specific posts to be whitelisted stick 'em in here whitelist_ids: [] +# If you want to whitelist or blacklist specific multireddits, add them here +multi_blacklist: [] +multi_whitelist: [] + # If you set this then no editing or deleting will be done # but the output from the program will be shown as an example trial_run: False diff --git a/shreddit/__init__.py b/shreddit/__init__.py index c5e676e..3ef0c40 100644 --- a/shreddit/__init__.py +++ b/shreddit/__init__.py @@ -4,6 +4,8 @@ default_config = {"username": None, "save_directory": "/tmp", "whitelist": [], "whitelist_ids": [], + "multi_blacklist": [], + "multi_whitelist": [], "item": "overview", "sort": "new", "whitelist_distinguished": True, diff --git a/shreddit/shredder.py b/shreddit/shredder.py index a59b001..4b7f91d 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -39,12 +39,29 @@ class Shredder(object): self._limit = None self._api_calls = [] + # Add any multireddit subreddits to the whitelist + self._whitelist = set([s.lower() for s in self._whitelist]) + for username, multiname in self._multi_whitelist: + multireddit = self._r.get_multireddit(username, multiname) + for subreddit in multireddit.subreddits: + self._whitelist.add(str(subreddit).lower()) + + # Add any multireddit subreddits to the blacklist + self._blacklist = set() + for username, multiname in self._multi_blacklist: + multireddit = self._r.get_multireddit(username, multiname) + for subreddit in multireddit.subreddits: + self._blacklist.add(str(subreddit).lower()) + + self._logger.info("Deleting ALL items before {}".format(self._nuke_cutoff)) self._logger.info("Deleting items not whitelisted until {}".format(self._recent_cutoff)) self._logger.info("Ignoring ALL items after {}".format(self._recent_cutoff)) self._logger.info("Targeting {} sorted by {}".format(self._item, self._sort)) + if self._blacklist: + self._logger.info("Deleting ALL items from subreddits {}".format(", ".join(list(self._blacklist)))) if self._whitelist: - self._logger.info("Keeping items from subreddits {}".format(", ".join(self._whitelist))) + self._logger.info("Keeping items from subreddits {}".format(", ".join(list(self._whitelist)))) if self._keep_a_copy and self._save_directory: self._logger.info("Saving deleted items to: {}".format(self._save_directory)) if self._trial_run: @@ -89,7 +106,7 @@ class Shredder(object): raise RateLimitExceeded("You're doing that too much.", e) self._logger.info("Logged in as {user}.".format(user=self._r.user)) - def _check_item(self, item): + def _check_whitelist(self, item): """Returns True if the item is whitelisted, False otherwise. """ if str(item.subreddit).lower() in self._whitelist or item.id in self._whitelist_ids: @@ -157,15 +174,18 @@ class Shredder(object): time.sleep(10) self._logger.debug("Examining item {}: {}".format(idx + 1, item)) created = arrow.get(item.created_utc) + if str(item.subreddit).lower() in self._blacklist: + self._logger.debug("Deleting due to blacklist") + self._remove(item) + elif self._check_whitelist(item): + self._logger.debug("Skipping due to: whitelisted") + continue if created <= self._nuke_cutoff: self._logger.debug("Item occurs prior to nuke cutoff") self._remove(item) elif created > self._recent_cutoff: self._logger.debug("Skipping due to: too recent") continue - elif self._check_item(item): - self._logger.debug("Skipping due to: whitelisted") - continue else: self._remove(item) return idx + 1 diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example index 38088a8..498e98f 100644 --- a/shreddit/shreddit.yml.example +++ b/shreddit/shreddit.yml.example @@ -38,6 +38,10 @@ whitelist: [AskScience, TheCulture, redditdev, programming, charity, netsec] # If you want any specific posts to be whitelisted stick 'em in here whitelist_ids: [] +# If you want to whitelist or blacklist specific multireddits, add them here +multi_blacklist: [] +multi_whitelist: [] + # If you set this then no editing or deleting will be done # but the output from the program will be shown as an example trial_run: False From 6cb808c4032fd83ada87d0d38f733567f4aedb74 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 28 Nov 2016 20:56:08 -0600 Subject: [PATCH 18/62] Improves config files self documentation --- shreddit.yml.example | 6 ++++++ shreddit/shreddit.yml.example | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/shreddit.yml.example b/shreddit.yml.example index 498e98f..3fe276b 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -1,6 +1,7 @@ --- # Login details for Reddit. Fill out if you don't wish # to be prompted for a login every time you run Shreddit. +# If your username or password has characters that could cause problems, surround them in quotes. ### NOTE: This may be deprecated as you can specify in praw.ini instead username: password: @@ -39,6 +40,11 @@ whitelist: [AskScience, TheCulture, redditdev, programming, charity, netsec] whitelist_ids: [] # If you want to whitelist or blacklist specific multireddits, add them here +# Each one must be a list of 2 elements: username, multireddit +# Example: +# multi_blacklist: +# - [myusername, mymulti] +# - [someotheruser, theirmulti] multi_blacklist: [] multi_whitelist: [] diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example index 498e98f..3fe276b 100644 --- a/shreddit/shreddit.yml.example +++ b/shreddit/shreddit.yml.example @@ -1,6 +1,7 @@ --- # Login details for Reddit. Fill out if you don't wish # to be prompted for a login every time you run Shreddit. +# If your username or password has characters that could cause problems, surround them in quotes. ### NOTE: This may be deprecated as you can specify in praw.ini instead username: password: @@ -39,6 +40,11 @@ whitelist: [AskScience, TheCulture, redditdev, programming, charity, netsec] whitelist_ids: [] # If you want to whitelist or blacklist specific multireddits, add them here +# Each one must be a list of 2 elements: username, multireddit +# Example: +# multi_blacklist: +# - [myusername, mymulti] +# - [someotheruser, theirmulti] multi_blacklist: [] multi_whitelist: [] From ca83643526daff7e4a8fede4eeeb4d850b5ba5fa Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 28 Nov 2016 20:57:16 -0600 Subject: [PATCH 19/62] Fast forwards version number so that PyPI and PRAW agent agree --- setup.py | 2 +- shreddit/shredder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3b5497c..190179c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "3.0.0" +VERSION = "5.0.0" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 4b7f91d..80a9d23 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -79,7 +79,7 @@ class Shredder(object): self.shred() def _connect(self, praw_ini, username, password): - self._r = praw.Reddit(user_agent="shreddit/4.4") + self._r = praw.Reddit(user_agent="shreddit/5.0") if praw_ini: # PRAW won't panic if the file is invalid, so check first if not os.path.exists(praw_ini): From 91cd47f139e8fb6e5c3343fe18f60e67b0ccc0dd Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 29 Nov 2016 18:34:41 -0600 Subject: [PATCH 20/62] Improves lorem ipsum to protect against lazy automoderation --- setup.py | 2 +- shreddit/util.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 190179c..03b7f6b 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "5.0.0" +VERSION = "5.0.1" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) diff --git a/shreddit/util.py b/shreddit/util.py index 28baec0..9d8931f 100644 --- a/shreddit/util.py +++ b/shreddit/util.py @@ -3,8 +3,10 @@ import random -WORDLIST = "/usr/share/dict/words" -STATIC_TEXT = "I have been Shreddited for privacy!" +LOREM = """Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sit amet nulla id mi imperdiet condimentum. \ + Vestibulum ac consequat lorem, eu tempus felis. Maecenas eget purus in nisi blandit volutpat. Aenean \ + bibendum eros sit amet ex rhoncus, eu porta magna sodales. Sed venenatis sapien sit amet tempor euismod. \ + Ut a neque purus. Vestibulum quis tortor non leo eleifend interdum.""" try: @@ -16,9 +18,6 @@ except ImportError: replacement. The current solution is to return a static text string unless the operating system has a word list. If that is the case, use it instead. """ - try: - lines = [line.strip() for line in open(WORDLIST).readlines()] - return " ".join(random.sample(lines, random.randint(50, 150))) - except IOError: - # The word list wasn't available... - return STATIC_TEXT + words = LOREM.split() + random.shuffle(words) + return " ".join(words) From c6d7631def86fae5d35ff500bdc7c91a4e446ffd Mon Sep 17 00:00:00 2001 From: skath Date: Mon, 5 Dec 2016 20:21:03 -0800 Subject: [PATCH 21/62] Change requirements for praw since praw>=4.0.0 is not backward compatible --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 92b2c37..b667be5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ arrow==0.9.0 decorator==4.0.10 -praw==3.6.0 +praw<4 PyYAML==3.12 requests==2.12.1 six==1.10.0 diff --git a/setup.py b/setup.py index 03b7f6b..7a19497 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( "Intended Audience :: End Users/Desktop", "Programming Language :: Python"], packages=["shreddit"], - install_requires=["arrow", "backports-abc", "praw", "PyYAML", "requests", "six", "tornado"], + install_requires=["arrow", "backports-abc", "praw<4", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["shreddit.yml.example"]}, entry_points={ "console_scripts": [ From 378d88e8b7aa142c1e8afe02900df0820b0d5efb Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 5 Dec 2016 23:57:23 -0600 Subject: [PATCH 22/62] Increments version to support requirements update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a19497..c9af536 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "5.0.1" +VERSION = "5.0.2" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) From 643937dcb6936eb1e7a8d0e2d0aba8bbcefa3110 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 00:29:49 -0600 Subject: [PATCH 23/62] Updates PRAW deps to V4 --- requirements.txt | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index b667be5..b76a730 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ arrow==0.9.0 decorator==4.0.10 -praw<4 +praw>=4.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 diff --git a/setup.py b/setup.py index c9af536..1a00f15 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "5.0.2" +VERSION = "6.0.0" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) @@ -24,7 +24,7 @@ setup( "Intended Audience :: End Users/Desktop", "Programming Language :: Python"], packages=["shreddit"], - install_requires=["arrow", "backports-abc", "praw<4", "PyYAML", "requests", "six", "tornado"], + install_requires=["arrow", "backports-abc", "praw>=4", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["shreddit.yml.example"]}, entry_points={ "console_scripts": [ From 2b90ba8aea6267a4f9bb74f0f0ed11ab2dfbedcd Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 00:35:00 -0600 Subject: [PATCH 24/62] Removes outdated OAuth helper scripts --- get_secret.py | 68 ----------------------------------------------- shreddit/oauth.py | 22 --------------- 2 files changed, 90 deletions(-) delete mode 100755 get_secret.py delete mode 100644 shreddit/oauth.py diff --git a/get_secret.py b/get_secret.py deleted file mode 100755 index 7d340a4..0000000 --- a/get_secret.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -''' -So I heard you want to use OAuth2? This is a helper tool that gets the -authenticaton code for you and fires it into praw.ini. - -How to use: - - Visit: https://www.reddit.com/prefs/apps - - Create new "script", under "redirect uri" put http://127.0.0.1:65010 - - Open praw.ini - - oauth_client_id = { The ID displayed under the icon thingy } - - oauth_client_secret = { The secret } - - oauth_redirect_uri = http://127.0.0.1:65010 - - Run this script - - Your browser will open to a page on Reddit listing requested perms - - Click permit -''' - -import praw -import webbrowser -from warnings import warn -from praw.errors import HTTPException, OAuthAppRequired -from tornado import gen, web -from tornado.ioloop import IOLoop -from tornado.httpserver import HTTPServer - -r = praw.Reddit('Shreddit refresh token grabber') - - -class Page(web.RequestHandler): - def get(self): - code = self.get_argument("code", default=None, strip=False) - self.write("Success! Your code: %s
\ - It will now be appended to praw.ini and you \ - should be able to enjoy Shreddit without storing \ - your user / pass anywhere." % code) - IOLoop.current().stop() - self.login(code) - - def login(self, code): - deets = r.get_access_information(code) - print("oauth_refresh_token: %s" % deets['refresh_token']) - r.set_access_credentials(**deets) - with open('praw.ini', mode='a') as fh: - fh.write('oauth_refresh_token = %s' % deets['refresh_token']) - print("Refresh token written to praw.ini") - -application = web.Application([(r"/", Page)]) - -try: - r.refresh_access_information() -except HTTPException: - url = r.get_authorize_url('uniqueKey', ['identity', 'read', 'vote', 'edit', 'history'], True) - try: - print("Opening url: %s" % url) - webbrowser.open(url, new=2) - except NameError: - warn('''Couldn't open URL: %s\n please do so manually''' % url) - server = HTTPServer(application) - server.listen(65010) - IOLoop.current().start() - -if r.user == None: - print("Failed to log in. Something went wrong!") -else: - print("Logged in as %s." % r.user) - -print() - diff --git a/shreddit/oauth.py b/shreddit/oauth.py deleted file mode 100644 index 8276c43..0000000 --- a/shreddit/oauth.py +++ /dev/null @@ -1,22 +0,0 @@ -"""This module contains a function that tests OAuth session validity. -""" -import os -import praw - - -def oauth_test(praw_ini): - if praw_ini: - # PRAW won't panic if the file is invalid, so check first - if not os.path.exists(praw_ini): - print("PRAW configuration file \"{}\" not found.".format(praw_ini)) - return - praw.settings.CONFIG.read(praw_ini) - r = praw.Reddit("Shreddit oauth test") - try: - r.refresh_access_information() - if r.is_oauth_session(): - print("Session is valid.") - else: - print("Session is not a valid OAuth session.") - except Exception as e: - print("Error encountered while checking credentials:\n{}".format(e)) From 8171da78284f731e1d51f34adea770d934604ac9 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 00:35:29 -0600 Subject: [PATCH 25/62] Adds OAuth creds to config example --- shreddit.yml.example | 8 ++++---- shreddit/shreddit.yml.example | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shreddit.yml.example b/shreddit.yml.example index 3fe276b..0637304 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -1,8 +1,8 @@ --- -# Login details for Reddit. Fill out if you don't wish -# to be prompted for a login every time you run Shreddit. -# If your username or password has characters that could cause problems, surround them in quotes. -### NOTE: This may be deprecated as you can specify in praw.ini instead +# Credentials go here. If you are unsure what to put in client_id and client_secret, please consult the README for more +# information. +client_id: +client_secret: username: password: diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example index 3fe276b..0637304 100644 --- a/shreddit/shreddit.yml.example +++ b/shreddit/shreddit.yml.example @@ -1,8 +1,8 @@ --- -# Login details for Reddit. Fill out if you don't wish -# to be prompted for a login every time you run Shreddit. -# If your username or password has characters that could cause problems, surround them in quotes. -### NOTE: This may be deprecated as you can specify in praw.ini instead +# Credentials go here. If you are unsure what to put in client_id and client_secret, please consult the README for more +# information. +client_id: +client_secret: username: password: From c8c3dcc52dbce81118bc169b6267e703ceabdfc1 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 01:03:46 -0600 Subject: [PATCH 26/62] Moves credentials to praw.ini --- praw.ini.example | 20 +++++++------------- shreddit.yml.example | 7 ------- shreddit/shreddit.yml.example | 7 ------- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/praw.ini.example b/praw.ini.example index 3f608b5..47ee85e 100644 --- a/praw.ini.example +++ b/praw.ini.example @@ -1,13 +1,7 @@ -[DEFAULT] -# Will be deprecated sometime in 2015 (probably) -user = -pswd = - -## OAuth2 settings: -# Client / Secret from your own app -oauth_client_id = -oauth_client_secret = -# Corresponds with the callback URL in the Reddit app -oauth_redirect_uri = http://127.0.0.1:65010 -log_requests = 0 -# After running get_secret.py you should find your auth ID below +# Credentials go here. Fill out default, or provide one or more names and call shreddit with the -u option to specify +# which set to use. +[default] +client_id= +client_secret= +username= +password= diff --git a/shreddit.yml.example b/shreddit.yml.example index 0637304..ddb6a64 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -1,11 +1,4 @@ --- -# Credentials go here. If you are unsure what to put in client_id and client_secret, please consult the README for more -# information. -client_id: -client_secret: -username: -password: - # How many hours of comments you want to keep # 24 hours in a day, # 168 hours in a week, diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example index 0637304..ddb6a64 100644 --- a/shreddit/shreddit.yml.example +++ b/shreddit/shreddit.yml.example @@ -1,11 +1,4 @@ --- -# Credentials go here. If you are unsure what to put in client_id and client_secret, please consult the README for more -# information. -client_id: -client_secret: -username: -password: - # How many hours of comments you want to keep # 24 hours in a day, # 168 hours in a week, From 9fdfb9a32a5225b426f8fd98c82aa03db2354c67 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 02:56:33 -0600 Subject: [PATCH 27/62] Removes edit only from configs --- shreddit.yml.example | 4 ---- shreddit/shreddit.yml.example | 4 ---- 2 files changed, 8 deletions(-) diff --git a/shreddit.yml.example b/shreddit.yml.example index ddb6a64..f7271ea 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -45,10 +45,6 @@ multi_whitelist: [] # but the output from the program will be shown as an example trial_run: False -# Don't delete but *do* edit, could prove... interesting to see a comment -# with 5000 upvotes and it's just a lorem ipsum! -edit_only: False - # Ignore distinguished comments. whitelist_distinguished: True diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example index ddb6a64..f7271ea 100644 --- a/shreddit/shreddit.yml.example +++ b/shreddit/shreddit.yml.example @@ -45,10 +45,6 @@ multi_whitelist: [] # but the output from the program will be shown as an example trial_run: False -# Don't delete but *do* edit, could prove... interesting to see a comment -# with 5000 upvotes and it's just a lorem ipsum! -edit_only: False - # Ignore distinguished comments. whitelist_distinguished: True From 83abfe9f8817603ac4950ead0827a4845f31e181 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 02:56:59 -0600 Subject: [PATCH 28/62] Makes prawcore dep explicit --- requirements.txt | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b76a730..6ba44e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ arrow==0.9.0 decorator==4.0.10 praw>=4.0 +prawcore PyYAML==3.12 requests==2.12.1 six==1.10.0 diff --git a/setup.py b/setup.py index 1a00f15..d11cfb4 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( "Intended Audience :: End Users/Desktop", "Programming Language :: Python"], packages=["shreddit"], - install_requires=["arrow", "backports-abc", "praw>=4", "PyYAML", "requests", "six", "tornado"], + install_requires=["arrow", "backports-abc", "praw>=4", "prawcore", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["shreddit.yml.example"]}, entry_points={ "console_scripts": [ From 0271b2e74123d89136ec754a0eadabe69cf2bb8b Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 02:57:47 -0600 Subject: [PATCH 29/62] Updates core functionality to PRAW v4 --- shreddit/app.py | 11 +---- shreddit/shredder.py | 96 ++++++++++++++++---------------------------- shreddit/util.py | 8 ++++ 3 files changed, 45 insertions(+), 70 deletions(-) diff --git a/shreddit/app.py b/shreddit/app.py index b25c2d5..d16b0e8 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -6,21 +6,15 @@ import logging import os import pkg_resources from shreddit import default_config -from shreddit.oauth import oauth_test from shreddit.shredder import Shredder def main(): parser = argparse.ArgumentParser(description="Command-line frontend to the shreddit library.") parser.add_argument("-c", "--config", help="Config file to use instead of the default shreddit.yml") - parser.add_argument("-p", "--praw", help="PRAW config (if not ./praw.ini)") - parser.add_argument("-t", "--test-oauth", help="Perform OAuth test and exit", action="store_true") + parser.add_argument("-u", "--user", help="User section from praw.ini if not default", default="default") args = parser.parse_args() - if args.test_oauth: - oauth_test(args.praw) - return - config_filename = args.config or "shreddit.yml" if not os.path.isfile(config_filename): print("No configuration file could be found. Paste the following into a file called \"shreddit.yml\" and " \ @@ -36,8 +30,7 @@ def main(): if option in user_config: default_config[option] = user_config[option] - # TODO: Validate config options - shredder = Shredder(default_config, args.praw) + shredder = Shredder(default_config, args.user) shredder.shred() diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 80a9d23..8a9f208 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -1,32 +1,30 @@ -import os -import sys -import logging +import arrow import argparse import json -import arrow -import yaml +import logging +import os import praw +import sys import time -from re import sub +import yaml from datetime import datetime, timedelta -from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, - HTTPException, OAuthAppRequired) -from praw.objects import Comment, Submission -from shreddit.util import get_sentence +from praw.models import Comment, Submission +from prawcore.exceptions import ResponseException, OAuthException +from re import sub +from shreddit.util import get_sentence, ShredditError class Shredder(object): """This class stores state for configuration, API objects, logging, etc. It exposes a shred() method that application code can call to start it. """ - def __init__(self, config, praw_ini=None): + def __init__(self, config, user): logging.basicConfig() self._logger = logging.getLogger("shreddit") self._logger.setLevel(level=logging.DEBUG if config.get("verbose", True) else logging.INFO) self.__dict__.update({"_{}".format(k): config[k] for k in config}) - self._praw_ini = praw_ini - self._connect(praw_ini, self._username, self._password) + self._connect(user) if self._save_directory: self._r.config.store_json_result = True @@ -36,8 +34,6 @@ class Shredder(object): if self._save_directory: if not os.path.exists(self._save_directory): os.makedirs(self._save_directory) - self._limit = None - self._api_calls = [] # Add any multireddit subreddits to the whitelist self._whitelist = set([s.lower() for s in self._whitelist]) @@ -68,7 +64,7 @@ class Shredder(object): self._logger.info("Trial run - no deletion will be performed") def shred(self): - deleted = self._remove_things(self._get_things()) + deleted = self._remove_things(self._build_iterator()) self._logger.info("Finished deleting {} items. ".format(deleted)) if deleted >= 1000: # This user has more than 1000 items to handle, which angers the gods of the Reddit API. So chill for a @@ -78,33 +74,14 @@ class Shredder(object): self._connect(None, self._username, self._password) self.shred() - def _connect(self, praw_ini, username, password): - self._r = praw.Reddit(user_agent="shreddit/5.0") - if praw_ini: - # PRAW won't panic if the file is invalid, so check first - if not os.path.exists(praw_ini): - print("PRAW configuration file \"{}\" not found.".format(praw_ini)) - return - praw.settings.CONFIG.read(praw_ini) + def _connect(self, user): try: - # Try to login with OAuth2 - self._r.refresh_access_information() - self._logger.debug("Logged in with OAuth.") - except (HTTPException, OAuthAppRequired) as e: - self._logger.warning("You should migrate to OAuth2 using get_secret.py before Reddit disables this login " - "method.") - try: - try: - self._r.login(username, password) - except InvalidUserPass: - self._r.login() # Supply details on the command line - except InvalidUser as e: - raise InvalidUser("User does not exist.", e) - except InvalidUserPass as e: - raise InvalidUserPass("Specified an incorrect password.", e) - except RateLimitExceeded as e: - raise RateLimitExceeded("You're doing that too much.", e) - self._logger.info("Logged in as {user}.".format(user=self._r.user)) + self._r = praw.Reddit(user, user_agent="python:shreddit:v6.0.0") + self._logger.info("Logged in as {user}.".format(user=self._r.user.me())) + except ResponseException: + raise ShredditError("Bad OAuth credentials") + except OAuthException: + raise ShredditError("Bad username or password") def _check_whitelist(self, item): """Returns True if the item is whitelisted, False otherwise. @@ -137,13 +114,9 @@ class Shredder(object): short_text = sub(b"\n\r\t", " ", comment.body[:35].encode("utf-8")) msg = "/r/{}/ #{} ({}) with: {}".format(comment.subreddit, comment.id, short_text, replacement_text) - if self._edit_only: - self._logger.debug("Editing (not removing) {msg}".format(msg=msg)) - else: - self._logger.debug("Editing and deleting {msg}".format(msg=msg)) + self._logger.debug("Editing and deleting {msg}".format(msg=msg)) if not self._trial_run: comment.edit(replacement_text) - self._api_calls.append(int(time.time())) def _remove(self, item): if self._keep_a_copy and self._save_directory: @@ -151,27 +124,20 @@ class Shredder(object): if self._clear_vote: try: item.clear_vote() - self._api_calls.append(int(time.time())) except HTTPException: self._logger.debug("Couldn't clear vote on {item}".format(item=item)) if isinstance(item, Submission): self._remove_submission(item) elif isinstance(item, Comment): self._remove_comment(item) - if not self._edit_only and not self._trial_run: + if not self._trial_run: item.delete() - self._api_calls.append(int(time.time())) def _remove_things(self, items): self._logger.info("Loading items to delete...") to_delete = [item for item in items] self._logger.info("Done. Starting on batch of {} items...".format(len(to_delete))) for idx, item in enumerate(to_delete): - minute_ago = arrow.now().replace(minutes=-1).timestamp - self._api_calls = [api_call for api_call in self._api_calls if api_call >= minute_ago] - if len(self._api_calls) >= 60: - self._logger.info("Sleeping 10 seconds to wait out API cooldown...") - time.sleep(10) self._logger.debug("Examining item {}: {}".format(idx + 1, item)) created = arrow.get(item.created_utc) if str(item.subreddit).lower() in self._blacklist: @@ -190,12 +156,20 @@ class Shredder(object): self._remove(item) return idx + 1 - def _get_things(self): + def _build_iterator(self): + item = self._r.user.me() if self._item == "comments": - return self._r.user.get_comments(limit=self._limit, sort=self._sort) + item = item.comments elif self._item == "submitted": - return self._r.user.get_submitted(limit=self._limit, sort=self._sort) - elif self._item == "overview": - return self._r.user.get_overview(limit=self._limit, sort=self._sort) + item = item.submissions + + if self._sort == "new": + return item.new(limit=None) + elif self._sort == "top": + return item.top(limit=None) + elif self._sort == "hot": + return item.hot(limit=None) + elif self._sort == "controversial": + return item.controversial(limit=None) else: - raise Exception("Your deletion section is wrong") + raise ShredditError("Sorting \"{}\" not recognized.".format(self._sort)) diff --git a/shreddit/util.py b/shreddit/util.py index 9d8931f..dea428a 100644 --- a/shreddit/util.py +++ b/shreddit/util.py @@ -21,3 +21,11 @@ except ImportError: words = LOREM.split() random.shuffle(words) return " ".join(words) + + +class ShredditError(Exception): + def __init__(self, value=None): + self.value = value if value else "No information provided" + + def __str__(self): + return repr(self.value) From 20205b4126cbe3cc541df49d20be468f998e614b Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:19:56 -0600 Subject: [PATCH 30/62] Removes more outdated OAuth code --- helpers/oauth_check.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 helpers/oauth_check.py diff --git a/helpers/oauth_check.py b/helpers/oauth_check.py deleted file mode 100644 index 563d0c6..0000000 --- a/helpers/oauth_check.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -''' -Simple script to check if your oauth is working. -''' -import praw -import sys - -r = praw.Reddit('Shreddit oauth test') -try: - r.refresh_access_information() - if r.is_oauth_session(): - sys.exit(0) - else: - sys.exit(2) -except: - sys.exit(1) - From e35a2aad10d7496ead81877b45c0f435439c0ff6 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:20:42 -0600 Subject: [PATCH 31/62] Removes deployment configs and files until documented --- helpers/ansible-shreddit.yml | 18 ------------------ lambda_handler.py | 12 ------------ 2 files changed, 30 deletions(-) delete mode 100644 helpers/ansible-shreddit.yml delete mode 100644 lambda_handler.py diff --git a/helpers/ansible-shreddit.yml b/helpers/ansible-shreddit.yml deleted file mode 100644 index 76de711..0000000 --- a/helpers/ansible-shreddit.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: ro - vars: - user: napalm - install_dir: "/home/{{ user }}/Shreddit" - tasks: - - name: Update the Shreddit repository - git: repo=https://github.com/x89/Shreddit.git dest={{ install_dir }} - - name: Ensure shreddit.cfg is correct - copy: src=shreddit.cfg dest={{ install_dir }}/shreddit.cfg owner={{ user }} group={{ user }} mode=0600 - - name: Ensure praw.ini is correct - copy: src=praw.ini dest={{ install_dir }}/praw.ini owner={{ user }} group={{ user }} mode=0600 - - name: Run get_secret.py to check that we can login with oauth - shell: bash -c "source .venv/bin/activate && python get_secret.py" chdir={{ install_dir }} - - name: Ensure Shreddit cron job - cron: user={{ user }} name="Shreddit" minute="0" job="cd {{ install_dir }} && bash -c 'source .venv/bin/activate && python shreddit.py' 2>/dev/null" - - name: Run a test Shreddit run - shell: bash -c "source .venv/bin/activate && python shreddit.py" chdir={{ install_dir }} diff --git a/lambda_handler.py b/lambda_handler.py deleted file mode 100644 index e7e7d97..0000000 --- a/lambda_handler.py +++ /dev/null @@ -1,12 +0,0 @@ -"""This module contains the handler function called by AWS. -""" -from shreddit.shredder import shred -import yaml - - -def lambda_handler(event, context): - with open("shreddit.yml") as fh: - config = yaml.safe_load(fh) - if not config: - raise Exception("No config options passed!") - shred(config) From 74bba08955c79b3d7abbb324ed54559570561d97 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:28:28 -0600 Subject: [PATCH 32/62] Allows user to generate praw and shreddit configs --- setup.py | 2 +- shreddit/app.py | 20 +++++++++++++++++--- shreddit/praw.ini.example | 7 +++++++ 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 shreddit/praw.ini.example diff --git a/setup.py b/setup.py index d11cfb4..abd4453 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( "Programming Language :: Python"], packages=["shreddit"], install_requires=["arrow", "backports-abc", "praw>=4", "prawcore", "PyYAML", "requests", "six", "tornado"], - package_data={"shreddit": ["shreddit.yml.example"]}, + package_data={"shreddit": ["*.example"]}, entry_points={ "console_scripts": [ "shreddit=shreddit.app:main" diff --git a/shreddit/app.py b/shreddit/app.py index d16b0e8..1678805 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -12,14 +12,28 @@ from shreddit.shredder import Shredder def main(): parser = argparse.ArgumentParser(description="Command-line frontend to the shreddit library.") parser.add_argument("-c", "--config", help="Config file to use instead of the default shreddit.yml") + parser.add_argument("-g", "--generate-configs", help="Write shreddit and praw config files to current directory.", + action="store_true") parser.add_argument("-u", "--user", help="User section from praw.ini if not default", default="default") args = parser.parse_args() + if args.generate_configs: + if not os.path.isfile("shreddit.yml"): + print("Writing shreddit.yml file...") + with open("shreddit.yml", "w") as fout: + fout.write(pkg_resources.resource_string("shreddit", "shreddit.yml.example")) + if not os.path.isfile("praw.ini"): + print("Writing praw.ini file...") + with open("praw.ini", "w") as fout: + fout.write(pkg_resources.resource_string("shreddit", "praw.ini.example")) + return + config_filename = args.config or "shreddit.yml" if not os.path.isfile(config_filename): - print("No configuration file could be found. Paste the following into a file called \"shreddit.yml\" and " \ - "try running shreddit again:\n\n") - print(pkg_resources.resource_string("shreddit", "shreddit.yml.example")) + print("No shreddit configuration file was found or provided. Run this script with -g to generate one.") + return + if not os.path.isfile("praw.ini"): + print("No praw configuration file was found. Run this script with -g to generate one.") return with open(config_filename) as fh: diff --git a/shreddit/praw.ini.example b/shreddit/praw.ini.example new file mode 100644 index 0000000..47ee85e --- /dev/null +++ b/shreddit/praw.ini.example @@ -0,0 +1,7 @@ +# Credentials go here. Fill out default, or provide one or more names and call shreddit with the -u option to specify +# which set to use. +[default] +client_id= +client_secret= +username= +password= From b0c95d35161d8c394a16407d8d636ad670686dd9 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:50:27 -0600 Subject: [PATCH 33/62] Removes check for praw.ini to allow global config --- shreddit/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/shreddit/app.py b/shreddit/app.py index 1678805..3064b1a 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -32,9 +32,6 @@ def main(): if not os.path.isfile(config_filename): print("No shreddit configuration file was found or provided. Run this script with -g to generate one.") return - if not os.path.isfile("praw.ini"): - print("No praw configuration file was found. Run this script with -g to generate one.") - return with open(config_filename) as fh: # Not doing a simple update() here because it's preferable to only set attributes that are "whitelisted" as From a1e611553afc5d8ccb862b7fc5ba384e26ec6ea3 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:50:52 -0600 Subject: [PATCH 34/62] Fixes inaccurate count when no items are deleted --- shreddit/shredder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 8a9f208..e54d432 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -137,8 +137,10 @@ class Shredder(object): self._logger.info("Loading items to delete...") to_delete = [item for item in items] self._logger.info("Done. Starting on batch of {} items...".format(len(to_delete))) - for idx, item in enumerate(to_delete): - self._logger.debug("Examining item {}: {}".format(idx + 1, item)) + count = 0 + for item in to_delete: + count += 1 + self._logger.debug("Examining item {}: {}".format(count, item)) created = arrow.get(item.created_utc) if str(item.subreddit).lower() in self._blacklist: self._logger.debug("Deleting due to blacklist") @@ -154,7 +156,7 @@ class Shredder(object): continue else: self._remove(item) - return idx + 1 + return count def _build_iterator(self): item = self._r.user.me() From ddf33498cb97d4ac2e89516e79ebdefbf86c2b76 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:54:22 -0600 Subject: [PATCH 35/62] Updates README for PRAW 4 operation --- README.md | 87 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 74dce83..89e9ee7 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,11 @@ prior to deletion. In fact you can actually turn off deletion all together and j about Shreddit) but this will increase how long it takes the script to run as it will be going over all of your messages every run. -## User Login deprecation +## Important New Changes (as of Dec 2016) -Reddit intends to disable username-password based authentication to access its APIs in the near future. You can specify -your username and password in the `shreddit.yml` or the `praw.ini` to make it work **FOR NOW**. But consider looking at -the [OAuth2 instructions](#oauth2-instructions) if you intend to use this program in the future. +Due to deprecation of the PRAW 3.x library, Shreddit is using PRAW 4. This requires that OAuth be used to authenticate. +Thankfully, however, it is much easier than in previous versions. If you are upgrading, [please review the usage section +to ensure that you have set up credentials correctly.](#configuring-credentials) ## Pip Installation @@ -31,10 +31,52 @@ installation. ## Usage -After installing the `shreddit` command line utility, the first step is setting up the tool's configuration file. Simply -typing `shreddit` will print a message with an example config. Copy the message from `---` onwards and save it as -`shreddit.yml`. Now, the tool may be used by simply typing `shreddit` from this directory. Alternatively, if you named -the configuration file something different such as `config.yml`, you may use it with `shreddit -c config.yml`. +After installing the `shreddit` command line utility, the first step is setting up the tool's configuration files. +Simply typing `shreddit -g` will generate configs. After configuring credentials, running the tool with the `shreddit` +command will begin the tool's operation. + +### Configuring Credentials + +Running `shreddit -g` will generate a blank praw.ini file that looks like this: + +``` +# Credentials go here. Fill out default, or provide one or more names and call shreddit with the -u option to specify +# which set to use. +[default] +client_id= +client_secret= +username= +password= +``` + +Username and password are simply your Reddit login credentials for the account that will be used. However, to obtain the +client ID and secret, follow these steps (taken from +[PRAW documentation](http://praw.readthedocs.io/en/latest/getting_started/authentication.html#script-application): + +1. Open your Reddit application preferences by clicking [here](https://www.reddit.com/prefs/apps/) +2. Add a new application. It doesn't matter what it's named, but calling it "shreddit" makes it easier to remember +3. Select "script" +4. Redirect URL does not matter, so enter something like http://127.0.0.1:8080 +5. Once created, you should see the name of your application followed by 14 character string. Enter this 14 character + string as your `client_id`. +6. Copy the 27 character "secret" string into the `client_secret` field. + +Finally, your praw.ini should look like this (with fake data provided here): + +``` +[default] +client_id=f3FaKeD4t40PsJ +client_secret=dfK3pfMoReFAkEDaTa123456789 +username=testuser +password=123passwordgoeshere123 +``` + +Keep your praw.ini either in the current directory when running `shreddit`, or in one of the config folders +[described here](http://praw.readthedocs.io/en/latest/getting_started/configuration/prawini.html) such as +`~/.config` in Linux or `%APPDATA%` in Windows. + +To use more than one account, you can add multiple profiles instead of just `[default]` and use the `-u` option to +`shreddit` to choose which one each time. ### Automating @@ -43,6 +85,9 @@ user's crontab settings. **Examples:** +The following examples require that the PRAW configuration file is located in the config directory. See [this PRAW +documentation](http://praw.readthedocs.io/en/latest/getting_started/configuration/prawini.html) for more information. + - Run every hour on the hour `0 * * * * shreddit -c ` @@ -60,7 +105,7 @@ If virtualenv was used, be sure to add `source /full/path/to/venv/bin/activate & ``` $ shreddit --help -usage: shreddit [-h] [-c CONFIG] [-p PRAW] [-t] +usage: app.py [-h] [-c CONFIG] [-g] [-u USER] Command-line frontend to the shreddit library. @@ -68,8 +113,10 @@ optional arguments: -h, --help show this help message and exit -c CONFIG, --config CONFIG Config file to use instead of the default shreddit.yml - -p PRAW, --praw PRAW PRAW config (if not ./praw.ini) - -t, --test-oauth Perform OAuth test and exit + -g, --generate-configs + Write shreddit and praw config files to current + directory. + -u USER, --user USER User section from praw.ini if not default ``` ## For Windows users @@ -81,23 +128,6 @@ optional arguments: 3. Open a new command prompt and verify that the `shreddit` command works before moving on to the [usage](#usage) section. -## OAuth2 Instructions - -1. Visit: https://www.reddit.com/prefs/apps -2. Click on 'Create app'. - - Fill in the name and select the 'script' option - - Under "redirect uri" put http://127.0.0.1:65010 -3. Copy from or rename `praw.ini.example` to `praw.ini` and open it. Enter the values from the Reddit page. - - oauth\_client\_id = { The ID displayed next to the icon thingy (under - "personal use script") } - - oauth\_client\_secret = { The secret } - - oauth\_redirect\_uri = http://127.0.0.1:65010 - - Save the file. -4. Run `python get_secret.py` in the command prompt. -5. Your browser will open to a page on Reddit listing requested permissions. -6. Click 'Allow'. - - ## Caveats - Certain limitations in the Reddit API and the PRAW library make it difficult to delete more than 1,000 comments. @@ -105,3 +135,4 @@ optional arguments: - We are relying on Reddit admin words that they do not store edits, deleted posts are still stored in the database they are merely inaccessible to the public. + From bdc83a44828d01592a6dd6bdefb45b85c0ef4494 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 02:00:10 -0600 Subject: [PATCH 36/62] Updates license information to be more standardized --- LICENCE => LICENSE | 0 setup.py | 2 ++ 2 files changed, 2 insertions(+) rename LICENCE => LICENSE (100%) diff --git a/LICENCE b/LICENSE similarity index 100% rename from LICENCE rename to LICENSE diff --git a/setup.py b/setup.py index abd4453..24c7d71 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,9 @@ setup( author_email="david@vaunt.eu", classifiers=["Development Status :: 4 - Beta", "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: BSD License", "Programming Language :: Python"], + license="FreeBSD License", packages=["shreddit"], install_requires=["arrow", "backports-abc", "praw>=4", "prawcore", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["*.example"]}, From d2ba5a7e17103ce0fef597e85fb524bc49e2fbf2 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 02:02:43 -0600 Subject: [PATCH 37/62] Tweaks README for style and typo corrections --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 89e9ee7..290c795 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,11 @@ password= Username and password are simply your Reddit login credentials for the account that will be used. However, to obtain the client ID and secret, follow these steps (taken from -[PRAW documentation](http://praw.readthedocs.io/en/latest/getting_started/authentication.html#script-application): +[PRAW documentation](http://praw.readthedocs.io/en/latest/getting_started/authentication.html#script-application)): -1. Open your Reddit application preferences by clicking [here](https://www.reddit.com/prefs/apps/) -2. Add a new application. It doesn't matter what it's named, but calling it "shreddit" makes it easier to remember -3. Select "script" +1. Open your Reddit application preferences by clicking [here](https://www.reddit.com/prefs/apps/). +2. Add a new application. It doesn't matter what it's named, but calling it "shreddit" makes it easier to remember. +3. Select "script". 4. Redirect URL does not matter, so enter something like http://127.0.0.1:8080 5. Once created, you should see the name of your application followed by 14 character string. Enter this 14 character string as your `client_id`. From 87742563df686bc915b6e980b94dc28b2a1aaed8 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 12 Dec 2016 12:09:20 -0600 Subject: [PATCH 38/62] Locks down a few more requirement numbers --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6ba44e3..16f7344 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ arrow==0.9.0 decorator==4.0.10 -praw>=4.0 -prawcore +praw==4.0.0 +prawcore==0.4.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 From f1fa1b638d736c7de7e7dccba62d9f9b90be0a43 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 12 Dec 2016 12:21:46 -0600 Subject: [PATCH 39/62] Freezes prawcore to 0.3.0 because praw 4.0.0 requires it --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 16f7344..c164c9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ arrow==0.9.0 decorator==4.0.10 praw==4.0.0 -prawcore==0.4.0 +prawcore==0.3.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 From 284be846784fc228a832614eb2db308f85e24ecb Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 12 Dec 2016 12:26:39 -0600 Subject: [PATCH 40/62] Removes prawcore as a requirement, lets praw manage it --- requirements.txt | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c164c9d..cf08091 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ arrow==0.9.0 decorator==4.0.10 praw==4.0.0 -prawcore==0.3.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 diff --git a/setup.py b/setup.py index 24c7d71..47eed11 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( "Programming Language :: Python"], license="FreeBSD License", packages=["shreddit"], - install_requires=["arrow", "backports-abc", "praw>=4", "prawcore", "PyYAML", "requests", "six", "tornado"], + install_requires=["arrow", "backports-abc", "praw>=4", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["*.example"]}, entry_points={ "console_scripts": [ From 2b6a6ff51e38036fe662319dd8332c9dfe812bbf Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Dec 2016 11:07:36 -0600 Subject: [PATCH 41/62] Handles pkg_resources byte strings in Python 3 correctly --- shreddit/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shreddit/app.py b/shreddit/app.py index 3064b1a..82521da 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -20,11 +20,11 @@ def main(): if args.generate_configs: if not os.path.isfile("shreddit.yml"): print("Writing shreddit.yml file...") - with open("shreddit.yml", "w") as fout: + with open("shreddit.yml", "wb") as fout: fout.write(pkg_resources.resource_string("shreddit", "shreddit.yml.example")) if not os.path.isfile("praw.ini"): print("Writing praw.ini file...") - with open("praw.ini", "w") as fout: + with open("praw.ini", "wb") as fout: fout.write(pkg_resources.resource_string("shreddit", "praw.ini.example")) return From c40e56e6b1a96ae181228669d9892f9ff1960663 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Dec 2016 12:17:28 -0600 Subject: [PATCH 42/62] Makes README more clear about filling out credentials. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 290c795..b53fc3c 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ username= password= ``` +**You must provide values for each of these.** As strange as it may seem to provide both a username/password pair *and* +a client id/secret pair, that is how the Reddit API does "OAuth" script applications. + Username and password are simply your Reddit login credentials for the account that will be used. However, to obtain the client ID and secret, follow these steps (taken from [PRAW documentation](http://praw.readthedocs.io/en/latest/getting_started/authentication.html#script-application)): @@ -56,7 +59,7 @@ client ID and secret, follow these steps (taken from 1. Open your Reddit application preferences by clicking [here](https://www.reddit.com/prefs/apps/). 2. Add a new application. It doesn't matter what it's named, but calling it "shreddit" makes it easier to remember. 3. Select "script". -4. Redirect URL does not matter, so enter something like http://127.0.0.1:8080 +4. Redirect URL does not matter for script applications, so enter something like http://127.0.0.1:8080 5. Once created, you should see the name of your application followed by 14 character string. Enter this 14 character string as your `client_id`. 6. Copy the 27 character "secret" string into the `client_secret` field. From e84d64f93c78e9155fd3a71b39b958af5b4d638d Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Dec 2016 13:14:53 -0600 Subject: [PATCH 43/62] Adds a MANIFEST file with README and LICENSE --- MANIFEST.in | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..72ca36c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE README.md diff --git a/setup.py b/setup.py index 47eed11..632fb27 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "6.0.0" +VERSION = "6.0.2" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) From 67ccc3944265742b5416a65aea3778ca9ae315e3 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Dec 2016 15:41:22 -0600 Subject: [PATCH 44/62] Fixes JSON output regression in PRAW4 --- setup.py | 2 +- shreddit/shredder.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 632fb27..212480b 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "6.0.2" +VERSION = "6.0.3" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index e54d432..9b456c8 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -97,8 +97,12 @@ class Shredder(object): return False def _save_item(self, item): - with open(os.path.join(self._save_directory, item.id), "w") as fh: - json.dump(item.json_dict, fh) + with open(os.path.join(self._save_directory, "{}.json".format(item.id)), "w") as fh: + # This is a temporary replacement for the old .json_dict property: + output = {k: item.__dict__[k] for k in item.__dict__ if not k.startswith("_")} + output["subreddit"] = output["subreddit"].title + output["author"] = output["author"].name + json.dump(output, fh) def _remove_submission(self, sub): self._logger.info("Deleting submission: #{id} {url}".format(id=sub.id, url=sub.url.encode("utf-8"))) From 5a49c99defeddc0f516107f7d6eb80f757e48e7f Mon Sep 17 00:00:00 2001 From: David Trail Date: Tue, 13 Dec 2016 21:56:38 +0000 Subject: [PATCH 45/62] Correct INFO output when skipped deletions --- shreddit/shredder.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 9b456c8..08c6d9f 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -141,26 +141,29 @@ class Shredder(object): self._logger.info("Loading items to delete...") to_delete = [item for item in items] self._logger.info("Done. Starting on batch of {} items...".format(len(to_delete))) - count = 0 + count, count_removed = 0, 0 for item in to_delete: count += 1 self._logger.debug("Examining item {}: {}".format(count, item)) created = arrow.get(item.created_utc) if str(item.subreddit).lower() in self._blacklist: self._logger.debug("Deleting due to blacklist") + count_removed += 1 self._remove(item) elif self._check_whitelist(item): self._logger.debug("Skipping due to: whitelisted") continue if created <= self._nuke_cutoff: self._logger.debug("Item occurs prior to nuke cutoff") + count_removed += 1 self._remove(item) elif created > self._recent_cutoff: self._logger.debug("Skipping due to: too recent") continue else: + count_removed += 1 self._remove(item) - return count + return count_removed def _build_iterator(self): item = self._r.user.me() From 705c076abcef7997e3fae80fff01bb505948bda2 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Dec 2016 17:01:29 -0600 Subject: [PATCH 46/62] Fixes reconnect regression --- setup.py | 2 +- shreddit/shredder.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 212480b..6c4b7ff 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "6.0.3" +VERSION = "6.0.4" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 08c6d9f..05c1065 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -24,7 +24,8 @@ class Shredder(object): self._logger.setLevel(level=logging.DEBUG if config.get("verbose", True) else logging.INFO) self.__dict__.update({"_{}".format(k): config[k] for k in config}) - self._connect(user) + self._user = user + self._connect() if self._save_directory: self._r.config.store_json_result = True @@ -71,12 +72,12 @@ class Shredder(object): # while and do it again. self._logger.info("Waiting {} seconds and continuing...".format(self._batch_cooldown)) time.sleep(self._batch_cooldown) - self._connect(None, self._username, self._password) + self._connect() self.shred() - def _connect(self, user): + def _connect(self): try: - self._r = praw.Reddit(user, user_agent="python:shreddit:v6.0.0") + self._r = praw.Reddit(self._user, user_agent="python:shreddit:v6.0.4") self._logger.info("Logged in as {user}.".format(user=self._r.user.me())) except ResponseException: raise ShredditError("Bad OAuth credentials") From 1d67dc7c76de7097cf446c4c54fbb31fa1bcca40 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Dec 2016 21:05:48 -0600 Subject: [PATCH 47/62] Fixes change to PRAW exception on vote clear --- setup.py | 2 +- shreddit/shredder.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6c4b7ff..cafe16a 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "6.0.4" +VERSION = "6.0.5" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 05c1065..3c0891e 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -9,7 +9,7 @@ import time import yaml from datetime import datetime, timedelta from praw.models import Comment, Submission -from prawcore.exceptions import ResponseException, OAuthException +from prawcore.exceptions import ResponseException, OAuthException, BadRequest from re import sub from shreddit.util import get_sentence, ShredditError @@ -129,7 +129,7 @@ class Shredder(object): if self._clear_vote: try: item.clear_vote() - except HTTPException: + except BadRequest: self._logger.debug("Couldn't clear vote on {item}".format(item=item)) if isinstance(item, Submission): self._remove_submission(item) From 1fd248e80a1e10eadaa8185bd947731064f03eea Mon Sep 17 00:00:00 2001 From: Scott Date: Sat, 7 Jan 2017 15:37:55 -0600 Subject: [PATCH 48/62] Removed nasty update message for minor version --- shreddit/shredder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 3c0891e..3b8fdaf 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -77,7 +77,7 @@ class Shredder(object): def _connect(self): try: - self._r = praw.Reddit(self._user, user_agent="python:shreddit:v6.0.4") + self._r = praw.Reddit(self._user, check_for_updates=False, user_agent="python:shreddit:v6.0.4") self._logger.info("Logged in as {user}.".format(user=self._r.user.me())) except ResponseException: raise ShredditError("Bad OAuth credentials") From ac8d91be208c469d2d22ebb17e1e8cc9671d1b14 Mon Sep 17 00:00:00 2001 From: crypticgeek Date: Wed, 11 Jan 2017 08:15:29 +0000 Subject: [PATCH 49/62] fix nuke_cutoff not being respected --- shreddit/shredder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 3c0891e..b4940af 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -151,14 +151,14 @@ class Shredder(object): self._logger.debug("Deleting due to blacklist") count_removed += 1 self._remove(item) - elif self._check_whitelist(item): - self._logger.debug("Skipping due to: whitelisted") - continue - if created <= self._nuke_cutoff: + elif created <= self._nuke_cutoff: self._logger.debug("Item occurs prior to nuke cutoff") count_removed += 1 self._remove(item) - elif created > self._recent_cutoff: + elif self._check_whitelist(item): + self._logger.debug("Skipping due to: whitelisted") + continue + elif created > self._recent_cutoff: self._logger.debug("Skipping due to: too recent") continue else: From dd69f2bc4e9b81c0c5b598776a979de26702ecee Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 11 Jan 2017 14:37:44 -0600 Subject: [PATCH 50/62] Bumping PRAW version up --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cf08091..56c70cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ arrow==0.9.0 decorator==4.0.10 -praw==4.0.0 +praw==4.2.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 From ec2e2f45c8eeeef053baeac0767d8834ed31f2c0 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 11 Jan 2017 14:37:55 -0600 Subject: [PATCH 51/62] Fixing tab inconsistency --- shreddit/shredder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 55e7e8b..bdbe68d 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -158,7 +158,7 @@ class Shredder(object): elif self._check_whitelist(item): self._logger.debug("Skipping due to: whitelisted") continue - elif created > self._recent_cutoff: + elif created > self._recent_cutoff: self._logger.debug("Skipping due to: too recent") continue else: From ec595b9a6a2702aed5a7e5a54b1e69263602fdd9 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 11 Jan 2017 14:38:56 -0600 Subject: [PATCH 52/62] Bumping version fix number for PyPI --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cafe16a..29f1467 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "6.0.5" +VERSION = "6.0.6" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) From 11d2babe5cb7ff7125c666e964bd47f2b357626c Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 9 Feb 2017 20:46:36 -0600 Subject: [PATCH 53/62] Migrates multireddit function to PRAWv4 version --- shreddit/shredder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index bdbe68d..32c524e 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -39,14 +39,14 @@ class Shredder(object): # Add any multireddit subreddits to the whitelist self._whitelist = set([s.lower() for s in self._whitelist]) for username, multiname in self._multi_whitelist: - multireddit = self._r.get_multireddit(username, multiname) + multireddit = self._r.multireddit(username, multiname) for subreddit in multireddit.subreddits: self._whitelist.add(str(subreddit).lower()) # Add any multireddit subreddits to the blacklist self._blacklist = set() for username, multiname in self._multi_blacklist: - multireddit = self._r.get_multireddit(username, multiname) + multireddit = self._r.multireddit(username, multiname) for subreddit in multireddit.subreddits: self._blacklist.add(str(subreddit).lower()) From 5562672acedbb9d45040793af031e970ceb2ee83 Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 9 Feb 2017 21:49:24 -0600 Subject: [PATCH 54/62] Bumps minor version for PyPI --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 29f1467..564372a 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "6.0.6" +VERSION = "6.0.7" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) From cdeaa1495d21cc141c3b10c13c41827358692404 Mon Sep 17 00:00:00 2001 From: manu Date: Thu, 4 May 2017 06:01:52 +0200 Subject: [PATCH 55/62] check for global shreddit.yml file in user config dir --- requirements.txt | 1 + shreddit/app.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 56c70cd..1dfc88e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ backports-abc==0.4 tornado==4.3 update-checker==0.15 wheel==0.24.0 +appdirs==1.4.3 diff --git a/shreddit/app.py b/shreddit/app.py index 82521da..d98b4c1 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -5,6 +5,7 @@ import yaml import logging import os import pkg_resources +from appdirs import user_config_dir from shreddit import default_config from shreddit.shredder import Shredder @@ -28,7 +29,15 @@ def main(): fout.write(pkg_resources.resource_string("shreddit", "praw.ini.example")) return - config_filename = args.config or "shreddit.yml" + config_dir = user_config_dir("shreddit/shreddit.yml") + + if args.config: + config_filename = args.config + elif os.path.exists(config_dir): + config_filename = config_dir + else: + config_filename = "shreddit.yml" + if not os.path.isfile(config_filename): print("No shreddit configuration file was found or provided. Run this script with -g to generate one.") return From e56cfbf6a812a946b254415522401b3d33c49415 Mon Sep 17 00:00:00 2001 From: manu Date: Wed, 10 May 2017 21:05:54 +0200 Subject: [PATCH 56/62] better save path --- shreddit/shredder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 32c524e..278a228 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -98,7 +98,11 @@ class Shredder(object): return False def _save_item(self, item): - with open(os.path.join(self._save_directory, "{}.json".format(item.id)), "w") as fh: + name = item.subreddit_name_prefixed[2:].lower() + path = "{}/{}/{}.json".format(item.author, name, item.id) + if not os.path.exists(os.path.join(self._save_directory, os.path.dirname(path))): + os.makedirs(os.path.join(self._save_directory, os.path.dirname(path))) + with open(os.path.join(self._save_directory, path), "w") as fh: # This is a temporary replacement for the old .json_dict property: output = {k: item.__dict__[k] for k in item.__dict__ if not k.startswith("_")} output["subreddit"] = output["subreddit"].title From 310e38a76c00d80f1b8e230d01a3416d0f4ca0a5 Mon Sep 17 00:00:00 2001 From: manu Date: Wed, 10 May 2017 21:31:23 +0200 Subject: [PATCH 57/62] no need to lowercase subreddits --- shreddit/shredder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 278a228..f778da3 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -98,7 +98,7 @@ class Shredder(object): return False def _save_item(self, item): - name = item.subreddit_name_prefixed[2:].lower() + name = item.subreddit_name_prefixed[2:] path = "{}/{}/{}.json".format(item.author, name, item.id) if not os.path.exists(os.path.join(self._save_directory, os.path.dirname(path))): os.makedirs(os.path.join(self._save_directory, os.path.dirname(path))) From d483de1dfde97c980bcc01d8a9ff40efc1e87a39 Mon Sep 17 00:00:00 2001 From: manu Date: Fri, 12 May 2017 09:45:32 +0200 Subject: [PATCH 58/62] catch keyboard interrupt --- shreddit/app.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shreddit/app.py b/shreddit/app.py index d98b4c1..abfed3c 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -55,4 +55,8 @@ def main(): if __name__ == "__main__": - main() + try: + main() + except KeyboardInterrupt: + print("Shreddit aborted by user") + quit() From 2dcd2de3a8c1478eab8d1ec0ec3488ed05e039e2 Mon Sep 17 00:00:00 2001 From: manu Date: Sat, 13 May 2017 07:33:07 +0200 Subject: [PATCH 59/62] prettify json --- shreddit/shredder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 32c524e..1a82b05 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -103,7 +103,7 @@ class Shredder(object): output = {k: item.__dict__[k] for k in item.__dict__ if not k.startswith("_")} output["subreddit"] = output["subreddit"].title output["author"] = output["author"].name - json.dump(output, fh) + json.dump(output, fh, indent=2) def _remove_submission(self, sub): self._logger.info("Deleting submission: #{id} {url}".format(id=sub.id, url=sub.url.encode("utf-8"))) From e550fa179c4983535ad8a30bdbbf83063fab1f12 Mon Sep 17 00:00:00 2001 From: David John Date: Fri, 19 May 2017 12:14:52 +0100 Subject: [PATCH 60/62] Add donation section --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index b53fc3c..6ccc0c3 100644 --- a/README.md +++ b/README.md @@ -139,3 +139,7 @@ optional arguments: - We are relying on Reddit admin words that they do not store edits, deleted posts are still stored in the database they are merely inaccessible to the public. +## Donate + +A few people have asked about donating so here's a Bitcoin address, any amount is appreciated, spread amongst recent +contributors and if there's enough interest a web service may be on the horizon! `1PbeYK7FonjVmgWxf4ieKmvwtomZR1K1Qu` From 8f0087e5c7a530f6904f93c743cdf5342ed86396 Mon Sep 17 00:00:00 2001 From: David John Date: Sun, 9 Jul 2017 16:12:50 +0100 Subject: [PATCH 61/62] Latest requirements, appdirs added --- requirements.txt | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1dfc88e..c970413 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,16 @@ -arrow==0.9.0 -decorator==4.0.10 -praw==4.2.0 +appdirs==1.4.3 +arrow==0.10.0 +backports-abc==0.5 +certifi==2017.4.17 +chardet==3.0.4 +idna==2.5 +praw==5.0.0 +prawcore==0.11.0 +python-dateutil==2.6.0 PyYAML==3.12 -requests==2.12.1 +requests==2.18.1 +shreddit==6.0.7 six==1.10.0 -backports-abc==0.4 -tornado==4.3 -update-checker==0.15 -wheel==0.24.0 -appdirs==1.4.3 +tornado==4.5.1 +update-checker==0.16 +urllib3==1.21.1 From 772df35c68a6782b2bd733801458023545977166 Mon Sep 17 00:00:00 2001 From: David John Date: Sun, 9 Jul 2017 16:21:45 +0100 Subject: [PATCH 62/62] Protect clear_vote when trial_run is in effect --- shreddit/shredder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 782edb1..c4633a0 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -130,11 +130,12 @@ class Shredder(object): def _remove(self, item): if self._keep_a_copy and self._save_directory: self._save_item(item) - if self._clear_vote: - try: - item.clear_vote() - except BadRequest: - self._logger.debug("Couldn't clear vote on {item}".format(item=item)) + if not self._trial_run: + if self._clear_vote: + try: + item.clear_vote() + except BadRequest: + self._logger.debug("Couldn't clear vote on {item}".format(item=item)) if isinstance(item, Submission): self._remove_submission(item) elif isinstance(item, Comment):