From 643937dcb6936eb1e7a8d0e2d0aba8bbcefa3110 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 00:29:49 -0600 Subject: [PATCH 01/20] Updates PRAW deps to V4 --- requirements.txt | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index b667be5..b76a730 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ arrow==0.9.0 decorator==4.0.10 -praw<4 +praw>=4.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 diff --git a/setup.py b/setup.py index c9af536..1a00f15 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from codecs import open from os import path -VERSION = "5.0.2" +VERSION = "6.0.0" DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so." here = path.abspath(path.dirname(__file__)) @@ -24,7 +24,7 @@ setup( "Intended Audience :: End Users/Desktop", "Programming Language :: Python"], packages=["shreddit"], - install_requires=["arrow", "backports-abc", "praw<4", "PyYAML", "requests", "six", "tornado"], + install_requires=["arrow", "backports-abc", "praw>=4", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["shreddit.yml.example"]}, entry_points={ "console_scripts": [ From 2b90ba8aea6267a4f9bb74f0f0ed11ab2dfbedcd Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 00:35:00 -0600 Subject: [PATCH 02/20] Removes outdated OAuth helper scripts --- get_secret.py | 68 ----------------------------------------------- shreddit/oauth.py | 22 --------------- 2 files changed, 90 deletions(-) delete mode 100755 get_secret.py delete mode 100644 shreddit/oauth.py diff --git a/get_secret.py b/get_secret.py deleted file mode 100755 index 7d340a4..0000000 --- a/get_secret.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -''' -So I heard you want to use OAuth2? This is a helper tool that gets the -authenticaton code for you and fires it into praw.ini. - -How to use: - - Visit: https://www.reddit.com/prefs/apps - - Create new "script", under "redirect uri" put http://127.0.0.1:65010 - - Open praw.ini - - oauth_client_id = { The ID displayed under the icon thingy } - - oauth_client_secret = { The secret } - - oauth_redirect_uri = http://127.0.0.1:65010 - - Run this script - - Your browser will open to a page on Reddit listing requested perms - - Click permit -''' - -import praw -import webbrowser -from warnings import warn -from praw.errors import HTTPException, OAuthAppRequired -from tornado import gen, web -from tornado.ioloop import IOLoop -from tornado.httpserver import HTTPServer - -r = praw.Reddit('Shreddit refresh token grabber') - - -class Page(web.RequestHandler): - def get(self): - code = self.get_argument("code", default=None, strip=False) - self.write("Success! Your code: %s
\ - It will now be appended to praw.ini and you \ - should be able to enjoy Shreddit without storing \ - your user / pass anywhere." % code) - IOLoop.current().stop() - self.login(code) - - def login(self, code): - deets = r.get_access_information(code) - print("oauth_refresh_token: %s" % deets['refresh_token']) - r.set_access_credentials(**deets) - with open('praw.ini', mode='a') as fh: - fh.write('oauth_refresh_token = %s' % deets['refresh_token']) - print("Refresh token written to praw.ini") - -application = web.Application([(r"/", Page)]) - -try: - r.refresh_access_information() -except HTTPException: - url = r.get_authorize_url('uniqueKey', ['identity', 'read', 'vote', 'edit', 'history'], True) - try: - print("Opening url: %s" % url) - webbrowser.open(url, new=2) - except NameError: - warn('''Couldn't open URL: %s\n please do so manually''' % url) - server = HTTPServer(application) - server.listen(65010) - IOLoop.current().start() - -if r.user == None: - print("Failed to log in. Something went wrong!") -else: - print("Logged in as %s." % r.user) - -print() - diff --git a/shreddit/oauth.py b/shreddit/oauth.py deleted file mode 100644 index 8276c43..0000000 --- a/shreddit/oauth.py +++ /dev/null @@ -1,22 +0,0 @@ -"""This module contains a function that tests OAuth session validity. -""" -import os -import praw - - -def oauth_test(praw_ini): - if praw_ini: - # PRAW won't panic if the file is invalid, so check first - if not os.path.exists(praw_ini): - print("PRAW configuration file \"{}\" not found.".format(praw_ini)) - return - praw.settings.CONFIG.read(praw_ini) - r = praw.Reddit("Shreddit oauth test") - try: - r.refresh_access_information() - if r.is_oauth_session(): - print("Session is valid.") - else: - print("Session is not a valid OAuth session.") - except Exception as e: - print("Error encountered while checking credentials:\n{}".format(e)) From 8171da78284f731e1d51f34adea770d934604ac9 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 00:35:29 -0600 Subject: [PATCH 03/20] Adds OAuth creds to config example --- shreddit.yml.example | 8 ++++---- shreddit/shreddit.yml.example | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shreddit.yml.example b/shreddit.yml.example index 3fe276b..0637304 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -1,8 +1,8 @@ --- -# Login details for Reddit. Fill out if you don't wish -# to be prompted for a login every time you run Shreddit. -# If your username or password has characters that could cause problems, surround them in quotes. -### NOTE: This may be deprecated as you can specify in praw.ini instead +# Credentials go here. If you are unsure what to put in client_id and client_secret, please consult the README for more +# information. +client_id: +client_secret: username: password: diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example index 3fe276b..0637304 100644 --- a/shreddit/shreddit.yml.example +++ b/shreddit/shreddit.yml.example @@ -1,8 +1,8 @@ --- -# Login details for Reddit. Fill out if you don't wish -# to be prompted for a login every time you run Shreddit. -# If your username or password has characters that could cause problems, surround them in quotes. -### NOTE: This may be deprecated as you can specify in praw.ini instead +# Credentials go here. If you are unsure what to put in client_id and client_secret, please consult the README for more +# information. +client_id: +client_secret: username: password: From c8c3dcc52dbce81118bc169b6267e703ceabdfc1 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 01:03:46 -0600 Subject: [PATCH 04/20] Moves credentials to praw.ini --- praw.ini.example | 20 +++++++------------- shreddit.yml.example | 7 ------- shreddit/shreddit.yml.example | 7 ------- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/praw.ini.example b/praw.ini.example index 3f608b5..47ee85e 100644 --- a/praw.ini.example +++ b/praw.ini.example @@ -1,13 +1,7 @@ -[DEFAULT] -# Will be deprecated sometime in 2015 (probably) -user = -pswd = - -## OAuth2 settings: -# Client / Secret from your own app -oauth_client_id = -oauth_client_secret = -# Corresponds with the callback URL in the Reddit app -oauth_redirect_uri = http://127.0.0.1:65010 -log_requests = 0 -# After running get_secret.py you should find your auth ID below +# Credentials go here. Fill out default, or provide one or more names and call shreddit with the -u option to specify +# which set to use. +[default] +client_id= +client_secret= +username= +password= diff --git a/shreddit.yml.example b/shreddit.yml.example index 0637304..ddb6a64 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -1,11 +1,4 @@ --- -# Credentials go here. If you are unsure what to put in client_id and client_secret, please consult the README for more -# information. -client_id: -client_secret: -username: -password: - # How many hours of comments you want to keep # 24 hours in a day, # 168 hours in a week, diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example index 0637304..ddb6a64 100644 --- a/shreddit/shreddit.yml.example +++ b/shreddit/shreddit.yml.example @@ -1,11 +1,4 @@ --- -# Credentials go here. If you are unsure what to put in client_id and client_secret, please consult the README for more -# information. -client_id: -client_secret: -username: -password: - # How many hours of comments you want to keep # 24 hours in a day, # 168 hours in a week, From 9fdfb9a32a5225b426f8fd98c82aa03db2354c67 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 02:56:33 -0600 Subject: [PATCH 05/20] Removes edit only from configs --- shreddit.yml.example | 4 ---- shreddit/shreddit.yml.example | 4 ---- 2 files changed, 8 deletions(-) diff --git a/shreddit.yml.example b/shreddit.yml.example index ddb6a64..f7271ea 100644 --- a/shreddit.yml.example +++ b/shreddit.yml.example @@ -45,10 +45,6 @@ multi_whitelist: [] # but the output from the program will be shown as an example trial_run: False -# Don't delete but *do* edit, could prove... interesting to see a comment -# with 5000 upvotes and it's just a lorem ipsum! -edit_only: False - # Ignore distinguished comments. whitelist_distinguished: True diff --git a/shreddit/shreddit.yml.example b/shreddit/shreddit.yml.example index ddb6a64..f7271ea 100644 --- a/shreddit/shreddit.yml.example +++ b/shreddit/shreddit.yml.example @@ -45,10 +45,6 @@ multi_whitelist: [] # but the output from the program will be shown as an example trial_run: False -# Don't delete but *do* edit, could prove... interesting to see a comment -# with 5000 upvotes and it's just a lorem ipsum! -edit_only: False - # Ignore distinguished comments. whitelist_distinguished: True From 83abfe9f8817603ac4950ead0827a4845f31e181 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 02:56:59 -0600 Subject: [PATCH 06/20] Makes prawcore dep explicit --- requirements.txt | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b76a730..6ba44e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ arrow==0.9.0 decorator==4.0.10 praw>=4.0 +prawcore PyYAML==3.12 requests==2.12.1 six==1.10.0 diff --git a/setup.py b/setup.py index 1a00f15..d11cfb4 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( "Intended Audience :: End Users/Desktop", "Programming Language :: Python"], packages=["shreddit"], - install_requires=["arrow", "backports-abc", "praw>=4", "PyYAML", "requests", "six", "tornado"], + install_requires=["arrow", "backports-abc", "praw>=4", "prawcore", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["shreddit.yml.example"]}, entry_points={ "console_scripts": [ From 0271b2e74123d89136ec754a0eadabe69cf2bb8b Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 6 Dec 2016 02:57:47 -0600 Subject: [PATCH 07/20] Updates core functionality to PRAW v4 --- shreddit/app.py | 11 +---- shreddit/shredder.py | 96 ++++++++++++++++---------------------------- shreddit/util.py | 8 ++++ 3 files changed, 45 insertions(+), 70 deletions(-) diff --git a/shreddit/app.py b/shreddit/app.py index b25c2d5..d16b0e8 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -6,21 +6,15 @@ import logging import os import pkg_resources from shreddit import default_config -from shreddit.oauth import oauth_test from shreddit.shredder import Shredder def main(): parser = argparse.ArgumentParser(description="Command-line frontend to the shreddit library.") parser.add_argument("-c", "--config", help="Config file to use instead of the default shreddit.yml") - parser.add_argument("-p", "--praw", help="PRAW config (if not ./praw.ini)") - parser.add_argument("-t", "--test-oauth", help="Perform OAuth test and exit", action="store_true") + parser.add_argument("-u", "--user", help="User section from praw.ini if not default", default="default") args = parser.parse_args() - if args.test_oauth: - oauth_test(args.praw) - return - config_filename = args.config or "shreddit.yml" if not os.path.isfile(config_filename): print("No configuration file could be found. Paste the following into a file called \"shreddit.yml\" and " \ @@ -36,8 +30,7 @@ def main(): if option in user_config: default_config[option] = user_config[option] - # TODO: Validate config options - shredder = Shredder(default_config, args.praw) + shredder = Shredder(default_config, args.user) shredder.shred() diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 80a9d23..8a9f208 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -1,32 +1,30 @@ -import os -import sys -import logging +import arrow import argparse import json -import arrow -import yaml +import logging +import os import praw +import sys import time -from re import sub +import yaml from datetime import datetime, timedelta -from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, - HTTPException, OAuthAppRequired) -from praw.objects import Comment, Submission -from shreddit.util import get_sentence +from praw.models import Comment, Submission +from prawcore.exceptions import ResponseException, OAuthException +from re import sub +from shreddit.util import get_sentence, ShredditError class Shredder(object): """This class stores state for configuration, API objects, logging, etc. It exposes a shred() method that application code can call to start it. """ - def __init__(self, config, praw_ini=None): + def __init__(self, config, user): logging.basicConfig() self._logger = logging.getLogger("shreddit") self._logger.setLevel(level=logging.DEBUG if config.get("verbose", True) else logging.INFO) self.__dict__.update({"_{}".format(k): config[k] for k in config}) - self._praw_ini = praw_ini - self._connect(praw_ini, self._username, self._password) + self._connect(user) if self._save_directory: self._r.config.store_json_result = True @@ -36,8 +34,6 @@ class Shredder(object): if self._save_directory: if not os.path.exists(self._save_directory): os.makedirs(self._save_directory) - self._limit = None - self._api_calls = [] # Add any multireddit subreddits to the whitelist self._whitelist = set([s.lower() for s in self._whitelist]) @@ -68,7 +64,7 @@ class Shredder(object): self._logger.info("Trial run - no deletion will be performed") def shred(self): - deleted = self._remove_things(self._get_things()) + deleted = self._remove_things(self._build_iterator()) self._logger.info("Finished deleting {} items. ".format(deleted)) if deleted >= 1000: # This user has more than 1000 items to handle, which angers the gods of the Reddit API. So chill for a @@ -78,33 +74,14 @@ class Shredder(object): self._connect(None, self._username, self._password) self.shred() - def _connect(self, praw_ini, username, password): - self._r = praw.Reddit(user_agent="shreddit/5.0") - if praw_ini: - # PRAW won't panic if the file is invalid, so check first - if not os.path.exists(praw_ini): - print("PRAW configuration file \"{}\" not found.".format(praw_ini)) - return - praw.settings.CONFIG.read(praw_ini) + def _connect(self, user): try: - # Try to login with OAuth2 - self._r.refresh_access_information() - self._logger.debug("Logged in with OAuth.") - except (HTTPException, OAuthAppRequired) as e: - self._logger.warning("You should migrate to OAuth2 using get_secret.py before Reddit disables this login " - "method.") - try: - try: - self._r.login(username, password) - except InvalidUserPass: - self._r.login() # Supply details on the command line - except InvalidUser as e: - raise InvalidUser("User does not exist.", e) - except InvalidUserPass as e: - raise InvalidUserPass("Specified an incorrect password.", e) - except RateLimitExceeded as e: - raise RateLimitExceeded("You're doing that too much.", e) - self._logger.info("Logged in as {user}.".format(user=self._r.user)) + self._r = praw.Reddit(user, user_agent="python:shreddit:v6.0.0") + self._logger.info("Logged in as {user}.".format(user=self._r.user.me())) + except ResponseException: + raise ShredditError("Bad OAuth credentials") + except OAuthException: + raise ShredditError("Bad username or password") def _check_whitelist(self, item): """Returns True if the item is whitelisted, False otherwise. @@ -137,13 +114,9 @@ class Shredder(object): short_text = sub(b"\n\r\t", " ", comment.body[:35].encode("utf-8")) msg = "/r/{}/ #{} ({}) with: {}".format(comment.subreddit, comment.id, short_text, replacement_text) - if self._edit_only: - self._logger.debug("Editing (not removing) {msg}".format(msg=msg)) - else: - self._logger.debug("Editing and deleting {msg}".format(msg=msg)) + self._logger.debug("Editing and deleting {msg}".format(msg=msg)) if not self._trial_run: comment.edit(replacement_text) - self._api_calls.append(int(time.time())) def _remove(self, item): if self._keep_a_copy and self._save_directory: @@ -151,27 +124,20 @@ class Shredder(object): if self._clear_vote: try: item.clear_vote() - self._api_calls.append(int(time.time())) except HTTPException: self._logger.debug("Couldn't clear vote on {item}".format(item=item)) if isinstance(item, Submission): self._remove_submission(item) elif isinstance(item, Comment): self._remove_comment(item) - if not self._edit_only and not self._trial_run: + if not self._trial_run: item.delete() - self._api_calls.append(int(time.time())) def _remove_things(self, items): self._logger.info("Loading items to delete...") to_delete = [item for item in items] self._logger.info("Done. Starting on batch of {} items...".format(len(to_delete))) for idx, item in enumerate(to_delete): - minute_ago = arrow.now().replace(minutes=-1).timestamp - self._api_calls = [api_call for api_call in self._api_calls if api_call >= minute_ago] - if len(self._api_calls) >= 60: - self._logger.info("Sleeping 10 seconds to wait out API cooldown...") - time.sleep(10) self._logger.debug("Examining item {}: {}".format(idx + 1, item)) created = arrow.get(item.created_utc) if str(item.subreddit).lower() in self._blacklist: @@ -190,12 +156,20 @@ class Shredder(object): self._remove(item) return idx + 1 - def _get_things(self): + def _build_iterator(self): + item = self._r.user.me() if self._item == "comments": - return self._r.user.get_comments(limit=self._limit, sort=self._sort) + item = item.comments elif self._item == "submitted": - return self._r.user.get_submitted(limit=self._limit, sort=self._sort) - elif self._item == "overview": - return self._r.user.get_overview(limit=self._limit, sort=self._sort) + item = item.submissions + + if self._sort == "new": + return item.new(limit=None) + elif self._sort == "top": + return item.top(limit=None) + elif self._sort == "hot": + return item.hot(limit=None) + elif self._sort == "controversial": + return item.controversial(limit=None) else: - raise Exception("Your deletion section is wrong") + raise ShredditError("Sorting \"{}\" not recognized.".format(self._sort)) diff --git a/shreddit/util.py b/shreddit/util.py index 9d8931f..dea428a 100644 --- a/shreddit/util.py +++ b/shreddit/util.py @@ -21,3 +21,11 @@ except ImportError: words = LOREM.split() random.shuffle(words) return " ".join(words) + + +class ShredditError(Exception): + def __init__(self, value=None): + self.value = value if value else "No information provided" + + def __str__(self): + return repr(self.value) From 20205b4126cbe3cc541df49d20be468f998e614b Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:19:56 -0600 Subject: [PATCH 08/20] Removes more outdated OAuth code --- helpers/oauth_check.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 helpers/oauth_check.py diff --git a/helpers/oauth_check.py b/helpers/oauth_check.py deleted file mode 100644 index 563d0c6..0000000 --- a/helpers/oauth_check.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -''' -Simple script to check if your oauth is working. -''' -import praw -import sys - -r = praw.Reddit('Shreddit oauth test') -try: - r.refresh_access_information() - if r.is_oauth_session(): - sys.exit(0) - else: - sys.exit(2) -except: - sys.exit(1) - From e35a2aad10d7496ead81877b45c0f435439c0ff6 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:20:42 -0600 Subject: [PATCH 09/20] Removes deployment configs and files until documented --- helpers/ansible-shreddit.yml | 18 ------------------ lambda_handler.py | 12 ------------ 2 files changed, 30 deletions(-) delete mode 100644 helpers/ansible-shreddit.yml delete mode 100644 lambda_handler.py diff --git a/helpers/ansible-shreddit.yml b/helpers/ansible-shreddit.yml deleted file mode 100644 index 76de711..0000000 --- a/helpers/ansible-shreddit.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: ro - vars: - user: napalm - install_dir: "/home/{{ user }}/Shreddit" - tasks: - - name: Update the Shreddit repository - git: repo=https://github.com/x89/Shreddit.git dest={{ install_dir }} - - name: Ensure shreddit.cfg is correct - copy: src=shreddit.cfg dest={{ install_dir }}/shreddit.cfg owner={{ user }} group={{ user }} mode=0600 - - name: Ensure praw.ini is correct - copy: src=praw.ini dest={{ install_dir }}/praw.ini owner={{ user }} group={{ user }} mode=0600 - - name: Run get_secret.py to check that we can login with oauth - shell: bash -c "source .venv/bin/activate && python get_secret.py" chdir={{ install_dir }} - - name: Ensure Shreddit cron job - cron: user={{ user }} name="Shreddit" minute="0" job="cd {{ install_dir }} && bash -c 'source .venv/bin/activate && python shreddit.py' 2>/dev/null" - - name: Run a test Shreddit run - shell: bash -c "source .venv/bin/activate && python shreddit.py" chdir={{ install_dir }} diff --git a/lambda_handler.py b/lambda_handler.py deleted file mode 100644 index e7e7d97..0000000 --- a/lambda_handler.py +++ /dev/null @@ -1,12 +0,0 @@ -"""This module contains the handler function called by AWS. -""" -from shreddit.shredder import shred -import yaml - - -def lambda_handler(event, context): - with open("shreddit.yml") as fh: - config = yaml.safe_load(fh) - if not config: - raise Exception("No config options passed!") - shred(config) From 74bba08955c79b3d7abbb324ed54559570561d97 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:28:28 -0600 Subject: [PATCH 10/20] Allows user to generate praw and shreddit configs --- setup.py | 2 +- shreddit/app.py | 20 +++++++++++++++++--- shreddit/praw.ini.example | 7 +++++++ 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 shreddit/praw.ini.example diff --git a/setup.py b/setup.py index d11cfb4..abd4453 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( "Programming Language :: Python"], packages=["shreddit"], install_requires=["arrow", "backports-abc", "praw>=4", "prawcore", "PyYAML", "requests", "six", "tornado"], - package_data={"shreddit": ["shreddit.yml.example"]}, + package_data={"shreddit": ["*.example"]}, entry_points={ "console_scripts": [ "shreddit=shreddit.app:main" diff --git a/shreddit/app.py b/shreddit/app.py index d16b0e8..1678805 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -12,14 +12,28 @@ from shreddit.shredder import Shredder def main(): parser = argparse.ArgumentParser(description="Command-line frontend to the shreddit library.") parser.add_argument("-c", "--config", help="Config file to use instead of the default shreddit.yml") + parser.add_argument("-g", "--generate-configs", help="Write shreddit and praw config files to current directory.", + action="store_true") parser.add_argument("-u", "--user", help="User section from praw.ini if not default", default="default") args = parser.parse_args() + if args.generate_configs: + if not os.path.isfile("shreddit.yml"): + print("Writing shreddit.yml file...") + with open("shreddit.yml", "w") as fout: + fout.write(pkg_resources.resource_string("shreddit", "shreddit.yml.example")) + if not os.path.isfile("praw.ini"): + print("Writing praw.ini file...") + with open("praw.ini", "w") as fout: + fout.write(pkg_resources.resource_string("shreddit", "praw.ini.example")) + return + config_filename = args.config or "shreddit.yml" if not os.path.isfile(config_filename): - print("No configuration file could be found. Paste the following into a file called \"shreddit.yml\" and " \ - "try running shreddit again:\n\n") - print(pkg_resources.resource_string("shreddit", "shreddit.yml.example")) + print("No shreddit configuration file was found or provided. Run this script with -g to generate one.") + return + if not os.path.isfile("praw.ini"): + print("No praw configuration file was found. Run this script with -g to generate one.") return with open(config_filename) as fh: diff --git a/shreddit/praw.ini.example b/shreddit/praw.ini.example new file mode 100644 index 0000000..47ee85e --- /dev/null +++ b/shreddit/praw.ini.example @@ -0,0 +1,7 @@ +# Credentials go here. Fill out default, or provide one or more names and call shreddit with the -u option to specify +# which set to use. +[default] +client_id= +client_secret= +username= +password= From b0c95d35161d8c394a16407d8d636ad670686dd9 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:50:27 -0600 Subject: [PATCH 11/20] Removes check for praw.ini to allow global config --- shreddit/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/shreddit/app.py b/shreddit/app.py index 1678805..3064b1a 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -32,9 +32,6 @@ def main(): if not os.path.isfile(config_filename): print("No shreddit configuration file was found or provided. Run this script with -g to generate one.") return - if not os.path.isfile("praw.ini"): - print("No praw configuration file was found. Run this script with -g to generate one.") - return with open(config_filename) as fh: # Not doing a simple update() here because it's preferable to only set attributes that are "whitelisted" as From a1e611553afc5d8ccb862b7fc5ba384e26ec6ea3 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:50:52 -0600 Subject: [PATCH 12/20] Fixes inaccurate count when no items are deleted --- shreddit/shredder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/shreddit/shredder.py b/shreddit/shredder.py index 8a9f208..e54d432 100644 --- a/shreddit/shredder.py +++ b/shreddit/shredder.py @@ -137,8 +137,10 @@ class Shredder(object): self._logger.info("Loading items to delete...") to_delete = [item for item in items] self._logger.info("Done. Starting on batch of {} items...".format(len(to_delete))) - for idx, item in enumerate(to_delete): - self._logger.debug("Examining item {}: {}".format(idx + 1, item)) + count = 0 + for item in to_delete: + count += 1 + self._logger.debug("Examining item {}: {}".format(count, item)) created = arrow.get(item.created_utc) if str(item.subreddit).lower() in self._blacklist: self._logger.debug("Deleting due to blacklist") @@ -154,7 +156,7 @@ class Shredder(object): continue else: self._remove(item) - return idx + 1 + return count def _build_iterator(self): item = self._r.user.me() From ddf33498cb97d4ac2e89516e79ebdefbf86c2b76 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 01:54:22 -0600 Subject: [PATCH 13/20] Updates README for PRAW 4 operation --- README.md | 87 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 74dce83..89e9ee7 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,11 @@ prior to deletion. In fact you can actually turn off deletion all together and j about Shreddit) but this will increase how long it takes the script to run as it will be going over all of your messages every run. -## User Login deprecation +## Important New Changes (as of Dec 2016) -Reddit intends to disable username-password based authentication to access its APIs in the near future. You can specify -your username and password in the `shreddit.yml` or the `praw.ini` to make it work **FOR NOW**. But consider looking at -the [OAuth2 instructions](#oauth2-instructions) if you intend to use this program in the future. +Due to deprecation of the PRAW 3.x library, Shreddit is using PRAW 4. This requires that OAuth be used to authenticate. +Thankfully, however, it is much easier than in previous versions. If you are upgrading, [please review the usage section +to ensure that you have set up credentials correctly.](#configuring-credentials) ## Pip Installation @@ -31,10 +31,52 @@ installation. ## Usage -After installing the `shreddit` command line utility, the first step is setting up the tool's configuration file. Simply -typing `shreddit` will print a message with an example config. Copy the message from `---` onwards and save it as -`shreddit.yml`. Now, the tool may be used by simply typing `shreddit` from this directory. Alternatively, if you named -the configuration file something different such as `config.yml`, you may use it with `shreddit -c config.yml`. +After installing the `shreddit` command line utility, the first step is setting up the tool's configuration files. +Simply typing `shreddit -g` will generate configs. After configuring credentials, running the tool with the `shreddit` +command will begin the tool's operation. + +### Configuring Credentials + +Running `shreddit -g` will generate a blank praw.ini file that looks like this: + +``` +# Credentials go here. Fill out default, or provide one or more names and call shreddit with the -u option to specify +# which set to use. +[default] +client_id= +client_secret= +username= +password= +``` + +Username and password are simply your Reddit login credentials for the account that will be used. However, to obtain the +client ID and secret, follow these steps (taken from +[PRAW documentation](http://praw.readthedocs.io/en/latest/getting_started/authentication.html#script-application): + +1. Open your Reddit application preferences by clicking [here](https://www.reddit.com/prefs/apps/) +2. Add a new application. It doesn't matter what it's named, but calling it "shreddit" makes it easier to remember +3. Select "script" +4. Redirect URL does not matter, so enter something like http://127.0.0.1:8080 +5. Once created, you should see the name of your application followed by 14 character string. Enter this 14 character + string as your `client_id`. +6. Copy the 27 character "secret" string into the `client_secret` field. + +Finally, your praw.ini should look like this (with fake data provided here): + +``` +[default] +client_id=f3FaKeD4t40PsJ +client_secret=dfK3pfMoReFAkEDaTa123456789 +username=testuser +password=123passwordgoeshere123 +``` + +Keep your praw.ini either in the current directory when running `shreddit`, or in one of the config folders +[described here](http://praw.readthedocs.io/en/latest/getting_started/configuration/prawini.html) such as +`~/.config` in Linux or `%APPDATA%` in Windows. + +To use more than one account, you can add multiple profiles instead of just `[default]` and use the `-u` option to +`shreddit` to choose which one each time. ### Automating @@ -43,6 +85,9 @@ user's crontab settings. **Examples:** +The following examples require that the PRAW configuration file is located in the config directory. See [this PRAW +documentation](http://praw.readthedocs.io/en/latest/getting_started/configuration/prawini.html) for more information. + - Run every hour on the hour `0 * * * * shreddit -c ` @@ -60,7 +105,7 @@ If virtualenv was used, be sure to add `source /full/path/to/venv/bin/activate & ``` $ shreddit --help -usage: shreddit [-h] [-c CONFIG] [-p PRAW] [-t] +usage: app.py [-h] [-c CONFIG] [-g] [-u USER] Command-line frontend to the shreddit library. @@ -68,8 +113,10 @@ optional arguments: -h, --help show this help message and exit -c CONFIG, --config CONFIG Config file to use instead of the default shreddit.yml - -p PRAW, --praw PRAW PRAW config (if not ./praw.ini) - -t, --test-oauth Perform OAuth test and exit + -g, --generate-configs + Write shreddit and praw config files to current + directory. + -u USER, --user USER User section from praw.ini if not default ``` ## For Windows users @@ -81,23 +128,6 @@ optional arguments: 3. Open a new command prompt and verify that the `shreddit` command works before moving on to the [usage](#usage) section. -## OAuth2 Instructions - -1. Visit: https://www.reddit.com/prefs/apps -2. Click on 'Create app'. - - Fill in the name and select the 'script' option - - Under "redirect uri" put http://127.0.0.1:65010 -3. Copy from or rename `praw.ini.example` to `praw.ini` and open it. Enter the values from the Reddit page. - - oauth\_client\_id = { The ID displayed next to the icon thingy (under - "personal use script") } - - oauth\_client\_secret = { The secret } - - oauth\_redirect\_uri = http://127.0.0.1:65010 - - Save the file. -4. Run `python get_secret.py` in the command prompt. -5. Your browser will open to a page on Reddit listing requested permissions. -6. Click 'Allow'. - - ## Caveats - Certain limitations in the Reddit API and the PRAW library make it difficult to delete more than 1,000 comments. @@ -105,3 +135,4 @@ optional arguments: - We are relying on Reddit admin words that they do not store edits, deleted posts are still stored in the database they are merely inaccessible to the public. + From bdc83a44828d01592a6dd6bdefb45b85c0ef4494 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 02:00:10 -0600 Subject: [PATCH 14/20] Updates license information to be more standardized --- LICENCE => LICENSE | 0 setup.py | 2 ++ 2 files changed, 2 insertions(+) rename LICENCE => LICENSE (100%) diff --git a/LICENCE b/LICENSE similarity index 100% rename from LICENCE rename to LICENSE diff --git a/setup.py b/setup.py index abd4453..24c7d71 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,9 @@ setup( author_email="david@vaunt.eu", classifiers=["Development Status :: 4 - Beta", "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: BSD License", "Programming Language :: Python"], + license="FreeBSD License", packages=["shreddit"], install_requires=["arrow", "backports-abc", "praw>=4", "prawcore", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["*.example"]}, From d2ba5a7e17103ce0fef597e85fb524bc49e2fbf2 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 7 Dec 2016 02:02:43 -0600 Subject: [PATCH 15/20] Tweaks README for style and typo corrections --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 89e9ee7..290c795 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,11 @@ password= Username and password are simply your Reddit login credentials for the account that will be used. However, to obtain the client ID and secret, follow these steps (taken from -[PRAW documentation](http://praw.readthedocs.io/en/latest/getting_started/authentication.html#script-application): +[PRAW documentation](http://praw.readthedocs.io/en/latest/getting_started/authentication.html#script-application)): -1. Open your Reddit application preferences by clicking [here](https://www.reddit.com/prefs/apps/) -2. Add a new application. It doesn't matter what it's named, but calling it "shreddit" makes it easier to remember -3. Select "script" +1. Open your Reddit application preferences by clicking [here](https://www.reddit.com/prefs/apps/). +2. Add a new application. It doesn't matter what it's named, but calling it "shreddit" makes it easier to remember. +3. Select "script". 4. Redirect URL does not matter, so enter something like http://127.0.0.1:8080 5. Once created, you should see the name of your application followed by 14 character string. Enter this 14 character string as your `client_id`. From 87742563df686bc915b6e980b94dc28b2a1aaed8 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 12 Dec 2016 12:09:20 -0600 Subject: [PATCH 16/20] Locks down a few more requirement numbers --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6ba44e3..16f7344 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ arrow==0.9.0 decorator==4.0.10 -praw>=4.0 -prawcore +praw==4.0.0 +prawcore==0.4.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 From f1fa1b638d736c7de7e7dccba62d9f9b90be0a43 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 12 Dec 2016 12:21:46 -0600 Subject: [PATCH 17/20] Freezes prawcore to 0.3.0 because praw 4.0.0 requires it --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 16f7344..c164c9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ arrow==0.9.0 decorator==4.0.10 praw==4.0.0 -prawcore==0.4.0 +prawcore==0.3.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 From 284be846784fc228a832614eb2db308f85e24ecb Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 12 Dec 2016 12:26:39 -0600 Subject: [PATCH 18/20] Removes prawcore as a requirement, lets praw manage it --- requirements.txt | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c164c9d..cf08091 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ arrow==0.9.0 decorator==4.0.10 praw==4.0.0 -prawcore==0.3.0 PyYAML==3.12 requests==2.12.1 six==1.10.0 diff --git a/setup.py b/setup.py index 24c7d71..47eed11 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( "Programming Language :: Python"], license="FreeBSD License", packages=["shreddit"], - install_requires=["arrow", "backports-abc", "praw>=4", "prawcore", "PyYAML", "requests", "six", "tornado"], + install_requires=["arrow", "backports-abc", "praw>=4", "PyYAML", "requests", "six", "tornado"], package_data={"shreddit": ["*.example"]}, entry_points={ "console_scripts": [ From 2b6a6ff51e38036fe662319dd8332c9dfe812bbf Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Dec 2016 11:07:36 -0600 Subject: [PATCH 19/20] Handles pkg_resources byte strings in Python 3 correctly --- shreddit/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shreddit/app.py b/shreddit/app.py index 3064b1a..82521da 100644 --- a/shreddit/app.py +++ b/shreddit/app.py @@ -20,11 +20,11 @@ def main(): if args.generate_configs: if not os.path.isfile("shreddit.yml"): print("Writing shreddit.yml file...") - with open("shreddit.yml", "w") as fout: + with open("shreddit.yml", "wb") as fout: fout.write(pkg_resources.resource_string("shreddit", "shreddit.yml.example")) if not os.path.isfile("praw.ini"): print("Writing praw.ini file...") - with open("praw.ini", "w") as fout: + with open("praw.ini", "wb") as fout: fout.write(pkg_resources.resource_string("shreddit", "praw.ini.example")) return From c40e56e6b1a96ae181228669d9892f9ff1960663 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Dec 2016 12:17:28 -0600 Subject: [PATCH 20/20] Makes README more clear about filling out credentials. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 290c795..b53fc3c 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ username= password= ``` +**You must provide values for each of these.** As strange as it may seem to provide both a username/password pair *and* +a client id/secret pair, that is how the Reddit API does "OAuth" script applications. + Username and password are simply your Reddit login credentials for the account that will be used. However, to obtain the client ID and secret, follow these steps (taken from [PRAW documentation](http://praw.readthedocs.io/en/latest/getting_started/authentication.html#script-application)): @@ -56,7 +59,7 @@ client ID and secret, follow these steps (taken from 1. Open your Reddit application preferences by clicking [here](https://www.reddit.com/prefs/apps/). 2. Add a new application. It doesn't matter what it's named, but calling it "shreddit" makes it easier to remember. 3. Select "script". -4. Redirect URL does not matter, so enter something like http://127.0.0.1:8080 +4. Redirect URL does not matter for script applications, so enter something like http://127.0.0.1:8080 5. Once created, you should see the name of your application followed by 14 character string. Enter this 14 character string as your `client_id`. 6. Copy the 27 character "secret" string into the `client_secret` field.