mirror of https://github.com/x89/Shreddit
David John
8 years ago
commit
2e31d78f56
@ -1,13 +1,61 @@
|
|||||||
.venv
|
|
||||||
pip-selfcheck.json
|
|
||||||
.errors
|
|
||||||
2015*.txt
|
|
||||||
2016*.txt
|
|
||||||
2017*.txt
|
|
||||||
*.cfg
|
|
||||||
*.pyc
|
|
||||||
shreddit.conf
|
|
||||||
shreddit.yml
|
shreddit.yml
|
||||||
praw.ini
|
praw.ini
|
||||||
|
|
||||||
|
# Docs
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
__pycache__/
|
__pycache__/
|
||||||
.*.swp
|
*.py[cod]
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
env/
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.coverage
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Can be unstaged
|
||||||
|
.errors
|
||||||
|
pip-selfcheck.json
|
||||||
|
|||||||
@ -1,11 +0,0 @@
|
|||||||
#!/usr/bin/env sh
|
|
||||||
|
|
||||||
virtualenv .
|
|
||||||
source ./bin/activate
|
|
||||||
pip install -r requirements.txt
|
|
||||||
|
|
||||||
if [ ! -f "shreddit.yml" ]; then
|
|
||||||
cp "shreddit.yml.example" "shreddit.yml"
|
|
||||||
$EDITOR shreddit.yml
|
|
||||||
fi
|
|
||||||
|
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
"""This module contains the handler function called by AWS.
|
||||||
|
"""
|
||||||
|
from shreddit.shredder import shred
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
def lambda_handler(event, context):
|
||||||
|
with open("shreddit.yml") as fh:
|
||||||
|
config = yaml.safe_load(fh)
|
||||||
|
if not config:
|
||||||
|
raise Exception("No config options passed!")
|
||||||
|
shred(config)
|
||||||
@ -1,9 +1,10 @@
|
|||||||
|
arrow
|
||||||
|
decorator
|
||||||
|
praw==3.6.0
|
||||||
|
PyYAML
|
||||||
|
requests
|
||||||
|
six
|
||||||
backports-abc==0.4
|
backports-abc==0.4
|
||||||
decorator==4.0.6
|
|
||||||
praw==3.5.0
|
|
||||||
PyYAML==3.11
|
|
||||||
requests==2.8.1
|
|
||||||
six==1.10.0
|
|
||||||
tornado==4.3
|
tornado==4.3
|
||||||
update-checker==0.11
|
update-checker==0.11
|
||||||
wheel==0.24.0
|
wheel==0.24.0
|
||||||
|
|||||||
@ -1,8 +0,0 @@
|
|||||||
#!/usr/bin/env sh
|
|
||||||
if [[ -f './bin/activate' ]]; then
|
|
||||||
source ./bin/activate
|
|
||||||
elif [[ -f '.venv/bin/activate' ]]; then
|
|
||||||
source '.venv/bin/activate'
|
|
||||||
fi
|
|
||||||
pip install --upgrade praw
|
|
||||||
python ./shreddit.py
|
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
"""Setup script for shreddit.
|
||||||
|
"""
|
||||||
|
from setuptools import setup
|
||||||
|
from codecs import open
|
||||||
|
from os import path
|
||||||
|
|
||||||
|
VERSION = "2.0.0"
|
||||||
|
DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so."
|
||||||
|
|
||||||
|
here = path.abspath(path.dirname(__file__))
|
||||||
|
|
||||||
|
with open(path.join(here, "README.md"), encoding='utf-8') as filein:
|
||||||
|
long_description = filein.read()
|
||||||
|
|
||||||
|
with open(path.join(here, "requirements.txt"), encoding="utf-8") as filein:
|
||||||
|
requirements = [line.strip() for line in filein.readlines()]
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name="shreddit",
|
||||||
|
version=VERSION,
|
||||||
|
description=DESCRIPTION,
|
||||||
|
long_description=long_description,
|
||||||
|
url="https://github.com/scott-hand/Shreddit",
|
||||||
|
author="Scott Hand",
|
||||||
|
author_email="scott@vkgfx.com",
|
||||||
|
classifiers=["Development Status :: 4 - Beta",
|
||||||
|
"Intended Audience :: End Users/Desktop",
|
||||||
|
"Programming Language :: Python"],
|
||||||
|
packages=["shreddit"],
|
||||||
|
install_requires=["arrow", "backports-abc", "decorator", "praw", "PyYAML",
|
||||||
|
"requests", "six", "tornado", "update-checker", "wheel"],
|
||||||
|
entry_points={
|
||||||
|
"console_scripts": [
|
||||||
|
"shreddit=shreddit.app:main"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
@ -1,195 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import logging
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import yaml
|
|
||||||
import praw
|
|
||||||
import random
|
|
||||||
|
|
||||||
from re import sub
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded,
|
|
||||||
HTTPException, OAuthAppRequired)
|
|
||||||
from praw.objects import Comment, Submission
|
|
||||||
|
|
||||||
logging.basicConfig(stream=sys.stdout)
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
log.setLevel(level=logging.DEBUG)
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument(
|
|
||||||
'-c',
|
|
||||||
'--config',
|
|
||||||
help="config file to use instead of the default shreddit.cfg"
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.config:
|
|
||||||
config_file = args.config
|
|
||||||
else:
|
|
||||||
config_file = 'shreddit.yml'
|
|
||||||
|
|
||||||
with open(config_file, 'r') as fh:
|
|
||||||
config = yaml.safe_load(fh)
|
|
||||||
if config is None:
|
|
||||||
raise Exception("No config options passed!")
|
|
||||||
|
|
||||||
save_directory = config.get('save_directory', '.')
|
|
||||||
|
|
||||||
r = praw.Reddit(user_agent="shreddit/4.3")
|
|
||||||
if save_directory:
|
|
||||||
r.config.store_json_result = True
|
|
||||||
|
|
||||||
if config.get('verbose', True):
|
|
||||||
log_level = config.get('debug', 'DEBUG')
|
|
||||||
log.setLevel(level=getattr(logging, log_level))
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Try to login with OAuth2
|
|
||||||
r.refresh_access_information()
|
|
||||||
log.debug("Logged in with OAuth.")
|
|
||||||
except (HTTPException, OAuthAppRequired) as e:
|
|
||||||
log.warning('''You should migrate to OAuth2 using get_secret.py before
|
|
||||||
Reddit disables this login method.''')
|
|
||||||
try:
|
|
||||||
try:
|
|
||||||
r.login(config['username'], config['password'])
|
|
||||||
except InvalidUserPass:
|
|
||||||
r.login() # Supply details on the command line
|
|
||||||
except InvalidUser as e:
|
|
||||||
raise InvalidUser("User does not exist.", e)
|
|
||||||
except InvalidUserPass as e:
|
|
||||||
raise InvalidUserPass("Specified an incorrect password.", e)
|
|
||||||
except RateLimitExceeded as e:
|
|
||||||
raise RateLimitExceeded("You're doing that too much.", e)
|
|
||||||
|
|
||||||
log.info("Logged in as {user}.".format(user=r.user))
|
|
||||||
log.debug("Deleting messages before {time}.".format(
|
|
||||||
time=datetime.now() - timedelta(hours=config['hours'])))
|
|
||||||
|
|
||||||
whitelist = config.get('whitelist', [])
|
|
||||||
whitelist_ids = config.get('whitelist_ids', [])
|
|
||||||
|
|
||||||
if whitelist:
|
|
||||||
whitelist = set([subr.lower() for subr in whitelist])
|
|
||||||
log.debug("Keeping messages from subreddits {subs}".format(
|
|
||||||
subs=', '.join(whitelist))
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_sentence():
|
|
||||||
return '''I have been Shreddited for privacy!'''
|
|
||||||
try:
|
|
||||||
# Provide a method that works on windows
|
|
||||||
from loremipsum import get_sentence
|
|
||||||
except ImportError:
|
|
||||||
# Module unavailable, use the default phrase
|
|
||||||
pass
|
|
||||||
|
|
||||||
if config.get('replacement_format') == 'random':
|
|
||||||
wordlist = config.get('wordlist')
|
|
||||||
if not wordlist:
|
|
||||||
os_wordlist = '/usr/share/dict/words'
|
|
||||||
if os.name == 'posix' and os.path.isfile(os_wordlist):
|
|
||||||
# Generate a random string of words from our system's dictionary
|
|
||||||
with open(os_wordlist) as fh:
|
|
||||||
wordlist = fh.read().splitlines()
|
|
||||||
if wordlist:
|
|
||||||
def get_sentence():
|
|
||||||
return ' '.join(random.sample(wordlist, min(len(wordlist),
|
|
||||||
random.randint(50,75))))
|
|
||||||
|
|
||||||
|
|
||||||
def get_things(after=None):
|
|
||||||
limit = None
|
|
||||||
item = config.get('item', 'comments')
|
|
||||||
sort = config.get('sort', 'new')
|
|
||||||
log.debug("Deleting items: {item}".format(item=item))
|
|
||||||
if item == "comments":
|
|
||||||
return r.user.get_comments(limit=limit, sort=sort)
|
|
||||||
elif item == "submitted":
|
|
||||||
return r.user.get_submitted(limit=limit, sort=sort)
|
|
||||||
elif item == "overview":
|
|
||||||
return r.user.get_overview(limit=limit, sort=sort)
|
|
||||||
else:
|
|
||||||
raise Exception("Your deletion section is wrong")
|
|
||||||
|
|
||||||
|
|
||||||
def remove_things(things):
|
|
||||||
for thing in things:
|
|
||||||
log.debug('Starting remove function on: {thing}'.format(thing=thing))
|
|
||||||
# Seems to be in users's timezone. Unclear.
|
|
||||||
thing_time = datetime.fromtimestamp(thing.created_utc)
|
|
||||||
# Exclude items from being deleted unless past X hours.
|
|
||||||
after_time = datetime.now() - timedelta(hours=config.get('hours', 24))
|
|
||||||
if thing_time > after_time:
|
|
||||||
if thing_time + timedelta(hours=config.get('nuke_hours', 4320)) < datetime.utcnow():
|
|
||||||
pass
|
|
||||||
continue
|
|
||||||
# For edit_only we're assuming that the hours aren't altered.
|
|
||||||
# This saves time when deleting (you don't edit already edited posts).
|
|
||||||
if config.get('edit_only'):
|
|
||||||
end_time = after_time - timedelta(hours=config.get('hours', 24))
|
|
||||||
if thing_time < end_time:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if str(thing.subreddit).lower() in whitelist \
|
|
||||||
or thing.id in config.get('whitelist_ids', []):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if config.get('whitelist_distinguished') and thing.distinguished:
|
|
||||||
continue
|
|
||||||
if config.get('whitelist_gilded') and thing.gilded:
|
|
||||||
continue
|
|
||||||
if 'max_score' in config and thing.score > config['max_score']:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if config.get('save_directory'):
|
|
||||||
save_directory = config['save_directory']
|
|
||||||
if not os.path.exists(save_directory):
|
|
||||||
os.makedirs(save_directory)
|
|
||||||
with open("%s/%s.json" % (save_directory, thing.id), "w") as fh:
|
|
||||||
json.dump(thing.json_dict, fh)
|
|
||||||
|
|
||||||
if config.get('trial_run'): # Don't do anything, trial mode!
|
|
||||||
log.debug("Would have deleted {thing}: '{content}'".format(
|
|
||||||
thing=thing.id, content=thing))
|
|
||||||
continue
|
|
||||||
|
|
||||||
if config.get('clear_vote'):
|
|
||||||
thing.clear_vote()
|
|
||||||
|
|
||||||
if isinstance(thing, Submission):
|
|
||||||
log.info('Deleting submission: #{id} {url}'.format(
|
|
||||||
id=thing.id,
|
|
||||||
url=thing.url.encode('utf-8'))
|
|
||||||
)
|
|
||||||
elif isinstance(thing, Comment):
|
|
||||||
rep_format = config.get('replacement_format')
|
|
||||||
if rep_format == 'random':
|
|
||||||
replacement_text = get_sentence()
|
|
||||||
elif rep_format == 'dot':
|
|
||||||
replacement_text = '.'
|
|
||||||
else:
|
|
||||||
replacement_text = rep_format
|
|
||||||
|
|
||||||
msg = '/r/{3}/ #{0} with:\n\t"{1}" to\n\t"{2}"'.format(
|
|
||||||
thing.id,
|
|
||||||
sub(b'\n\r\t', ' ', thing.body[:78].encode('utf-8')),
|
|
||||||
replacement_text[:78],
|
|
||||||
thing.subreddit
|
|
||||||
)
|
|
||||||
|
|
||||||
if config.get('edit_only'):
|
|
||||||
log.info('Editing (not removing) {msg}'.format(msg=msg))
|
|
||||||
else:
|
|
||||||
log.info('Editing and deleting {msg}'.format(msg=msg))
|
|
||||||
|
|
||||||
thing.edit(replacement_text)
|
|
||||||
if not config.get('edit_only'):
|
|
||||||
thing.delete()
|
|
||||||
|
|
||||||
remove_things(get_things())
|
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
default_config = {"username": None,
|
||||||
|
"password": None,
|
||||||
|
"verbose": True,
|
||||||
|
"save_directory": "/tmp",
|
||||||
|
"whitelist": [],
|
||||||
|
"whitelist_ids": [],
|
||||||
|
"item": "overview",
|
||||||
|
"sort": "new",
|
||||||
|
"whitelist_distinguished": True,
|
||||||
|
"whitelist_gilded": True,
|
||||||
|
"max_score": 100,
|
||||||
|
"hours": 24,
|
||||||
|
"nuke_hours": 4320,
|
||||||
|
"keep_a_copy": False,
|
||||||
|
"save_directory": None,
|
||||||
|
"trial_run": False,
|
||||||
|
"clear_vote": False,
|
||||||
|
"replacement_format": "random",
|
||||||
|
"edit_only": False,
|
||||||
|
"batch_cooldown": 10}
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
"""This module contains script entrypoints for shreddit.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import yaml
|
||||||
|
import logging
|
||||||
|
from shreddit import default_config
|
||||||
|
from shreddit.oauth import oauth_test
|
||||||
|
from shreddit.shredder import Shredder
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Command-line frontend to the shreddit library.")
|
||||||
|
parser.add_argument("-c", "--config", help="Config file to use instead of the default shreddit.yml")
|
||||||
|
parser.add_argument("-p", "--praw", help="PRAW config (if not ./praw.ini)")
|
||||||
|
parser.add_argument("-t", "--test-oauth", help="Perform OAuth test and exit", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.test_oauth:
|
||||||
|
oauth_test(args.praw)
|
||||||
|
return
|
||||||
|
|
||||||
|
with open(args.config or "shreddit.yml") as fh:
|
||||||
|
# Not doing a simple update() here because it's preferable to only set attributes that are "whitelisted" as
|
||||||
|
# configuration options in the form of default values.
|
||||||
|
user_config = yaml.safe_load(fh)
|
||||||
|
for option in default_config:
|
||||||
|
if option in user_config:
|
||||||
|
default_config[option] = user_config[option]
|
||||||
|
|
||||||
|
# TODO: Validate config options
|
||||||
|
shredder = Shredder(default_config, args.praw)
|
||||||
|
shredder.shred()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -0,0 +1,22 @@
|
|||||||
|
"""This module contains a function that tests OAuth session validity.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import praw
|
||||||
|
|
||||||
|
|
||||||
|
def oauth_test(praw_ini):
|
||||||
|
if praw_ini:
|
||||||
|
# PRAW won't panic if the file is invalid, so check first
|
||||||
|
if not os.path.exists(praw_ini):
|
||||||
|
print("PRAW configuration file \"{}\" not found.".format(praw_ini))
|
||||||
|
return
|
||||||
|
praw.settings.CONFIG.read(praw_ini)
|
||||||
|
r = praw.Reddit("Shreddit oauth test")
|
||||||
|
try:
|
||||||
|
r.refresh_access_information()
|
||||||
|
if r.is_oauth_session():
|
||||||
|
print("Session is valid.")
|
||||||
|
else:
|
||||||
|
print("Session is not a valid OAuth session.")
|
||||||
|
except Exception as e:
|
||||||
|
print("Error encountered while checking credentials:\n{}".format(e))
|
||||||
@ -0,0 +1,177 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import logging
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import arrow
|
||||||
|
import yaml
|
||||||
|
import praw
|
||||||
|
import time
|
||||||
|
from re import sub
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, HTTPException, OAuthAppRequired)
|
||||||
|
from praw.objects import Comment, Submission
|
||||||
|
from shreddit.util import get_sentence
|
||||||
|
|
||||||
|
|
||||||
|
class Shredder(object):
|
||||||
|
"""This class stores state for configuration, API objects, logging, etc. It exposes a shred() method that
|
||||||
|
application code can call to start it.
|
||||||
|
"""
|
||||||
|
def __init__(self, config, praw_ini=None):
|
||||||
|
logging.basicConfig()
|
||||||
|
self._logger = logging.getLogger("shreddit")
|
||||||
|
self._logger.setLevel(level=logging.DEBUG if config.get("verbose", True) else logging.INFO)
|
||||||
|
self.__dict__.update({"_{}".format(k): config[k] for k in config})
|
||||||
|
|
||||||
|
self._praw_ini = praw_ini
|
||||||
|
self._connect(praw_ini, self._username, self._password)
|
||||||
|
|
||||||
|
if self._save_directory:
|
||||||
|
self._r.config.store_json_result = True
|
||||||
|
|
||||||
|
self._recent_cutoff = arrow.now().replace(hours=-self._hours)
|
||||||
|
self._nuke_cutoff = arrow.now().replace(hours=-self._nuke_hours)
|
||||||
|
if self._save_directory:
|
||||||
|
if not os.path.exists(self._save_directory):
|
||||||
|
os.makedirs(self._save_directory)
|
||||||
|
self._limit = None
|
||||||
|
self._api_calls = []
|
||||||
|
|
||||||
|
self._logger.info("Deleting ALL items before {}".format(self._nuke_cutoff))
|
||||||
|
self._logger.info("Deleting items not whitelisted until {}".format(self._recent_cutoff))
|
||||||
|
self._logger.info("Ignoring ALL items after {}".format(self._recent_cutoff))
|
||||||
|
self._logger.info("Targeting {} sorted by {}".format(self._item, self._sort))
|
||||||
|
if self._whitelist:
|
||||||
|
self._logger.info("Keeping items from subreddits {}".format(", ".join(self._whitelist)))
|
||||||
|
if self._keep_a_copy and self._save_directory:
|
||||||
|
self._logger.info("Saving deleted items to: {}".format(self._save_directory))
|
||||||
|
if self._trial_run:
|
||||||
|
self._logger.info("Trial run - no deletion will be performed")
|
||||||
|
|
||||||
|
def shred(self):
|
||||||
|
deleted = self._remove_things(self._get_things())
|
||||||
|
self._logger.info("Finished deleting {} items. ".format(deleted))
|
||||||
|
if deleted >= 1000:
|
||||||
|
# This user has more than 1000 items to handle, which angers the gods of the Reddit API. So chill for a
|
||||||
|
# while and do it again.
|
||||||
|
self._logger.info("Waiting {} seconds and continuing...".format(self._batch_cooldown))
|
||||||
|
time.sleep(self._batch_cooldown)
|
||||||
|
self._connect(None, self._username, self._password)
|
||||||
|
self.shred()
|
||||||
|
|
||||||
|
def _connect(self, praw_ini, username, password):
|
||||||
|
self._r = praw.Reddit(user_agent="shreddit/4.2")
|
||||||
|
if praw_ini:
|
||||||
|
# PRAW won't panic if the file is invalid, so check first
|
||||||
|
if not os.path.exists(praw_ini):
|
||||||
|
print("PRAW configuration file \"{}\" not found.".format(praw_ini))
|
||||||
|
return
|
||||||
|
praw.settings.CONFIG.read(praw_ini)
|
||||||
|
try:
|
||||||
|
# Try to login with OAuth2
|
||||||
|
self._r.refresh_access_information()
|
||||||
|
self._logger.debug("Logged in with OAuth.")
|
||||||
|
except (HTTPException, OAuthAppRequired) as e:
|
||||||
|
self._logger.warning("You should migrate to OAuth2 using get_secret.py before Reddit disables this login "
|
||||||
|
"method.")
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
self._r.login(username, password)
|
||||||
|
except InvalidUserPass:
|
||||||
|
self._r.login() # Supply details on the command line
|
||||||
|
except InvalidUser as e:
|
||||||
|
raise InvalidUser("User does not exist.", e)
|
||||||
|
except InvalidUserPass as e:
|
||||||
|
raise InvalidUserPass("Specified an incorrect password.", e)
|
||||||
|
except RateLimitExceeded as e:
|
||||||
|
raise RateLimitExceeded("You're doing that too much.", e)
|
||||||
|
self._logger.info("Logged in as {user}.".format(user=self._r.user))
|
||||||
|
|
||||||
|
def _check_item(self, item):
|
||||||
|
"""Returns True if the item is whitelisted, False otherwise.
|
||||||
|
"""
|
||||||
|
if str(item.subreddit).lower() in self._whitelist or item.id in self._whitelist_ids:
|
||||||
|
return True
|
||||||
|
if self._whitelist_distinguished and item.distinguished:
|
||||||
|
return True
|
||||||
|
if self._whitelist_gilded and item.gilded:
|
||||||
|
return True
|
||||||
|
if self._max_score is not None and item.score > self._max_score:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _save_item(self, item):
|
||||||
|
with open(os.path.join(self._save_directory, item.id), "w") as fh:
|
||||||
|
json.dump(item.json_dict, fh)
|
||||||
|
|
||||||
|
def _remove_submission(self, sub):
|
||||||
|
self._logger.info("Deleting submission: #{id} {url}".format(id=sub.id, url=sub.url.encode("utf-8")))
|
||||||
|
|
||||||
|
def _remove_comment(self, comment):
|
||||||
|
if self._replacement_format == "random":
|
||||||
|
replacement_text = get_sentence()
|
||||||
|
elif self._replacement_format == "dot":
|
||||||
|
replacement_text = "."
|
||||||
|
else:
|
||||||
|
replacement_text = self._replacement_format
|
||||||
|
|
||||||
|
short_text = sub(b"\n\r\t", " ", comment.body[:35].encode("utf-8"))
|
||||||
|
msg = "/r/{}/ #{} ({}) with: {}".format(comment.subreddit, comment.id, short_text, replacement_text)
|
||||||
|
|
||||||
|
if self._edit_only:
|
||||||
|
self._logger.debug("Editing (not removing) {msg}".format(msg=msg))
|
||||||
|
else:
|
||||||
|
self._logger.debug("Editing and deleting {msg}".format(msg=msg))
|
||||||
|
if not self._trial_run:
|
||||||
|
comment.edit(replacement_text)
|
||||||
|
self._api_calls.append(int(time.time()))
|
||||||
|
|
||||||
|
def _remove(self, item):
|
||||||
|
if self._keep_a_copy and self._save_directory:
|
||||||
|
self._save_item(item)
|
||||||
|
if self._clear_vote:
|
||||||
|
item.clear_vote()
|
||||||
|
self._api_calls.append(int(time.time()))
|
||||||
|
if isinstance(item, Submission):
|
||||||
|
self._remove_submission(item)
|
||||||
|
elif isinstance(item, Comment):
|
||||||
|
self._remove_comment(item)
|
||||||
|
if not self._edit_only and not self._trial_run:
|
||||||
|
item.delete()
|
||||||
|
self._api_calls.append(int(time.time()))
|
||||||
|
|
||||||
|
def _remove_things(self, items):
|
||||||
|
self._logger.info("Loading items to delete...")
|
||||||
|
to_delete = [item for item in items]
|
||||||
|
self._logger.info("Done. Starting on batch of {} items...".format(len(to_delete)))
|
||||||
|
for idx, item in enumerate(to_delete):
|
||||||
|
minute_ago = arrow.now().replace(minutes=-1).timestamp
|
||||||
|
self._api_calls = [api_call for api_call in self._api_calls if api_call >= minute_ago]
|
||||||
|
if len(self._api_calls) >= 60:
|
||||||
|
self._logger.info("Sleeping 10 seconds to wait out API cooldown...")
|
||||||
|
time.sleep(10)
|
||||||
|
self._logger.debug("Examining item {}: {}".format(idx + 1, item))
|
||||||
|
created = arrow.get(item.created_utc)
|
||||||
|
if created <= self._nuke_cutoff:
|
||||||
|
self._logger.debug("Item occurs prior to nuke cutoff")
|
||||||
|
self._remove(item)
|
||||||
|
elif created > self._recent_cutoff:
|
||||||
|
self._logger.debug("Skipping due to: too recent")
|
||||||
|
continue
|
||||||
|
elif self._check_item(item):
|
||||||
|
self._logger.debug("Skipping due to: whitelisted")
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
self._remove(item)
|
||||||
|
return idx + 1
|
||||||
|
|
||||||
|
def _get_things(self):
|
||||||
|
if self._item == "comments":
|
||||||
|
return self._r.user.get_comments(limit=self._limit, sort=self._sort)
|
||||||
|
elif self._item == "submitted":
|
||||||
|
return self._r.user.get_submitted(limit=self._limit, sort=self._sort)
|
||||||
|
elif self._item == "overview":
|
||||||
|
return self._r.user.get_overview(limit=self._limit, sort=self._sort)
|
||||||
|
else:
|
||||||
|
raise Exception("Your deletion section is wrong")
|
||||||
@ -0,0 +1,24 @@
|
|||||||
|
"""This module contains common utilities for the rest of the package.
|
||||||
|
"""
|
||||||
|
import random
|
||||||
|
|
||||||
|
|
||||||
|
WORDLIST = "/usr/share/dict/words"
|
||||||
|
STATIC_TEXT = "I have been Shreddited for privacy!"
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
from loremipsum import get_sentence
|
||||||
|
except ImportError:
|
||||||
|
def get_sentence():
|
||||||
|
"""This keeps the mess of dealing with the loremipsum library out of the shredding code. Until the maintainer of
|
||||||
|
the loremipsum package uploads a version that works with Python 3 to pypi, it is necessary to provide a drop-in
|
||||||
|
replacement. The current solution is to return a static text string unless the operating system has a word list.
|
||||||
|
If that is the case, use it instead.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
lines = [line.strip() for line in open(WORDLIST).readlines()]
|
||||||
|
return " ".join(random.sample(lines, random.randint(50, 150)))
|
||||||
|
except IOError:
|
||||||
|
# The word list wasn't available...
|
||||||
|
return STATIC_TEXT
|
||||||
Loading…
Reference in new issue