Tools
/
Shreddit
mirror of https://github.com/x89/Shreddit
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Changed name for fork in setup

Browse Source
pull/69/head
Scott 9 years ago
parent 3d9b19ab6a
commit 48047a7884
12 changed files with 342 additions and 233 deletions
Show Stats Download Patch File Download Diff File

66
.gitignore vendored
Unescape View File

@ -1,13 +1,55 @@
.venv
pip-selfcheck.json
.errors
2015*.txt
2016*.txt
2017*.txt
*.cfg
*.pyc
shreddit.conf
shreddit.yml
praw.ini
# Docs
docs/_build/
# Byte-compiled / optimized / DLL files
__pycache__/
.*.swp
*.py[cod]
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
*.pot
# Django stuff:
*.log
# PyBuilder
target/

17
Makefile
Unescape View File

@ -0,0 +1,17 @@
help:
@echo "build - Build package"
@echo "install - Install package to local system"
@echo "clean - Clean built artifacts"
@echo "test - Run test suite with coverage"
build:
python setup.py build
python setup.py bdist_wheel
install:
pip install dist/*.whl --upgrade --force-reinstall --no-deps
python setup.py clean
clean:
find . -type f -name "*.pyc" -delete
rm -rf ./build ./dist ./*.egg-info

11
install.sh
Unescape View File

@ -1,11 +0,0 @@
#!/usr/bin/env sh
virtualenv .
source ./bin/activate
pip install -r requirements.txt
if [ ! -f "shreddit.yml" ]; then
cp "shreddit.yml.example" "shreddit.yml"
$EDITOR shreddit.yml
fi

12
lambda_handler.py
Unescape View File

@ -0,0 +1,12 @@
"""This module contains the handler function called by AWS.
"""
from shreddit.shredder import shred
import yaml
def lambda_handler(event, context):
with open("shreddit.yml") as fh:
config = yaml.safe_load(fh)
if not config:
raise Exception("No config options passed!")
shred(config)

17
oauth_test.py
Unescape View File

@ -1,17 +0,0 @@
#!/usr/bin/env python
'''
Simple script to check if your oauth is working.
'''
import praw
import sys
r = praw.Reddit('Shreddit oauth test')
try:
r.refresh_access_information()
if r.is_oauth_session():
sys.exit(0)
else:
sys.exit(2)
except:
sys.exit(1)

4
run.sh
Unescape View File

@ -1,4 +0,0 @@
#!/usr/bin/env sh
source ./bin/activate
pip install --upgrade praw
python ./shreddit.py

36
setup.py
Unescape View File

@ -0,0 +1,36 @@
"""Setup script for shreddit.
"""
from setuptools import setup
from codecs import open
from os import path
VERSION = "2.0.0"
DESCRIPTION = " Remove your comment history on Reddit as deleting an account does not do so."
here = path.abspath(path.dirname(__file__))
with open(path.join(here, "README.md"), encoding='utf-8') as filein:
long_description = filein.read()
with open(path.join(here, "requirements.txt"), encoding="utf-8") as filein:
requirements = [line.strip() for line in filein.readlines()]
setup(
name="shreddit",
version=VERSION,
description=DESCRIPTION,
long_description=long_description,
url="https://github.com/scott-hand/Shreddit",
author="Scott Hand",
author_email="scott@vkgfx.com",
classifiers=["Development Status :: 3 - Alpha",
"Intended Audience :: End Users/Desktop",
"Programming Language :: Python :: 2"],
packages=["shreddit"],
install_requires=requirements,
entry_points={
"console_scripts": [
"shreddit=shreddit.app:main"
]
}
)

189
shreddit.py
Unescape View File

@ -1,189 +0,0 @@
#!/usr/bin/env python
import os
import sys
import logging
import argparse
import json
import yaml
import praw
from re import sub
from random import shuffle, randint
from datetime import datetime, timedelta
from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded,
HTTPException, OAuthAppRequired)
from praw.objects import Comment, Submission
logging.basicConfig(stream=sys.stdout)
log = logging.getLogger(__name__)
log.setLevel(level=logging.WARNING)
try:
from loremipsum import get_sentence # This only works on Python 2
except ImportError:
def get_sentence():
return '''I have been Shreddited for privacy!'''
os_wordlist = '/usr/share/dict/words'
if os.name == 'posix' and os.path.isfile(os_wordlist):
# Generate a random string of words from our system's dictionary
fh = open(os_wordlist)
words = fh.read().splitlines()
fh.close()
shuffle(words)
def get_sentence():
return ' '.join(words[:randint(50, 150)])
assert get_sentence
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--config',
help="config file to use instead of the default shreddit.cfg"
)
args = parser.parse_args()
if args.config:
config_file = args.config
else:
config_file = 'shreddit.yml'
with open(config_file, 'r') as fh:
config = yaml.safe_load(fh)
if config is None:
raise Exception("No config options passed!")
save_directory = config.get('save_directory', '.')
r = praw.Reddit(user_agent="shreddit/4.2")
if save_directory:
r.config.store_json_result = True
if config.get('verbose', True):
log.setLevel(level=logging.DEBUG)
try:
# Try to login with OAuth2
r.refresh_access_information()
log.debug("Logged in with OAuth.")
except (HTTPException, OAuthAppRequired) as e:
log.warning('''You should migrate to OAuth2 using get_secret.py before
Reddit disables this login method.''')
try:
try:
r.login(config['username'], config['password'])
except InvalidUserPass:
r.login() # Supply details on the command line
except InvalidUser as e:
raise InvalidUser("User does not exist.", e)
except InvalidUserPass as e:
raise InvalidUserPass("Specified an incorrect password.", e)
except RateLimitExceeded as e:
raise RateLimitExceeded("You're doing that too much.", e)
log.info("Logged in as {user}.".format(user=r.user))
log.debug("Deleting messages before {time}.".format(
time=datetime.now() - timedelta(hours=config['hours'])))
whitelist = config.get('whitelist', [])
whitelist_ids = config.get('whitelist_ids', [])
if config.get('whitelist'):
log.debug("Keeping messages from subreddits {subs}".format(
subs=', '.join(whitelist))
)
def get_things(after=None):
limit = None
item = config.get('item', 'comments')
sort = config.get('sort', 'new')
log.debug("Deleting items: {item}".format(item=item))
if item == "comments":
return r.user.get_comments(limit=limit, sort=sort)
elif item == "submitted":
return r.user.get_submitted(limit=limit, sort=sort)
elif item == "overview":
return r.user.get_overview(limit=limit, sort=sort)
else:
raise Exception("Your deletion section is wrong")
def remove_things(things):
for thing in things:
log.debug('Starting remove function on: {thing}'.format(thing=thing))
# Seems to be in users's timezone. Unclear.
thing_time = datetime.fromtimestamp(thing.created_utc)
# Exclude items from being deleted unless past X hours.
after_time = datetime.now() - timedelta(hours=config.get('hours', 24))
if thing_time > after_time:
if thing_time + timedelta(hours=config.get('nuke_hours', 4320)) < datetime.utcnow():
pass
continue
# For edit_only we're assuming that the hours aren't altered.
# This saves time when deleting (you don't edit already edited posts).
if config.get('edit_only'):
end_time = after_time - timedelta(hours=config.get('hours', 24))
if thing_time < end_time:
continue
if str(thing.subreddit).lower() in config.get('whitelist', []) \
or thing.id in config.get('whitelist_ids', []):
continue
if config.get('whitelist_distinguished') and thing.distinguished:
continue
if config.get('whitelist_gilded') and thing.gilded:
continue
if 'max_score' in config and thing.score > config['max_score']:
continue
if config.get('save_directory'):
save_directory = config['save_directory']
if not os.path.exists(save_directory):
os.makedirs(save_directory)
with open("%s/%s.json" % (save_directory, thing.id), "w") as fh:
json.dump(thing.json_dict, fh)
if config.get('trial_run'): # Don't do anything, trial mode!
log.debug("Would have deleted {thing}: '{content}'".format(
thing=thing.id, content=thing))
continue
if config.get('clear_vote'):
thing.clear_vote()
if isinstance(thing, Submission):
log.info('Deleting submission: #{id} {url}'.format(
id=thing.id,
url=thing.url.encode('utf-8'))
)
elif isinstance(thing, Comment):
rep_format = config.get('replacement_format')
if rep_format == 'random':
replacement_text = get_sentence()
elif rep_format == 'dot':
replacement_text = '.'
else:
replacement_text = rep_format
msg = '/r/{3}/ #{0} with:\n\t"{1}" to\n\t"{2}"'.format(
thing.id,
sub(b'\n\r\t', ' ', thing.body[:78].encode('utf-8')),
replacement_text[:78],
thing.subreddit
)
if config.get('edit_only'):
log.info('Editing (not removing) {msg}'.format(msg=msg))
else:
log.info('Editing and deleting {msg}'.format(msg=msg))
thing.edit(replacement_text)
if not config.get('edit_only'):
thing.delete()
remove_things(get_things())

0
shreddit/__init__.py
Unescape View File

29
shreddit/app.py
Unescape View File

@ -0,0 +1,29 @@
"""This module contains script entrypoints for shreddit.
"""
import argparse
import yaml
from shreddit.oauth import oauth_test
from shreddit.shredder import shred
def main():
parser = argparse.ArgumentParser(description="Command-line frontend to the shreddit library.")
parser.add_argument("-c", "--config", help="Config file to use instead of the default shreddit.cfg")
parser.add_argument("-p", "--praw", help="PRAW config (if not ./praw.ini)")
parser.add_argument("-t", "--test-oauth", help="Perform OAuth test and exit", action="store_true")
args = parser.parse_args()
if args.test_oauth:
oauth_test(args.praw)
return
with open(args.config or "shreddit.yml") as fh:
config = yaml.safe_load(fh)
if not config:
raise Exception("No config options passed!")
shred(config, args.praw)
if __name__ == "__main__":
main()

22
shreddit/oauth.py
Unescape View File

@ -0,0 +1,22 @@
"""This module contains a function that tests OAuth session validity.
"""
import os
import praw
def oauth_test(praw_ini):
if praw_ini:
# PRAW won't panic if the file is invalid, so check first
if not os.path.exists(praw_ini):
print("PRAW configuration file \"{}\" not found.".format(praw_ini))
return
praw.settings.CONFIG.read(praw_ini)
r = praw.Reddit("Shreddit oauth test")
try:
r.refresh_access_information()
if r.is_oauth_session():
print("Session is valid.")
else:
print("Session is not a valid OAuth session.")
except Exception as e:
print("Error encountered while checking credentials:\n{}".format(e))

172
shreddit/shredder.py
Unescape View File

@ -0,0 +1,172 @@
import os
import sys
import logging
import argparse
import json
import yaml
import praw
from re import sub
from random import shuffle, randint
from datetime import datetime, timedelta
from praw.errors import (InvalidUser, InvalidUserPass, RateLimitExceeded, HTTPException, OAuthAppRequired)
from praw.objects import Comment, Submission
try:
from loremipsum import get_sentence # This only works on Python 2
except ImportError:
def get_sentence():
return "I have been Shreddited for privacy!"
os_wordlist = "/usr/share/dict/words"
if os.name == "posix" and os.path.isfile(os_wordlist):
# Generate a random string of words from our system's dictionary
fh = open(os_wordlist)
words = fh.read().splitlines()
fh.close()
shuffle(words)
def get_sentence():
return " ".join(words[:randint(50, 150)])
assert get_sentence
def shred(config, praw_ini=None):
logging.basicConfig(stream=sys.stdout)
log = logging.getLogger("shreddit")
log.setLevel(level=logging.WARNING)
if praw_ini:
# PRAW won't panic if the file is invalid, so check first
if not os.path.exists(praw_ini):
print("PRAW configuration file \"{}\" not found.".format(praw_ini))
return
praw.settings.CONFIG.read(praw_ini)
save_directory = config.get("save_directory", ".")
r = praw.Reddit(user_agent="shreddit/4.2")
if save_directory:
r.config.store_json_result = True
if config.get("verbose", True):
log.setLevel(level=logging.DEBUG)
try:
# Try to login with OAuth2
r.refresh_access_information()
log.debug("Logged in with OAuth.")
except (HTTPException, OAuthAppRequired) as e:
log.warning("You should migrate to OAuth2 using get_secret.py before Reddit disables this login method.")
try:
try:
r.login(config["username"], config["password"])
except InvalidUserPass:
r.login() # Supply details on the command line
except InvalidUser as e:
raise InvalidUser("User does not exist.", e)
except InvalidUserPass as e:
raise InvalidUserPass("Specified an incorrect password.", e)
except RateLimitExceeded as e:
raise RateLimitExceeded("You're doing that too much.", e)
log.info("Logged in as {user}.".format(user=r.user))
log.debug("Deleting messages before {time}.".format(
time=datetime.now() - timedelta(hours=config["hours"])))
whitelist = config.get("whitelist", [])
whitelist_ids = config.get("whitelist_ids", [])
if config.get("whitelist"):
log.debug("Keeping messages from subreddits {subs}".format(subs=", ".join(whitelist)))
remove_things(r, config, log, get_things(r, config, log))
def get_things(r, config, log, after=None):
limit = None
item = config.get("item", "comments")
sort = config.get("sort", "new")
log.debug("Deleting items: {item}".format(item=item))
if item == "comments":
return r.user.get_comments(limit=limit, sort=sort)
elif item == "submitted":
return r.user.get_submitted(limit=limit, sort=sort)
elif item == "overview":
return r.user.get_overview(limit=limit, sort=sort)
else:
raise Exception("Your deletion section is wrong")
def remove_things(r, config, log, things):
for thing in things:
log.debug("Starting remove function on: {thing}".format(thing=thing))
# Seems to be in users's timezone. Unclear.
thing_time = datetime.fromtimestamp(thing.created_utc)
# Exclude items from being deleted unless past X hours.
after_time = datetime.now() - timedelta(hours=config.get("hours", 24))
if thing_time > after_time:
if thing_time + timedelta(hours=config.get("nuke_hours", 4320)) < datetime.utcnow():
pass
continue
# For edit_only we're assuming that the hours aren't altered.
# This saves time when deleting (you don't edit already edited posts).
if config.get("edit_only"):
end_time = after_time - timedelta(hours=config.get("hours", 24))
if thing_time < end_time:
continue
if str(thing.subreddit).lower() in config.get("whitelist", []) \
or thing.id in config.get("whitelist_ids", []):
continue
if config.get("whitelist_distinguished") and thing.distinguished:
continue
if config.get("whitelist_gilded") and thing.gilded:
continue
if "max_score" in config and thing.score > config["max_score"]:
continue
if config.get("save_directory"):
save_directory = config["save_directory"]
if not os.path.exists(save_directory):
os.makedirs(save_directory)
with open("%s/%s.json" % (save_directory, thing.id), "w") as fh:
json.dump(thing.json_dict, fh)
if config.get("trial_run"): # Don't do anything, trial mode!
log.debug("Would have deleted {thing}: '{content}'".format(
thing=thing.id, content=thing))
continue
if config.get("clear_vote"):
thing.clear_vote()
if isinstance(thing, Submission):
log.info("Deleting submission: #{id} {url}".format(
id=thing.id,
url=thing.url.encode("utf-8"))
)
elif isinstance(thing, Comment):
rep_format = config.get("replacement_format")
if rep_format == "random":
replacement_text = get_sentence()
elif rep_format == "dot":
replacement_text = "."
else:
replacement_text = rep_format
msg = '/r/{3}/ #{0} with:\n\t"{1}" to\n\t"{2}"'.format(
thing.id,
sub(b"\n\r\t", " ", thing.body[:78].encode("utf-8")),
replacement_text[:78],
thing.subreddit
)
if config.get("edit_only"):
log.info("Editing (not removing) {msg}".format(msg=msg))
else:
log.info("Editing and deleting {msg}".format(msg=msg))
thing.edit(replacement_text)
if not config.get("edit_only"):
thing.delete()
Write Preview
Loading…
Cancel
Save