Download and browse content you love!
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

72 lines
2.6 KiB

import os
from utils import logger
import redditScraper
import tumblrScraper
import submission
import settings
import LikedSavedDatabase
# Unfortunately, this can only ever get 1000 (reddit's imposed limit)
def AddAllFromReddit(database, settings):
if not settings.hasRedditSettings():
logger.log('Reddit settings are not provided!')
submissions = []
logger.log('Adding last 1000 liked/saved submissions from Reddit. This will take a long time.')
redditSubmissions, redditComments, earlyOutPoints = redditScraper.getRedditUserLikedSavedSubmissions(
settings.settings['Username'], settings.settings['Password'],
settings.settings['Client_id'], settings.settings['Client_secret'],
request_limit = None, # No limit = request as many as possible (1000)
saveLiked = settings.settings['Reddit_Save_Liked'],
saveSaved = settings.settings['Reddit_Save_Saved'],
earlyOutPointSaved = None,
earlyOutPointLiked = None,
unlikeLiked = False,
unsaveSaved = False)
logger.log('Retrieved submissions, adding to database...')
for submission in redditSubmissions:
for comment in redditComments:
logger.log('Done! Saved {} submissions and {} comments'.format(len(redditSubmissions), len(redditComments)))
def AddAllFromDirectory(fileDatabase, directory):
logger.log('Scanning {} to create FileCollectionDatabase'.format(directory))
numFiles = 0
for root, dirs, files in os.walk(directory):
for file in files:
fileDatabase.addFileNoSave(file, os.path.relpath(os.path.join(root, file), directory))
numFiles += 1
logger.log('Done; {} files in database'.format(numFiles))
if __name__ == '__main__':
createSubmissionsDatabase = False
createFileCollectionDatabase = True
if createSubmissionsDatabase:
databaseFilename = 'LikedSavedDatabase.db'
isNewDatabase = not os.path.exists(databaseFilename)
if isNewDatabase:
db = LikedSavedDatabase.LikedSavedDatabase(databaseFilename)
AddAllFromReddit(db, settings)
if createFileCollectionDatabase:
databaseFilename = 'FileCollectionDatabase.db'
isNewDatabase = not os.path.exists(databaseFilename)
if isNewDatabase:
db = LikedSavedDatabase.FileCollectionDatabase(databaseFilename)
AddAllFromDirectory(db, settings.settings['Output_dir'])