Browse Source

Close #53: Added explicit URL download field

This allows the user to download things to their Content Collection by
explicitly pasting the URLs.

* Filter out playlists from Youtube-DL because it does not support it
  yet
* Better runScript output
* Fix Gfycat exception output
* Output how many unsupported submissions there are
pull/74/head
Macoy Madson 1 year ago
parent
commit
a258697018
8 changed files with 154 additions and 37 deletions
  1. +43
    -32
      LikedSavedDownloaderServer.py
  2. +6
    -3
      imageSaver.py
  3. +39
    -0
      redditUserImageScraper.py
  4. +1
    -0
      templates/UnsupportedSubmissions.html
  5. +2
    -0
      videoDownloader.py
  6. +20
    -2
      webInterface/runScript.html
  7. +39
    -0
      webInterface/runScript.js
  8. +4
    -0
      webInterfaceNoAuth/index.css

+ 43
- 32
LikedSavedDownloaderServer.py View File

@ -284,7 +284,8 @@ class UnsupportedSubmissionsHandler(AuthHandler):
unsupportedSubmissionsListHtml += '</tbody>\n'
self.render("templates/UnsupportedSubmissions.html",
unsupported_submissions_html=unsupportedSubmissionsListHtml)
unsupported_submissions_html=unsupportedSubmissionsListHtml,
length_unsupported_submissions=len(unsupportedSubmissions))
class SettingsHandler(AuthHandler):
def doSettings(self, afterSubmit):
@ -601,48 +602,58 @@ class RunScriptWebSocket(tornado.websocket.WebSocketHandler):
command = parsedMessage['command']
print('RunScriptWebSocket: Command ', command)
if scriptProcess and scriptProcess.is_alive():
print('RunScriptWebSocket: Script already running')
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('A download process is already running. Please wait until it completes.\\n',
'printMessage'))
self.write_message(responseMessage)
if command == 'runScript':
if scriptProcess and scriptProcess.is_alive():
print('RunScriptWebSocket: Script already running')
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('Script already running\\n', 'printMessage'))
self.write_message(responseMessage)
else:
print('RunScriptWebSocket: Starting script')
print('RunScriptWebSocket: Starting script')
startScript(redditUserImageScraper.runLikedSavedDownloader)
startScript(redditUserImageScraper.runLikedSavedDownloader)
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('Running downloader.\\n', 'printMessage'))
self.write_message(responseMessage)
elif command == 'retrySubmissions':
print('RunScriptWebSocket: Starting script')
if parsedMessage['submissionsToRetry']:
submissionIds = []
for submissionId in parsedMessage['submissionsToRetry']:
submissionIds.append(int(submissionId))
startScript(redditUserImageScraper.saveRequestedSubmissions,
submissionIds)
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('Running downloader.\\n', 'printMessage'))
self.write_message(responseMessage)
elif command == 'retrySubmissions':
if scriptProcess and scriptProcess.is_alive():
print('RunScriptWebSocket: A downloader script is already running')
else:
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('A download process is already running. Note that you cannot have '
'Unsupported Submissions downloading at the same time that the normal'
'Download Content process is running.\\n', 'printMessage'))
.format('No content selected.\\n', 'printMessage'))
self.write_message(responseMessage)
elif command == 'explicitDownloadUrls':
print('RunScriptWebSocket: Starting script')
if parsedMessage['urls']:
urls = []
urlLines = parsedMessage['urls'].split('\n')
for line in urlLines:
# TODO: It would be a good idea to do some validation here, and maybe even regex extract URLs
urls.append(line)
print(urls)
startScript(redditUserImageScraper.saveRequestedUrls, urls)
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('Running downloader.\\n', 'printMessage'))
self.write_message(responseMessage)
else:
print('RunScriptWebSocket: Starting script')
if parsedMessage['submissionsToRetry']:
submissionIds = []
for submissionId in parsedMessage['submissionsToRetry']:
submissionIds.append(int(submissionId))
startScript(redditUserImageScraper.saveRequestedSubmissions,
submissionIds)
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('Running downloader.\\n', 'printMessage'))
self.write_message(responseMessage)
else:
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('No content selected.\\n', 'printMessage'))
self.write_message(responseMessage)
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('No URLs provided.\\n', 'printMessage'))
self.write_message(responseMessage)
else:
print('RunScriptWebSocket: Error: Received command not understood')


+ 6
- 3
imageSaver.py View File

@ -174,13 +174,16 @@ def convertGfycatUrlToWebM(url):
# Get the gfyname from the url
matches = re.findall(r'gfycat\.com.*/([a-zA-Z]+)', url)
if not matches:
logger.log("Gfycat URL {} doesn't seem to match expected URL format")
errorMessage = "Gfycat URL {} doesn't seem to match expected URL format".format(url)
logger.log(errorMessage)
LikedSavedDatabase.db.addUnsupportedSubmission(submission, errorMessage)
else:
try:
gfycatUrlInfo = gfycatClient.query_gfy(matches[0])
except Exception as e:
logger.log('[ERROR] Exception: Url {0} raised exception:\n\t {1}'
.format(url, e))
errorMessage = '[ERROR] Exception: Url {0} raised exception:\n\t {1}'.format(url, e)
logger.log(errorMessage)
LikedSavedDatabase.db.addUnsupportedSubmission(submission, errorMessage)
return None
return gfycatUrlInfo['gfyItem']['mp4Url']


+ 39
- 0
redditUserImageScraper.py View File

@ -181,5 +181,44 @@ def saveRequestedSubmissions(pipeConnection, submissionIds):
logger.log(scriptFinishedSentinel)
pipeConnection.close()
def saveRequestedUrls(pipeConnection, urls):
if pipeConnection:
logger.setPipe(pipeConnection)
initialize()
logger.log('Attempting to save {} requested urls. This may take several minutes...'
.format(len(urls)))
submissions = []
# Create Submission for each URL
for url in urls:
convertedSubmission = submission.Submission()
convertedSubmission.source = "UserRequested"
convertedSubmission.title = "UserRequested"
convertedSubmission.author = "(Requested by user)"
convertedSubmission.subreddit = "Requested_Downloads"
convertedSubmission.subredditTitle = "Requested Downloads"
convertedSubmission.body = "(Requested by user)"
convertedSubmission.bodyUrl= url
convertedSubmission.postUrl= url
submissions.append(convertedSubmission)
if len(submissions) != len(urls):
logger.log('Could not parse {} URLs!'.format(len(urls) - len(submissions)))
unsupportedSubmissions = imageSaver.saveAllImages(settings.settings['Output_dir'], submissions,
imgur_auth = imgurDownloader.getImgurAuth(),
only_download_albums = settings.settings['Only_download_albums'],
skip_n_percent_submissions = settings.settings['Skip_n_percent_submissions'],
soft_retrieve_imgs = settings.settings['Should_soft_retrieve'],
only_important_messages = settings.settings['Only_important_messages'])
logger.log('Download finished. Output to \'Requested Downloads\' directory')
if pipeConnection:
logger.log(scriptFinishedSentinel)
pipeConnection.close()
if __name__ == '__main__':
runLikedSavedDownloader(None)

+ 1
- 0
templates/UnsupportedSubmissions.html View File

@ -25,6 +25,7 @@
<h2>Failed to Download Content</h2>
<p>{{ length_unsupported_submissions }} submissions</p>
<button id="RetrySelected" onclick="retrySelected()">Retry selected</button>
<button id="ToggleAll" onclick="toggleRetryAll()">Toggle all</button>


+ 2
- 0
videoDownloader.py View File

@ -87,6 +87,8 @@ def downloadVideo(outputPath, url):
youtubeDlLogger = YoutubeDlLogger()
youtubeDL_options = {"outtmpl": "{}/{}".format(outputPath, youtubeDL_filenameFormat),
# TODO: Support playlists?
"noplaylist": True,
"writethumbnail": True,
"writeinfojson": True,
"logger": youtubeDlLogger}


+ 20
- 2
webInterface/runScript.html View File

@ -18,9 +18,27 @@
<a href="/">Back to Homepage</a><br /><br />
<button id="runScriptButton" onclick="onRunScriptClicked()">Check for new content</button>
<p id="serverStatus">Not connected to server.</p>
<h3>Output Log</h3>
<article class="OutputScrollbox"><p id="messages"></p></article>
<h2>Download from Content Directories</h2>
<p>Check your content directory providers (Reddit, Tumblr) for new content, and download it.
This downloads things you have Liked, Upvoted, Hearted, or Saved (depending on your settings).</p>
<button id="runScriptButton" onclick="onRunScriptClicked()">Download new content</button>
<h2>Download URLs</h2>
<p>Paste URLs to images or videos. The script will attempt to download the content at the URL.
This only works with direct links or on supported websites, e.g. YouTube.<br />
Paste one URL per line.</p>
<button id="explicitDownloadButton" onclick="onExplicitDownloadClicked()">Download URLs</button>
<button id="clearExplicitDownload" onclick="clearExplicitDownloadClicked()">Clear all</button>
<br /><label for="explicitDownloadUrls">URLs to download</label>
<textarea rows="10" cols="100" id="explicitDownloadUrls"
name="explicitDownloadUrls" onfocus="smartClearBox()">Enter URLs here</textarea>
<p id="messages"></p>
</body>
</html>

+ 39
- 0
webInterface/runScript.js View File

@ -1,6 +1,20 @@
var ws = new WebSocket("wss://" + window.location.host + "/runScriptWebSocket");
var username = "likedSavedBrowserClient";
// As soon as the websocket opens, request the initial image
ws.onopen = function(event) {
var serverStatus = document.getElementById("serverStatus");
// Hide it if the socket is open, so it doesn't get in the way
serverStatus.innerHTML = "";
}
// As soon as the websocket opens, request the initial image
ws.onclose = function(event) {
var serverStatus = document.getElementById("serverStatus");
// Hide it if the socket is open, so it doesn't get in the way
serverStatus.innerHTML = "Connection to server lost. Reload the page to attempt to reconnect.";
}
function sendMessage(message) {
var payload = {
"command": message,
@ -26,3 +40,28 @@ function onRunScriptClicked() {
sendMessage("runScript");
document.getElementById("runScriptButton").style.display = "none";
}
function onExplicitDownloadClicked() {
document.getElementById("messages").innerText = "";
// document.getElementById("runScriptButton").style.display = "none";
explicitDownloadText = document.getElementById("explicitDownloadUrls").value;
var payload = {
"command": "explicitDownloadUrls",
"urls": explicitDownloadText
}
// Make the request to the WebSocket.
ws.send(JSON.stringify(payload));
}
function clearExplicitDownloadClicked() {
urls = document.getElementById("explicitDownloadUrls")
urls.value = '';
}
function smartClearBox() {
urls = document.getElementById("explicitDownloadUrls")
if (urls.value =="Enter URLs here")
urls.value = '';
}

+ 4
- 0
webInterfaceNoAuth/index.css View File

@ -93,6 +93,7 @@ li,
tr,
td,
input,
textarea,
button {
color: #95aec7;
}
@ -128,11 +129,13 @@ img {
color: #000000;
}
textarea,
input {
background-color: #22282d;
font-family: Arial;
}
textarea,
input[type=text],
input[type=password],
input[type=number] {
@ -144,6 +147,7 @@ input[type=number] {
box-sizing: border-box;
}
textarea:focus,
input[type=text]:focus,
input[type=password]:focus,
input[type=number]:focus,


Loading…
Cancel
Save