Browse Source

Close #81: Fix Pixiv filenames not being unique

A button will appear on the "Content which failed to download" page
which should allow you to download all the missing Pixiv content.
pull/84/head
Macoy Madson 6 months ago
parent
commit
664a986a6f
6 changed files with 53 additions and 10 deletions
  1. +8
    -2
      LikedSavedDatabase.py
  2. +31
    -2
      LikedSavedDownloaderServer.py
  3. +2
    -1
      imageSaver.py
  4. +1
    -4
      submission.py
  5. +1
    -1
      templates/UnsupportedSubmissions.html
  6. +10
    -0
      webInterface/UnsupportedSubmissions.js

+ 8
- 2
LikedSavedDatabase.py View File

@ -257,8 +257,14 @@ class LikedSavedDatabase:
cursor.execute("select * from Submissions, RequestedSubmissions "
"where Submissions.id = RequestedSubmissions.id")
return cursor.fetchall()
def getMissingPixivSubmissionIds(self):
cursor = self.dbConnection.cursor()
cursor.execute('select Submissions.id from Submissions where Submissions.source = "Pixiv"'
'and Submissions.id not in (select submissionKey from FilesToSubmissions)')
return cursor.fetchall()
def initializeFromSettings(userSettings):
global db
if not db:


+ 31
- 2
LikedSavedDownloaderServer.py View File

@ -271,7 +271,20 @@ class UnsupportedSubmissionsHandler(AuthHandler):
tableHeaderHtml += '</tr>\n</thead>\n<tbody>\n'
return tableHeaderHtml
def getPendingFixups(self):
fixupHtml = ''
missingPixivSubmissions = LikedSavedDatabase.db.getMissingPixivSubmissionIds()
if len(missingPixivSubmissions):
if not fixupHtml:
fixupHtml += "<h2>Download missing content</h2>"
fixupHtml += '<p>There was an error which caused {} Pixiv submissions to not be downloaded.</p>'.format(len(missingPixivSubmissions))
fixupHtml += '<button id="FixupPixiv" onclick="fixupPixiv()">Download missing Pixiv submissions</button>'
fixupHtml += '<p>You should only need to do this once. The code error has been fixed.</p>'
return fixupHtml
@tornado.web.authenticated
def get(self):
unsupportedSubmissionsListHtml = self.createTableHeader()
@ -287,7 +300,8 @@ class UnsupportedSubmissionsHandler(AuthHandler):
self.render("templates/UnsupportedSubmissions.html",
unsupported_submissions_html=unsupportedSubmissionsListHtml,
length_unsupported_submissions=len(unsupportedSubmissions))
length_unsupported_submissions=len(unsupportedSubmissions),
fixup_html=self.getPendingFixups())
class SettingsHandler(AuthHandler):
def doSettings(self, afterSubmit):
@ -637,6 +651,21 @@ class RunScriptWebSocket(tornado.websocket.WebSocketHandler):
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('No content selected.\\n', 'printMessage'))
self.write_message(responseMessage)
# Fix the non-unique filenames error
elif command == 'fixupPixivSubmissions':
print('RunScriptWebSocket: Starting pixiv fixup')
missingPixivSubmissions = LikedSavedDatabase.db.getMissingPixivSubmissionIds()
missingPixivSubmissionIds = []
for missingPixivSubmission in missingPixivSubmissions:
missingPixivSubmissionIds.append(int(missingPixivSubmission['id']))
# print(missingPixivSubmissionIds)
startScript(redditUserImageScraper.saveRequestedSubmissions, missingPixivSubmissionIds)
responseMessage = ('{{"message":"{}", "action":"{}"}}'
.format('Running downloader to download {} missing pixiv submissions.\\n'
.format(len(missingPixivSubmissions)),
'printMessage'))
elif command == 'explicitDownloadUrls':
print('RunScriptWebSocket: Starting script')
if parsedMessage['urls']:


+ 2
- 1
imageSaver.py View File

@ -359,7 +359,8 @@ def saveAllImages(outputDir, submissions, imgur_auth = None, only_download_album
submissionOutputDir = outputDir + u'/' + subredditDir
utilities.makeDirIfNonexistant(submissionOutputDir)
# Weird webp format...
pixivFilename = "{}.riff".format(safeFileName(submissionTitle))
pixivFilename = "{}_{}.riff".format(str(signedCrc32(submission.postUrl.encode())),
safeFileName(submissionTitle))
saveFilePath = "{}/{}".format(submissionOutputDir, pixivFilename)
if os.path.isfile(saveFilePath):
if not only_important_messages:


+ 1
- 4
submission.py View File

@ -54,8 +54,7 @@ class Submission:
self.body, self.bodyUrl, self.postUrl]
def initFromDict(self, dictEntry):
# Old logs might not have source field; assume reddit
self.source = dictEntry['source'] if 'source' in dictEntry else 'reddit'
self.source = dictEntry['source']
self.title = dictEntry['title']
self.author = dictEntry['author']
@ -67,8 +66,6 @@ class Submission:
self.bodyUrl = dictEntry['bodyUrl']
self.postUrl = dictEntry['postUrl']
def getAsList_generator(submissions):
for submission in submissions:
yield submission.getAsList()


+ 1
- 1
templates/UnsupportedSubmissions.html View File

@ -22,8 +22,8 @@
<h3>Output Log</h3>
<article class="OutputScrollbox"><p id="messages"></p></article>
{% raw fixup_html %}
<h2>Failed to Download Content</h2>
<p>{{ length_unsupported_submissions }} submissions</p>
<button id="RetrySelected" onclick="retrySelected()">Retry selected</button>


+ 10
- 0
webInterface/UnsupportedSubmissions.js View File

@ -58,3 +58,13 @@ function retrySelected()
// Make the request to the WebSocket.
ws.send(JSON.stringify(payload));
}
function fixupPixiv()
{
var payload = {
"command": "fixupPixivSubmissions"
};
// Make the request to the WebSocket.
ws.send(JSON.stringify(payload));
}

Loading…
Cancel
Save