Close #81: Fix Pixiv filenames not being unique

A button will appear on the "Content which failed to download" page
which should allow you to download all the missing Pixiv content.

LikedSavedDatabase.py

@@ -257,8 +257,14 @@ class LikedSavedDatabase:
         cursor.execute("select * from Submissions, RequestedSubmissions "
                        "where Submissions.id = RequestedSubmissions.id")
         return cursor.fetchall()
-        
-    
+
+    def getMissingPixivSubmissionIds(self):
+        cursor = self.dbConnection.cursor()
+        cursor.execute('select Submissions.id from Submissions where Submissions.source = "Pixiv"'
+                       'and Submissions.id not in (select submissionKey from FilesToSubmissions)')
+        return cursor.fetchall()
+
+
 def initializeFromSettings(userSettings):
     global db
     if not db:

LikedSavedDownloaderServer.py

@@ -271,7 +271,20 @@ class UnsupportedSubmissionsHandler(AuthHandler):
 
         tableHeaderHtml += '</tr>\n</thead>\n<tbody>\n'
         return tableHeaderHtml
-    
+
+    def getPendingFixups(self):
+        fixupHtml = ''
+
+        missingPixivSubmissions = LikedSavedDatabase.db.getMissingPixivSubmissionIds()
+        if len(missingPixivSubmissions):
+            if not fixupHtml:
+                fixupHtml += "<h2>Download missing content</h2>"
+            fixupHtml += '<p>There was an error which caused {} Pixiv submissions to not be downloaded.</p>'.format(len(missingPixivSubmissions))
+            fixupHtml += '<button id="FixupPixiv" onclick="fixupPixiv()">Download missing Pixiv submissions</button>'
+            fixupHtml += '<p>You should only need to do this once. The code error has been fixed.</p>'
+
+        return fixupHtml
+
     @tornado.web.authenticated
     def get(self):
         unsupportedSubmissionsListHtml = self.createTableHeader()
@@ -287,7 +300,8 @@ class UnsupportedSubmissionsHandler(AuthHandler):
 
         self.render("templates/UnsupportedSubmissions.html",
                     unsupported_submissions_html=unsupportedSubmissionsListHtml,
-                    length_unsupported_submissions=len(unsupportedSubmissions))
+                    length_unsupported_submissions=len(unsupportedSubmissions),
+                    fixup_html=self.getPendingFixups())
 
 class SettingsHandler(AuthHandler):
     def doSettings(self, afterSubmit):
@@ -637,6 +651,21 @@ class RunScriptWebSocket(tornado.websocket.WebSocketHandler):
                 responseMessage = ('{{"message":"{}", "action":"{}"}}'
                                    .format('No content selected.\\n', 'printMessage'))
                 self.write_message(responseMessage)
+        # Fix the non-unique filenames error
+        elif command == 'fixupPixivSubmissions':
+            print('RunScriptWebSocket: Starting pixiv fixup')
+            missingPixivSubmissions = LikedSavedDatabase.db.getMissingPixivSubmissionIds()
+            missingPixivSubmissionIds = []
+            for missingPixivSubmission in missingPixivSubmissions:
+                missingPixivSubmissionIds.append(int(missingPixivSubmission['id']))
+
+            # print(missingPixivSubmissionIds)
+            startScript(redditUserImageScraper.saveRequestedSubmissions, missingPixivSubmissionIds)
+
+            responseMessage = ('{{"message":"{}", "action":"{}"}}'
+                               .format('Running downloader to download {} missing pixiv submissions.\\n'
+                                       .format(len(missingPixivSubmissions)),
+                                       'printMessage'))
         elif command == 'explicitDownloadUrls':
             print('RunScriptWebSocket: Starting script')
             if parsedMessage['urls']:

imageSaver.py

@@ -359,7 +359,8 @@ def saveAllImages(outputDir, submissions, imgur_auth = None, only_download_album
             submissionOutputDir = outputDir + u'/' + subredditDir
             utilities.makeDirIfNonexistant(submissionOutputDir)
             # Weird webp format...
-            pixivFilename = "{}.riff".format(safeFileName(submissionTitle))
+            pixivFilename = "{}_{}.riff".format(str(signedCrc32(submission.postUrl.encode())),
+                                                safeFileName(submissionTitle))
             saveFilePath = "{}/{}".format(submissionOutputDir, pixivFilename)
             if os.path.isfile(saveFilePath):
                 if not only_important_messages:

submission.py

@@ -54,8 +54,7 @@ class Submission:
                 self.body, self.bodyUrl, self.postUrl]
     
     def initFromDict(self, dictEntry):
-        # Old logs might not have source field; assume reddit
-        self.source = dictEntry['source'] if 'source' in dictEntry else 'reddit'
+        self.source = dictEntry['source']
 
         self.title = dictEntry['title']
         self.author = dictEntry['author']
@@ -67,8 +66,6 @@ class Submission:
         self.bodyUrl = dictEntry['bodyUrl']
         self.postUrl = dictEntry['postUrl']
 
-    
-
 def getAsList_generator(submissions):
     for submission in submissions:
         yield submission.getAsList()

templates/UnsupportedSubmissions.html

@@ -22,8 +22,8 @@
 
 		<h3>Output Log</h3>
 		<article class="OutputScrollbox"><p id="messages"></p></article>
-		
 
+		{% raw fixup_html %}
 		<h2>Failed to Download Content</h2>
 		<p>{{ length_unsupported_submissions }} submissions</p>
 		<button id="RetrySelected" onclick="retrySelected()">Retry selected</button>

webInterface/UnsupportedSubmissions.js

@@ -58,3 +58,13 @@ function retrySelected()
     // Make the request to the WebSocket.
     ws.send(JSON.stringify(payload));
 }
+
+function fixupPixiv()
+{
+	var payload = {
+        "command": "fixupPixivSubmissions"
+    };
+
+	// Make the request to the WebSocket.
+    ws.send(JSON.stringify(payload));
+}