Full cleanup functionality

Cleanup script now does everything it needs to do to make sure folders
don't get too big on space constrained devices.

CleanUpPhonePhotos.py

@@ -1,41 +1,113 @@
-!/bin/python3
+#!/bin/python3
 
 import os
+import Settings
 
-photosPath = "/home/macoy/Camera S9 Test"
-archivePhotosPath = "/home/macoy/Archive-Photos-Sync"
-
-# One gibibyte max size
-maxPhotosSize = 1024 * 1024 * 1024 * 1
+# Don't actually do anything
+softRun = True
 
 def getAllFilesInPath(path):
     fileList = []
     for root, dirs, files in os.walk(path):
+        # Ignore hidden folders (lazy programming detected!)
+        # Note that this can mean thumbnails build up forever if they're still running google photos
+        if "/." in root:
+            continue
         for file in files:
             # if file.endswith(renderedContentExtensions):
             filePath = os.path.join(root, file)
             fileList.append(filePath)
+    return fileList
+
+# From https://stackoverflow.com/questions/1392413/calculating-a-directorys-size-using-python
+def getFolderSize(path='.'):
+    total = 0
+    for entry in os.scandir(path):
+        if entry.is_file():
+            total += entry.stat().st_size
+        elif entry.is_dir():
+            total += getFolderSize(entry.path)
+    return total
+
+def makeRelativePath(fullPath, sourceDirectory):
+    if sourceDirectory not in fullPath:
+        print("ERROR: Path '{}' cannot be made relative.\n"
+              "Make sure '{}' is an absolute path"
+              .format(fullPath, sourceDirectory))
+        exit(1)
+    return fullPath[len(sourceDirectory):]
 
+def bytesToKiBString(byteCount):
+     return "{:,}".format(int(byteCount / 1024))
 
-def main():
-    print("Checking Photos for cleanup...")
+def bytesToMiBString(byteCount):
+     return "{:,}".format(int(byteCount / 1024 / 1024))
+
+def CleanUpFolder(photosPath, archivePhotosPath, maxPhotosSize):
+    print("Checking folder {} against {} for cleanup..."
+          .format(photosPath, archivePhotosPath))
     
     # Is the folder too big?
-    photosSize = os.path.getsize(photosPath)
-    print("Photos folder size = {} GiB".format(photosSize / (1024 * 1024)))
+    photosSize = getFolderSize(photosPath)
+    print("Folder size = {} MiB".format(bytesToMiBString(photosSize)))
     if photosSize < maxPhotosSize:
-        print("Photos folder within size. Nothing to do")
+        print("Folder within size. Nothing to do")
         return
+
+    print("Going to try to free up {} MiB"
+          .format(bytesToMiBString(photosSize - maxPhotosSize)))
     
     # Build file lists
     photosFiles = getAllFilesInPath(photosPath)
-    archivePhotosFiles = getAllFilesInPath(archivePhotosPath)
+    archivePhotosFilesList = getAllFilesInPath(archivePhotosPath)
+    archivePhotosFilesDict = {}
+    # For faster lookups
+    for archivePhoto in archivePhotosFilesList:
+        archivePhotosFilesDict[makeRelativePath(archivePhoto, archivePhotosPath)] = None
 
     print("Found {} photos and {} archive photos"
-          .format(len(photosFiles), len(archivePhotosPath))
-    # Sort photos by age
+          .format(len(photosFiles), len(archivePhotosFilesList)))
+
+    # Sort photos by age (oldest = first)
+    photosFiles.sort(key=os.path.getmtime)
+    
+    sizeLeftToFree = photosSize - maxPhotosSize
+    totalFreed = 0
+    totalDeletedFiles = 0
     # Are there any files we can delete?
-    # Delete files
+    for photoFile in photosFiles:
+        if makeRelativePath(photoFile, photosPath) in archivePhotosFilesDict:
+            # It's already archived. Delete it!
+            photoSize = os.stat(photoFile).st_size
+            sizeLeftToFree -= photoSize
+            totalFreed += photoSize
+            totalDeletedFiles += 1
+
+            print("Deleting {}".format(photoFile))
+            if not softRun:
+                os.remove(photoFile)
+            
+            if sizeLeftToFree > 0:
+                print("Removed {} KiB, {} MiB to go..."
+                      .format(bytesToKiBString(photoSize), bytesToMiBString(sizeLeftToFree)))
+            # Finished deleting!
+            else:
+                print("Removed {} KiB"
+                      .format(bytesToKiBString(photoSize)))
+                break
+        else:
+            # While I could do the copying here, I wanted this script to be for cleanup only
+            print("WARNING: {} was not found in {}. It will not eligible for deletion."
+                  .format(photoFile, archivePhotosPath))
+            
+    print("Deleted {} files, reclaiming {} MiB"
+          .format(totalDeletedFiles, bytesToMiBString(totalFreed)))
+
+    if sizeLeftToFree > 0:
+        print("FAILED to free {} MiB".format(bytesToMiBString(sizeLeftToFree)))
 
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    for cleanSetting in Settings.settings:
+        CleanUpFolder(cleanSetting["folder"],
+                      cleanSetting["archive"],
+                      cleanSetting["maxSizeGiB"] * 1024 * 1024 * 1024)

ReadMe.org

@@ -8,15 +8,17 @@ Server-side management of photos. Intended to replace the core functionality of
 ** Network
 I'm using [[https://syncthing.net][Syncthing]] to get files to/from my phone's Photo folder. The network looks like this:
 
-| Device        | Folder       | Purpose                                                                                          |
-|---------------+--------------+--------------------------------------------------------------------------------------------------|
-| Phone         | Photos       | New photos saved here by the camera. Sync both ways so that servers can remove photos easily     |
-| Amazon Server | Photos       | Direct sync with Phone Photos. Server controls which photos are on phone by changing this folder |
-|               | Archive      | Photos folder is synced via ~rync~ to the archive. Nothing is ever removed from the Archive      |
-| Home Pi       | Photos       | Redundant Photos sync (all devices sync to each other)                                           |
-|               | Archive      | Redundant Archive. Phone can download from Pi or Amazon servers                                  |
-|               | Cold Storage | Because Amazon is limited in space, only keep the full collection of photos on the Pi            |
+| Device        | Folder       | Purpose                                                                                                                          |
+|---------------+--------------+----------------------------------------------------------------------------------------------------------------------------------|
+| Phone         | Photos       | New photos saved here by the camera. Sync both ways so that servers can remove photos easily                                     |
+| Amazon Server | Photos       | Direct sync with Phone Photos. Server controls which photos are on phone by changing this folder                                 |
+|               | Archive      | Photos folder is synced via ~rync~ to the archive                                                                                |
+| Home Pi       | Photos       | Redundant Photos sync (all devices sync to each other)                                                                           |
+|               | Archive      | Redundant Archive. Phone can download from Pi or Amazon servers                                                                  |
+|               | Cold Storage | Because Amazon is limited in space, only keep the full collection of photos on the Pi. Nothing is ever removed from Cold Storage |
 
 ** Setup
 
 You will need to set up Syncthing (or some other file sync service) to suit your requirements.
+
+I use ~rsync -a~ to sync Photos with Archive and Archive with Cold Storage. The former is on a cronjob every 30 minutes or so, and the latter is a job which should run every couple hours. Cleanup (reclaiming storage on phone and limited-space server) will happen every morning at 4 AM.

Settings.py

@@ -0,0 +1,23 @@
+import os
+import json
+
+settings = None
+
+def loadSettings():
+    global settings
+    
+    settingsFilename = "settings.json"
+    # Load git ignored settings, if it exists (so I don't check in my API tokens)
+    if os.path.isfile("LOCAL_settings.json"):
+        settingsFilename = "LOCAL_settings.json"
+        
+    print("Loading settings from {}".format(settingsFilename))
+        
+    settingsFile = open(settingsFilename, "r")
+    settingsLines = settingsFile.readlines()
+    settingsFile.close()
+
+    settings = json.loads("".join(settingsLines))
+    print(settings)
+    
+loadSettings()

settings.json

@@ -0,0 +1,12 @@
+[
+    {
+        "folder": "folder/to/clean",
+        "archive": "archive/to/compare",
+        "maxSizeGiB": 1.5
+    },
+    {
+        "folder": "another/folder/to/clean",
+        "archive": "another/archive/to/compare",
+        "maxSizeGiB": 5
+    }
+]