Browse Source

Full cleanup functionality

Cleanup script now does everything it needs to do to make sure folders
don't get too big on space constrained devices.
master
Macoy Madson 2 years ago
parent
commit
c81eef3054
4 changed files with 134 additions and 25 deletions
  1. +89
    -17
      CleanUpPhonePhotos.py
  2. +10
    -8
      ReadMe.org
  3. +23
    -0
      Settings.py
  4. +12
    -0
      settings.json

+ 89
- 17
CleanUpPhonePhotos.py View File

@ -1,41 +1,113 @@
!/bin/python3
#!/bin/python3
import os
import Settings
photosPath = "/home/macoy/Camera S9 Test"
archivePhotosPath = "/home/macoy/Archive-Photos-Sync"
# One gibibyte max size
maxPhotosSize = 1024 * 1024 * 1024 * 1
# Don't actually do anything
softRun = True
def getAllFilesInPath(path):
fileList = []
for root, dirs, files in os.walk(path):
# Ignore hidden folders (lazy programming detected!)
# Note that this can mean thumbnails build up forever if they're still running google photos
if "/." in root:
continue
for file in files:
# if file.endswith(renderedContentExtensions):
filePath = os.path.join(root, file)
fileList.append(filePath)
return fileList
# From https://stackoverflow.com/questions/1392413/calculating-a-directorys-size-using-python
def getFolderSize(path='.'):
total = 0
for entry in os.scandir(path):
if entry.is_file():
total += entry.stat().st_size
elif entry.is_dir():
total += getFolderSize(entry.path)
return total
def makeRelativePath(fullPath, sourceDirectory):
if sourceDirectory not in fullPath:
print("ERROR: Path '{}' cannot be made relative.\n"
"Make sure '{}' is an absolute path"
.format(fullPath, sourceDirectory))
exit(1)
return fullPath[len(sourceDirectory):]
def bytesToKiBString(byteCount):
return "{:,}".format(int(byteCount / 1024))
def main():
print("Checking Photos for cleanup...")
def bytesToMiBString(byteCount):
return "{:,}".format(int(byteCount / 1024 / 1024))
def CleanUpFolder(photosPath, archivePhotosPath, maxPhotosSize):
print("Checking folder {} against {} for cleanup..."
.format(photosPath, archivePhotosPath))
# Is the folder too big?
photosSize = os.path.getsize(photosPath)
print("Photos folder size = {} GiB".format(photosSize / (1024 * 1024)))
photosSize = getFolderSize(photosPath)
print("Folder size = {} MiB".format(bytesToMiBString(photosSize)))
if photosSize < maxPhotosSize:
print("Photos folder within size. Nothing to do")
print("Folder within size. Nothing to do")
return
print("Going to try to free up {} MiB"
.format(bytesToMiBString(photosSize - maxPhotosSize)))
# Build file lists
photosFiles = getAllFilesInPath(photosPath)
archivePhotosFiles = getAllFilesInPath(archivePhotosPath)
archivePhotosFilesList = getAllFilesInPath(archivePhotosPath)
archivePhotosFilesDict = {}
# For faster lookups
for archivePhoto in archivePhotosFilesList:
archivePhotosFilesDict[makeRelativePath(archivePhoto, archivePhotosPath)] = None
print("Found {} photos and {} archive photos"
.format(len(photosFiles), len(archivePhotosPath))
# Sort photos by age
.format(len(photosFiles), len(archivePhotosFilesList)))
# Sort photos by age (oldest = first)
photosFiles.sort(key=os.path.getmtime)
sizeLeftToFree = photosSize - maxPhotosSize
totalFreed = 0
totalDeletedFiles = 0
# Are there any files we can delete?
# Delete files
for photoFile in photosFiles:
if makeRelativePath(photoFile, photosPath) in archivePhotosFilesDict:
# It's already archived. Delete it!
photoSize = os.stat(photoFile).st_size
sizeLeftToFree -= photoSize
totalFreed += photoSize
totalDeletedFiles += 1
print("Deleting {}".format(photoFile))
if not softRun:
os.remove(photoFile)
if sizeLeftToFree > 0:
print("Removed {} KiB, {} MiB to go..."
.format(bytesToKiBString(photoSize), bytesToMiBString(sizeLeftToFree)))
# Finished deleting!
else:
print("Removed {} KiB"
.format(bytesToKiBString(photoSize)))
break
else:
# While I could do the copying here, I wanted this script to be for cleanup only
print("WARNING: {} was not found in {}. It will not eligible for deletion."
.format(photoFile, archivePhotosPath))
print("Deleted {} files, reclaiming {} MiB"
.format(totalDeletedFiles, bytesToMiBString(totalFreed)))
if sizeLeftToFree > 0:
print("FAILED to free {} MiB".format(bytesToMiBString(sizeLeftToFree)))
if __name__ == "__main__":
main()
if __name__ == '__main__':
for cleanSetting in Settings.settings:
CleanUpFolder(cleanSetting["folder"],
cleanSetting["archive"],
cleanSetting["maxSizeGiB"] * 1024 * 1024 * 1024)

+ 10
- 8
ReadMe.org View File

@ -8,15 +8,17 @@ Server-side management of photos. Intended to replace the core functionality of
** Network
I'm using [[https://syncthing.net][Syncthing]] to get files to/from my phone's Photo folder. The network looks like this:
| Device | Folder | Purpose |
|---------------+--------------+--------------------------------------------------------------------------------------------------|
| Phone | Photos | New photos saved here by the camera. Sync both ways so that servers can remove photos easily |
| Amazon Server | Photos | Direct sync with Phone Photos. Server controls which photos are on phone by changing this folder |
| | Archive | Photos folder is synced via ~rync~ to the archive. Nothing is ever removed from the Archive |
| Home Pi | Photos | Redundant Photos sync (all devices sync to each other) |
| | Archive | Redundant Archive. Phone can download from Pi or Amazon servers |
| | Cold Storage | Because Amazon is limited in space, only keep the full collection of photos on the Pi |
| Device | Folder | Purpose |
|---------------+--------------+----------------------------------------------------------------------------------------------------------------------------------|
| Phone | Photos | New photos saved here by the camera. Sync both ways so that servers can remove photos easily |
| Amazon Server | Photos | Direct sync with Phone Photos. Server controls which photos are on phone by changing this folder |
| | Archive | Photos folder is synced via ~rync~ to the archive |
| Home Pi | Photos | Redundant Photos sync (all devices sync to each other) |
| | Archive | Redundant Archive. Phone can download from Pi or Amazon servers |
| | Cold Storage | Because Amazon is limited in space, only keep the full collection of photos on the Pi. Nothing is ever removed from Cold Storage |
** Setup
You will need to set up Syncthing (or some other file sync service) to suit your requirements.
I use ~rsync -a~ to sync Photos with Archive and Archive with Cold Storage. The former is on a cronjob every 30 minutes or so, and the latter is a job which should run every couple hours. Cleanup (reclaiming storage on phone and limited-space server) will happen every morning at 4 AM.

+ 23
- 0
Settings.py View File

@ -0,0 +1,23 @@
import os
import json
settings = None
def loadSettings():
global settings
settingsFilename = "settings.json"
# Load git ignored settings, if it exists (so I don't check in my API tokens)
if os.path.isfile("LOCAL_settings.json"):
settingsFilename = "LOCAL_settings.json"
print("Loading settings from {}".format(settingsFilename))
settingsFile = open(settingsFilename, "r")
settingsLines = settingsFile.readlines()
settingsFile.close()
settings = json.loads("".join(settingsLines))
print(settings)
loadSettings()

+ 12
- 0
settings.json View File

@ -0,0 +1,12 @@
[
{
"folder": "folder/to/clean",
"archive": "archive/to/compare",
"maxSizeGiB": 1.5
},
{
"folder": "another/folder/to/clean",
"archive": "another/archive/to/compare",
"maxSizeGiB": 5
}
]

Loading…
Cancel
Save