Add avatar to prune script

1 files changed, 78 insertions, 32 deletions

diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts
index 4953a7439..d6dff8247 100755
--- a/scripts/prune-storage.ts
+++ b/scripts/prune-storage.ts
@@ -3,9 +3,12 @@ import { join } from 'path'
 import { CONFIG } from '../server/initializers/config'
 import { VideoModel } from '../server/models/video/video'
 import { initDatabaseModels } from '../server/initializers'
-import { remove, readdir } from 'fs-extra'
+import { readdir, remove } from 'fs-extra'
 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
+import * as Bluebird from 'bluebird'
 import { getUUIDFromFilename } from '../server/helpers/utils'
+import { ThumbnailModel } from '../server/models/video/thumbnail'
+import { AvatarModel } from '../server/models/avatar/avatar'
 
 run()
   .then(() => process.exit(0))
@@ -17,25 +20,19 @@ run()
 async function run () {
   await initDatabaseModels(true)
 
-  const storageOnlyOwnedToPrune = [
-    CONFIG.STORAGE.VIDEOS_DIR,
-    CONFIG.STORAGE.TORRENTS_DIR,
-    CONFIG.STORAGE.REDUNDANCY_DIR
-  ]
+  let toDelete: string[] = []
 
-  const storageForAllToPrune = [
-    CONFIG.STORAGE.PREVIEWS_DIR,
-    CONFIG.STORAGE.THUMBNAILS_DIR
-  ]
+  toDelete = toDelete.concat(
+    await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
+    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
 
-  let toDelete: string[] = []
-  for (const directory of storageOnlyOwnedToPrune) {
-    toDelete = toDelete.concat(await pruneDirectory(directory, true))
-  }
+    await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
 
-  for (const directory of storageForAllToPrune) {
-    toDelete = toDelete.concat(await pruneDirectory(directory, false))
-  }
+    await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
+    await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
+
+    await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
+  )
 
   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
@@ -61,30 +58,79 @@ async function run () {
   }
 }
 
-async function pruneDirectory (directory: string, onlyOwned = false) {
+type ExistFun = (file: string) => Promise<boolean>
+async function pruneDirectory (directory: string, existFun: ExistFun) {
   const files = await readdir(directory)
 
   const toDelete: string[] = []
-  for (const file of files) {
+  await Bluebird.map(files, async file => {
+    if (await existFun(file) !== true) {
+      toDelete.push(join(directory, file))
+    }
+  }, { concurrency: 20 })
+
+  return toDelete
+}
+
+function doesVideoExist (keepOnlyOwned: boolean) {
+  return async (file: string) => {
     const uuid = getUUIDFromFilename(file)
-    let video: VideoModel
-    let localRedundancy: boolean
+    const video = await VideoModel.loadByUUID(uuid)
 
-    if (uuid) {
-      video = await VideoModel.loadByUUIDWithFile(uuid)
-      localRedundancy = await VideoRedundancyModel.isLocalByVideoUUIDExists(uuid)
-    }
+    return video && (keepOnlyOwned === false || video.isOwned())
+  }
+}
 
-    if (
-      !uuid ||
-      !video ||
-      (onlyOwned === true && (video.isOwned() === false && localRedundancy === false))
-    ) {
-      toDelete.push(join(directory, file))
+function doesThumbnailExist (keepOnlyOwned: boolean) {
+  return async (file: string) => {
+    const thumbnail = await ThumbnailModel.loadByName(file)
+    if (!thumbnail) return false
+
+    if (keepOnlyOwned) {
+      const video = await VideoModel.load(thumbnail.videoId)
+      if (video.isOwned() === false) return false
     }
+
+    return true
   }
+}
 
-  return toDelete
+async function doesAvatarExist (file: string) {
+  const avatar = await AvatarModel.loadByName(file)
+
+  return !!avatar
+}
+
+async function doesRedundancyExist (file: string) {
+  const uuid = getUUIDFromFilename(file)
+  const video = await VideoModel.loadWithFiles(uuid)
+
+  if (!video) return false
+
+  const isPlaylist = file.includes('.') === false
+
+  if (isPlaylist) {
+    const p = video.getHLSPlaylist()
+    if (!p) return false
+
+    const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
+    return !!redundancy
+  }
+
+  const resolution = parseInt(file.split('-')[5], 10)
+  if (isNaN(resolution)) {
+    console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
+    return true
+  }
+
+  const videoFile = video.getFile(resolution)
+  if (!videoFile) {
+    console.error('Cannot find file of video %s - %d', video.url, resolution)
+    return true
+  }
+
+  const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
+  return !!redundancy
 }
 
 async function askConfirmation () {