X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=scripts%2Fprune-storage.ts;h=bb1e8e024041e74c4df41d72988db9bf040ea033;hb=10d2be9e64c9dce2ac181487ee8134a5b0b4cde3;hp=1973725c86ebda92de21a3b14f966fa18d63d5a5;hpb=a9729e21d1cc5046c1a788448e347b62e0c45c53;p=github%2FChocobozzz%2FPeerTube.git

diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts
index 1973725c8..bb1e8e024 100755
--- a/scripts/prune-storage.ts
+++ b/scripts/prune-storage.ts
@@ -1,11 +1,19 @@
-import * as prompt from 'prompt'
-import { createReadStream } from 'fs'
-import { join } from 'path'
-import { createInterface } from 'readline'
-import { readdirPromise, unlinkPromise } from '../server/helpers/core-utils'
-import { CONFIG } from '../server/initializers/constants'
+import { map } from 'bluebird'
+import { readdir, remove, stat } from 'fs-extra'
+import { uniq, values } from 'lodash'
+import { basename, join } from 'path'
+import { get, start } from 'prompt'
+import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
+import { VideoFileModel } from '@server/models/video/video-file'
+import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
+import { ThumbnailType } from '@shared/models'
+import { getUUIDFromFilename } from '../server/helpers/utils'
+import { CONFIG } from '../server/initializers/config'
+import { initDatabaseModels } from '../server/initializers/database'
+import { ActorImageModel } from '../server/models/actor/actor-image'
+import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
+import { ThumbnailModel } from '../server/models/video/thumbnail'
 import { VideoModel } from '../server/models/video/video'
-import { initDatabaseModels } from '../server/initializers'
 
 run()
   .then(() => process.exit(0))
@@ -15,19 +23,36 @@ run()
   })
 
 async function run () {
-  await initDatabaseModels(true)
+  const dirs = values(CONFIG.STORAGE)
 
-  const storageToPrune = [
-    CONFIG.STORAGE.VIDEOS_DIR,
-    CONFIG.STORAGE.PREVIEWS_DIR,
-    CONFIG.STORAGE.THUMBNAILS_DIR,
-    CONFIG.STORAGE.TORRENTS_DIR
-  ]
+  if (uniq(dirs).length !== dirs.length) {
+    console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
+    process.exit(0)
+  }
+
+  await initDatabaseModels(true)
 
   let toDelete: string[] = []
-  for (const directory of storageToPrune) {
-    toDelete = toDelete.concat(await pruneDirectory(directory))
-  }
+
+  console.log('Detecting files to remove, it could take a while...')
+
+  toDelete = toDelete.concat(
+    await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
+
+    await pruneDirectory(HLS_STREAMING_PLAYLIST_DIRECTORY, doesHLSPlaylistExist()),
+
+    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
+
+    await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
+
+    await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
+    await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
+
+    await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES, doesActorImageExist)
+  )
+
+  const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
+  toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
 
   if (toDelete.length === 0) {
     console.log('No files to delete.')
@@ -41,7 +66,7 @@ async function run () {
     console.log('Processing delete...\n')
 
     for (const path of toDelete) {
-      await unlinkPromise(path)
+      await remove(path)
     }
 
     console.log('Done!')
@@ -50,47 +75,98 @@ async function run () {
   }
 }
 
-async function pruneDirectory (directory: string) {
-  const files = await readdirPromise(directory)
+type ExistFun = (file: string) => Promise<boolean>
+async function pruneDirectory (directory: string, existFun: ExistFun) {
+  const files = await readdir(directory)
 
   const toDelete: string[] = []
-  for (const file of files) {
-    const uuid = getUUIDFromFilename(file)
-    let video: VideoModel
+  await map(files, async file => {
+    const filePath = join(directory, file)
 
-    if (uuid) video = await VideoModel.loadByUUID(uuid)
+    if (await existFun(filePath) !== true) {
+      toDelete.push(filePath)
+    }
+  }, { concurrency: 20 })
 
-    if (!uuid || !video) toDelete.push(join(directory, file))
+  return toDelete
+}
+
+function doesWebTorrentFileExist () {
+  return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
+}
+
+function doesHLSPlaylistExist () {
+  return (hlsPath: string) => VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
+}
+
+function doesTorrentFileExist () {
+  return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
+}
+
+function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
+  return async (filePath: string) => {
+    const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
+    if (!thumbnail) return false
+
+    if (keepOnlyOwned) {
+      const video = await VideoModel.load(thumbnail.videoId)
+      if (video.isOwned() === false) return false
+    }
+
+    return true
   }
+}
 
-  return toDelete
+async function doesActorImageExist (filePath: string) {
+  const image = await ActorImageModel.loadByName(basename(filePath))
+
+  return !!image
 }
 
-function getUUIDFromFilename (filename: string) {
-  const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
-  const result = filename.match(regex)
+async function doesRedundancyExist (filePath: string) {
+  const isPlaylist = (await stat(filePath)).isDirectory()
+
+  if (isPlaylist) {
+    // Don't delete HLS directory
+    if (filePath === HLS_REDUNDANCY_DIRECTORY) return true
+
+    const uuid = getUUIDFromFilename(filePath)
+    const video = await VideoModel.loadWithFiles(uuid)
+    if (!video) return false
 
-  if (!result || Array.isArray(result) === false) return null
+    const p = video.getHLSPlaylist()
+    if (!p) return false
 
-  return result[0]
+    const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
+    return !!redundancy
+  }
+
+  const file = await VideoFileModel.loadByFilename(basename(filePath))
+  if (!file) return false
+
+  const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
+  return !!redundancy
 }
 
 async function askConfirmation () {
   return new Promise((res, rej) => {
-    prompt.start()
+    start()
     const schema = {
       properties: {
         confirm: {
           type: 'string',
-          description: 'Are you sure you want to delete these files? Please check carefully',
+          description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
+            ' Notice PeerTube must have been stopped when your ran this script.' +
+            ' Can we delete these files?',
           default: 'n',
           required: true
         }
       }
     }
-    prompt.get(schema, function (err, result) {
+    get(schema, function (err, result) {
       if (err) return rej(err)
-      return res(result.confirm && result.confirm.match(/y/) !== null)
+
+      return res(result.confirm?.match(/y/) !== null)
     })
   })
 }