scripts/prune-storage.ts

   1 import * as prompt from 'prompt'
   2 import { join } from 'path'
   3 import { CONFIG } from '../server/initializers/config'
   4 import { VideoModel } from '../server/models/video/video'
   5 import { initDatabaseModels } from '../server/initializers'
   6 import { readdir, remove } from 'fs-extra'
   7 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
   8 import * as Bluebird from 'bluebird'
   9 import { getUUIDFromFilename } from '../server/helpers/utils'
  10 import { ThumbnailModel } from '../server/models/video/thumbnail'
  11 import { AvatarModel } from '../server/models/avatar/avatar'
  12
  13 run()
  14   .then(() => process.exit(0))
  15   .catch(err => {
  16     console.error(err)
  17     process.exit(-1)
  18   })
  19
  20 async function run () {
  21   await initDatabaseModels(true)
  22
  23   let toDelete: string[] = []
  24
  25   toDelete = toDelete.concat(
  26     await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
  27     await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
  28
  29     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
  30
  31     await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
  32     await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
  33
  34     await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
  35   )
  36
  37   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  38   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
  39
  40   if (toDelete.length === 0) {
  41     console.log('No files to delete.')
  42     return
  43   }
  44
  45   console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
  46
  47   const res = await askConfirmation()
  48   if (res === true) {
  49     console.log('Processing delete...\n')
  50
  51     for (const path of toDelete) {
  52       await remove(path)
  53     }
  54
  55     console.log('Done!')
  56   } else {
  57     console.log('Exiting without deleting files.')
  58   }
  59 }
  60
  61 type ExistFun = (file: string) => Promise<boolean>
  62 async function pruneDirectory (directory: string, existFun: ExistFun) {
  63   const files = await readdir(directory)
  64
  65   const toDelete: string[] = []
  66   await Bluebird.map(files, async file => {
  67     if (await existFun(file) !== true) {
  68       toDelete.push(join(directory, file))
  69     }
  70   }, { concurrency: 20 })
  71
  72   return toDelete
  73 }
  74
  75 function doesVideoExist (keepOnlyOwned: boolean) {
  76   return async (file: string) => {
  77     const uuid = getUUIDFromFilename(file)
  78     const video = await VideoModel.loadByUUID(uuid)
  79
  80     return video && (keepOnlyOwned === false || video.isOwned())
  81   }
  82 }
  83
  84 function doesThumbnailExist (keepOnlyOwned: boolean) {
  85   return async (file: string) => {
  86     const thumbnail = await ThumbnailModel.loadByName(file)
  87     if (!thumbnail) return false
  88
  89     if (keepOnlyOwned) {
  90       const video = await VideoModel.load(thumbnail.videoId)
  91       if (video.isOwned() === false) return false
  92     }
  93
  94     return true
  95   }
  96 }
  97
  98 async function doesAvatarExist (file: string) {
  99   const avatar = await AvatarModel.loadByName(file)
 100
 101   return !!avatar
 102 }
 103
 104 async function doesRedundancyExist (file: string) {
 105   const uuid = getUUIDFromFilename(file)
 106   const video = await VideoModel.loadWithFiles(uuid)
 107
 108   if (!video) return false
 109
 110   const isPlaylist = file.includes('.') === false
 111
 112   if (isPlaylist) {
 113     const p = video.getHLSPlaylist()
 114     if (!p) return false
 115
 116     const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
 117     return !!redundancy
 118   }
 119
 120   const resolution = parseInt(file.split('-')[5], 10)
 121   if (isNaN(resolution)) {
 122     console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
 123     return true
 124   }
 125
 126   const videoFile = video.getFile(resolution)
 127   if (!videoFile) {
 128     console.error('Cannot find file of video %s - %d', video.url, resolution)
 129     return true
 130   }
 131
 132   const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
 133   return !!redundancy
 134 }
 135
 136 async function askConfirmation () {
 137   return new Promise((res, rej) => {
 138     prompt.start()
 139     const schema = {
 140       properties: {
 141         confirm: {
 142           type: 'string',
 143           description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
 144             ' Notice PeerTube must have been stopped when your ran this script.' +
 145             ' Can we delete these files?',
 146           default: 'n',
 147           required: true
 148         }
 149       }
 150     }
 151     prompt.get(schema, function (err, result) {
 152       if (err) return rej(err)
 153       return res(result.confirm && result.confirm.match(/y/) !== null)
 154     })
 155   })
 156 }