scripts/prune-storage.ts

   1 import * as prompt from 'prompt'
   2 import { join } from 'path'
   3 import { CONFIG } from '../server/initializers/config'
   4 import { VideoModel } from '../server/models/video/video'
   5 import { initDatabaseModels } from '../server/initializers'
   6 import { readdir, remove } from 'fs-extra'
   7 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
   8 import * as Bluebird from 'bluebird'
   9 import { getUUIDFromFilename } from '../server/helpers/utils'
  10 import { ThumbnailModel } from '../server/models/video/thumbnail'
  11 import { AvatarModel } from '../server/models/avatar/avatar'
  12 import { uniq, values } from 'lodash'
  13
  14 run()
  15   .then(() => process.exit(0))
  16   .catch(err => {
  17     console.error(err)
  18     process.exit(-1)
  19   })
  20
  21 async function run () {
  22   const dirs = values(CONFIG.STORAGE)
  23
  24   if (uniq(dirs).length !== dirs.length) {
  25     console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
  26     process.exit(0)
  27   }
  28
  29   await initDatabaseModels(true)
  30
  31   let toDelete: string[] = []
  32
  33   toDelete = toDelete.concat(
  34     await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
  35     await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
  36
  37     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
  38
  39     await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
  40     await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
  41
  42     await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
  43   )
  44
  45   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  46   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
  47
  48   if (toDelete.length === 0) {
  49     console.log('No files to delete.')
  50     return
  51   }
  52
  53   console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
  54
  55   const res = await askConfirmation()
  56   if (res === true) {
  57     console.log('Processing delete...\n')
  58
  59     for (const path of toDelete) {
  60       await remove(path)
  61     }
  62
  63     console.log('Done!')
  64   } else {
  65     console.log('Exiting without deleting files.')
  66   }
  67 }
  68
  69 type ExistFun = (file: string) => Promise<boolean>
  70 async function pruneDirectory (directory: string, existFun: ExistFun) {
  71   const files = await readdir(directory)
  72
  73   const toDelete: string[] = []
  74   await Bluebird.map(files, async file => {
  75     if (await existFun(file) !== true) {
  76       toDelete.push(join(directory, file))
  77     }
  78   }, { concurrency: 20 })
  79
  80   return toDelete
  81 }
  82
  83 function doesVideoExist (keepOnlyOwned: boolean) {
  84   return async (file: string) => {
  85     const uuid = getUUIDFromFilename(file)
  86     const video = await VideoModel.loadByUUID(uuid)
  87
  88     return video && (keepOnlyOwned === false || video.isOwned())
  89   }
  90 }
  91
  92 function doesThumbnailExist (keepOnlyOwned: boolean) {
  93   return async (file: string) => {
  94     const thumbnail = await ThumbnailModel.loadByName(file)
  95     if (!thumbnail) return false
  96
  97     if (keepOnlyOwned) {
  98       const video = await VideoModel.load(thumbnail.videoId)
  99       if (video.isOwned() === false) return false
 100     }
 101
 102     return true
 103   }
 104 }
 105
 106 async function doesAvatarExist (file: string) {
 107   const avatar = await AvatarModel.loadByName(file)
 108
 109   return !!avatar
 110 }
 111
 112 async function doesRedundancyExist (file: string) {
 113   const uuid = getUUIDFromFilename(file)
 114   const video = await VideoModel.loadWithFiles(uuid)
 115
 116   if (!video) return false
 117
 118   const isPlaylist = file.includes('.') === false
 119
 120   if (isPlaylist) {
 121     const p = video.getHLSPlaylist()
 122     if (!p) return false
 123
 124     const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
 125     return !!redundancy
 126   }
 127
 128   const resolution = parseInt(file.split('-')[5], 10)
 129   if (isNaN(resolution)) {
 130     console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
 131     return true
 132   }
 133
 134   const videoFile = video.getFile(resolution)
 135   if (!videoFile) {
 136     console.error('Cannot find file of video %s - %d', video.url, resolution)
 137     return true
 138   }
 139
 140   const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
 141   return !!redundancy
 142 }
 143
 144 async function askConfirmation () {
 145   return new Promise((res, rej) => {
 146     prompt.start()
 147     const schema = {
 148       properties: {
 149         confirm: {
 150           type: 'string',
 151           description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
 152             ' Notice PeerTube must have been stopped when your ran this script.' +
 153             ' Can we delete these files?',
 154           default: 'n',
 155           required: true
 156         }
 157       }
 158     }
 159     prompt.get(schema, function (err, result) {
 160       if (err) return rej(err)
 161       return res(result.confirm && result.confirm.match(/y/) !== null)
 162     })
 163   })
 164 }