scripts/prune-storage.ts

   1 import { registerTSPaths } from '../server/helpers/register-ts-paths'
   2 registerTSPaths()
   3
   4 import * as prompt from 'prompt'
   5 import { join } from 'path'
   6 import { CONFIG } from '../server/initializers/config'
   7 import { VideoModel } from '../server/models/video/video'
   8 import { initDatabaseModels } from '../server/initializers'
   9 import { readdir, remove } from 'fs-extra'
  10 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
  11 import * as Bluebird from 'bluebird'
  12 import { getUUIDFromFilename } from '../server/helpers/utils'
  13 import { ThumbnailModel } from '../server/models/video/thumbnail'
  14 import { AvatarModel } from '../server/models/avatar/avatar'
  15 import { uniq, values } from 'lodash'
  16
  17 run()
  18   .then(() => process.exit(0))
  19   .catch(err => {
  20     console.error(err)
  21     process.exit(-1)
  22   })
  23
  24 async function run () {
  25   const dirs = values(CONFIG.STORAGE)
  26
  27   if (uniq(dirs).length !== dirs.length) {
  28     console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
  29     process.exit(0)
  30   }
  31
  32   await initDatabaseModels(true)
  33
  34   let toDelete: string[] = []
  35
  36   toDelete = toDelete.concat(
  37     await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
  38     await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
  39
  40     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
  41
  42     await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
  43     await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
  44
  45     await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
  46   )
  47
  48   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  49   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
  50
  51   if (toDelete.length === 0) {
  52     console.log('No files to delete.')
  53     return
  54   }
  55
  56   console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
  57
  58   const res = await askConfirmation()
  59   if (res === true) {
  60     console.log('Processing delete...\n')
  61
  62     for (const path of toDelete) {
  63       await remove(path)
  64     }
  65
  66     console.log('Done!')
  67   } else {
  68     console.log('Exiting without deleting files.')
  69   }
  70 }
  71
  72 type ExistFun = (file: string) => Promise<boolean>
  73 async function pruneDirectory (directory: string, existFun: ExistFun) {
  74   const files = await readdir(directory)
  75
  76   const toDelete: string[] = []
  77   await Bluebird.map(files, async file => {
  78     if (await existFun(file) !== true) {
  79       toDelete.push(join(directory, file))
  80     }
  81   }, { concurrency: 20 })
  82
  83   return toDelete
  84 }
  85
  86 function doesVideoExist (keepOnlyOwned: boolean) {
  87   return async (file: string) => {
  88     const uuid = getUUIDFromFilename(file)
  89     const video = await VideoModel.loadByUUID(uuid)
  90
  91     return video && (keepOnlyOwned === false || video.isOwned())
  92   }
  93 }
  94
  95 function doesThumbnailExist (keepOnlyOwned: boolean) {
  96   return async (file: string) => {
  97     const thumbnail = await ThumbnailModel.loadByName(file)
  98     if (!thumbnail) return false
  99
 100     if (keepOnlyOwned) {
 101       const video = await VideoModel.load(thumbnail.videoId)
 102       if (video.isOwned() === false) return false
 103     }
 104
 105     return true
 106   }
 107 }
 108
 109 async function doesAvatarExist (file: string) {
 110   const avatar = await AvatarModel.loadByName(file)
 111
 112   return !!avatar
 113 }
 114
 115 async function doesRedundancyExist (file: string) {
 116   const uuid = getUUIDFromFilename(file)
 117   const video = await VideoModel.loadWithFiles(uuid)
 118
 119   if (!video) return false
 120
 121   const isPlaylist = file.includes('.') === false
 122
 123   if (isPlaylist) {
 124     const p = video.getHLSPlaylist()
 125     if (!p) return false
 126
 127     const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
 128     return !!redundancy
 129   }
 130
 131   const resolution = parseInt(file.split('-')[5], 10)
 132   if (isNaN(resolution)) {
 133     console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
 134     return true
 135   }
 136
 137   const videoFile = video.getWebTorrentFile(resolution)
 138   if (!videoFile) {
 139     console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution)
 140     return true
 141   }
 142
 143   const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
 144   return !!redundancy
 145 }
 146
 147 async function askConfirmation () {
 148   return new Promise((res, rej) => {
 149     prompt.start()
 150     const schema = {
 151       properties: {
 152         confirm: {
 153           type: 'string',
 154           description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
 155             ' Notice PeerTube must have been stopped when your ran this script.' +
 156             ' Can we delete these files?',
 157           default: 'n',
 158           required: true
 159         }
 160       }
 161     }
 162     prompt.get(schema, function (err, result) {
 163       if (err) return rej(err)
 164       return res(result.confirm && result.confirm.match(/y/) !== null)
 165     })
 166   })
 167 }