scripts/prune-storage.ts

   1 import { registerTSPaths } from '../server/helpers/register-ts-paths'
   2 registerTSPaths()
   3
   4 import * as prompt from 'prompt'
   5 import { join } from 'path'
   6 import { CONFIG } from '../server/initializers/config'
   7 import { VideoModel } from '../server/models/video/video'
   8 import { initDatabaseModels } from '../server/initializers/database'
   9 import { readdir, remove } from 'fs-extra'
  10 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
  11 import * as Bluebird from 'bluebird'
  12 import { getUUIDFromFilename } from '../server/helpers/utils'
  13 import { ThumbnailModel } from '../server/models/video/thumbnail'
  14 import { ActorImageModel } from '../server/models/actor/actor-image'
  15 import { uniq, values } from 'lodash'
  16 import { ThumbnailType } from '@shared/models'
  17
  18 run()
  19   .then(() => process.exit(0))
  20   .catch(err => {
  21     console.error(err)
  22     process.exit(-1)
  23   })
  24
  25 async function run () {
  26   const dirs = values(CONFIG.STORAGE)
  27
  28   if (uniq(dirs).length !== dirs.length) {
  29     console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
  30     process.exit(0)
  31   }
  32
  33   await initDatabaseModels(true)
  34
  35   let toDelete: string[] = []
  36
  37   console.log('Detecting files to remove, it could take a while...')
  38
  39   toDelete = toDelete.concat(
  40     await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
  41     await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
  42
  43     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
  44
  45     await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
  46     await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
  47
  48     await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES, doesActorImageExist)
  49   )
  50
  51   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  52   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
  53
  54   if (toDelete.length === 0) {
  55     console.log('No files to delete.')
  56     return
  57   }
  58
  59   console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
  60
  61   const res = await askConfirmation()
  62   if (res === true) {
  63     console.log('Processing delete...\n')
  64
  65     for (const path of toDelete) {
  66       await remove(path)
  67     }
  68
  69     console.log('Done!')
  70   } else {
  71     console.log('Exiting without deleting files.')
  72   }
  73 }
  74
  75 type ExistFun = (file: string) => Promise<boolean>
  76 async function pruneDirectory (directory: string, existFun: ExistFun) {
  77   const files = await readdir(directory)
  78
  79   const toDelete: string[] = []
  80   await Bluebird.map(files, async file => {
  81     if (await existFun(file) !== true) {
  82       toDelete.push(join(directory, file))
  83     }
  84   }, { concurrency: 20 })
  85
  86   return toDelete
  87 }
  88
  89 function doesVideoExist (keepOnlyOwned: boolean) {
  90   return async (file: string) => {
  91     const uuid = getUUIDFromFilename(file)
  92     const video = await VideoModel.load(uuid)
  93
  94     return video && (keepOnlyOwned === false || video.isOwned())
  95   }
  96 }
  97
  98 function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
  99   return async (file: string) => {
 100     const thumbnail = await ThumbnailModel.loadByFilename(file, type)
 101     if (!thumbnail) return false
 102
 103     if (keepOnlyOwned) {
 104       const video = await VideoModel.load(thumbnail.videoId)
 105       if (video.isOwned() === false) return false
 106     }
 107
 108     return true
 109   }
 110 }
 111
 112 async function doesActorImageExist (file: string) {
 113   const image = await ActorImageModel.loadByName(file)
 114
 115   return !!image
 116 }
 117
 118 async function doesRedundancyExist (file: string) {
 119   const uuid = getUUIDFromFilename(file)
 120   const video = await VideoModel.loadWithFiles(uuid)
 121
 122   if (!video) return false
 123
 124   const isPlaylist = file.includes('.') === false
 125
 126   if (isPlaylist) {
 127     const p = video.getHLSPlaylist()
 128     if (!p) return false
 129
 130     const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
 131     return !!redundancy
 132   }
 133
 134   const resolution = parseInt(file.split('-')[5], 10)
 135   if (isNaN(resolution)) {
 136     console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
 137     return true
 138   }
 139
 140   const videoFile = video.getWebTorrentFile(resolution)
 141   if (!videoFile) {
 142     console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution)
 143     return true
 144   }
 145
 146   const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
 147   return !!redundancy
 148 }
 149
 150 async function askConfirmation () {
 151   return new Promise((res, rej) => {
 152     prompt.start()
 153     const schema = {
 154       properties: {
 155         confirm: {
 156           type: 'string',
 157           description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
 158             ' Notice PeerTube must have been stopped when your ran this script.' +
 159             ' Can we delete these files?',
 160           default: 'n',
 161           required: true
 162         }
 163       }
 164     }
 165     prompt.get(schema, function (err, result) {
 166       if (err) return rej(err)
 167
 168       return res(result.confirm?.match(/y/) !== null)
 169     })
 170   })
 171 }