scripts/prune-storage.ts

   1 import { registerTSPaths } from '../server/helpers/register-ts-paths'
   2 registerTSPaths()
   3
   4 import { start, get } from 'prompt'
   5 import { join, basename } from 'path'
   6 import { CONFIG } from '../server/initializers/config'
   7 import { VideoModel } from '../server/models/video/video'
   8 import { initDatabaseModels } from '../server/initializers/database'
   9 import { readdir, remove, stat } from 'fs-extra'
  10 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
  11 import { map } from 'bluebird'
  12 import { getUUIDFromFilename } from '../server/helpers/utils'
  13 import { ThumbnailModel } from '../server/models/video/thumbnail'
  14 import { ActorImageModel } from '../server/models/actor/actor-image'
  15 import { uniq, values } from 'lodash'
  16 import { ThumbnailType } from '@shared/models'
  17 import { VideoFileModel } from '@server/models/video/video-file'
  18
  19 run()
  20   .then(() => process.exit(0))
  21   .catch(err => {
  22     console.error(err)
  23     process.exit(-1)
  24   })
  25
  26 async function run () {
  27   const dirs = values(CONFIG.STORAGE)
  28
  29   if (uniq(dirs).length !== dirs.length) {
  30     console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
  31     process.exit(0)
  32   }
  33
  34   await initDatabaseModels(true)
  35
  36   let toDelete: string[] = []
  37
  38   console.log('Detecting files to remove, it could take a while...')
  39
  40   toDelete = toDelete.concat(
  41     await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
  42     await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
  43
  44     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
  45
  46     await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
  47     await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
  48
  49     await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES, doesActorImageExist)
  50   )
  51
  52   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  53   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
  54
  55   if (toDelete.length === 0) {
  56     console.log('No files to delete.')
  57     return
  58   }
  59
  60   console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
  61
  62   const res = await askConfirmation()
  63   if (res === true) {
  64     console.log('Processing delete...\n')
  65
  66     for (const path of toDelete) {
  67       await remove(path)
  68     }
  69
  70     console.log('Done!')
  71   } else {
  72     console.log('Exiting without deleting files.')
  73   }
  74 }
  75
  76 type ExistFun = (file: string) => Promise<boolean>
  77 async function pruneDirectory (directory: string, existFun: ExistFun) {
  78   const files = await readdir(directory)
  79
  80   const toDelete: string[] = []
  81   await map(files, async file => {
  82     const filePath = join(directory, file)
  83
  84     if (await existFun(filePath) !== true) {
  85       toDelete.push(filePath)
  86     }
  87   }, { concurrency: 20 })
  88
  89   return toDelete
  90 }
  91
  92 function doesWebTorrentFileExist () {
  93   return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
  94 }
  95
  96 function doesTorrentFileExist () {
  97   return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
  98 }
  99
 100 function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
 101   return async (filePath: string) => {
 102     const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
 103     if (!thumbnail) return false
 104
 105     if (keepOnlyOwned) {
 106       const video = await VideoModel.load(thumbnail.videoId)
 107       if (video.isOwned() === false) return false
 108     }
 109
 110     return true
 111   }
 112 }
 113
 114 async function doesActorImageExist (filePath: string) {
 115   const image = await ActorImageModel.loadByName(basename(filePath))
 116
 117   return !!image
 118 }
 119
 120 async function doesRedundancyExist (filePath: string) {
 121   const isPlaylist = (await stat(filePath)).isDirectory()
 122
 123   if (isPlaylist) {
 124     const uuid = getUUIDFromFilename(filePath)
 125     const video = await VideoModel.loadWithFiles(uuid)
 126     if (!video) return false
 127
 128     const p = video.getHLSPlaylist()
 129     if (!p) return false
 130
 131     const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
 132     return !!redundancy
 133   }
 134
 135   const file = await VideoFileModel.loadByFilename(basename(filePath))
 136   if (!file) return false
 137
 138   const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
 139   return !!redundancy
 140 }
 141
 142 async function askConfirmation () {
 143   return new Promise((res, rej) => {
 144     start()
 145     const schema = {
 146       properties: {
 147         confirm: {
 148           type: 'string',
 149           description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
 150             ' Notice PeerTube must have been stopped when your ran this script.' +
 151             ' Can we delete these files?',
 152           default: 'n',
 153           required: true
 154         }
 155       }
 156     }
 157     get(schema, function (err, result) {
 158       if (err) return rej(err)
 159
 160       return res(result.confirm?.match(/y/) !== null)
 161     })
 162   })
 163 }