scripts/prune-storage.ts

   1 import { registerTSPaths } from '../server/helpers/register-ts-paths'
   2 registerTSPaths()
   3
   4 import { start, get } from 'prompt'
   5 import { join, basename } from 'path'
   6 import { CONFIG } from '../server/initializers/config'
   7 import { VideoModel } from '../server/models/video/video'
   8 import { initDatabaseModels } from '../server/initializers/database'
   9 import { readdir, remove, stat } from 'fs-extra'
  10 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
  11 import { map } from 'bluebird'
  12 import { getUUIDFromFilename } from '../server/helpers/utils'
  13 import { ThumbnailModel } from '../server/models/video/thumbnail'
  14 import { ActorImageModel } from '../server/models/actor/actor-image'
  15 import { uniq, values } from 'lodash'
  16 import { ThumbnailType } from '@shared/models'
  17 import { VideoFileModel } from '@server/models/video/video-file'
  18 import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
  19 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
  20
  21 run()
  22   .then(() => process.exit(0))
  23   .catch(err => {
  24     console.error(err)
  25     process.exit(-1)
  26   })
  27
  28 async function run () {
  29   const dirs = values(CONFIG.STORAGE)
  30
  31   if (uniq(dirs).length !== dirs.length) {
  32     console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
  33     process.exit(0)
  34   }
  35
  36   await initDatabaseModels(true)
  37
  38   let toDelete: string[] = []
  39
  40   console.log('Detecting files to remove, it could take a while...')
  41
  42   toDelete = toDelete.concat(
  43     await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
  44
  45     await pruneDirectory(HLS_STREAMING_PLAYLIST_DIRECTORY, doesHLSPlaylistExist()),
  46
  47     await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
  48
  49     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
  50
  51     await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
  52     await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
  53
  54     await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES, doesActorImageExist)
  55   )
  56
  57   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  58   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
  59
  60   if (toDelete.length === 0) {
  61     console.log('No files to delete.')
  62     return
  63   }
  64
  65   console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
  66
  67   const res = await askConfirmation()
  68   if (res === true) {
  69     console.log('Processing delete...\n')
  70
  71     for (const path of toDelete) {
  72       await remove(path)
  73     }
  74
  75     console.log('Done!')
  76   } else {
  77     console.log('Exiting without deleting files.')
  78   }
  79 }
  80
  81 type ExistFun = (file: string) => Promise<boolean>
  82 async function pruneDirectory (directory: string, existFun: ExistFun) {
  83   const files = await readdir(directory)
  84
  85   const toDelete: string[] = []
  86   await map(files, async file => {
  87     const filePath = join(directory, file)
  88
  89     if (await existFun(filePath) !== true) {
  90       toDelete.push(filePath)
  91     }
  92   }, { concurrency: 20 })
  93
  94   return toDelete
  95 }
  96
  97 function doesWebTorrentFileExist () {
  98   return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
  99 }
 100
 101 function doesHLSPlaylistExist () {
 102   return (hlsPath: string) => VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
 103 }
 104
 105 function doesTorrentFileExist () {
 106   return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
 107 }
 108
 109 function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
 110   return async (filePath: string) => {
 111     const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
 112     if (!thumbnail) return false
 113
 114     if (keepOnlyOwned) {
 115       const video = await VideoModel.load(thumbnail.videoId)
 116       if (video.isOwned() === false) return false
 117     }
 118
 119     return true
 120   }
 121 }
 122
 123 async function doesActorImageExist (filePath: string) {
 124   const image = await ActorImageModel.loadByName(basename(filePath))
 125
 126   return !!image
 127 }
 128
 129 async function doesRedundancyExist (filePath: string) {
 130   const isPlaylist = (await stat(filePath)).isDirectory()
 131
 132   if (isPlaylist) {
 133     // Don't delete HLS directory
 134     if (filePath === HLS_REDUNDANCY_DIRECTORY) return true
 135
 136     const uuid = getUUIDFromFilename(filePath)
 137     const video = await VideoModel.loadWithFiles(uuid)
 138     if (!video) return false
 139
 140     const p = video.getHLSPlaylist()
 141     if (!p) return false
 142
 143     const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
 144     return !!redundancy
 145   }
 146
 147   const file = await VideoFileModel.loadByFilename(basename(filePath))
 148   if (!file) return false
 149
 150   const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
 151   return !!redundancy
 152 }
 153
 154 async function askConfirmation () {
 155   return new Promise((res, rej) => {
 156     start()
 157     const schema = {
 158       properties: {
 159         confirm: {
 160           type: 'string',
 161           description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
 162             ' Notice PeerTube must have been stopped when your ran this script.' +
 163             ' Can we delete these files?',
 164           default: 'n',
 165           required: true
 166         }
 167       }
 168     }
 169     get(schema, function (err, result) {
 170       if (err) return rej(err)
 171
 172       return res(result.confirm?.match(/y/) !== null)
 173     })
 174   })
 175 }