scripts/prune-storage.ts

   1 import { map } from 'bluebird'
   2 import { readdir, remove, stat } from 'fs-extra'
   3 import { basename, join } from 'path'
   4 import { get, start } from 'prompt'
   5 import { DIRECTORIES } from '@server/initializers/constants'
   6 import { VideoFileModel } from '@server/models/video/video-file'
   7 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
   8 import { uniqify } from '@shared/core-utils'
   9 import { ThumbnailType } from '@shared/models'
  10 import { getUUIDFromFilename } from '../server/helpers/utils'
  11 import { CONFIG } from '../server/initializers/config'
  12 import { initDatabaseModels } from '../server/initializers/database'
  13 import { ActorImageModel } from '../server/models/actor/actor-image'
  14 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
  15 import { ThumbnailModel } from '../server/models/video/thumbnail'
  16 import { VideoModel } from '../server/models/video/video'
  17
  18 run()
  19   .then(() => process.exit(0))
  20   .catch(err => {
  21     console.error(err)
  22     process.exit(-1)
  23   })
  24
  25 async function run () {
  26   const dirs = Object.values(CONFIG.STORAGE)
  27
  28   if (uniqify(dirs).length !== dirs.length) {
  29     console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
  30     process.exit(0)
  31   }
  32
  33   await initDatabaseModels(true)
  34
  35   let toDelete: string[] = []
  36
  37   console.log('Detecting files to remove, it could take a while...')
  38
  39   toDelete = toDelete.concat(
  40     await pruneDirectory(DIRECTORIES.VIDEOS.PUBLIC, doesWebTorrentFileExist()),
  41     await pruneDirectory(DIRECTORIES.VIDEOS.PRIVATE, doesWebTorrentFileExist()),
  42
  43     await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()),
  44     await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()),
  45
  46     await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
  47
  48     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
  49
  50     await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
  51     await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
  52
  53     await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES, doesActorImageExist)
  54   )
  55
  56   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  57   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
  58
  59   if (toDelete.length === 0) {
  60     console.log('No files to delete.')
  61     return
  62   }
  63
  64   console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
  65
  66   const res = await askConfirmation()
  67   if (res === true) {
  68     console.log('Processing delete...\n')
  69
  70     for (const path of toDelete) {
  71       await remove(path)
  72     }
  73
  74     console.log('Done!')
  75   } else {
  76     console.log('Exiting without deleting files.')
  77   }
  78 }
  79
  80 type ExistFun = (file: string) => Promise<boolean> | boolean
  81 async function pruneDirectory (directory: string, existFun: ExistFun) {
  82   const files = await readdir(directory)
  83
  84   const toDelete: string[] = []
  85   await map(files, async file => {
  86     const filePath = join(directory, file)
  87
  88     if (await existFun(filePath) !== true) {
  89       toDelete.push(filePath)
  90     }
  91   }, { concurrency: 20 })
  92
  93   return toDelete
  94 }
  95
  96 function doesWebTorrentFileExist () {
  97   return (filePath: string) => {
  98     // Don't delete private directory
  99     if (filePath === DIRECTORIES.VIDEOS.PRIVATE) return true
 100
 101     return VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
 102   }
 103 }
 104
 105 function doesHLSPlaylistExist () {
 106   return (hlsPath: string) => {
 107     // Don't delete private directory
 108     if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
 109
 110     return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
 111   }
 112 }
 113
 114 function doesTorrentFileExist () {
 115   return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
 116 }
 117
 118 function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
 119   return async (filePath: string) => {
 120     const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
 121     if (!thumbnail) return false
 122
 123     if (keepOnlyOwned) {
 124       const video = await VideoModel.load(thumbnail.videoId)
 125       if (video.isOwned() === false) return false
 126     }
 127
 128     return true
 129   }
 130 }
 131
 132 async function doesActorImageExist (filePath: string) {
 133   const image = await ActorImageModel.loadByName(basename(filePath))
 134
 135   return !!image
 136 }
 137
 138 async function doesRedundancyExist (filePath: string) {
 139   const isPlaylist = (await stat(filePath)).isDirectory()
 140
 141   if (isPlaylist) {
 142     // Don't delete HLS redundancy directory
 143     if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
 144
 145     const uuid = getUUIDFromFilename(filePath)
 146     const video = await VideoModel.loadWithFiles(uuid)
 147     if (!video) return false
 148
 149     const p = video.getHLSPlaylist()
 150     if (!p) return false
 151
 152     const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
 153     return !!redundancy
 154   }
 155
 156   const file = await VideoFileModel.loadByFilename(basename(filePath))
 157   if (!file) return false
 158
 159   const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
 160   return !!redundancy
 161 }
 162
 163 async function askConfirmation () {
 164   return new Promise((res, rej) => {
 165     start()
 166     const schema = {
 167       properties: {
 168         confirm: {
 169           type: 'string',
 170           description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
 171             ' Notice PeerTube must have been stopped when your ran this script.' +
 172             ' Can we delete these files?',
 173           default: 'n',
 174           required: true
 175         }
 176       }
 177     }
 178     get(schema, function (err, result) {
 179       if (err) return rej(err)
 180
 181       return res(result.confirm?.match(/y/) !== null)
 182     })
 183   })
 184 }