]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blob - scripts/prune-storage.ts
bb1e8e024041e74c4df41d72988db9bf040ea033
[github/Chocobozzz/PeerTube.git] / scripts / prune-storage.ts
1 import { map } from 'bluebird'
2 import { readdir, remove, stat } from 'fs-extra'
3 import { uniq, values } from 'lodash'
4 import { basename, join } from 'path'
5 import { get, start } from 'prompt'
6 import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
7 import { VideoFileModel } from '@server/models/video/video-file'
8 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
9 import { ThumbnailType } from '@shared/models'
10 import { getUUIDFromFilename } from '../server/helpers/utils'
11 import { CONFIG } from '../server/initializers/config'
12 import { initDatabaseModels } from '../server/initializers/database'
13 import { ActorImageModel } from '../server/models/actor/actor-image'
14 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
15 import { ThumbnailModel } from '../server/models/video/thumbnail'
16 import { VideoModel } from '../server/models/video/video'
17
18 run()
19 .then(() => process.exit(0))
20 .catch(err => {
21 console.error(err)
22 process.exit(-1)
23 })
24
25 async function run () {
26 const dirs = values(CONFIG.STORAGE)
27
28 if (uniq(dirs).length !== dirs.length) {
29 console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
30 process.exit(0)
31 }
32
33 await initDatabaseModels(true)
34
35 let toDelete: string[] = []
36
37 console.log('Detecting files to remove, it could take a while...')
38
39 toDelete = toDelete.concat(
40 await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
41
42 await pruneDirectory(HLS_STREAMING_PLAYLIST_DIRECTORY, doesHLSPlaylistExist()),
43
44 await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
45
46 await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
47
48 await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
49 await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
50
51 await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES, doesActorImageExist)
52 )
53
54 const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
55 toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
56
57 if (toDelete.length === 0) {
58 console.log('No files to delete.')
59 return
60 }
61
62 console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
63
64 const res = await askConfirmation()
65 if (res === true) {
66 console.log('Processing delete...\n')
67
68 for (const path of toDelete) {
69 await remove(path)
70 }
71
72 console.log('Done!')
73 } else {
74 console.log('Exiting without deleting files.')
75 }
76 }
77
78 type ExistFun = (file: string) => Promise<boolean>
79 async function pruneDirectory (directory: string, existFun: ExistFun) {
80 const files = await readdir(directory)
81
82 const toDelete: string[] = []
83 await map(files, async file => {
84 const filePath = join(directory, file)
85
86 if (await existFun(filePath) !== true) {
87 toDelete.push(filePath)
88 }
89 }, { concurrency: 20 })
90
91 return toDelete
92 }
93
94 function doesWebTorrentFileExist () {
95 return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
96 }
97
98 function doesHLSPlaylistExist () {
99 return (hlsPath: string) => VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
100 }
101
102 function doesTorrentFileExist () {
103 return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
104 }
105
106 function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
107 return async (filePath: string) => {
108 const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
109 if (!thumbnail) return false
110
111 if (keepOnlyOwned) {
112 const video = await VideoModel.load(thumbnail.videoId)
113 if (video.isOwned() === false) return false
114 }
115
116 return true
117 }
118 }
119
120 async function doesActorImageExist (filePath: string) {
121 const image = await ActorImageModel.loadByName(basename(filePath))
122
123 return !!image
124 }
125
126 async function doesRedundancyExist (filePath: string) {
127 const isPlaylist = (await stat(filePath)).isDirectory()
128
129 if (isPlaylist) {
130 // Don't delete HLS directory
131 if (filePath === HLS_REDUNDANCY_DIRECTORY) return true
132
133 const uuid = getUUIDFromFilename(filePath)
134 const video = await VideoModel.loadWithFiles(uuid)
135 if (!video) return false
136
137 const p = video.getHLSPlaylist()
138 if (!p) return false
139
140 const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
141 return !!redundancy
142 }
143
144 const file = await VideoFileModel.loadByFilename(basename(filePath))
145 if (!file) return false
146
147 const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
148 return !!redundancy
149 }
150
151 async function askConfirmation () {
152 return new Promise((res, rej) => {
153 start()
154 const schema = {
155 properties: {
156 confirm: {
157 type: 'string',
158 description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
159 ' Notice PeerTube must have been stopped when your ran this script.' +
160 ' Can we delete these files?',
161 default: 'n',
162 required: true
163 }
164 }
165 }
166 get(schema, function (err, result) {
167 if (err) return rej(err)
168
169 return res(result.confirm?.match(/y/) !== null)
170 })
171 })
172 }