aboutsummaryrefslogtreecommitdiffhomepage
path: root/scripts/prune-storage.ts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/prune-storage.ts')
-rwxr-xr-xscripts/prune-storage.ts110
1 files changed, 78 insertions, 32 deletions
diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts
index 4953a7439..d6dff8247 100755
--- a/scripts/prune-storage.ts
+++ b/scripts/prune-storage.ts
@@ -3,9 +3,12 @@ import { join } from 'path'
3import { CONFIG } from '../server/initializers/config' 3import { CONFIG } from '../server/initializers/config'
4import { VideoModel } from '../server/models/video/video' 4import { VideoModel } from '../server/models/video/video'
5import { initDatabaseModels } from '../server/initializers' 5import { initDatabaseModels } from '../server/initializers'
6import { remove, readdir } from 'fs-extra' 6import { readdir, remove } from 'fs-extra'
7import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy' 7import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
8import * as Bluebird from 'bluebird'
8import { getUUIDFromFilename } from '../server/helpers/utils' 9import { getUUIDFromFilename } from '../server/helpers/utils'
10import { ThumbnailModel } from '../server/models/video/thumbnail'
11import { AvatarModel } from '../server/models/avatar/avatar'
9 12
10run() 13run()
11 .then(() => process.exit(0)) 14 .then(() => process.exit(0))
@@ -17,25 +20,19 @@ run()
17async function run () { 20async function run () {
18 await initDatabaseModels(true) 21 await initDatabaseModels(true)
19 22
20 const storageOnlyOwnedToPrune = [ 23 let toDelete: string[] = []
21 CONFIG.STORAGE.VIDEOS_DIR,
22 CONFIG.STORAGE.TORRENTS_DIR,
23 CONFIG.STORAGE.REDUNDANCY_DIR
24 ]
25 24
26 const storageForAllToPrune = [ 25 toDelete = toDelete.concat(
27 CONFIG.STORAGE.PREVIEWS_DIR, 26 await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
28 CONFIG.STORAGE.THUMBNAILS_DIR 27 await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
29 ]
30 28
31 let toDelete: string[] = [] 29 await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
32 for (const directory of storageOnlyOwnedToPrune) {
33 toDelete = toDelete.concat(await pruneDirectory(directory, true))
34 }
35 30
36 for (const directory of storageForAllToPrune) { 31 await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
37 toDelete = toDelete.concat(await pruneDirectory(directory, false)) 32 await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
38 } 33
34 await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
35 )
39 36
40 const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR) 37 const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
41 toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t))) 38 toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
@@ -61,30 +58,79 @@ async function run () {
61 } 58 }
62} 59}
63 60
64async function pruneDirectory (directory: string, onlyOwned = false) { 61type ExistFun = (file: string) => Promise<boolean>
62async function pruneDirectory (directory: string, existFun: ExistFun) {
65 const files = await readdir(directory) 63 const files = await readdir(directory)
66 64
67 const toDelete: string[] = [] 65 const toDelete: string[] = []
68 for (const file of files) { 66 await Bluebird.map(files, async file => {
67 if (await existFun(file) !== true) {
68 toDelete.push(join(directory, file))
69 }
70 }, { concurrency: 20 })
71
72 return toDelete
73}
74
75function doesVideoExist (keepOnlyOwned: boolean) {
76 return async (file: string) => {
69 const uuid = getUUIDFromFilename(file) 77 const uuid = getUUIDFromFilename(file)
70 let video: VideoModel 78 const video = await VideoModel.loadByUUID(uuid)
71 let localRedundancy: boolean
72 79
73 if (uuid) { 80 return video && (keepOnlyOwned === false || video.isOwned())
74 video = await VideoModel.loadByUUIDWithFile(uuid) 81 }
75 localRedundancy = await VideoRedundancyModel.isLocalByVideoUUIDExists(uuid) 82}
76 }
77 83
78 if ( 84function doesThumbnailExist (keepOnlyOwned: boolean) {
79 !uuid || 85 return async (file: string) => {
80 !video || 86 const thumbnail = await ThumbnailModel.loadByName(file)
81 (onlyOwned === true && (video.isOwned() === false && localRedundancy === false)) 87 if (!thumbnail) return false
82 ) { 88
83 toDelete.push(join(directory, file)) 89 if (keepOnlyOwned) {
90 const video = await VideoModel.load(thumbnail.videoId)
91 if (video.isOwned() === false) return false
84 } 92 }
93
94 return true
85 } 95 }
96}
86 97
87 return toDelete 98async function doesAvatarExist (file: string) {
99 const avatar = await AvatarModel.loadByName(file)
100
101 return !!avatar
102}
103
104async function doesRedundancyExist (file: string) {
105 const uuid = getUUIDFromFilename(file)
106 const video = await VideoModel.loadWithFiles(uuid)
107
108 if (!video) return false
109
110 const isPlaylist = file.includes('.') === false
111
112 if (isPlaylist) {
113 const p = video.getHLSPlaylist()
114 if (!p) return false
115
116 const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
117 return !!redundancy
118 }
119
120 const resolution = parseInt(file.split('-')[5], 10)
121 if (isNaN(resolution)) {
122 console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
123 return true
124 }
125
126 const videoFile = video.getFile(resolution)
127 if (!videoFile) {
128 console.error('Cannot find file of video %s - %d', video.url, resolution)
129 return true
130 }
131
132 const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
133 return !!redundancy
88} 134}
89 135
90async function askConfirmation () { 136async function askConfirmation () {