From e2600d8b261994abbbeb1ff921edaefd267fc122 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 9 Aug 2019 15:04:36 +0200 Subject: Add avatar to prune script --- scripts/create-import-video-file-job.ts | 2 +- scripts/create-transcoding-job.ts | 2 +- scripts/prune-storage.ts | 110 ++++++++++++++++++++++---------- 3 files changed, 80 insertions(+), 34 deletions(-) (limited to 'scripts') diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts index c8c6c6429..2b636014a 100644 --- a/scripts/create-import-video-file-job.ts +++ b/scripts/create-import-video-file-job.ts @@ -25,7 +25,7 @@ run() async function run () { await initDatabaseModels(true) - const video = await VideoModel.loadByUUIDWithFile(program['video']) + const video = await VideoModel.loadByUUID(program['video']) if (!video) throw new Error('Video not found.') if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.') diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts index 2b7cb5177..2eb872169 100755 --- a/scripts/create-transcoding-job.ts +++ b/scripts/create-transcoding-job.ts @@ -29,7 +29,7 @@ run() async function run () { await initDatabaseModels(true) - const video = await VideoModel.loadByUUIDWithFile(program['video']) + const video = await VideoModel.loadByUUID(program['video']) if (!video) throw new Error('Video not found.') const dataInput: VideoTranscodingPayload = program.resolution !== undefined diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts index 4953a7439..d6dff8247 100755 --- a/scripts/prune-storage.ts +++ b/scripts/prune-storage.ts @@ -3,9 +3,12 @@ import { join } from 'path' import { CONFIG } from '../server/initializers/config' import { VideoModel } from '../server/models/video/video' import { initDatabaseModels } from '../server/initializers' -import { remove, readdir } from 'fs-extra' +import { readdir, remove } from 'fs-extra' import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy' +import * as Bluebird from 'bluebird' import { getUUIDFromFilename } from '../server/helpers/utils' +import { ThumbnailModel } from '../server/models/video/thumbnail' +import { AvatarModel } from '../server/models/avatar/avatar' run() .then(() => process.exit(0)) @@ -17,25 +20,19 @@ run() async function run () { await initDatabaseModels(true) - const storageOnlyOwnedToPrune = [ - CONFIG.STORAGE.VIDEOS_DIR, - CONFIG.STORAGE.TORRENTS_DIR, - CONFIG.STORAGE.REDUNDANCY_DIR - ] + let toDelete: string[] = [] - const storageForAllToPrune = [ - CONFIG.STORAGE.PREVIEWS_DIR, - CONFIG.STORAGE.THUMBNAILS_DIR - ] + toDelete = toDelete.concat( + await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)), + await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)), - let toDelete: string[] = [] - for (const directory of storageOnlyOwnedToPrune) { - toDelete = toDelete.concat(await pruneDirectory(directory, true)) - } + await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist), - for (const directory of storageForAllToPrune) { - toDelete = toDelete.concat(await pruneDirectory(directory, false)) - } + await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)), + await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)), + + await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist) + ) const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR) toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t))) @@ -61,30 +58,79 @@ async function run () { } } -async function pruneDirectory (directory: string, onlyOwned = false) { +type ExistFun = (file: string) => Promise +async function pruneDirectory (directory: string, existFun: ExistFun) { const files = await readdir(directory) const toDelete: string[] = [] - for (const file of files) { + await Bluebird.map(files, async file => { + if (await existFun(file) !== true) { + toDelete.push(join(directory, file)) + } + }, { concurrency: 20 }) + + return toDelete +} + +function doesVideoExist (keepOnlyOwned: boolean) { + return async (file: string) => { const uuid = getUUIDFromFilename(file) - let video: VideoModel - let localRedundancy: boolean + const video = await VideoModel.loadByUUID(uuid) - if (uuid) { - video = await VideoModel.loadByUUIDWithFile(uuid) - localRedundancy = await VideoRedundancyModel.isLocalByVideoUUIDExists(uuid) - } + return video && (keepOnlyOwned === false || video.isOwned()) + } +} - if ( - !uuid || - !video || - (onlyOwned === true && (video.isOwned() === false && localRedundancy === false)) - ) { - toDelete.push(join(directory, file)) +function doesThumbnailExist (keepOnlyOwned: boolean) { + return async (file: string) => { + const thumbnail = await ThumbnailModel.loadByName(file) + if (!thumbnail) return false + + if (keepOnlyOwned) { + const video = await VideoModel.load(thumbnail.videoId) + if (video.isOwned() === false) return false } + + return true } +} - return toDelete +async function doesAvatarExist (file: string) { + const avatar = await AvatarModel.loadByName(file) + + return !!avatar +} + +async function doesRedundancyExist (file: string) { + const uuid = getUUIDFromFilename(file) + const video = await VideoModel.loadWithFiles(uuid) + + if (!video) return false + + const isPlaylist = file.includes('.') === false + + if (isPlaylist) { + const p = video.getHLSPlaylist() + if (!p) return false + + const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id) + return !!redundancy + } + + const resolution = parseInt(file.split('-')[5], 10) + if (isNaN(resolution)) { + console.error('Cannot prune %s because we cannot guess guess the resolution.', file) + return true + } + + const videoFile = video.getFile(resolution) + if (!videoFile) { + console.error('Cannot find file of video %s - %d', video.url, resolution) + return true + } + + const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) + return !!redundancy } async function askConfirmation () { -- cgit v1.2.3