X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=scripts%2Fprune-storage.ts;h=d19594a60ed45122fb497e83344422a44cd43b5f;hb=7dcd45a9a289c2da38eafd9a0b293453981b0da4;hp=c8968013b8dbfd0573c17af4d35d73013aa6bcf9;hpb=2aaa1a3fdc49be77aec5309dab5507865c38d392;p=github%2FChocobozzz%2FPeerTube.git diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts index c8968013b..d19594a60 100755 --- a/scripts/prune-storage.ts +++ b/scripts/prune-storage.ts @@ -1,18 +1,19 @@ -import { registerTSPaths } from '../server/helpers/register-ts-paths' -registerTSPaths() - -import * as prompt from 'prompt' -import { join } from 'path' +import { map } from 'bluebird' +import { readdir, remove, stat } from 'fs-extra' +import { basename, join } from 'path' +import { get, start } from 'prompt' +import { DIRECTORIES } from '@server/initializers/constants' +import { VideoFileModel } from '@server/models/video/video-file' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' +import { uniqify } from '@shared/core-utils' +import { ThumbnailType } from '@shared/models' +import { getUUIDFromFilename } from '../server/helpers/utils' import { CONFIG } from '../server/initializers/config' -import { VideoModel } from '../server/models/video/video' -import { initDatabaseModels } from '../server/initializers' -import { readdir, remove } from 'fs-extra' +import { initDatabaseModels } from '../server/initializers/database' +import { ActorImageModel } from '../server/models/actor/actor-image' import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy' -import * as Bluebird from 'bluebird' -import { getUUIDFromFilename } from '../server/helpers/utils' import { ThumbnailModel } from '../server/models/video/thumbnail' -import { AvatarModel } from '../server/models/avatar/avatar' -import { uniq, values } from 'lodash' +import { VideoModel } from '../server/models/video/video' run() .then(() => process.exit(0)) @@ -22,9 +23,9 @@ run() }) async function run () { - const dirs = values(CONFIG.STORAGE) + const dirs = Object.values(CONFIG.STORAGE) - if (uniq(dirs).length !== dirs.length) { + if (uniqify(dirs).length !== dirs.length) { console.error('Cannot prune storage because you put multiple storage keys in the same directory.') process.exit(0) } @@ -33,16 +34,23 @@ async function run () { let toDelete: string[] = [] + console.log('Detecting files to remove, it could take a while...') + toDelete = toDelete.concat( - await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)), - await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)), + await pruneDirectory(DIRECTORIES.VIDEOS.PUBLIC, doesWebTorrentFileExist()), + await pruneDirectory(DIRECTORIES.VIDEOS.PRIVATE, doesWebTorrentFileExist()), + + await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()), + await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()), + + await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()), await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist), - await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)), - await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)), + await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)), + await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)), - await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist) + await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES, doesActorImageExist) ) const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR) @@ -69,32 +77,47 @@ async function run () { } } -type ExistFun = (file: string) => Promise +type ExistFun = (file: string) => Promise | boolean async function pruneDirectory (directory: string, existFun: ExistFun) { const files = await readdir(directory) const toDelete: string[] = [] - await Bluebird.map(files, async file => { - if (await existFun(file) !== true) { - toDelete.push(join(directory, file)) + await map(files, async file => { + const filePath = join(directory, file) + + if (await existFun(filePath) !== true) { + toDelete.push(filePath) } }, { concurrency: 20 }) return toDelete } -function doesVideoExist (keepOnlyOwned: boolean) { - return async (file: string) => { - const uuid = getUUIDFromFilename(file) - const video = await VideoModel.loadByUUID(uuid) +function doesWebTorrentFileExist () { + return (filePath: string) => { + // Don't delete private directory + if (filePath === DIRECTORIES.VIDEOS.PRIVATE) return true - return video && (keepOnlyOwned === false || video.isOwned()) + return VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath)) } } -function doesThumbnailExist (keepOnlyOwned: boolean) { - return async (file: string) => { - const thumbnail = await ThumbnailModel.loadByName(file) +function doesHLSPlaylistExist () { + return (hlsPath: string) => { + // Don't delete private directory + if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true + + return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath)) + } +} + +function doesTorrentFileExist () { + return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath)) +} + +function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) { + return async (filePath: string) => { + const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type) if (!thumbnail) return false if (keepOnlyOwned) { @@ -106,21 +129,23 @@ function doesThumbnailExist (keepOnlyOwned: boolean) { } } -async function doesAvatarExist (file: string) { - const avatar = await AvatarModel.loadByName(file) +async function doesActorImageExist (filePath: string) { + const image = await ActorImageModel.loadByName(basename(filePath)) - return !!avatar + return !!image } -async function doesRedundancyExist (file: string) { - const uuid = getUUIDFromFilename(file) - const video = await VideoModel.loadWithFiles(uuid) +async function doesRedundancyExist (filePath: string) { + const isPlaylist = (await stat(filePath)).isDirectory() - if (!video) return false + if (isPlaylist) { + // Don't delete HLS redundancy directory + if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true - const isPlaylist = file.includes('.') === false + const uuid = getUUIDFromFilename(filePath) + const video = await VideoModel.loadWithFiles(uuid) + if (!video) return false - if (isPlaylist) { const p = video.getHLSPlaylist() if (!p) return false @@ -128,25 +153,16 @@ async function doesRedundancyExist (file: string) { return !!redundancy } - const resolution = parseInt(file.split('-')[5], 10) - if (isNaN(resolution)) { - console.error('Cannot prune %s because we cannot guess guess the resolution.', file) - return true - } + const file = await VideoFileModel.loadByFilename(basename(filePath)) + if (!file) return false - const videoFile = video.getFile(resolution) - if (!videoFile) { - console.error('Cannot find file of video %s - %d', video.url, resolution) - return true - } - - const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) + const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) return !!redundancy } async function askConfirmation () { return new Promise((res, rej) => { - prompt.start() + start() const schema = { properties: { confirm: { @@ -159,9 +175,10 @@ async function askConfirmation () { } } } - prompt.get(schema, function (err, result) { + get(schema, function (err, result) { if (err) return rej(err) - return res(result.confirm && result.confirm.match(/y/) !== null) + + return res(result.confirm?.match(/y/) !== null) }) }) }