From 764b1a14fc494f2cfd7ea590d2f07b01df65c7ad Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 23 Jul 2021 11:20:00 +0200 Subject: Use random names for VOD HLS playlists --- scripts/optimize-old-videos.ts | 32 ++++++++++++---------- scripts/prune-storage.ts | 62 ++++++++++++++++++------------------------ scripts/update-host.ts | 13 +++++---- 3 files changed, 52 insertions(+), 55 deletions(-) (limited to 'scripts') diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts index 9692d76ba..bde9d1e01 100644 --- a/scripts/optimize-old-videos.ts +++ b/scripts/optimize-old-videos.ts @@ -19,13 +19,13 @@ run() process.exit(-1) }) -let currentVideoId = null -let currentFile = null +let currentVideoId: string +let currentFilePath: string process.on('SIGINT', async function () { console.log('Cleaning up temp files') - await remove(`${currentFile}_backup`) - await remove(`${dirname(currentFile)}/${currentVideoId}-transcoded.mp4`) + await remove(`${currentFilePath}_backup`) + await remove(`${dirname(currentFilePath)}/${currentVideoId}-transcoded.mp4`) process.exit(0) }) @@ -40,12 +40,12 @@ async function run () { currentVideoId = video.id for (const file of video.VideoFiles) { - currentFile = getVideoFilePath(video, file) + currentFilePath = getVideoFilePath(video, file) const [ videoBitrate, fps, resolution ] = await Promise.all([ - getVideoFileBitrate(currentFile), - getVideoFileFPS(currentFile), - getVideoFileResolution(currentFile) + getVideoFileBitrate(currentFilePath), + getVideoFileFPS(currentFilePath), + getVideoFileResolution(currentFilePath) ]) const maxBitrate = getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS) @@ -53,25 +53,27 @@ async function run () { if (isMaxBitrateExceeded) { console.log( 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)', - basename(currentFile), videoBitrate / 1000, maxBitrate / 1000 + basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000 ) - const backupFile = `${currentFile}_backup` - await copy(currentFile, backupFile) + const backupFile = `${currentFilePath}_backup` + await copy(currentFilePath, backupFile) await optimizeOriginalVideofile(video, file) + // Update file path, the video filename changed + currentFilePath = getVideoFilePath(video, file) const originalDuration = await getDurationFromVideoFile(backupFile) - const newDuration = await getDurationFromVideoFile(currentFile) + const newDuration = await getDurationFromVideoFile(currentFilePath) if (originalDuration === newDuration) { - console.log('Finished optimizing %s', basename(currentFile)) + console.log('Finished optimizing %s', basename(currentFilePath)) await remove(backupFile) continue } - console.log('Failed to optimize %s, restoring original', basename(currentFile)) - await move(backupFile, currentFile, { overwrite: true }) + console.log('Failed to optimize %s, restoring original', basename(currentFilePath)) + await move(backupFile, currentFilePath, { overwrite: true }) await createTorrentAndSetInfoHash(video, file) await file.save() } diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts index 58d24816e..5b029d215 100755 --- a/scripts/prune-storage.ts +++ b/scripts/prune-storage.ts @@ -2,11 +2,11 @@ import { registerTSPaths } from '../server/helpers/register-ts-paths' registerTSPaths() import * as prompt from 'prompt' -import { join } from 'path' +import { join, basename } from 'path' import { CONFIG } from '../server/initializers/config' import { VideoModel } from '../server/models/video/video' import { initDatabaseModels } from '../server/initializers/database' -import { readdir, remove } from 'fs-extra' +import { readdir, remove, stat } from 'fs-extra' import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy' import * as Bluebird from 'bluebird' import { getUUIDFromFilename } from '../server/helpers/utils' @@ -14,6 +14,7 @@ import { ThumbnailModel } from '../server/models/video/thumbnail' import { ActorImageModel } from '../server/models/actor/actor-image' import { uniq, values } from 'lodash' import { ThumbnailType } from '@shared/models' +import { VideoFileModel } from '@server/models/video/video-file' run() .then(() => process.exit(0)) @@ -37,8 +38,8 @@ async function run () { console.log('Detecting files to remove, it could take a while...') toDelete = toDelete.concat( - await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)), - await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)), + await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()), + await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()), await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist), @@ -78,26 +79,27 @@ async function pruneDirectory (directory: string, existFun: ExistFun) { const toDelete: string[] = [] await Bluebird.map(files, async file => { - if (await existFun(file) !== true) { - toDelete.push(join(directory, file)) + const filePath = join(directory, file) + + if (await existFun(filePath) !== true) { + toDelete.push(filePath) } }, { concurrency: 20 }) return toDelete } -function doesVideoExist (keepOnlyOwned: boolean) { - return async (file: string) => { - const uuid = getUUIDFromFilename(file) - const video = await VideoModel.load(uuid) +function doesWebTorrentFileExist () { + return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath)) +} - return video && (keepOnlyOwned === false || video.isOwned()) - } +function doesTorrentFileExist () { + return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath)) } function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) { - return async (file: string) => { - const thumbnail = await ThumbnailModel.loadByFilename(file, type) + return async (filePath: string) => { + const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type) if (!thumbnail) return false if (keepOnlyOwned) { @@ -109,21 +111,20 @@ function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) { } } -async function doesActorImageExist (file: string) { - const image = await ActorImageModel.loadByName(file) +async function doesActorImageExist (filePath: string) { + const image = await ActorImageModel.loadByName(basename(filePath)) return !!image } -async function doesRedundancyExist (file: string) { - const uuid = getUUIDFromFilename(file) - const video = await VideoModel.loadWithFiles(uuid) - - if (!video) return false - - const isPlaylist = file.includes('.') === false +async function doesRedundancyExist (filePath: string) { + const isPlaylist = (await stat(filePath)).isDirectory() if (isPlaylist) { + const uuid = getUUIDFromFilename(filePath) + const video = await VideoModel.loadWithFiles(uuid) + if (!video) return false + const p = video.getHLSPlaylist() if (!p) return false @@ -131,19 +132,10 @@ async function doesRedundancyExist (file: string) { return !!redundancy } - const resolution = parseInt(file.split('-')[5], 10) - if (isNaN(resolution)) { - console.error('Cannot prune %s because we cannot guess guess the resolution.', file) - return true - } - - const videoFile = video.getWebTorrentFile(resolution) - if (!videoFile) { - console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution) - return true - } + const file = await VideoFileModel.loadByFilename(basename(filePath)) + if (!file) return false - const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) + const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) return !!redundancy } diff --git a/scripts/update-host.ts b/scripts/update-host.ts index 592684225..9e8dd41ca 100755 --- a/scripts/update-host.ts +++ b/scripts/update-host.ts @@ -16,7 +16,6 @@ import { VideoShareModel } from '../server/models/video/video-share' import { VideoCommentModel } from '../server/models/video/video-comment' import { AccountModel } from '../server/models/account/account' import { VideoChannelModel } from '../server/models/video/video-channel' -import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist' import { initDatabaseModels } from '../server/initializers/database' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getServerActor } from '@server/models/application/application' @@ -128,13 +127,17 @@ async function run () { for (const file of video.VideoFiles) { console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) await createTorrentAndSetInfoHash(video, file) + + await file.save() } - for (const playlist of video.VideoStreamingPlaylists) { - playlist.playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) - playlist.segmentsSha256Url = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive) + const playlist = video.getHLSPlaylist() + for (const file of (playlist?.VideoFiles || [])) { + console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid) + + await createTorrentAndSetInfoHash(video, file) - await playlist.save() + await file.save() } } } -- cgit v1.2.3