X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Fschedulers%2Fvideos-redundancy-scheduler.ts;h=59b55ccccdfee16a68cb253b70c8b0a4f0d21928;hb=5ec3cbdf22fc88ebe57f370fc0bc0e3df7453458;hp=1a48f2bd041319008be2bd28f10b49d492940a76;hpb=28f3d1b36a70426795240c9370e47b6c4ba847f8;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 1a48f2bd0..59b55cccc 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,31 +1,50 @@ -import { AbstractScheduler } from './abstract-scheduler' -import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' +import { move } from 'fs-extra' +import { join } from 'path' +import { getServerActor } from '@server/models/application/application' +import { TrackerModel } from '@server/models/server/tracker' +import { VideoModel } from '@server/models/video/video' +import { + MStreamingPlaylist, + MStreamingPlaylistFiles, + MStreamingPlaylistVideo, + MVideoAccountLight, + MVideoFile, + MVideoFileVideo, + MVideoRedundancyFileVideo, + MVideoRedundancyStreamingPlaylistVideo, + MVideoRedundancyVideo, + MVideoWithAllFiles +} from '@server/types/models' +import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' import { logger } from '../../helpers/logger' -import { VideosRedundancy } from '../../../shared/models/redundancy' +import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' +import { CONFIG } from '../../initializers/config' +import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' -import { VideoFileModel } from '../../models/video/video-file' -import { downloadWebTorrentVideo } from '../../helpers/webtorrent' -import { join } from 'path' -import { move } from 'fs-extra' -import { getServerActor } from '../../helpers/utils' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' -import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' -import { removeVideoRedundancy } from '../redundancy' -import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' -import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' -import { VideoModel } from '../../models/video/video' +import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' +import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' import { downloadPlaylistSegments } from '../hls' +import { removeVideoRedundancy } from '../redundancy' +import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' +import { AbstractScheduler } from './abstract-scheduler' type CandidateToDuplicate = { - redundancy: VideosRedundancy, - video: VideoModel, - files: VideoFileModel[], - streamingPlaylists: VideoStreamingPlaylistModel[] + redundancy: VideosRedundancyStrategy + video: MVideoWithAllFiles + files: MVideoFile[] + streamingPlaylists: MStreamingPlaylistFiles[] +} + +function isMVideoRedundancyFileVideo ( + o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo +): o is MVideoRedundancyFileVideo { + return !!(o as MVideoRedundancyFileVideo).VideoFile } export class VideosRedundancyScheduler extends AbstractScheduler { - private static instance: AbstractScheduler + private static instance: VideosRedundancyScheduler protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL @@ -33,6 +52,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler { super() } + async createManualRedundancy (videoId: number) { + const videoToDuplicate = await VideoModel.loadWithFiles(videoId) + + if (!videoToDuplicate) { + logger.warn('Video to manually duplicate %d does not exist anymore.', videoId) + return + } + + return this.createVideoRedundancies({ + video: videoToDuplicate, + redundancy: null, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + }) + } + protected async internalExecute () { for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) @@ -78,7 +113,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { for (const redundancyModel of expired) { try { const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - const candidate = { + const candidate: CandidateToDuplicate = { redundancy: redundancyConfig, video: null, files: [], @@ -101,10 +136,13 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { + private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) { const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) // Redundancy strategy disabled, remove our redundancy instead of extending expiration - if (!redundancy) await removeVideoRedundancy(redundancyModel) + if (!redundancy) { + await removeVideoRedundancy(redundancyModel) + return + } await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) } @@ -121,7 +159,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private findVideoToDuplicate (cache: VideosRedundancy) { + private findVideoToDuplicate (cache: VideosRedundancyStrategy) { if (cache.strategy === 'most-views') { return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) } @@ -168,26 +206,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { + private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + + const file = fileArg as MVideoFileVideo file.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) + const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) + const magnetUri = generateMagnetUri(video, file, trackerUrls) const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) - await move(tmpPath, destPath) + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) + await move(tmpPath, destPath, { overwrite: true }) - const createdModel = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), - url: getVideoCacheFileActivityPubUrl(file), - fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), - strategy: redundancy.strategy, + const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ + expiresOn, + url: getLocalVideoCacheFileActivityPubUrl(file), + fileUrl: generateWebTorrentRedundancyUrl(file), + strategy, videoFileId: file.id, actorId: serverActor.id }) @@ -199,21 +246,34 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) } - private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { + private async createStreamingPlaylistRedundancy ( + redundancy: VideosRedundancyStrategy, + video: MVideoAccountLight, + playlistArg: MStreamingPlaylist + ) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + + const playlist = playlistArg as MStreamingPlaylistVideo playlist.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) + logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) - const createdModel = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), - url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), - fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL), - strategy: redundancy.strategy, + const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ + expiresOn, + url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), + fileUrl: generateHLSRedundancyUrl(video, playlistArg), + strategy, videoStreamingPlaylistId: playlist.id, actorId: serverActor.id }) @@ -225,7 +285,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) } - private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { + private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { logger.info('Extending expiration of %s.', redundancy.url) const serverActor = await getServerActor() @@ -237,20 +297,28 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { - while (this.isTooHeavy(candidateToDuplicate)) { + while (await this.isTooHeavy(candidateToDuplicate)) { const redundancy = candidateToDuplicate.redundancy - const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) + const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime) if (!toDelete) return - await removeVideoRedundancy(toDelete) + const videoId = toDelete.VideoFile + ? toDelete.VideoFile.videoId + : toDelete.VideoStreamingPlaylist.videoId + + const redundancies = await VideoRedundancyModel.listLocalByVideoId(videoId) + + for (const redundancy of redundancies) { + await removeVideoRedundancy(redundancy) + } } } private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { const maxSize = candidateToDuplicate.redundancy.size - const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) - const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) + const { totalUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) + const totalWillDuplicate = totalUsed + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) return totalWillDuplicate > maxSize } @@ -259,16 +327,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return new Date(Date.now() + expiresAfterMs) } - private buildEntryLogId (object: VideoRedundancyModel) { - if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` + private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { + if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` return `${object.VideoStreamingPlaylist.playlistUrl}` } - private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { - const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size + private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { + const fileReducer = (previous: number, current: MVideoFile) => previous + current.size + + let allFiles = files + for (const p of playlists) { + allFiles = allFiles.concat(p.VideoFiles) + } - return files.reduce(fileReducer, 0) * playlists.length + return allFiles.reduce(fileReducer, 0) } private async loadAndRefreshVideo (videoUrl: string) {