X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Fschedulers%2Fvideos-redundancy-scheduler.ts;h=59b55ccccdfee16a68cb253b70c8b0a4f0d21928;hb=5ec3cbdf22fc88ebe57f370fc0bc0e3df7453458;hp=de8fc075b29a2f5a10ee48ac8570d827112fe544;hpb=453e83ea5d81d203ba34bc43cd5c2c750ba40568;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index de8fc075b..59b55cccc 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,20 +1,11 @@ -import { AbstractScheduler } from './abstract-scheduler' -import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' -import { logger } from '../../helpers/logger' -import { VideosRedundancy } from '../../../shared/models/redundancy' -import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' -import { downloadWebTorrentVideo } from '../../helpers/webtorrent' -import { join } from 'path' import { move } from 'fs-extra' -import { getServerActor } from '../../helpers/utils' -import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' -import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' -import { removeVideoRedundancy } from '../redundancy' -import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' -import { downloadPlaylistSegments } from '../hls' -import { CONFIG } from '../../initializers/config' +import { join } from 'path' +import { getServerActor } from '@server/models/application/application' +import { TrackerModel } from '@server/models/server/tracker' +import { VideoModel } from '@server/models/video/video' import { MStreamingPlaylist, + MStreamingPlaylistFiles, MStreamingPlaylistVideo, MVideoAccountLight, MVideoFile, @@ -23,22 +14,37 @@ import { MVideoRedundancyStreamingPlaylistVideo, MVideoRedundancyVideo, MVideoWithAllFiles -} from '@server/typings/models' +} from '@server/types/models' +import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' +import { logger } from '../../helpers/logger' +import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' +import { CONFIG } from '../../initializers/config' +import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' +import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' +import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' +import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' +import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' +import { downloadPlaylistSegments } from '../hls' +import { removeVideoRedundancy } from '../redundancy' +import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' +import { AbstractScheduler } from './abstract-scheduler' type CandidateToDuplicate = { - redundancy: VideosRedundancy, - video: MVideoWithAllFiles, - files: MVideoFile[], - streamingPlaylists: MStreamingPlaylist[] + redundancy: VideosRedundancyStrategy + video: MVideoWithAllFiles + files: MVideoFile[] + streamingPlaylists: MStreamingPlaylistFiles[] } -function isMVideoRedundancyFileVideo (o: MVideoRedundancyVideo): o is MVideoRedundancyFileVideo { +function isMVideoRedundancyFileVideo ( + o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo +): o is MVideoRedundancyFileVideo { return !!(o as MVideoRedundancyFileVideo).VideoFile } export class VideosRedundancyScheduler extends AbstractScheduler { - private static instance: AbstractScheduler + private static instance: VideosRedundancyScheduler protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL @@ -46,6 +52,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler { super() } + async createManualRedundancy (videoId: number) { + const videoToDuplicate = await VideoModel.loadWithFiles(videoId) + + if (!videoToDuplicate) { + logger.warn('Video to manually duplicate %d does not exist anymore.', videoId) + return + } + + return this.createVideoRedundancies({ + video: videoToDuplicate, + redundancy: null, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + }) + } + protected async internalExecute () { for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) @@ -91,7 +113,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { for (const redundancyModel of expired) { try { const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - const candidate = { + const candidate: CandidateToDuplicate = { redundancy: redundancyConfig, video: null, files: [], @@ -137,7 +159,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private findVideoToDuplicate (cache: VideosRedundancy) { + private findVideoToDuplicate (cache: VideosRedundancyStrategy) { if (cache.strategy === 'most-views') { return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) } @@ -184,27 +206,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { + private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + const file = fileArg as MVideoFileVideo file.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) + const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) + const magnetUri = generateMagnetUri(video, file, trackerUrls) const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) - await move(tmpPath, destPath) + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) + await move(tmpPath, destPath, { overwrite: true }) const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), - url: getVideoCacheFileActivityPubUrl(file), - fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), - strategy: redundancy.strategy, + expiresOn, + url: getLocalVideoCacheFileActivityPubUrl(file), + fileUrl: generateWebTorrentRedundancyUrl(file), + strategy, videoFileId: file.id, actorId: serverActor.id }) @@ -217,25 +247,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private async createStreamingPlaylistRedundancy ( - redundancy: VideosRedundancy, + redundancy: VideosRedundancyStrategy, video: MVideoAccountLight, playlistArg: MStreamingPlaylist ) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + const playlist = playlistArg as MStreamingPlaylistVideo playlist.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) + logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), - url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), - fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), - strategy: redundancy.strategy, + expiresOn, + url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), + fileUrl: generateHLSRedundancyUrl(video, playlistArg), + strategy, videoStreamingPlaylistId: playlist.id, actorId: serverActor.id }) @@ -259,20 +297,28 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { - while (this.isTooHeavy(candidateToDuplicate)) { + while (await this.isTooHeavy(candidateToDuplicate)) { const redundancy = candidateToDuplicate.redundancy const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime) if (!toDelete) return - await removeVideoRedundancy(toDelete) + const videoId = toDelete.VideoFile + ? toDelete.VideoFile.videoId + : toDelete.VideoStreamingPlaylist.videoId + + const redundancies = await VideoRedundancyModel.listLocalByVideoId(videoId) + + for (const redundancy of redundancies) { + await removeVideoRedundancy(redundancy) + } } } private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { const maxSize = candidateToDuplicate.redundancy.size - const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) - const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) + const { totalUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) + const totalWillDuplicate = totalUsed + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) return totalWillDuplicate > maxSize } @@ -287,13 +333,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return `${object.VideoStreamingPlaylist.playlistUrl}` } - private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylist[]) { + private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { const fileReducer = (previous: number, current: MVideoFile) => previous + current.size - const totalSize = files.reduce(fileReducer, 0) - if (playlists.length === 0) return totalSize + let allFiles = files + for (const p of playlists) { + allFiles = allFiles.concat(p.VideoFiles) + } - return totalSize * playlists.length + return allFiles.reduce(fileReducer, 0) } private async loadAndRefreshVideo (videoUrl: string) {