From b764380ac23f4e9d4677d08acdc3474c2931a16d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 10 Jan 2020 10:11:28 +0100 Subject: Add ability to list redundancies --- server/lib/activitypub/cache-file.ts | 4 +- server/lib/job-queue/handlers/video-redundancy.ts | 20 ++++++++ server/lib/job-queue/job-queue.ts | 13 +++-- server/lib/redundancy.ts | 8 +-- server/lib/schedulers/update-videos-scheduler.ts | 1 - .../lib/schedulers/videos-redundancy-scheduler.ts | 59 +++++++++++++++++----- 6 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 server/lib/job-queue/handlers/video-redundancy.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 65b2dcb49..8252e95e9 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts @@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) return { - expiresOn: new Date(cacheFileObject.expires), + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, url: cacheFileObject.id, fileUrl: url.href, strategy: null, @@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) return { - expiresOn: new Date(cacheFileObject.expires), + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, url: cacheFileObject.id, fileUrl: url.href, strategy: null, diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts new file mode 100644 index 000000000..319d7090e --- /dev/null +++ b/server/lib/job-queue/handlers/video-redundancy.ts @@ -0,0 +1,20 @@ +import * as Bull from 'bull' +import { logger } from '../../../helpers/logger' +import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler' + +export type VideoRedundancyPayload = { + videoId: number +} + +async function processVideoRedundancy (job: Bull.Job) { + const payload = job.data as VideoRedundancyPayload + logger.info('Processing video redundancy in job %d.', job.id) + + return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId) +} + +// --------------------------------------------------------------------------- + +export { + processVideoRedundancy +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index ec601e9ea..8bbf58f2b 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import' import { processVideosViews } from './handlers/video-views' import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' +import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -24,20 +25,21 @@ type CreateJobArgument = { type: 'email', payload: EmailPayload } | { type: 'video-import', payload: VideoImportPayload } | { type: 'activitypub-refresher', payload: RefreshPayload } | - { type: 'videos-views', payload: {} } + { type: 'videos-views', payload: {} } | + { type: 'video-redundancy', payload: VideoRedundancyPayload } -const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise} = { +const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 'activitypub-http-unicast': processActivityPubHttpUnicast, 'activitypub-http-fetcher': processActivityPubHttpFetcher, 'activitypub-follow': processActivityPubFollow, 'video-file-import': processVideoFileImport, 'video-transcoding': processVideoTranscoding, - 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3) 'email': processEmail, 'video-import': processVideoImport, 'videos-views': processVideosViews, - 'activitypub-refresher': refreshAPObject + 'activitypub-refresher': refreshAPObject, + 'video-redundancy': processVideoRedundancy } const jobTypes: JobType[] = [ @@ -50,7 +52,8 @@ const jobTypes: JobType[] = [ 'video-file-import', 'video-import', 'videos-views', - 'activitypub-refresher' + 'activitypub-refresher', + 'video-redundancy' ] class JobQueue { diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts index 1b4ecd7c0..78d84e02e 100644 --- a/server/lib/redundancy.ts +++ b/server/lib/redundancy.ts @@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t? await videoRedundancy.destroy({ transaction: t }) } -async function removeRedundancyOf (serverId: number) { - const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId) +async function removeRedundanciesOfServer (serverId: number) { + const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId) - for (const redundancy of videosRedundancy) { + for (const redundancy of redundancies) { await removeVideoRedundancy(redundancy) } } @@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) { // --------------------------------------------------------------------------- export { - removeRedundancyOf, + removeRedundanciesOfServer, removeVideoRedundancy } diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 350a335d3..956780a77 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts @@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda import { retryTransactionWrapper } from '../../helpers/database-utils' import { federateVideoIfNeeded } from '../activitypub' import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { VideoPrivacy } from '../../../shared/models/videos' import { Notifier } from '../notifier' import { sequelizeTypescript } from '../../initializers/database' import { MVideoFullLight } from '@server/typings/models' diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c1c91b656..6e61cbe7d 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,7 +1,7 @@ import { AbstractScheduler } from './abstract-scheduler' import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' import { logger } from '../../helpers/logger' -import { VideosRedundancy } from '../../../shared/models/redundancy' +import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' import { join } from 'path' @@ -25,9 +25,10 @@ import { MVideoWithAllFiles } from '@server/typings/models' import { getVideoFilename } from '../video-paths' +import { VideoModel } from '@server/models/video/video' type CandidateToDuplicate = { - redundancy: VideosRedundancy, + redundancy: VideosRedundancyStrategy, video: MVideoWithAllFiles, files: MVideoFile[], streamingPlaylists: MStreamingPlaylistFiles[] @@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo ( export class VideosRedundancyScheduler extends AbstractScheduler { - private static instance: AbstractScheduler + private static instance: VideosRedundancyScheduler protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL @@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler { super() } + async createManualRedundancy (videoId: number) { + const videoToDuplicate = await VideoModel.loadWithFiles(videoId) + + if (!videoToDuplicate) { + logger.warn('Video to manually duplicate %d does not exist anymore.', videoId) + return + } + + return this.createVideoRedundancies({ + video: videoToDuplicate, + redundancy: null, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + }) + } + protected async internalExecute () { for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) @@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { for (const redundancyModel of expired) { try { const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - const candidate = { + const candidate: CandidateToDuplicate = { redundancy: redundancyConfig, video: null, files: [], @@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private findVideoToDuplicate (cache: VideosRedundancy) { + private findVideoToDuplicate (cache: VideosRedundancyStrategy) { if (cache.strategy === 'most-views') { return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) } @@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { + private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + const file = fileArg as MVideoFileVideo file.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) @@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { await move(tmpPath, destPath, { overwrite: true }) const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), + expiresOn, url: getVideoCacheFileActivityPubUrl(file), fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), - strategy: redundancy.strategy, + strategy, videoFileId: file.id, actorId: serverActor.id }) @@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private async createStreamingPlaylistRedundancy ( - redundancy: VideosRedundancy, + redundancy: VideosRedundancyStrategy, video: MVideoAccountLight, playlistArg: MStreamingPlaylist ) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + const playlist = playlistArg as MStreamingPlaylistVideo playlist.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) + logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), + expiresOn, url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), - strategy: redundancy.strategy, + strategy, videoStreamingPlaylistId: playlist.id, actorId: serverActor.id }) -- cgit v1.2.3