From b764380ac23f4e9d4677d08acdc3474c2931a16d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 10 Jan 2020 10:11:28 +0100 Subject: Add ability to list redundancies --- server/controllers/api/server/follows.ts | 4 +- server/controllers/api/server/redundancy.ts | 84 ++++- server/controllers/api/server/stats.ts | 10 +- .../custom-validators/activitypub/cache-file.ts | 2 +- .../custom-validators/video-redundancies.ts | 12 + server/helpers/webtorrent.ts | 4 +- server/initializers/config.ts | 4 +- server/initializers/constants.ts | 25 +- .../migrations/0475-redundancy-expires-on.ts | 27 ++ server/lib/activitypub/cache-file.ts | 4 +- server/lib/job-queue/handlers/video-redundancy.ts | 20 ++ server/lib/job-queue/job-queue.ts | 13 +- server/lib/redundancy.ts | 8 +- server/lib/schedulers/update-videos-scheduler.ts | 1 - .../lib/schedulers/videos-redundancy-scheduler.ts | 59 +++- server/middlewares/sort.ts | 23 +- server/middlewares/validators/redundancy.ts | 74 +++- server/middlewares/validators/sort.ts | 3 + server/models/redundancy/video-redundancy.ts | 186 +++++++++- server/tests/api/check-params/redundancy.ts | 141 +++++++- server/tests/api/redundancy/index.ts | 1 + server/tests/api/redundancy/manage-redundancy.ts | 373 +++++++++++++++++++++ server/tests/api/redundancy/redundancy.ts | 138 ++++++-- server/typings/models/video/video-file.ts | 5 +- .../models/video/video-streaming-playlist.ts | 6 +- server/typings/models/video/video.ts | 13 +- 26 files changed, 1137 insertions(+), 103 deletions(-) create mode 100644 server/helpers/custom-validators/video-redundancies.ts create mode 100644 server/initializers/migrations/0475-redundancy-expires-on.ts create mode 100644 server/lib/job-queue/handlers/video-redundancy.ts create mode 100644 server/tests/api/redundancy/manage-redundancy.ts (limited to 'server') diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts index 29a403a43..c69de21fb 100644 --- a/server/controllers/api/server/follows.ts +++ b/server/controllers/api/server/follows.ts @@ -24,7 +24,7 @@ import { } from '../../../middlewares/validators' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { JobQueue } from '../../../lib/job-queue' -import { removeRedundancyOf } from '../../../lib/redundancy' +import { removeRedundanciesOfServer } from '../../../lib/redundancy' import { sequelizeTypescript } from '../../../initializers/database' import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow' @@ -153,7 +153,7 @@ async function removeFollowing (req: express.Request, res: express.Response) { await server.save({ transaction: t }) // Async, could be long - removeRedundancyOf(server.id) + removeRedundanciesOfServer(server.id) .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err)) await follow.destroy({ transaction: t }) diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts index 4ea6164a3..a11c1a74f 100644 --- a/server/controllers/api/server/redundancy.ts +++ b/server/controllers/api/server/redundancy.ts @@ -1,9 +1,24 @@ import * as express from 'express' import { UserRight } from '../../../../shared/models/users' -import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' -import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy' -import { removeRedundancyOf } from '../../../lib/redundancy' +import { + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + setDefaultPagination, + setDefaultVideoRedundanciesSort, + videoRedundanciesSortValidator +} from '../../../middlewares' +import { + listVideoRedundanciesValidator, + updateServerRedundancyValidator, + addVideoRedundancyValidator, + removeVideoRedundancyValidator +} from '../../../middlewares/validators/redundancy' +import { removeRedundanciesOfServer, removeVideoRedundancy } from '../../../lib/redundancy' import { logger } from '../../../helpers/logger' +import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' +import { JobQueue } from '@server/lib/job-queue' const serverRedundancyRouter = express.Router() @@ -14,6 +29,31 @@ serverRedundancyRouter.put('/redundancy/:host', asyncMiddleware(updateRedundancy) ) +serverRedundancyRouter.get('/redundancy/videos', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + listVideoRedundanciesValidator, + paginationValidator, + videoRedundanciesSortValidator, + setDefaultVideoRedundanciesSort, + setDefaultPagination, + asyncMiddleware(listVideoRedundancies) +) + +serverRedundancyRouter.post('/redundancy/videos', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + addVideoRedundancyValidator, + asyncMiddleware(addVideoRedundancy) +) + +serverRedundancyRouter.delete('/redundancy/videos/:redundancyId', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + removeVideoRedundancyValidator, + asyncMiddleware(removeVideoRedundancyController) +) + // --------------------------------------------------------------------------- export { @@ -22,6 +62,42 @@ export { // --------------------------------------------------------------------------- +async function listVideoRedundancies (req: express.Request, res: express.Response) { + const resultList = await VideoRedundancyModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + target: req.query.target, + strategy: req.query.strategy + }) + + const result = { + total: resultList.total, + data: resultList.data.map(r => VideoRedundancyModel.toFormattedJSONStatic(r)) + } + + return res.json(result) +} + +async function addVideoRedundancy (req: express.Request, res: express.Response) { + const payload = { + videoId: res.locals.onlyVideo.id + } + + await JobQueue.Instance.createJob({ + type: 'video-redundancy', + payload + }) + + return res.sendStatus(204) +} + +async function removeVideoRedundancyController (req: express.Request, res: express.Response) { + await removeVideoRedundancy(res.locals.videoRedundancy) + + return res.sendStatus(204) +} + async function updateRedundancy (req: express.Request, res: express.Response) { const server = res.locals.server @@ -30,7 +106,7 @@ async function updateRedundancy (req: express.Request, res: express.Response) { await server.save() // Async, could be long - removeRedundancyOf(server.id) + removeRedundanciesOfServer(server.id) .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) return res.sendStatus(204) diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index 3616c074d..6d508a481 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts @@ -10,6 +10,7 @@ import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' import { cacheRoute } from '../../../middlewares/cache' import { VideoFileModel } from '../../../models/video/video-file' import { CONFIG } from '../../../initializers/config' +import { VideoRedundancyStrategyWithManual } from '@shared/models' const statsRouter = express.Router() @@ -25,8 +26,15 @@ async function getStats (req: express.Request, res: express.Response) { const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() + const strategies: { strategy: VideoRedundancyStrategyWithManual, size: number }[] = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES + .map(r => ({ + strategy: r.strategy, + size: r.size + })) + strategies.push({ strategy: 'manual', size: null }) + const videosRedundancyStats = await Promise.all( - CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => { + strategies.map(r => { return VideoRedundancyModel.getStats(r.strategy) .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) }) diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts index 21d5c53ca..c5b3b4d9f 100644 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ b/server/helpers/custom-validators/activitypub/cache-file.ts @@ -6,7 +6,7 @@ import { CacheFileObject } from '../../../../shared/models/activitypub/objects' function isCacheFileObjectValid (object: CacheFileObject) { return exists(object) && object.type === 'CacheFile' && - isDateValid(object.expires) && + (object.expires === null || isDateValid(object.expires)) && isActivityPubUrlValid(object.object) && (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) } diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts new file mode 100644 index 000000000..50a559c4f --- /dev/null +++ b/server/helpers/custom-validators/video-redundancies.ts @@ -0,0 +1,12 @@ +import { exists } from './misc' + +function isVideoRedundancyTarget (value: any) { + return exists(value) && + (value === 'my-videos' || value === 'remote-videos') +} + +// --------------------------------------------------------------------------- + +export { + isVideoRedundancyTarget +} diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 3a99518c6..8a5d030df 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -9,12 +9,12 @@ import { promisify2 } from './core-utils' import { MVideo } from '@server/typings/models/video/video' import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' -import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' +import { WEBSERVER } from '@server/initializers/constants' import * as parseTorrent from 'parse-torrent' import * as magnetUtil from 'magnet-uri' import { isArray } from '@server/helpers/custom-validators/misc' import { extractVideo } from '@server/lib/videos' -import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' +import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' const createTorrentPromise = promisify2(createTorrent) diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 7fd77f3e8..fd8bf09fc 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -1,6 +1,6 @@ import { IConfig } from 'config' import { dirname, join } from 'path' -import { VideosRedundancy } from '../../shared/models' +import { VideosRedundancyStrategy } from '../../shared/models' // Do not use barrels, remain constants as independent as possible import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' @@ -304,7 +304,7 @@ function getLocalConfigFilePath () { return join(dirname(configSources[ 0 ].name), filename + '.json') } -function buildVideosRedundancy (objs: any[]): VideosRedundancy[] { +function buildVideosRedundancy (objs: any[]): VideosRedundancyStrategy[] { if (!objs) return [] if (!Array.isArray(objs)) return objs diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 032f63c8f..e01ab8943 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 470 +const LAST_MIGRATION_VERSION = 475 // --------------------------------------------------------------------------- @@ -73,7 +73,9 @@ const SORTABLE_COLUMNS = { PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], - AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ] + AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ], + + VIDEO_REDUNDANCIES: [ 'name' ] } const OAUTH_LIFETIME = { @@ -117,45 +119,44 @@ const REMOTE_SCHEME = { WS: 'wss' } -// TODO: remove 'video-file' -const JOB_ATTEMPTS: { [id in (JobType | 'video-file')]: number } = { +const JOB_ATTEMPTS: { [id in JobType]: number } = { 'activitypub-http-broadcast': 5, 'activitypub-http-unicast': 5, 'activitypub-http-fetcher': 5, 'activitypub-follow': 5, 'video-file-import': 1, 'video-transcoding': 1, - 'video-file': 1, 'video-import': 1, 'email': 5, 'videos-views': 1, - 'activitypub-refresher': 1 + 'activitypub-refresher': 1, + 'video-redundancy': 1 } -const JOB_CONCURRENCY: { [id in (JobType | 'video-file')]: number } = { +const JOB_CONCURRENCY: { [id in JobType]: number } = { 'activitypub-http-broadcast': 1, 'activitypub-http-unicast': 5, 'activitypub-http-fetcher': 1, 'activitypub-follow': 1, 'video-file-import': 1, 'video-transcoding': 1, - 'video-file': 1, 'video-import': 1, 'email': 5, 'videos-views': 1, - 'activitypub-refresher': 1 + 'activitypub-refresher': 1, + 'video-redundancy': 1 } -const JOB_TTL: { [id in (JobType | 'video-file')]: number } = { +const JOB_TTL: { [id in JobType]: number } = { 'activitypub-http-broadcast': 60000 * 10, // 10 minutes 'activitypub-http-unicast': 60000 * 10, // 10 minutes 'activitypub-http-fetcher': 60000 * 10, // 10 minutes 'activitypub-follow': 60000 * 10, // 10 minutes 'video-file-import': 1000 * 3600, // 1 hour 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long - 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long 'video-import': 1000 * 3600 * 2, // hours 'email': 60000 * 10, // 10 minutes 'videos-views': undefined, // Unlimited - 'activitypub-refresher': 60000 * 10 // 10 minutes + 'activitypub-refresher': 60000 * 10, // 10 minutes + 'video-redundancy': 1000 * 3600 * 3 // 3 hours } const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { 'videos-views': { diff --git a/server/initializers/migrations/0475-redundancy-expires-on.ts b/server/initializers/migrations/0475-redundancy-expires-on.ts new file mode 100644 index 000000000..7e392c8c0 --- /dev/null +++ b/server/initializers/migrations/0475-redundancy-expires-on.ts @@ -0,0 +1,27 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + { + const data = { + type: Sequelize.DATE, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.changeColumn('videoRedundancy', 'expiresOn', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 65b2dcb49..8252e95e9 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts @@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) return { - expiresOn: new Date(cacheFileObject.expires), + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, url: cacheFileObject.id, fileUrl: url.href, strategy: null, @@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) return { - expiresOn: new Date(cacheFileObject.expires), + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, url: cacheFileObject.id, fileUrl: url.href, strategy: null, diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts new file mode 100644 index 000000000..319d7090e --- /dev/null +++ b/server/lib/job-queue/handlers/video-redundancy.ts @@ -0,0 +1,20 @@ +import * as Bull from 'bull' +import { logger } from '../../../helpers/logger' +import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler' + +export type VideoRedundancyPayload = { + videoId: number +} + +async function processVideoRedundancy (job: Bull.Job) { + const payload = job.data as VideoRedundancyPayload + logger.info('Processing video redundancy in job %d.', job.id) + + return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId) +} + +// --------------------------------------------------------------------------- + +export { + processVideoRedundancy +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index ec601e9ea..8bbf58f2b 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import' import { processVideosViews } from './handlers/video-views' import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' +import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -24,20 +25,21 @@ type CreateJobArgument = { type: 'email', payload: EmailPayload } | { type: 'video-import', payload: VideoImportPayload } | { type: 'activitypub-refresher', payload: RefreshPayload } | - { type: 'videos-views', payload: {} } + { type: 'videos-views', payload: {} } | + { type: 'video-redundancy', payload: VideoRedundancyPayload } -const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise} = { +const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 'activitypub-http-unicast': processActivityPubHttpUnicast, 'activitypub-http-fetcher': processActivityPubHttpFetcher, 'activitypub-follow': processActivityPubFollow, 'video-file-import': processVideoFileImport, 'video-transcoding': processVideoTranscoding, - 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3) 'email': processEmail, 'video-import': processVideoImport, 'videos-views': processVideosViews, - 'activitypub-refresher': refreshAPObject + 'activitypub-refresher': refreshAPObject, + 'video-redundancy': processVideoRedundancy } const jobTypes: JobType[] = [ @@ -50,7 +52,8 @@ const jobTypes: JobType[] = [ 'video-file-import', 'video-import', 'videos-views', - 'activitypub-refresher' + 'activitypub-refresher', + 'video-redundancy' ] class JobQueue { diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts index 1b4ecd7c0..78d84e02e 100644 --- a/server/lib/redundancy.ts +++ b/server/lib/redundancy.ts @@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t? await videoRedundancy.destroy({ transaction: t }) } -async function removeRedundancyOf (serverId: number) { - const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId) +async function removeRedundanciesOfServer (serverId: number) { + const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId) - for (const redundancy of videosRedundancy) { + for (const redundancy of redundancies) { await removeVideoRedundancy(redundancy) } } @@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) { // --------------------------------------------------------------------------- export { - removeRedundancyOf, + removeRedundanciesOfServer, removeVideoRedundancy } diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 350a335d3..956780a77 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts @@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda import { retryTransactionWrapper } from '../../helpers/database-utils' import { federateVideoIfNeeded } from '../activitypub' import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { VideoPrivacy } from '../../../shared/models/videos' import { Notifier } from '../notifier' import { sequelizeTypescript } from '../../initializers/database' import { MVideoFullLight } from '@server/typings/models' diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c1c91b656..6e61cbe7d 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,7 +1,7 @@ import { AbstractScheduler } from './abstract-scheduler' import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' import { logger } from '../../helpers/logger' -import { VideosRedundancy } from '../../../shared/models/redundancy' +import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' import { join } from 'path' @@ -25,9 +25,10 @@ import { MVideoWithAllFiles } from '@server/typings/models' import { getVideoFilename } from '../video-paths' +import { VideoModel } from '@server/models/video/video' type CandidateToDuplicate = { - redundancy: VideosRedundancy, + redundancy: VideosRedundancyStrategy, video: MVideoWithAllFiles, files: MVideoFile[], streamingPlaylists: MStreamingPlaylistFiles[] @@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo ( export class VideosRedundancyScheduler extends AbstractScheduler { - private static instance: AbstractScheduler + private static instance: VideosRedundancyScheduler protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL @@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler { super() } + async createManualRedundancy (videoId: number) { + const videoToDuplicate = await VideoModel.loadWithFiles(videoId) + + if (!videoToDuplicate) { + logger.warn('Video to manually duplicate %d does not exist anymore.', videoId) + return + } + + return this.createVideoRedundancies({ + video: videoToDuplicate, + redundancy: null, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + }) + } + protected async internalExecute () { for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) @@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { for (const redundancyModel of expired) { try { const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - const candidate = { + const candidate: CandidateToDuplicate = { redundancy: redundancyConfig, video: null, files: [], @@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private findVideoToDuplicate (cache: VideosRedundancy) { + private findVideoToDuplicate (cache: VideosRedundancyStrategy) { if (cache.strategy === 'most-views') { return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) } @@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { + private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + const file = fileArg as MVideoFileVideo file.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) @@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { await move(tmpPath, destPath, { overwrite: true }) const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), + expiresOn, url: getVideoCacheFileActivityPubUrl(file), fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), - strategy: redundancy.strategy, + strategy, videoFileId: file.id, actorId: serverActor.id }) @@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private async createStreamingPlaylistRedundancy ( - redundancy: VideosRedundancy, + redundancy: VideosRedundancyStrategy, video: MVideoAccountLight, playlistArg: MStreamingPlaylist ) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + const playlist = playlistArg as MStreamingPlaylistVideo playlist.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) + logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), + expiresOn, url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), - strategy: redundancy.strategy, + strategy, videoStreamingPlaylistId: playlist.id, actorId: serverActor.id }) diff --git a/server/middlewares/sort.ts b/server/middlewares/sort.ts index 8c27e8237..75238228f 100644 --- a/server/middlewares/sort.ts +++ b/server/middlewares/sort.ts @@ -1,17 +1,11 @@ import * as express from 'express' import { SortType } from '../models/utils' -function setDefaultSort (req: express.Request, res: express.Response, next: express.NextFunction) { - if (!req.query.sort) req.query.sort = '-createdAt' - - return next() -} +const setDefaultSort = setDefaultSortFactory('-createdAt') -function setDefaultSearchSort (req: express.Request, res: express.Response, next: express.NextFunction) { - if (!req.query.sort) req.query.sort = '-match' +const setDefaultVideoRedundanciesSort = setDefaultSortFactory('name') - return next() -} +const setDefaultSearchSort = setDefaultSortFactory('-match') function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) { let newSort: SortType = { sortModel: undefined, sortValue: '' } @@ -39,5 +33,16 @@ function setBlacklistSort (req: express.Request, res: express.Response, next: ex export { setDefaultSort, setDefaultSearchSort, + setDefaultVideoRedundanciesSort, setBlacklistSort } + +// --------------------------------------------------------------------------- + +function setDefaultSortFactory (sort: string) { + return (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (!req.query.sort) req.query.sort = sort + + return next() + } +} diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts index 8098e3a44..16b42fc0d 100644 --- a/server/middlewares/validators/redundancy.ts +++ b/server/middlewares/validators/redundancy.ts @@ -1,12 +1,13 @@ import * as express from 'express' -import { body, param } from 'express-validator' -import { exists, isBooleanValid, isIdOrUUIDValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' +import { body, param, query } from 'express-validator' +import { exists, isBooleanValid, isIdOrUUIDValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' import { logger } from '../../helpers/logger' import { areValidationErrors } from './utils' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { isHostValid } from '../../helpers/custom-validators/servers' import { ServerModel } from '../../models/server/server' import { doesVideoExist } from '../../helpers/middlewares' +import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies' const videoFileRedundancyGetValidator = [ param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), @@ -101,10 +102,77 @@ const updateServerRedundancyValidator = [ } ] +const listVideoRedundanciesValidator = [ + query('target') + .custom(isVideoRedundancyTarget).withMessage('Should have a valid video redundancies target'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking listVideoRedundanciesValidator parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + +const addVideoRedundancyValidator = [ + body('videoId') + .custom(isIdValid) + .withMessage('Should have a valid video id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking addVideoRedundancyValidator parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return + + if (res.locals.onlyVideo.remote === false) { + return res.status(400) + .json({ error: 'Cannot create a redundancy on a local video' }) + .end() + } + + const alreadyExists = await VideoRedundancyModel.isLocalByVideoUUIDExists(res.locals.onlyVideo.uuid) + if (alreadyExists) { + return res.status(409) + .json({ error: 'This video is already duplicated by your instance.' }) + } + + return next() + } +] + +const removeVideoRedundancyValidator = [ + param('redundancyId') + .custom(isIdValid) + .withMessage('Should have a valid redundancy id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking removeVideoRedundancyValidator parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + const redundancy = await VideoRedundancyModel.loadByIdWithVideo(parseInt(req.params.redundancyId, 10)) + if (!redundancy) { + return res.status(404) + .json({ error: 'Video redundancy not found' }) + .end() + } + + res.locals.videoRedundancy = redundancy + + return next() + } +] + // --------------------------------------------------------------------------- export { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator, - updateServerRedundancyValidator + updateServerRedundancyValidator, + listVideoRedundanciesValidator, + addVideoRedundancyValidator, + removeVideoRedundancyValidator } diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index c75e701d6..b76dab722 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts @@ -23,6 +23,7 @@ const SORTABLE_USER_NOTIFICATIONS_COLUMNS = createSortableColumns(SORTABLE_COLUM const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS) const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) +const SORTABLE_VIDEO_REDUNDANCIES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS) const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS) @@ -45,6 +46,7 @@ const userNotificationsSortValidator = checkSort(SORTABLE_USER_NOTIFICATIONS_COL const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS) const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS) const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS) +const videoRedundanciesSortValidator = checkSort(SORTABLE_VIDEO_REDUNDANCIES_COLUMNS) // --------------------------------------------------------------------------- @@ -69,5 +71,6 @@ export { serversBlocklistSortValidator, userNotificationsSortValidator, videoPlaylistsSortValidator, + videoRedundanciesSortValidator, pluginsSortValidator } diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 8c9a7eabf..4e66d72e3 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -13,13 +13,13 @@ import { UpdatedAt } from 'sequelize-typescript' import { ActorModel } from '../activitypub/actor' -import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' +import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' import { VideoFileModel } from '../video/video-file' import { getServerActor } from '../../helpers/utils' import { VideoModel } from '../video/video' -import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' +import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy' import { logger } from '../../helpers/logger' import { CacheFileObject, VideoPrivacy } from '../../../shared' import { VideoChannelModel } from '../video/video-channel' @@ -27,10 +27,16 @@ import { ServerModel } from '../server/server' import { sample } from 'lodash' import { isTestInstance } from '../../helpers/core-utils' import * as Bluebird from 'bluebird' -import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize' +import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize' import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' import { CONFIG } from '../../initializers/config' -import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models' +import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models' +import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model' +import { + FileRedundancyInformation, + StreamingPlaylistRedundancyInformation, + VideoRedundancy +} from '@shared/models/redundancy/video-redundancy.model' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -86,7 +92,7 @@ export class VideoRedundancyModel extends Model { @UpdatedAt updatedAt: Date - @AllowNull(false) + @AllowNull(true) @Column expiresOn: Date @@ -193,6 +199,15 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } + static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird { + const query = { + where: { id }, + transaction + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + static loadByUrl (url: string, transaction?: Transaction): Bluebird { const query = { where: { @@ -394,7 +409,8 @@ export class VideoRedundancyModel extends Model { [Op.ne]: actor.id }, expiresOn: { - [ Op.lt ]: new Date() + [ Op.lt ]: new Date(), + [ Op.ne ]: null } } } @@ -447,7 +463,112 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.findAll(query) } - static async getStats (strategy: VideoRedundancyStrategy) { + static listForApi (options: { + start: number, + count: number, + sort: string, + target: VideoRedundanciesTarget, + strategy?: string + }) { + const { start, count, sort, target, strategy } = options + let redundancyWhere: WhereOptions = {} + let videosWhere: WhereOptions = {} + let redundancySqlSuffix = '' + + if (target === 'my-videos') { + Object.assign(videosWhere, { remote: false }) + } else if (target === 'remote-videos') { + Object.assign(videosWhere, { remote: true }) + Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } }) + redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL' + } + + if (strategy) { + Object.assign(redundancyWhere, { strategy: strategy }) + } + + const videoFilterWhere = { + [Op.and]: [ + { + [ Op.or ]: [ + { + id: { + [ Op.in ]: literal( + '(' + + 'SELECT "videoId" FROM "videoFile" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' + + redundancySqlSuffix + + ')' + ) + } + }, + { + id: { + [ Op.in ]: literal( + '(' + + 'select "videoId" FROM "videoStreamingPlaylist" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' + + redundancySqlSuffix + + ')' + ) + } + } + ] + }, + + videosWhere + ] + } + + // /!\ On video model /!\ + const findOptions = { + offset: start, + limit: count, + order: getSort(sort), + include: [ + { + required: false, + model: VideoFileModel.unscoped(), + include: [ + { + model: VideoRedundancyModel.unscoped(), + required: false, + where: redundancyWhere + } + ] + }, + { + required: false, + model: VideoStreamingPlaylistModel.unscoped(), + include: [ + { + model: VideoRedundancyModel.unscoped(), + required: false, + where: redundancyWhere + }, + { + model: VideoFileModel.unscoped(), + required: false + } + ] + } + ], + where: videoFilterWhere + } + + // /!\ On video model /!\ + const countOptions = { + where: videoFilterWhere + } + + return Promise.all([ + VideoModel.findAll(findOptions), + + VideoModel.count(countOptions) + ]).then(([ data, total ]) => ({ total, data })) + } + + static async getStats (strategy: VideoRedundancyStrategyWithManual) { const actor = await getServerActor() const query: FindOptions = { @@ -478,6 +599,53 @@ export class VideoRedundancyModel extends Model { })) } + static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { + let filesRedundancies: FileRedundancyInformation[] = [] + let streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = [] + + for (const file of video.VideoFiles) { + for (const redundancy of file.RedundancyVideos) { + filesRedundancies.push({ + id: redundancy.id, + fileUrl: redundancy.fileUrl, + strategy: redundancy.strategy, + createdAt: redundancy.createdAt, + updatedAt: redundancy.updatedAt, + expiresOn: redundancy.expiresOn, + size: file.size + }) + } + } + + for (const playlist of video.VideoStreamingPlaylists) { + const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0) + + for (const redundancy of playlist.RedundancyVideos) { + streamingPlaylistsRedundancies.push({ + id: redundancy.id, + fileUrl: redundancy.fileUrl, + strategy: redundancy.strategy, + createdAt: redundancy.createdAt, + updatedAt: redundancy.updatedAt, + expiresOn: redundancy.expiresOn, + size + }) + } + } + + return { + id: video.id, + name: video.name, + url: video.url, + uuid: video.uuid, + + redundancies: { + files: filesRedundancies, + streamingPlaylists: streamingPlaylistsRedundancies + } + } + } + getVideo () { if (this.VideoFile) return this.VideoFile.Video @@ -494,7 +662,7 @@ export class VideoRedundancyModel extends Model { id: this.url, type: 'CacheFile' as 'CacheFile', object: this.VideoStreamingPlaylist.Video.url, - expires: this.expiresOn.toISOString(), + expires: this.expiresOn ? this.expiresOn.toISOString() : null, url: { type: 'Link', mediaType: 'application/x-mpegURL', @@ -507,7 +675,7 @@ export class VideoRedundancyModel extends Model { id: this.url, type: 'CacheFile' as 'CacheFile', object: this.VideoFile.Video.url, - expires: this.expiresOn.toISOString(), + expires: this.expiresOn ? this.expiresOn.toISOString() : null, url: { type: 'Link', mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts index 6471da840..7012a39ee 100644 --- a/server/tests/api/check-params/redundancy.ts +++ b/server/tests/api/check-params/redundancy.ts @@ -3,21 +3,25 @@ import 'mocha' import { + checkBadCountPagination, + checkBadSortPagination, + checkBadStartPagination, cleanupTests, createUser, doubleFollow, - flushAndRunMultipleServers, - flushTests, - killallServers, + flushAndRunMultipleServers, makeDeleteRequest, + makeGetRequest, makePostBodyRequest, makePutBodyRequest, ServerInfo, - setAccessTokensToServers, - userLogin + setAccessTokensToServers, uploadVideoAndGetId, + userLogin, waitJobs } from '../../../../shared/extra-utils' describe('Test server redundancy API validators', function () { let servers: ServerInfo[] let userAccessToken = null + let videoIdLocal: number + let videoIdRemote: number // --------------------------------------------------------------- @@ -36,9 +40,134 @@ describe('Test server redundancy API validators', function () { await createUser({ url: servers[ 0 ].url, accessToken: servers[ 0 ].accessToken, username: user.username, password: user.password }) userAccessToken = await userLogin(servers[0], user) + + videoIdLocal = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video' })).id + videoIdRemote = (await uploadVideoAndGetId({ server: servers[1], videoName: 'video' })).id + + await waitJobs(servers) + }) + + describe('When listing redundancies', function () { + const path = '/api/v1/server/redundancy/videos' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makeGetRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeGetRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with a bad target', async function () { + await makeGetRequest({ url, path, token, query: { target: 'bad target' } }) + }) + + it('Should fail without target', async function () { + await makeGetRequest({ url, path, token }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ url, path, token, query: { target: 'my-videos' }, statusCodeExpected: 200 }) + }) + }) + + describe('When manually adding a redundancy', function () { + const path = '/api/v1/server/redundancy/videos' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makePostBodyRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePostBodyRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 }) + }) + + it('Should fail without a video id', async function () { + await makePostBodyRequest({ url, path, token }) + }) + + it('Should fail with an incorrect video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: 'peertube' } }) + }) + + it('Should fail with a not found video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: 6565 }, statusCodeExpected: 404 }) + }) + + it('Should fail with a local a video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdLocal } }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 204 }) + }) + + it('Should fail if the video is already duplicated', async function () { + this.timeout(30000) + + await waitJobs(servers) + + await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 409 }) + }) + }) + + describe('When manually removing a redundancy', function () { + const path = '/api/v1/server/redundancy/videos/' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makeDeleteRequest({ url, path: path + '1', token: 'fake_token', statusCodeExpected: 401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeDeleteRequest({ url, path: path + '1', token: userAccessToken, statusCodeExpected: 403 }) + }) + + it('Should fail with an incorrect video id', async function () { + await makeDeleteRequest({ url, path: path + 'toto', token }) + }) + + it('Should fail with a not found video redundancy', async function () { + await makeDeleteRequest({ url, path: path + '454545', token, statusCodeExpected: 404 }) + }) }) - describe('When updating redundancy', function () { + describe('When updating server redundancy', function () { const path = '/api/v1/server/redundancy' it('Should fail with an invalid token', async function () { diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts index 8e69b95a6..5359055b0 100644 --- a/server/tests/api/redundancy/index.ts +++ b/server/tests/api/redundancy/index.ts @@ -1 +1,2 @@ import './redundancy' +import './manage-redundancy' diff --git a/server/tests/api/redundancy/manage-redundancy.ts b/server/tests/api/redundancy/manage-redundancy.ts new file mode 100644 index 000000000..6a8937f24 --- /dev/null +++ b/server/tests/api/redundancy/manage-redundancy.ts @@ -0,0 +1,373 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { + cleanupTests, + doubleFollow, + flushAndRunMultipleServers, + getLocalIdByUUID, + ServerInfo, + setAccessTokensToServers, + uploadVideo, + uploadVideoAndGetId, + waitUntilLog +} from '../../../../shared/extra-utils' +import { waitJobs } from '../../../../shared/extra-utils/server/jobs' +import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy, updateRedundancy } from '@shared/extra-utils/server/redundancy' +import { VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' + +const expect = chai.expect + +describe('Test manage videos redundancy', function () { + const targets: VideoRedundanciesTarget[] = [ 'my-videos', 'remote-videos' ] + + let servers: ServerInfo[] + let video1Server2UUID: string + let video2Server2UUID: string + let redundanciesToRemove: number[] = [] + + before(async function () { + this.timeout(120000) + + const config = { + transcoding: { + hls: { + enabled: true + } + }, + redundancy: { + videos: { + check_interval: '1 second', + strategies: [ + { + strategy: 'recently-added', + min_lifetime: '1 hour', + size: '10MB', + min_views: 0 + } + ] + } + } + } + servers = await flushAndRunMultipleServers(3, config) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) + video1Server2UUID = res.body.video.uuid + } + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) + video2Server2UUID = res.body.video.uuid + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[ 0 ], servers[ 1 ]) + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) + + await waitJobs(servers) + }) + + it('Should not have redundancies on server 3', async function () { + for (const target of targets) { + const res = await listVideoRedundancies({ + url: servers[2].url, + accessToken: servers[2].accessToken, + target + }) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + } + }) + + it('Should not have "remote-videos" redundancies on server 2', async function () { + this.timeout(120000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 10) + await waitJobs(servers) + + const res = await listVideoRedundancies({ + url: servers[1].url, + accessToken: servers[1].accessToken, + target: 'remote-videos' + }) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + }) + + it('Should have "my-videos" redundancies on server 2', async function () { + this.timeout(120000) + + const res = await listVideoRedundancies({ + url: servers[1].url, + accessToken: servers[1].accessToken, + target: 'my-videos' + }) + + expect(res.body.total).to.equal(2) + + const videos = res.body.data as VideoRedundancy[] + expect(videos).to.have.lengthOf(2) + + const videos1 = videos.find(v => v.uuid === video1Server2UUID) + const videos2 = videos.find(v => v.uuid === video2Server2UUID) + + expect(videos1.name).to.equal('video 1 server 2') + expect(videos2.name).to.equal('video 2 server 2') + + expect(videos1.redundancies.files).to.have.lengthOf(4) + expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.be.null + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.exist + } + }) + + it('Should not have "my-videos" redundancies on server 1', async function () { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'my-videos' + }) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + }) + + it('Should have "remote-videos" redundancies on server 1', async function () { + this.timeout(120000) + + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos' + }) + + expect(res.body.total).to.equal(2) + + const videos = res.body.data as VideoRedundancy[] + expect(videos).to.have.lengthOf(2) + + const videos1 = videos.find(v => v.uuid === video1Server2UUID) + const videos2 = videos.find(v => v.uuid === video2Server2UUID) + + expect(videos1.name).to.equal('video 1 server 2') + expect(videos2.name).to.equal('video 2 server 2') + + expect(videos1.redundancies.files).to.have.lengthOf(4) + expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.equal('recently-added') + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.exist + } + }) + + it('Should correctly paginate and sort results', async function () { + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: 'name', + start: 0, + count: 2 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 1 server 2') + expect(videos[ 1 ].name).to.equal('video 2 server 2') + } + + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 0, + count: 2 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 2 server 2') + expect(videos[ 1 ].name).to.equal('video 1 server 2') + } + + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 1, + count: 1 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 1 server 2') + } + }) + + it('Should manually add a redundancy and list it', async function () { + this.timeout(120000) + + const uuid = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video 3 server 2', privacy: VideoPrivacy.UNLISTED })).uuid + await waitJobs(servers) + const videoId = await getLocalIdByUUID(servers[0].url, uuid) + + await addVideoRedundancy({ + url: servers[0].url, + accessToken: servers[0].accessToken, + videoId + }) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 15) + await waitJobs(servers) + + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 3 server 2') + + const video = videos[ 0 ] + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + for (const r of redundancies) { + redundanciesToRemove.push(r.id) + + expect(r.strategy).to.equal('manual') + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.be.null + } + } + + const res = await listVideoRedundancies({ + url: servers[1].url, + accessToken: servers[1].accessToken, + target: 'my-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 3 server 2') + + const video = videos[ 0 ] + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.be.null + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.be.null + } + }) + + it('Should manually remove a redundancy and remove it from the list', async function () { + this.timeout(120000) + + for (const redundancyId of redundanciesToRemove) { + await removeVideoRedundancy({ + url: servers[ 0 ].url, + accessToken: servers[ 0 ].accessToken, + redundancyId + }) + } + + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = res.body.data + expect(videos).to.have.lengthOf(2) + + expect(videos[ 0 ].name).to.equal('video 2 server 2') + + redundanciesToRemove = [] + const video = videos[ 0 ] + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + for (const r of redundancies) { + redundanciesToRemove.push(r.id) + } + } + }) + + it('Should remove another (auto) redundancy', async function () { + { + for (const redundancyId of redundanciesToRemove) { + await removeVideoRedundancy({ + url: servers[ 0 ].url, + accessToken: servers[ 0 ].accessToken, + redundancyId + }) + } + + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 1 server 2') + expect(videos).to.have.lengthOf(1) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 1cdf93aa1..f5bf130d5 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -5,7 +5,8 @@ import 'mocha' import { VideoDetails } from '../../../../shared/models/videos' import { checkSegmentHash, - checkVideoFilesWereRemoved, cleanupTests, + checkVideoFilesWereRemoved, + cleanupTests, doubleFollow, flushAndRunMultipleServers, getFollowingListPaginationAndSort, @@ -28,11 +29,16 @@ import { import { waitJobs } from '../../../../shared/extra-utils/server/jobs' import * as magnetUtil from 'magnet-uri' -import { updateRedundancy } from '../../../../shared/extra-utils/server/redundancy' +import { + addVideoRedundancy, + listVideoRedundancies, + removeVideoRedundancy, + updateRedundancy +} from '../../../../shared/extra-utils/server/redundancy' import { ActorFollow } from '../../../../shared/models/actors' import { readdir } from 'fs-extra' import { join } from 'path' -import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' +import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy' import { getStats } from '../../../../shared/extra-utils/server/stats' import { ServerStats } from '../../../../shared/models/server/server-stats.model' @@ -40,6 +46,7 @@ const expect = chai.expect let servers: ServerInfo[] = [] let video1Server2UUID: string +let video1Server2Id: number function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { const parsed = magnetUtil.decode(file.magnetUri) @@ -52,7 +59,19 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) } -async function flushAndRunServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { +async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) { + const strategies: any[] = [] + + if (strategy !== null) { + strategies.push( + immutableAssign({ + min_lifetime: '1 hour', + strategy: strategy, + size: '400KB' + }, additionalParams) + ) + } + const config = { transcoding: { hls: { @@ -62,16 +81,11 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional redundancy: { videos: { check_interval: '5 seconds', - strategies: [ - immutableAssign({ - min_lifetime: '1 hour', - strategy: strategy, - size: '400KB' - }, additionalParams) - ] + strategies } } } + servers = await flushAndRunMultipleServers(3, config) // Get the access tokens @@ -80,6 +94,7 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional { const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) video1Server2UUID = res.body.video.uuid + video1Server2Id = res.body.video.id await viewVideo(servers[ 1 ].url, video1Server2UUID) } @@ -216,29 +231,38 @@ async function check1PlaylistRedundancies (videoUUID?: string) { } } -async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { +async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { + let totalSize: number = null + let statsLength = 1 + + if (strategy !== 'manual') { + totalSize = 409600 + statsLength = 2 + } + const res = await getStats(servers[0].url) const data: ServerStats = res.body - expect(data.videosRedundancy).to.have.lengthOf(1) - const stat = data.videosRedundancy[0] + expect(data.videosRedundancy).to.have.lengthOf(statsLength) + const stat = data.videosRedundancy[0] expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(409600) + expect(stat.totalSize).to.equal(totalSize) + + return stat +} + +async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) { + const stat = await checkStatsGlobal(strategy) + expect(stat.totalUsed).to.be.at.least(1).and.below(409601) expect(stat.totalVideoFiles).to.equal(4) expect(stat.totalVideos).to.equal(1) } -async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) +async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) { + const stat = await checkStatsGlobal(strategy) - const stat = data.videosRedundancy[0] - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(409600) expect(stat.totalUsed).to.equal(0) expect(stat.totalVideoFiles).to.equal(0) expect(stat.totalVideos).to.equal(0) @@ -446,6 +470,74 @@ describe('Test videos redundancy', function () { }) }) + describe('With manual strategy', function () { + before(function () { + this.timeout(120000) + + return flushAndRunServers(null) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed() + await check0PlaylistRedundancies() + await checkStatsWith1Webseed('manual') + }) + + it('Should create a redundancy on first video', async function () { + await addVideoRedundancy({ + url: servers[0].url, + accessToken: servers[0].accessToken, + videoId: video1Server2Id + }) + }) + + it('Should have 2 webseeds on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 5) + await waitJobs(servers) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith2Webseed('manual') + }) + + it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { + this.timeout(80000) + + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos' + }) + + const videos = res.body.data as VideoRedundancy[] + expect(videos).to.have.lengthOf(1) + + const video = videos[0] + for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) { + await removeVideoRedundancy({ + url: servers[0].url, + accessToken: servers[0].accessToken, + redundancyId: r.id + }) + } + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed() + await check0PlaylistRedundancies() + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + describe('Test expiration', function () { const strategy = 'recently-added' diff --git a/server/typings/models/video/video-file.ts b/server/typings/models/video/video-file.ts index 352fe3d32..139b22b2c 100644 --- a/server/typings/models/video/video-file.ts +++ b/server/typings/models/video/video-file.ts @@ -1,7 +1,7 @@ import { VideoFileModel } from '../../../models/video/video-file' import { PickWith, PickWithOpt } from '../../utils' import { MVideo, MVideoUUID } from './video' -import { MVideoRedundancyFileUrl } from './video-redundancy' +import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy' import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist' type Use = PickWith @@ -22,6 +22,9 @@ export type MVideoFileStreamingPlaylistVideo = MVideoFile & export type MVideoFileVideoUUID = MVideoFile & Use<'Video', MVideoUUID> +export type MVideoFileRedundanciesAll = MVideoFile & + PickWithOpt + export type MVideoFileRedundanciesOpt = MVideoFile & PickWithOpt diff --git a/server/typings/models/video/video-streaming-playlist.ts b/server/typings/models/video/video-streaming-playlist.ts index 436c0c072..6fd489945 100644 --- a/server/typings/models/video/video-streaming-playlist.ts +++ b/server/typings/models/video/video-streaming-playlist.ts @@ -1,6 +1,6 @@ import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist' import { PickWith, PickWithOpt } from '../../utils' -import { MVideoRedundancyFileUrl } from './video-redundancy' +import { MVideoRedundancyFileUrl, MVideoRedundancy } from './video-redundancy' import { MVideo } from './video' import { MVideoFile } from './video-file' @@ -20,6 +20,10 @@ export type MStreamingPlaylistFilesVideo = MStreamingPlaylist & Use<'VideoFiles', MVideoFile[]> & Use<'Video', MVideo> +export type MStreamingPlaylistRedundanciesAll = MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & + Use<'RedundancyVideos', MVideoRedundancy[]> + export type MStreamingPlaylistRedundancies = MStreamingPlaylist & Use<'VideoFiles', MVideoFile[]> & Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts index 7f69a91de..bcc5e5028 100644 --- a/server/typings/models/video/video.ts +++ b/server/typings/models/video/video.ts @@ -10,8 +10,13 @@ import { } from './video-channels' import { MTag } from './tag' import { MVideoCaptionLanguage } from './video-caption' -import { MStreamingPlaylistFiles, MStreamingPlaylistRedundancies, MStreamingPlaylistRedundanciesOpt } from './video-streaming-playlist' -import { MVideoFile, MVideoFileRedundanciesOpt } from './video-file' +import { + MStreamingPlaylistFiles, + MStreamingPlaylistRedundancies, + MStreamingPlaylistRedundanciesAll, + MStreamingPlaylistRedundanciesOpt +} from './video-streaming-playlist' +import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file' import { MThumbnail } from './thumbnail' import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' import { MScheduleVideoUpdate } from './schedule-video-update' @@ -158,6 +163,10 @@ export type MVideoForUser = MVideo & Use<'VideoBlacklist', MVideoBlacklistLight> & Use<'Thumbnails', MThumbnail[]> +export type MVideoForRedundancyAPI = MVideo & + Use<'VideoFiles', MVideoFileRedundanciesAll[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesAll[]> + // ############################################################################ // Format for API or AP object -- cgit v1.2.3