From 3f6b6a565dc98a658ec9d8f697252788c0faa46d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 11:05:38 +0200 Subject: [PATCH] Add recently added redundancy strategy --- config/default.yaml | 4 ++ config/production.yaml.example | 4 ++ config/test.yaml | 4 ++ .../custom-validators/activitypub/videos.ts | 2 - server/initializers/checker.ts | 13 +++- server/initializers/constants.ts | 11 +--- .../schedulers/videos-redundancy-scheduler.ts | 44 +++++-------- server/models/redundancy/video-redundancy.ts | 53 +++++++++++----- server/tests/api/server/redundancy.ts | 63 ++++++++++++++++--- .../redundancy/videos-redundancy.model.ts | 19 +++++- 10 files changed, 152 insertions(+), 65 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index ecb809c6a..adac9deeb 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -77,6 +77,10 @@ redundancy: # - # size: '10GB' # strategy: 'trending' # Cache trending videos +# - +# size: '10GB' +# strategy: 'recently-added' # Cache recently added videos +# minViews: 10 # Having at least x views cache: previews: diff --git a/config/production.yaml.example b/config/production.yaml.example index 48d69e987..ca7b936c2 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -78,6 +78,10 @@ redundancy: # - # size: '10GB' # strategy: 'trending' # Cache trending videos +# - +# size: '10GB' +# strategy: 'recently-added' # Cache recently added videos +# minViews: 10 # Having at least x views ############################################################################### # diff --git a/config/test.yaml b/config/test.yaml index 73bc5da98..517fc7449 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -29,6 +29,10 @@ redundancy: - size: '100KB' strategy: 'trending' + - + size: '100KB' + strategy: 'recently-added' + minViews: 10 cache: previews: diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index f76eba474..8772e74cf 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -171,5 +171,3 @@ function setRemoteVideoTruncatedContent (video: any) { return true } - - diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 6048151a3..29f4f3036 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -7,7 +7,7 @@ import { parse } from 'url' import { CONFIG } from './constants' import { logger } from '../helpers/logger' import { getServerActor } from '../helpers/utils' -import { VideosRedundancy } from '../../shared/models/redundancy' +import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy' import { isArray } from '../helpers/custom-validators/misc' import { uniq } from 'lodash' @@ -34,24 +34,31 @@ async function checkActivityPubUrls () { function checkConfig () { const defaultNSFWPolicy = config.get('instance.default_nsfw_policy') + // NSFW policy if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) { return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy } + // Redundancies const redundancyVideos = config.get('redundancy.videos') if (isArray(redundancyVideos)) { for (const r of redundancyVideos) { - if ([ 'most-views', 'trending' ].indexOf(r.strategy) === -1) { + if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) { return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy } } const filtered = uniq(redundancyVideos.map(r => r.strategy)) if (filtered.length !== redundancyVideos.length) { - return 'Redundancy video entries should have uniq strategies' + return 'Redundancy video entries should have unique strategies' } } + const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy + if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { + return 'Min views in recently added strategy is not a number' + } + return null } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6b4afbfd8..5f7bcbd48 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -1,6 +1,6 @@ import { IConfig } from 'config' import { dirname, join } from 'path' -import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models' +import { JobType, VideoRateType, VideoState, VideosRedundancy } from '../../shared/models' import { ActivityPubActorType } from '../../shared/models/activitypub' import { FollowState } from '../../shared/models/actors' import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' @@ -741,15 +741,10 @@ function updateWebserverConfig () { CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) } -function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] { +function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] { if (!objs) return [] - return objs.map(obj => { - return { - strategy: obj.strategy, - size: bytes.parse(obj.size) - } - }) + return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) })) } function buildLanguages () { diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c1e619249..8b91d750b 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,7 +1,7 @@ import { AbstractScheduler } from './abstract-scheduler' import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' import { logger } from '../../helpers/logger' -import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' +import { RecentlyAddedStrategy, VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { VideoFileModel } from '../../models/video/video-file' import { sortBy } from 'lodash' @@ -32,16 +32,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler { this.executing = true for (const obj of CONFIG.REDUNDANCY.VIDEOS) { - try { - const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) + const videoToDuplicate = await this.findVideoToDuplicate(obj) if (!videoToDuplicate) continue const videoFiles = videoToDuplicate.VideoFiles videoFiles.forEach(f => f.Video = videoToDuplicate) - const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) - if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) { + if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) { if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) continue } @@ -73,10 +71,19 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return this.instance || (this.instance = new this()) } - private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { - if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + private findVideoToDuplicate (cache: VideosRedundancy) { + if (cache.strategy === 'most-views') { + return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } + + if (cache.strategy === 'trending') { + return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } - if (strategy === 'trending') return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + if (cache.strategy === 'recently-added') { + const minViews = (cache as RecentlyAddedStrategy).minViews + return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) + } } private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { @@ -122,27 +129,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - // Unused, but could be useful in the future, with a custom strategy - private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) { - const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt') - - while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) { - const toDelete = sortedVideosRedundancy.shift() - - const videoFile = toDelete.VideoFile - logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution) - - await removeVideoRedundancy(toDelete, undefined) - } - - return sortedVideosRedundancy - } - - private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) { + private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) { const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) - const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size - const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0) + const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy) return totalDuplicated > maxSize } diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index b13ade0f4..b7454c617 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -27,6 +27,7 @@ import { VideoChannelModel } from '../video/video-channel' import { ServerModel } from '../server/server' import { sample } from 'lodash' import { isTestInstance } from '../../helpers/core-utils' +import * as Bluebird from 'bluebird' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -144,7 +145,8 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.findOne(query) } - static getVideoSample (rows: { id: number }[]) { + static async getVideoSample (p: Bluebird) { + const rows = await p const ids = rows.map(r => r.id) const id = sample(ids) @@ -164,9 +166,7 @@ export class VideoRedundancyModel extends Model { ] } - const rows = await VideoModel.unscoped().findAll(query) - - return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } static async findTrendingToDuplicate (randomizedFactor: number) { @@ -186,24 +186,49 @@ export class VideoRedundancyModel extends Model { ] } - const rows = await VideoModel.unscoped().findAll(query) + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } - return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) + static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { + // On VideoModel! + const query = { + attributes: [ 'id', 'publishedAt' ], + // logging: !isTestInstance(), + limit: randomizedFactor, + order: getVideoSort('-publishedAt'), + where: { + views: { + [ Sequelize.Op.gte ]: minViews + } + }, + include: [ + await VideoRedundancyModel.buildVideoFileForDuplication(), + VideoRedundancyModel.buildServerRedundancyInclude() + ] + } + + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } - static async getVideoFiles (strategy: VideoRedundancyStrategy) { + static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { const actor = await getServerActor() - const queryVideoFiles = { + const options = { logging: !isTestInstance(), - where: { - actorId: actor.id, - strategy - } + include: [ + { + attributes: [], + model: VideoRedundancyModel, + required: true, + where: { + actorId: actor.id, + strategy + } + } + ] } - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) - .findAll(queryVideoFiles) + return VideoFileModel.sum('size', options) } static listAllExpired () { diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index 211570d2f..6574e8ea9 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts @@ -14,7 +14,7 @@ import { setAccessTokensToServers, uploadVideo, wait, - root, viewVideo + root, viewVideo, immutableAssign } from '../../utils' import { waitJobs } from '../../utils/server/jobs' import * as magnetUtil from 'magnet-uri' @@ -39,14 +39,14 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe } } -async function runServers (strategy: VideoRedundancyStrategy) { +async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { const config = { redundancy: { videos: [ - { + immutableAssign({ strategy: strategy, size: '100KB' - } + }, additionalParams) ] } } @@ -153,11 +153,11 @@ describe('Test videos redundancy', function () { return check1WebSeed() }) - it('Should enable redundancy on server 1', async function () { + it('Should enable redundancy on server 1', function () { return enableRedundancy() }) - it('Should have 2 webseed on the first video', async function () { + it('Should have 2 webseed on the first video', function () { this.timeout(40000) return check2Webseeds() @@ -180,11 +180,58 @@ describe('Test videos redundancy', function () { return check1WebSeed() }) - it('Should enable redundancy on server 1', async function () { + it('Should enable redundancy on server 1', function () { return enableRedundancy() }) - it('Should have 2 webseed on the first video', async function () { + it('Should have 2 webseed on the first video', function () { + this.timeout(40000) + + return check2Webseeds() + }) + + after(function () { + return cleanServers() + }) + }) + + describe('With recently added strategy', function () { + + before(function () { + this.timeout(120000) + + return runServers('recently-added', { minViews: 3 }) + }) + + it('Should have 1 webseed on the first video', function () { + return check1WebSeed() + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancy() + }) + + it('Should still have 1 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + return check1WebSeed() + }) + + it('Should view 2 times the first video', async function () { + this.timeout(40000) + + await viewVideo(servers[ 0 ].url, video1Server2UUID) + await viewVideo(servers[ 2 ].url, video1Server2UUID) + + await wait(10000) + await waitJobs(servers) + }) + + it('Should have 2 webseed on the first video', function () { this.timeout(40000) return check2Webseeds() diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy.model.ts index 85982e5b3..436394c1e 100644 --- a/shared/models/redundancy/videos-redundancy.model.ts +++ b/shared/models/redundancy/videos-redundancy.model.ts @@ -1,6 +1,19 @@ -export type VideoRedundancyStrategy = 'most-views' | 'trending' +export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-added' -export interface VideosRedundancy { - strategy: VideoRedundancyStrategy +export type MostViewsRedundancyStrategy = { + strategy: 'most-views' size: number } + +export type TrendingRedundancyStrategy = { + strategy: 'trending' + size: number +} + +export type RecentlyAddedStrategy = { + strategy: 'recently-added' + size: number + minViews: number +} + +export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy -- 2.41.0