From b36f41ca09e92ecb30d367d91d1089a23d10d585 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 09:57:21 +0200 Subject: Add trending videos strategy --- server/lib/schedulers/videos-redundancy-scheduler.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'server/lib/schedulers') diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index ee9ba1766..c1e619249 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -75,6 +75,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler { private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + + if (strategy === 'trending') return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) } private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { -- cgit v1.2.3 From 3f6b6a565dc98a658ec9d8f697252788c0faa46d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 11:05:38 +0200 Subject: Add recently added redundancy strategy --- .../lib/schedulers/videos-redundancy-scheduler.ts | 44 +++++++++------------- 1 file changed, 17 insertions(+), 27 deletions(-) (limited to 'server/lib/schedulers') diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c1e619249..8b91d750b 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,7 +1,7 @@ import { AbstractScheduler } from './abstract-scheduler' import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' import { logger } from '../../helpers/logger' -import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' +import { RecentlyAddedStrategy, VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { VideoFileModel } from '../../models/video/video-file' import { sortBy } from 'lodash' @@ -32,16 +32,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler { this.executing = true for (const obj of CONFIG.REDUNDANCY.VIDEOS) { - try { - const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) + const videoToDuplicate = await this.findVideoToDuplicate(obj) if (!videoToDuplicate) continue const videoFiles = videoToDuplicate.VideoFiles videoFiles.forEach(f => f.Video = videoToDuplicate) - const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) - if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) { + if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) { if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) continue } @@ -73,10 +71,19 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return this.instance || (this.instance = new this()) } - private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { - if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + private findVideoToDuplicate (cache: VideosRedundancy) { + if (cache.strategy === 'most-views') { + return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } + + if (cache.strategy === 'trending') { + return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } - if (strategy === 'trending') return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + if (cache.strategy === 'recently-added') { + const minViews = (cache as RecentlyAddedStrategy).minViews + return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) + } } private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { @@ -122,27 +129,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - // Unused, but could be useful in the future, with a custom strategy - private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) { - const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt') - - while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) { - const toDelete = sortedVideosRedundancy.shift() - - const videoFile = toDelete.VideoFile - logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution) - - await removeVideoRedundancy(toDelete, undefined) - } - - return sortedVideosRedundancy - } - - private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) { + private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) { const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) - const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size - const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0) + const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy) return totalDuplicated > maxSize } -- cgit v1.2.3 From 7348b1fd84dee869b3c36554aea6797f09d4ceed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 11:52:23 +0200 Subject: Speed up overviews route --- server/lib/schedulers/videos-redundancy-scheduler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/lib/schedulers') diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 8b91d750b..7079600a9 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -81,7 +81,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } if (cache.strategy === 'recently-added') { - const minViews = (cache as RecentlyAddedStrategy).minViews + const minViews = cache.minViews return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) } } -- cgit v1.2.3 From d9bdd007d7a1368d2a13127ecb5c0a81a18a8c04 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 16:12:07 +0200 Subject: Put config redundancy strategies in "strategies" subkey --- server/lib/schedulers/videos-redundancy-scheduler.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'server/lib/schedulers') diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 7079600a9..5f9fd9911 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,10 +1,9 @@ import { AbstractScheduler } from './abstract-scheduler' import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' import { logger } from '../../helpers/logger' -import { RecentlyAddedStrategy, VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' +import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { VideoFileModel } from '../../models/video/video-file' -import { sortBy } from 'lodash' import { downloadWebTorrentVideo } from '../../helpers/webtorrent' import { join } from 'path' import { rename } from 'fs-extra' @@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' import { VideoModel } from '../../models/video/video' import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' -import { removeVideoRedundancy } from '../redundancy' import { isTestInstance } from '../../helpers/core-utils' export class VideosRedundancyScheduler extends AbstractScheduler { @@ -31,7 +29,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { this.executing = true - for (const obj of CONFIG.REDUNDANCY.VIDEOS) { + for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { try { const videoToDuplicate = await this.findVideoToDuplicate(obj) if (!videoToDuplicate) continue -- cgit v1.2.3 From f9f899b9f803eb5159a67781f10649a0cf040677 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 16:21:09 +0200 Subject: Add redundancy check interval in config --- server/lib/schedulers/videos-redundancy-scheduler.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'server/lib/schedulers') diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 5f9fd9911..960651712 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -18,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { private static instance: AbstractScheduler private executing = false - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy + protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL private constructor () { super() @@ -50,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } + await this.removeExpired() + + this.executing = false + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + private async removeExpired () { const expired = await VideoRedundancyModel.listAllExpired() for (const m of expired) { @@ -61,12 +71,6 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) } } - - this.executing = false - } - - static get Instance () { - return this.instance || (this.instance = new this()) } private findVideoToDuplicate (cache: VideosRedundancy) { -- cgit v1.2.3 From 606c946e74211c4123b16087288902226306198d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 16:24:24 +0200 Subject: Move youtubeDL upgrader in helpers/ --- .../lib/schedulers/youtube-dl-update-scheduler.ts | 60 ++-------------------- 1 file changed, 4 insertions(+), 56 deletions(-) (limited to 'server/lib/schedulers') diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index faadb4334..2fc8950fe 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts @@ -1,5 +1,4 @@ -// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js -// We rewrote it to avoid sync calls + import { AbstractScheduler } from './abstract-scheduler' import { SCHEDULER_INTERVALS_MS } from '../../initializers' @@ -8,6 +7,7 @@ import * as request from 'request' import { createWriteStream, ensureDir, writeFile } from 'fs-extra' import { join } from 'path' import { root } from '../../helpers/core-utils' +import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' export class YoutubeDlUpdateScheduler extends AbstractScheduler { @@ -19,60 +19,8 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler { super() } - async execute () { - logger.info('Updating youtubeDL binary.') - - const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') - const bin = join(binDirectory, 'youtube-dl') - const detailsPath = join(binDirectory, 'details') - const url = 'https://yt-dl.org/downloads/latest/youtube-dl' - - await ensureDir(binDirectory) - - return new Promise(res => { - request.get(url, { followRedirect: false }, (err, result) => { - if (err) { - logger.error('Cannot update youtube-dl.', { err }) - return res() - } - - if (result.statusCode !== 302) { - logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) - return res() - } - - const url = result.headers.location - const downloadFile = request.get(url) - const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ] - - downloadFile.on('response', result => { - if (result.statusCode !== 200) { - logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) - return res() - } - - downloadFile.pipe(createWriteStream(bin, { mode: 493 })) - }) - - downloadFile.on('error', err => { - logger.error('youtube-dl update error.', { err }) - return res() - }) - - downloadFile.on('end', () => { - const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) - writeFile(detailsPath, details, { encoding: 'utf8' }, err => { - if (err) { - logger.error('youtube-dl update error: cannot write details.', { err }) - return res() - } - - logger.info('youtube-dl updated to version %s.', newVersion) - return res() - }) - }) - }) - }) + execute () { + return updateYoutubeDLBinary() } static get Instance () { -- cgit v1.2.3 From 91411dba928678c15a5e99d9795ae061909e397d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Sep 2018 10:13:13 +0200 Subject: Limit associations fetch when loading token --- server/lib/schedulers/youtube-dl-update-scheduler.ts | 7 ------- 1 file changed, 7 deletions(-) (limited to 'server/lib/schedulers') diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index 2fc8950fe..461cd045e 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts @@ -1,12 +1,5 @@ - - import { AbstractScheduler } from './abstract-scheduler' import { SCHEDULER_INTERVALS_MS } from '../../initializers' -import { logger } from '../../helpers/logger' -import * as request from 'request' -import { createWriteStream, ensureDir, writeFile } from 'fs-extra' -import { join } from 'path' -import { root } from '../../helpers/core-utils' import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' export class YoutubeDlUpdateScheduler extends AbstractScheduler { -- cgit v1.2.3