aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/schedulers/videos-redundancy-scheduler.ts
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-09-20 16:24:31 +0200
committerGitHub <noreply@github.com>2018-09-20 16:24:31 +0200
commit0491173a61aed66205c017e0d7e0503ea316c144 (patch)
treece6621597505f9518cfdf0981977d097c63f9fad /server/lib/schedulers/videos-redundancy-scheduler.ts
parent8704acf49efc770d73bf07c10468ed8c74d28a83 (diff)
parent6247b2057b792cea155a1abd9788c363ae7d2cc2 (diff)
downloadPeerTube-0491173a61aed66205c017e0d7e0503ea316c144.tar.gz
PeerTube-0491173a61aed66205c017e0d7e0503ea316c144.tar.zst
PeerTube-0491173a61aed66205c017e0d7e0503ea316c144.zip
Merge branch 'develop' into cli-wrapper
Diffstat (limited to 'server/lib/schedulers/videos-redundancy-scheduler.ts')
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts64
1 files changed, 29 insertions, 35 deletions
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index ee9ba1766..960651712 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,10 +1,9 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' 4import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file' 6import { VideoFileModel } from '../../models/video/video-file'
7import { sortBy } from 'lodash'
8import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 7import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
9import { join } from 'path' 8import { join } from 'path'
10import { rename } from 'fs-extra' 9import { rename } from 'fs-extra'
@@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils'
12import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
13import { VideoModel } from '../../models/video/video' 12import { VideoModel } from '../../models/video/video'
14import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 13import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
15import { removeVideoRedundancy } from '../redundancy'
16import { isTestInstance } from '../../helpers/core-utils' 14import { isTestInstance } from '../../helpers/core-utils'
17 15
18export class VideosRedundancyScheduler extends AbstractScheduler { 16export class VideosRedundancyScheduler extends AbstractScheduler {
@@ -20,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
20 private static instance: AbstractScheduler 18 private static instance: AbstractScheduler
21 private executing = false 19 private executing = false
22 20
23 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy 21 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
24 22
25 private constructor () { 23 private constructor () {
26 super() 24 super()
@@ -31,17 +29,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
31 29
32 this.executing = true 30 this.executing = true
33 31
34 for (const obj of CONFIG.REDUNDANCY.VIDEOS) { 32 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
35
36 try { 33 try {
37 const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) 34 const videoToDuplicate = await this.findVideoToDuplicate(obj)
38 if (!videoToDuplicate) continue 35 if (!videoToDuplicate) continue
39 36
40 const videoFiles = videoToDuplicate.VideoFiles 37 const videoFiles = videoToDuplicate.VideoFiles
41 videoFiles.forEach(f => f.Video = videoToDuplicate) 38 videoFiles.forEach(f => f.Video = videoToDuplicate)
42 39
43 const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) 40 if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) {
44 if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
45 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 41 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
46 continue 42 continue
47 } 43 }
@@ -54,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
54 } 50 }
55 } 51 }
56 52
53 await this.removeExpired()
54
55 this.executing = false
56 }
57
58 static get Instance () {
59 return this.instance || (this.instance = new this())
60 }
61
62 private async removeExpired () {
57 const expired = await VideoRedundancyModel.listAllExpired() 63 const expired = await VideoRedundancyModel.listAllExpired()
58 64
59 for (const m of expired) { 65 for (const m of expired) {
@@ -65,16 +71,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
65 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) 71 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
66 } 72 }
67 } 73 }
68
69 this.executing = false
70 } 74 }
71 75
72 static get Instance () { 76 private findVideoToDuplicate (cache: VideosRedundancy) {
73 return this.instance || (this.instance = new this()) 77 if (cache.strategy === 'most-views') {
74 } 78 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
79 }
80
81 if (cache.strategy === 'trending') {
82 return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
83 }
75 84
76 private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { 85 if (cache.strategy === 'recently-added') {
77 if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 86 const minViews = cache.minViews
87 return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
88 }
78 } 89 }
79 90
80 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { 91 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
@@ -120,27 +131,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
120 } 131 }
121 } 132 }
122 133
123 // Unused, but could be useful in the future, with a custom strategy 134 private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
124 private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
125 const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
126
127 while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
128 const toDelete = sortedVideosRedundancy.shift()
129
130 const videoFile = toDelete.VideoFile
131 logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
132
133 await removeVideoRedundancy(toDelete, undefined)
134 }
135
136 return sortedVideosRedundancy
137 }
138
139 private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
140 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) 135 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
141 136
142 const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size 137 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy)
143 const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
144 138
145 return totalDuplicated > maxSize 139 return totalDuplicated > maxSize
146 } 140 }