aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-09-14 11:05:38 +0200
committerChocobozzz <me@florianbigard.com>2018-09-14 11:05:38 +0200
commit3f6b6a565dc98a658ec9d8f697252788c0faa46d (patch)
tree9549d3722b2d2d6501d85b4a56e6bfd3ea14fed3
parent780daa7e91336116a5163156f298c9ad875ec1e3 (diff)
downloadPeerTube-3f6b6a565dc98a658ec9d8f697252788c0faa46d.tar.gz
PeerTube-3f6b6a565dc98a658ec9d8f697252788c0faa46d.tar.zst
PeerTube-3f6b6a565dc98a658ec9d8f697252788c0faa46d.zip
Add recently added redundancy strategy
-rw-r--r--config/default.yaml4
-rw-r--r--config/production.yaml.example4
-rw-r--r--config/test.yaml4
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts2
-rw-r--r--server/initializers/checker.ts13
-rw-r--r--server/initializers/constants.ts11
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts44
-rw-r--r--server/models/redundancy/video-redundancy.ts53
-rw-r--r--server/tests/api/server/redundancy.ts63
-rw-r--r--shared/models/redundancy/videos-redundancy.model.ts19
10 files changed, 152 insertions, 65 deletions
diff --git a/config/default.yaml b/config/default.yaml
index ecb809c6a..adac9deeb 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -77,6 +77,10 @@ redundancy:
77# - 77# -
78# size: '10GB' 78# size: '10GB'
79# strategy: 'trending' # Cache trending videos 79# strategy: 'trending' # Cache trending videos
80# -
81# size: '10GB'
82# strategy: 'recently-added' # Cache recently added videos
83# minViews: 10 # Having at least x views
80 84
81cache: 85cache:
82 previews: 86 previews:
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 48d69e987..ca7b936c2 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -78,6 +78,10 @@ redundancy:
78# - 78# -
79# size: '10GB' 79# size: '10GB'
80# strategy: 'trending' # Cache trending videos 80# strategy: 'trending' # Cache trending videos
81# -
82# size: '10GB'
83# strategy: 'recently-added' # Cache recently added videos
84# minViews: 10 # Having at least x views
81 85
82############################################################################### 86###############################################################################
83# 87#
diff --git a/config/test.yaml b/config/test.yaml
index 73bc5da98..517fc7449 100644
--- a/config/test.yaml
+++ b/config/test.yaml
@@ -29,6 +29,10 @@ redundancy:
29 - 29 -
30 size: '100KB' 30 size: '100KB'
31 strategy: 'trending' 31 strategy: 'trending'
32 -
33 size: '100KB'
34 strategy: 'recently-added'
35 minViews: 10
32 36
33cache: 37cache:
34 previews: 38 previews:
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index f76eba474..8772e74cf 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -171,5 +171,3 @@ function setRemoteVideoTruncatedContent (video: any) {
171 171
172 return true 172 return true
173} 173}
174
175
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 6048151a3..29f4f3036 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -7,7 +7,7 @@ import { parse } from 'url'
7import { CONFIG } from './constants' 7import { CONFIG } from './constants'
8import { logger } from '../helpers/logger' 8import { logger } from '../helpers/logger'
9import { getServerActor } from '../helpers/utils' 9import { getServerActor } from '../helpers/utils'
10import { VideosRedundancy } from '../../shared/models/redundancy' 10import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
11import { isArray } from '../helpers/custom-validators/misc' 11import { isArray } from '../helpers/custom-validators/misc'
12import { uniq } from 'lodash' 12import { uniq } from 'lodash'
13 13
@@ -34,24 +34,31 @@ async function checkActivityPubUrls () {
34function checkConfig () { 34function checkConfig () {
35 const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy') 35 const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy')
36 36
37 // NSFW policy
37 if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) { 38 if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) {
38 return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy 39 return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
39 } 40 }
40 41
42 // Redundancies
41 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos') 43 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos')
42 if (isArray(redundancyVideos)) { 44 if (isArray(redundancyVideos)) {
43 for (const r of redundancyVideos) { 45 for (const r of redundancyVideos) {
44 if ([ 'most-views', 'trending' ].indexOf(r.strategy) === -1) { 46 if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) {
45 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy 47 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
46 } 48 }
47 } 49 }
48 50
49 const filtered = uniq(redundancyVideos.map(r => r.strategy)) 51 const filtered = uniq(redundancyVideos.map(r => r.strategy))
50 if (filtered.length !== redundancyVideos.length) { 52 if (filtered.length !== redundancyVideos.length) {
51 return 'Redundancy video entries should have uniq strategies' 53 return 'Redundancy video entries should have unique strategies'
52 } 54 }
53 } 55 }
54 56
57 const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
58 if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
59 return 'Min views in recently added strategy is not a number'
60 }
61
55 return null 62 return null
56} 63}
57 64
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6b4afbfd8..5f7bcbd48 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -1,6 +1,6 @@
1import { IConfig } from 'config' 1import { IConfig } from 'config'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models' 3import { JobType, VideoRateType, VideoState, VideosRedundancy } from '../../shared/models'
4import { ActivityPubActorType } from '../../shared/models/activitypub' 4import { ActivityPubActorType } from '../../shared/models/activitypub'
5import { FollowState } from '../../shared/models/actors' 5import { FollowState } from '../../shared/models/actors'
6import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' 6import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
@@ -741,15 +741,10 @@ function updateWebserverConfig () {
741 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) 741 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
742} 742}
743 743
744function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] { 744function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] {
745 if (!objs) return [] 745 if (!objs) return []
746 746
747 return objs.map(obj => { 747 return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) }))
748 return {
749 strategy: obj.strategy,
750 size: bytes.parse(obj.size)
751 }
752 })
753} 748}
754 749
755function buildLanguages () { 750function buildLanguages () {
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index c1e619249..8b91d750b 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,7 +1,7 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' 4import { RecentlyAddedStrategy, VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file' 6import { VideoFileModel } from '../../models/video/video-file'
7import { sortBy } from 'lodash' 7import { sortBy } from 'lodash'
@@ -32,16 +32,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
32 this.executing = true 32 this.executing = true
33 33
34 for (const obj of CONFIG.REDUNDANCY.VIDEOS) { 34 for (const obj of CONFIG.REDUNDANCY.VIDEOS) {
35
36 try { 35 try {
37 const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) 36 const videoToDuplicate = await this.findVideoToDuplicate(obj)
38 if (!videoToDuplicate) continue 37 if (!videoToDuplicate) continue
39 38
40 const videoFiles = videoToDuplicate.VideoFiles 39 const videoFiles = videoToDuplicate.VideoFiles
41 videoFiles.forEach(f => f.Video = videoToDuplicate) 40 videoFiles.forEach(f => f.Video = videoToDuplicate)
42 41
43 const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) 42 if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) {
44 if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
45 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 43 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
46 continue 44 continue
47 } 45 }
@@ -73,10 +71,19 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
73 return this.instance || (this.instance = new this()) 71 return this.instance || (this.instance = new this())
74 } 72 }
75 73
76 private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { 74 private findVideoToDuplicate (cache: VideosRedundancy) {
77 if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 75 if (cache.strategy === 'most-views') {
76 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
77 }
78
79 if (cache.strategy === 'trending') {
80 return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
81 }
78 82
79 if (strategy === 'trending') return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 83 if (cache.strategy === 'recently-added') {
84 const minViews = (cache as RecentlyAddedStrategy).minViews
85 return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
86 }
80 } 87 }
81 88
82 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { 89 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
@@ -122,27 +129,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
122 } 129 }
123 } 130 }
124 131
125 // Unused, but could be useful in the future, with a custom strategy 132 private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
126 private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
127 const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
128
129 while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
130 const toDelete = sortedVideosRedundancy.shift()
131
132 const videoFile = toDelete.VideoFile
133 logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
134
135 await removeVideoRedundancy(toDelete, undefined)
136 }
137
138 return sortedVideosRedundancy
139 }
140
141 private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
142 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) 133 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
143 134
144 const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size 135 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy)
145 const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
146 136
147 return totalDuplicated > maxSize 137 return totalDuplicated > maxSize
148 } 138 }
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index b13ade0f4..b7454c617 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -27,6 +27,7 @@ import { VideoChannelModel } from '../video/video-channel'
27import { ServerModel } from '../server/server' 27import { ServerModel } from '../server/server'
28import { sample } from 'lodash' 28import { sample } from 'lodash'
29import { isTestInstance } from '../../helpers/core-utils' 29import { isTestInstance } from '../../helpers/core-utils'
30import * as Bluebird from 'bluebird'
30 31
31export enum ScopeNames { 32export enum ScopeNames {
32 WITH_VIDEO = 'WITH_VIDEO' 33 WITH_VIDEO = 'WITH_VIDEO'
@@ -144,7 +145,8 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
144 return VideoRedundancyModel.findOne(query) 145 return VideoRedundancyModel.findOne(query)
145 } 146 }
146 147
147 static getVideoSample (rows: { id: number }[]) { 148 static async getVideoSample (p: Bluebird<VideoModel[]>) {
149 const rows = await p
148 const ids = rows.map(r => r.id) 150 const ids = rows.map(r => r.id)
149 const id = sample(ids) 151 const id = sample(ids)
150 152
@@ -164,9 +166,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
164 ] 166 ]
165 } 167 }
166 168
167 const rows = await VideoModel.unscoped().findAll(query) 169 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
168
169 return VideoRedundancyModel.getVideoSample(rows as { id: number }[])
170 } 170 }
171 171
172 static async findTrendingToDuplicate (randomizedFactor: number) { 172 static async findTrendingToDuplicate (randomizedFactor: number) {
@@ -186,24 +186,49 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
186 ] 186 ]
187 } 187 }
188 188
189 const rows = await VideoModel.unscoped().findAll(query) 189 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
190 }
190 191
191 return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) 192 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
193 // On VideoModel!
194 const query = {
195 attributes: [ 'id', 'publishedAt' ],
196 // logging: !isTestInstance(),
197 limit: randomizedFactor,
198 order: getVideoSort('-publishedAt'),
199 where: {
200 views: {
201 [ Sequelize.Op.gte ]: minViews
202 }
203 },
204 include: [
205 await VideoRedundancyModel.buildVideoFileForDuplication(),
206 VideoRedundancyModel.buildServerRedundancyInclude()
207 ]
208 }
209
210 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
192 } 211 }
193 212
194 static async getVideoFiles (strategy: VideoRedundancyStrategy) { 213 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
195 const actor = await getServerActor() 214 const actor = await getServerActor()
196 215
197 const queryVideoFiles = { 216 const options = {
198 logging: !isTestInstance(), 217 logging: !isTestInstance(),
199 where: { 218 include: [
200 actorId: actor.id, 219 {
201 strategy 220 attributes: [],
202 } 221 model: VideoRedundancyModel,
222 required: true,
223 where: {
224 actorId: actor.id,
225 strategy
226 }
227 }
228 ]
203 } 229 }
204 230
205 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) 231 return VideoFileModel.sum('size', options)
206 .findAll(queryVideoFiles)
207 } 232 }
208 233
209 static listAllExpired () { 234 static listAllExpired () {
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index 211570d2f..6574e8ea9 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -14,7 +14,7 @@ import {
14 setAccessTokensToServers, 14 setAccessTokensToServers,
15 uploadVideo, 15 uploadVideo,
16 wait, 16 wait,
17 root, viewVideo 17 root, viewVideo, immutableAssign
18} from '../../utils' 18} from '../../utils'
19import { waitJobs } from '../../utils/server/jobs' 19import { waitJobs } from '../../utils/server/jobs'
20import * as magnetUtil from 'magnet-uri' 20import * as magnetUtil from 'magnet-uri'
@@ -39,14 +39,14 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
39 } 39 }
40} 40}
41 41
42async function runServers (strategy: VideoRedundancyStrategy) { 42async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
43 const config = { 43 const config = {
44 redundancy: { 44 redundancy: {
45 videos: [ 45 videos: [
46 { 46 immutableAssign({
47 strategy: strategy, 47 strategy: strategy,
48 size: '100KB' 48 size: '100KB'
49 } 49 }, additionalParams)
50 ] 50 ]
51 } 51 }
52 } 52 }
@@ -153,11 +153,11 @@ describe('Test videos redundancy', function () {
153 return check1WebSeed() 153 return check1WebSeed()
154 }) 154 })
155 155
156 it('Should enable redundancy on server 1', async function () { 156 it('Should enable redundancy on server 1', function () {
157 return enableRedundancy() 157 return enableRedundancy()
158 }) 158 })
159 159
160 it('Should have 2 webseed on the first video', async function () { 160 it('Should have 2 webseed on the first video', function () {
161 this.timeout(40000) 161 this.timeout(40000)
162 162
163 return check2Webseeds() 163 return check2Webseeds()
@@ -180,11 +180,58 @@ describe('Test videos redundancy', function () {
180 return check1WebSeed() 180 return check1WebSeed()
181 }) 181 })
182 182
183 it('Should enable redundancy on server 1', async function () { 183 it('Should enable redundancy on server 1', function () {
184 return enableRedundancy() 184 return enableRedundancy()
185 }) 185 })
186 186
187 it('Should have 2 webseed on the first video', async function () { 187 it('Should have 2 webseed on the first video', function () {
188 this.timeout(40000)
189
190 return check2Webseeds()
191 })
192
193 after(function () {
194 return cleanServers()
195 })
196 })
197
198 describe('With recently added strategy', function () {
199
200 before(function () {
201 this.timeout(120000)
202
203 return runServers('recently-added', { minViews: 3 })
204 })
205
206 it('Should have 1 webseed on the first video', function () {
207 return check1WebSeed()
208 })
209
210 it('Should enable redundancy on server 1', function () {
211 return enableRedundancy()
212 })
213
214 it('Should still have 1 webseed on the first video', async function () {
215 this.timeout(40000)
216
217 await waitJobs(servers)
218 await wait(15000)
219 await waitJobs(servers)
220
221 return check1WebSeed()
222 })
223
224 it('Should view 2 times the first video', async function () {
225 this.timeout(40000)
226
227 await viewVideo(servers[ 0 ].url, video1Server2UUID)
228 await viewVideo(servers[ 2 ].url, video1Server2UUID)
229
230 await wait(10000)
231 await waitJobs(servers)
232 })
233
234 it('Should have 2 webseed on the first video', function () {
188 this.timeout(40000) 235 this.timeout(40000)
189 236
190 return check2Webseeds() 237 return check2Webseeds()
diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy.model.ts
index 85982e5b3..436394c1e 100644
--- a/shared/models/redundancy/videos-redundancy.model.ts
+++ b/shared/models/redundancy/videos-redundancy.model.ts
@@ -1,6 +1,19 @@
1export type VideoRedundancyStrategy = 'most-views' | 'trending' 1export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-added'
2 2
3export interface VideosRedundancy { 3export type MostViewsRedundancyStrategy = {
4 strategy: VideoRedundancyStrategy 4 strategy: 'most-views'
5 size: number 5 size: number
6} 6}
7
8export type TrendingRedundancyStrategy = {
9 strategy: 'trending'
10 size: number
11}
12
13export type RecentlyAddedStrategy = {
14 strategy: 'recently-added'
15 size: number
16 minViews: number
17}
18
19export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy