aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/initializers/checker.ts2
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts2
-rw-r--r--server/models/redundancy/video-redundancy.ts115
-rw-r--r--server/models/video/video.ts32
-rw-r--r--server/tests/api/server/redundancy.ts205
-rw-r--r--server/tests/utils/server/servers.ts4
6 files changed, 232 insertions, 128 deletions
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 6a2badd35..6048151a3 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -41,7 +41,7 @@ function checkConfig () {
41 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos') 41 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos')
42 if (isArray(redundancyVideos)) { 42 if (isArray(redundancyVideos)) {
43 for (const r of redundancyVideos) { 43 for (const r of redundancyVideos) {
44 if ([ 'most-views' ].indexOf(r.strategy) === -1) { 44 if ([ 'most-views', 'trending' ].indexOf(r.strategy) === -1) {
45 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy 45 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
46 } 46 }
47 } 47 }
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index ee9ba1766..c1e619249 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -75,6 +75,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
75 75
76 private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { 76 private findVideoToDuplicate (strategy: VideoRedundancyStrategy) {
77 if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 77 if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
78
79 if (strategy === 'trending') return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
78 } 80 }
79 81
80 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { 82 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 48ec77206..b13ade0f4 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -14,11 +14,10 @@ import {
14 UpdatedAt 14 UpdatedAt
15} from 'sequelize-typescript' 15} from 'sequelize-typescript'
16import { ActorModel } from '../activitypub/actor' 16import { ActorModel } from '../activitypub/actor'
17import { throwIfNotValid } from '../utils' 17import { getVideoSort, throwIfNotValid } from '../utils'
18import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' 18import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
19import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' 19import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
20import { VideoFileModel } from '../video/video-file' 20import { VideoFileModel } from '../video/video-file'
21import { isDateValid } from '../../helpers/custom-validators/misc'
22import { getServerActor } from '../../helpers/utils' 21import { getServerActor } from '../../helpers/utils'
23import { VideoModel } from '../video/video' 22import { VideoModel } from '../video/video'
24import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' 23import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
@@ -145,50 +144,51 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
145 return VideoRedundancyModel.findOne(query) 144 return VideoRedundancyModel.findOne(query)
146 } 145 }
147 146
147 static getVideoSample (rows: { id: number }[]) {
148 const ids = rows.map(r => r.id)
149 const id = sample(ids)
150
151 return VideoModel.loadWithFile(id, undefined, !isTestInstance())
152 }
153
148 static async findMostViewToDuplicate (randomizedFactor: number) { 154 static async findMostViewToDuplicate (randomizedFactor: number) {
149 // On VideoModel! 155 // On VideoModel!
150 const query = { 156 const query = {
157 attributes: [ 'id', 'views' ],
151 logging: !isTestInstance(), 158 logging: !isTestInstance(),
152 limit: randomizedFactor, 159 limit: randomizedFactor,
153 order: [ [ 'views', 'DESC' ] ], 160 order: getVideoSort('-views'),
154 include: [ 161 include: [
155 { 162 await VideoRedundancyModel.buildVideoFileForDuplication(),
156 model: VideoFileModel.unscoped(), 163 VideoRedundancyModel.buildServerRedundancyInclude()
157 required: true, 164 ]
158 where: { 165 }
159 id: { 166
160 [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn() 167 const rows = await VideoModel.unscoped().findAll(query)
161 } 168
162 } 169 return VideoRedundancyModel.getVideoSample(rows as { id: number }[])
163 }, 170 }
164 { 171
165 attributes: [], 172 static async findTrendingToDuplicate (randomizedFactor: number) {
166 model: VideoChannelModel.unscoped(), 173 // On VideoModel!
167 required: true, 174 const query = {
168 include: [ 175 attributes: [ 'id', 'views' ],
169 { 176 subQuery: false,
170 attributes: [], 177 logging: !isTestInstance(),
171 model: ActorModel.unscoped(), 178 group: 'VideoModel.id',
172 required: true, 179 limit: randomizedFactor,
173 include: [ 180 order: getVideoSort('-trending'),
174 { 181 include: [
175 attributes: [], 182 await VideoRedundancyModel.buildVideoFileForDuplication(),
176 model: ServerModel.unscoped(), 183 VideoRedundancyModel.buildServerRedundancyInclude(),
177 required: true, 184
178 where: { 185 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
179 redundancyAllowed: true
180 }
181 }
182 ]
183 }
184 ]
185 }
186 ] 186 ]
187 } 187 }
188 188
189 const rows = await VideoModel.unscoped().findAll(query) 189 const rows = await VideoModel.unscoped().findAll(query)
190 190
191 return sample(rows) 191 return VideoRedundancyModel.getVideoSample(rows as { id: number }[])
192 } 192 }
193 193
194 static async getVideoFiles (strategy: VideoRedundancyStrategy) { 194 static async getVideoFiles (strategy: VideoRedundancyStrategy) {
@@ -211,7 +211,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
211 logging: !isTestInstance(), 211 logging: !isTestInstance(),
212 where: { 212 where: {
213 expiresOn: { 213 expiresOn: {
214 [Sequelize.Op.lt]: new Date() 214 [ Sequelize.Op.lt ]: new Date()
215 } 215 }
216 } 216 }
217 } 217 }
@@ -237,13 +237,50 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
237 } 237 }
238 } 238 }
239 239
240 private static async buildExcludeIn () { 240 // Don't include video files we already duplicated
241 private static async buildVideoFileForDuplication () {
241 const actor = await getServerActor() 242 const actor = await getServerActor()
242 243
243 return Sequelize.literal( 244 const notIn = Sequelize.literal(
244 '(' + 245 '(' +
245 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + 246 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
246 ')' 247 ')'
247 ) 248 )
249
250 return {
251 attributes: [],
252 model: VideoFileModel.unscoped(),
253 required: true,
254 where: {
255 id: {
256 [ Sequelize.Op.notIn ]: notIn
257 }
258 }
259 }
260 }
261
262 private static buildServerRedundancyInclude () {
263 return {
264 attributes: [],
265 model: VideoChannelModel.unscoped(),
266 required: true,
267 include: [
268 {
269 attributes: [],
270 model: ActorModel.unscoped(),
271 required: true,
272 include: [
273 {
274 attributes: [],
275 model: ServerModel.unscoped(),
276 required: true,
277 where: {
278 redundancyAllowed: true
279 }
280 }
281 ]
282 }
283 ]
284 }
248 } 285 }
249} 286}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 27c631dcd..ef8be7c86 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -387,16 +387,7 @@ type AvailableForListIDsOptions = {
387 } 387 }
388 388
389 if (options.trendingDays) { 389 if (options.trendingDays) {
390 query.include.push({ 390 query.include.push(VideoModel.buildTrendingQuery(options.trendingDays))
391 attributes: [],
392 model: VideoViewModel,
393 required: false,
394 where: {
395 startDate: {
396 [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays)
397 }
398 }
399 })
400 391
401 query.subQuery = false 392 query.subQuery = false
402 } 393 }
@@ -1071,9 +1062,12 @@ export class VideoModel extends Model<VideoModel> {
1071 } 1062 }
1072 1063
1073 static load (id: number, t?: Sequelize.Transaction) { 1064 static load (id: number, t?: Sequelize.Transaction) {
1074 const options = t ? { transaction: t } : undefined 1065 return VideoModel.findById(id, { transaction: t })
1066 }
1075 1067
1076 return VideoModel.findById(id, options) 1068 static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) {
1069 return VideoModel.scope(ScopeNames.WITH_FILES)
1070 .findById(id, { transaction: t, logging })
1077 } 1071 }
1078 1072
1079 static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { 1073 static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) {
@@ -1191,6 +1185,20 @@ export class VideoModel extends Model<VideoModel> {
1191 .then(rows => rows.map(r => r[ field ])) 1185 .then(rows => rows.map(r => r[ field ]))
1192 } 1186 }
1193 1187
1188 static buildTrendingQuery (trendingDays: number) {
1189 return {
1190 attributes: [],
1191 subQuery: false,
1192 model: VideoViewModel,
1193 required: false,
1194 where: {
1195 startDate: {
1196 [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays)
1197 }
1198 }
1199 }
1200 }
1201
1194 private static buildActorWhereWithFilter (filter?: VideoFilter) { 1202 private static buildActorWhereWithFilter (filter?: VideoFilter) {
1195 if (filter && filter === 'local') { 1203 if (filter && filter === 'local') {
1196 return { 1204 return {
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index c0ec75a45..211570d2f 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -22,9 +22,14 @@ import { updateRedundancy } from '../../utils/server/redundancy'
22import { ActorFollow } from '../../../../shared/models/actors' 22import { ActorFollow } from '../../../../shared/models/actors'
23import { readdir } from 'fs-extra' 23import { readdir } from 'fs-extra'
24import { join } from 'path' 24import { join } from 'path'
25import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
25 26
26const expect = chai.expect 27const expect = chai.expect
27 28
29let servers: ServerInfo[] = []
30let video1Server2UUID: string
31let video2Server2UUID: string
32
28function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { 33function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
29 const parsed = magnetUtil.decode(file.magnetUri) 34 const parsed = magnetUtil.decode(file.magnetUri)
30 35
@@ -34,107 +39,159 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
34 } 39 }
35} 40}
36 41
37describe('Test videos redundancy', function () { 42async function runServers (strategy: VideoRedundancyStrategy) {
38 let servers: ServerInfo[] = [] 43 const config = {
39 let video1Server2UUID: string 44 redundancy: {
40 let video2Server2UUID: string 45 videos: [
46 {
47 strategy: strategy,
48 size: '100KB'
49 }
50 ]
51 }
52 }
53 servers = await flushAndRunMultipleServers(3, config)
41 54
42 before(async function () { 55 // Get the access tokens
43 this.timeout(120000) 56 await setAccessTokensToServers(servers)
44 57
45 servers = await flushAndRunMultipleServers(3) 58 {
59 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
60 video1Server2UUID = res.body.video.uuid
46 61
47 // Get the access tokens 62 await viewVideo(servers[ 1 ].url, video1Server2UUID)
48 await setAccessTokensToServers(servers) 63 }
49 64
50 { 65 {
51 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) 66 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
52 video1Server2UUID = res.body.video.uuid 67 video2Server2UUID = res.body.video.uuid
68 }
53 69
54 await viewVideo(servers[1].url, video1Server2UUID) 70 await waitJobs(servers)
55 }
56 71
57 { 72 // Server 1 and server 2 follow each other
58 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) 73 await doubleFollow(servers[ 0 ], servers[ 1 ])
59 video2Server2UUID = res.body.video.uuid 74 // Server 1 and server 3 follow each other
60 } 75 await doubleFollow(servers[ 0 ], servers[ 2 ])
76 // Server 2 and server 3 follow each other
77 await doubleFollow(servers[ 1 ], servers[ 2 ])
78
79 await waitJobs(servers)
80}
61 81
62 await waitJobs(servers) 82async function check1WebSeed () {
83 const webseeds = [
84 'http://localhost:9002/static/webseed/' + video1Server2UUID
85 ]
63 86
64 // Server 1 and server 2 follow each other 87 for (const server of servers) {
65 await doubleFollow(servers[0], servers[1]) 88 const res = await getVideo(server.url, video1Server2UUID)
66 // Server 1 and server 3 follow each other
67 await doubleFollow(servers[0], servers[2])
68 // Server 2 and server 3 follow each other
69 await doubleFollow(servers[1], servers[2])
70 89
71 await waitJobs(servers) 90 const video: VideoDetails = res.body
72 }) 91 video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
92 }
93}
94
95async function enableRedundancy () {
96 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
97
98 const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
99 const follows: ActorFollow[] = res.body.data
100 const server2 = follows.find(f => f.following.host === 'localhost:9002')
101 const server3 = follows.find(f => f.following.host === 'localhost:9003')
102
103 expect(server3).to.not.be.undefined
104 expect(server3.following.hostRedundancyAllowed).to.be.false
105
106 expect(server2).to.not.be.undefined
107 expect(server2.following.hostRedundancyAllowed).to.be.true
108}
73 109
74 it('Should have 1 webseed on the first video', async function () { 110async function check2Webseeds () {
75 const webseeds = [ 111 await waitJobs(servers)
76 'http://localhost:9002/static/webseed/' + video1Server2UUID 112 await wait(15000)
77 ] 113 await waitJobs(servers)
78 114
79 for (const server of servers) { 115 const webseeds = [
80 const res = await getVideo(server.url, video1Server2UUID) 116 'http://localhost:9001/static/webseed/' + video1Server2UUID,
117 'http://localhost:9002/static/webseed/' + video1Server2UUID
118 ]
81 119
82 const video: VideoDetails = res.body 120 for (const server of servers) {
83 video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) 121 const res = await getVideo(server.url, video1Server2UUID)
122
123 const video: VideoDetails = res.body
124
125 for (const file of video.files) {
126 checkMagnetWebseeds(file, webseeds)
84 } 127 }
85 }) 128 }
86 129
87 it('Should enable redundancy on server 1', async function () { 130 const files = await readdir(join(root(), 'test1', 'videos'))
88 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) 131 expect(files).to.have.lengthOf(4)
89 132
90 const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt') 133 for (const resolution of [ 240, 360, 480, 720 ]) {
91 const follows: ActorFollow[] = res.body.data 134 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
92 const server2 = follows.find(f => f.following.host === 'localhost:9002') 135 }
93 const server3 = follows.find(f => f.following.host === 'localhost:9003') 136}
94 137
95 expect(server3).to.not.be.undefined 138async function cleanServers () {
96 expect(server3.following.hostRedundancyAllowed).to.be.false 139 killallServers(servers)
140}
97 141
98 expect(server2).to.not.be.undefined 142describe('Test videos redundancy', function () {
99 expect(server2.following.hostRedundancyAllowed).to.be.true
100 })
101 143
102 it('Should have 2 webseed on the first video', async function () { 144 describe('With most-views strategy', function () {
103 this.timeout(40000)
104 145
105 await waitJobs(servers) 146 before(function () {
106 await wait(15000) 147 this.timeout(120000)
107 await waitJobs(servers)
108 148
109 const webseeds = [ 149 return runServers('most-views')
110 'http://localhost:9001/static/webseed/' + video1Server2UUID, 150 })
111 'http://localhost:9002/static/webseed/' + video1Server2UUID
112 ]
113 151
114 for (const server of servers) { 152 it('Should have 1 webseed on the first video', function () {
115 const res = await getVideo(server.url, video1Server2UUID) 153 return check1WebSeed()
154 })
116 155
117 const video: VideoDetails = res.body 156 it('Should enable redundancy on server 1', async function () {
157 return enableRedundancy()
158 })
118 159
119 for (const file of video.files) { 160 it('Should have 2 webseed on the first video', async function () {
120 checkMagnetWebseeds(file, webseeds) 161 this.timeout(40000)
121 }
122 }
123 162
124 const files = await readdir(join(root(), 'test1', 'videos')) 163 return check2Webseeds()
125 expect(files).to.have.lengthOf(4) 164 })
126 165
127 for (const resolution of [ 240, 360, 480, 720 ]) { 166 after(function () {
128 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined 167 return cleanServers()
129 } 168 })
130 }) 169 })
131 170
132 after(async function () { 171 describe('With trending strategy', function () {
133 killallServers(servers)
134 172
135 // Keep the logs if the test failed 173 before(function () {
136 if (this['ok']) { 174 this.timeout(120000)
137 await flushTests() 175
138 } 176 return runServers('trending')
177 })
178
179 it('Should have 1 webseed on the first video', function () {
180 return check1WebSeed()
181 })
182
183 it('Should enable redundancy on server 1', async function () {
184 return enableRedundancy()
185 })
186
187 it('Should have 2 webseed on the first video', async function () {
188 this.timeout(40000)
189
190 return check2Webseeds()
191 })
192
193 after(function () {
194 return cleanServers()
195 })
139 }) 196 })
140}) 197})
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts
index 1372c03c3..e95be4a16 100644
--- a/server/tests/utils/server/servers.ts
+++ b/server/tests/utils/server/servers.ts
@@ -35,7 +35,7 @@ interface ServerInfo {
35 } 35 }
36} 36}
37 37
38function flushAndRunMultipleServers (totalServers) { 38function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) {
39 let apps = [] 39 let apps = []
40 let i = 0 40 let i = 0
41 41
@@ -53,7 +53,7 @@ function flushAndRunMultipleServers (totalServers) {
53 for (let j = 1; j <= totalServers; j++) { 53 for (let j = 1; j <= totalServers; j++) {
54 // For the virtual buffer 54 // For the virtual buffer
55 setTimeout(() => { 55 setTimeout(() => {
56 runServer(j).then(app => anotherServerDone(j, app)) 56 runServer(j, configOverride).then(app => anotherServerDone(j, app))
57 }, 1000 * (j - 1)) 57 }, 1000 * (j - 1))
58 } 58 }
59 }) 59 })