aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2017-09-12 12:53:55 +0200
committerChocobozzz <florian.bigard@gmail.com>2017-09-12 13:12:35 +0200
commit6d33593a0829a7f041127d50d4c455456550a47f (patch)
treeda316570134a33f4f8cef1fda282c8ef2d0e30a6 /server
parent7ca86c864e102b65e4ff3224a06554a66148fef8 (diff)
downloadPeerTube-6d33593a0829a7f041127d50d4c455456550a47f.tar.gz
PeerTube-6d33593a0829a7f041127d50d4c455456550a47f.tar.zst
PeerTube-6d33593a0829a7f041127d50d4c455456550a47f.zip
Improve real world script
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/remote/videos.ts30
-rw-r--r--server/controllers/api/videos/index.ts11
-rw-r--r--server/middlewares/validators/videos.ts16
-rw-r--r--server/tests/real-world/real-world.ts102
4 files changed, 109 insertions, 50 deletions
diff --git a/server/controllers/api/remote/videos.ts b/server/controllers/api/remote/videos.ts
index 9dd4afdb5..23023211f 100644
--- a/server/controllers/api/remote/videos.ts
+++ b/server/controllers/api/remote/videos.ts
@@ -17,7 +17,7 @@ import {
17} from '../../../middlewares' 17} from '../../../middlewares'
18import { logger, retryTransactionWrapper } from '../../../helpers' 18import { logger, retryTransactionWrapper } from '../../../helpers'
19import { quickAndDirtyUpdatesVideoToFriends } from '../../../lib' 19import { quickAndDirtyUpdatesVideoToFriends } from '../../../lib'
20import { PodInstance } from '../../../models' 20import { PodInstance, VideoFileInstance } from '../../../models'
21import { 21import {
22 RemoteVideoRequest, 22 RemoteVideoRequest,
23 RemoteVideoCreateData, 23 RemoteVideoCreateData,
@@ -81,7 +81,7 @@ function remoteVideos (req: express.Request, res: express.Response, next: expres
81 // Get the function we need to call in order to process the request 81 // Get the function we need to call in order to process the request
82 const fun = functionsHash[request.type] 82 const fun = functionsHash[request.type]
83 if (fun === undefined) { 83 if (fun === undefined) {
84 logger.error('Unkown remote request type %s.', request.type) 84 logger.error('Unknown remote request type %s.', request.type)
85 return 85 return
86 } 86 }
87 87
@@ -176,7 +176,7 @@ function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInsta
176 return quickAndDirtyUpdatesVideoToFriends(qadusParams, t) 176 return quickAndDirtyUpdatesVideoToFriends(qadusParams, t)
177 }) 177 })
178 }) 178 })
179 .then(() => logger.info('Remote video event processed for video %s.', eventData.uuid)) 179 .then(() => logger.info('Remote video event processed for video with uuid %s.', eventData.uuid))
180 .catch(err => { 180 .catch(err => {
181 logger.debug('Cannot process a video event.', err) 181 logger.debug('Cannot process a video event.', err)
182 throw err 182 throw err
@@ -193,14 +193,14 @@ function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, f
193} 193}
194 194
195function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) { 195function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
196 let videoName 196 let videoUUID = ''
197 197
198 return db.sequelize.transaction(t => { 198 return db.sequelize.transaction(t => {
199 return fetchVideoByHostAndUUID(fromPod.host, videoData.uuid) 199 return fetchVideoByHostAndUUID(fromPod.host, videoData.uuid)
200 .then(videoInstance => { 200 .then(videoInstance => {
201 const options = { transaction: t } 201 const options = { transaction: t }
202 202
203 videoName = videoInstance.name 203 videoUUID = videoInstance.uuid
204 204
205 if (videoData.views) { 205 if (videoData.views) {
206 videoInstance.set('views', videoData.views) 206 videoInstance.set('views', videoData.views)
@@ -217,7 +217,7 @@ function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodI
217 return videoInstance.save(options) 217 return videoInstance.save(options)
218 }) 218 })
219 }) 219 })
220 .then(() => logger.info('Remote video %s quick and dirty updated', videoName)) 220 .then(() => logger.info('Remote video with uuid %s quick and dirty updated', videoUUID))
221 .catch(err => logger.debug('Cannot quick and dirty update the remote video.', err)) 221 .catch(err => logger.debug('Cannot quick and dirty update the remote video.', err))
222} 222}
223 223
@@ -315,7 +315,7 @@ function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodI
315 return videoCreated.setTags(tagInstances, options) 315 return videoCreated.setTags(tagInstances, options)
316 }) 316 })
317 }) 317 })
318 .then(() => logger.info('Remote video %s inserted.', videoToCreateData.name)) 318 .then(() => logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid))
319 .catch(err => { 319 .catch(err => {
320 logger.debug('Cannot insert the remote video.', err) 320 logger.debug('Cannot insert the remote video.', err)
321 throw err 321 throw err
@@ -361,7 +361,17 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from
361 return videoInstance.save(options).then(() => ({ videoInstance, tagInstances })) 361 return videoInstance.save(options).then(() => ({ videoInstance, tagInstances }))
362 }) 362 })
363 .then(({ tagInstances, videoInstance }) => { 363 .then(({ tagInstances, videoInstance }) => {
364 const tasks = [] 364 const tasks: Promise<void>[] = []
365
366 // Remove old video files
367 videoInstance.VideoFiles.forEach(videoFile => {
368 tasks.push(videoFile.destroy())
369 })
370
371 return Promise.all(tasks).then(() => ({ tagInstances, videoInstance }))
372 })
373 .then(({ tagInstances, videoInstance }) => {
374 const tasks: Promise<VideoFileInstance>[] = []
365 const options = { 375 const options = {
366 transaction: t 376 transaction: t
367 } 377 }
@@ -386,7 +396,7 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from
386 return videoInstance.setTags(tagInstances, options) 396 return videoInstance.setTags(tagInstances, options)
387 }) 397 })
388 }) 398 })
389 .then(() => logger.info('Remote video %s updated', videoAttributesToUpdate.name)) 399 .then(() => logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid))
390 .catch(err => { 400 .catch(err => {
391 // This is just a debug because we will retry the insert 401 // This is just a debug because we will retry the insert
392 logger.debug('Cannot update the remote video.', err) 402 logger.debug('Cannot update the remote video.', err)
@@ -398,7 +408,7 @@ function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: P
398 // We need the instance because we have to remove some other stuffs (thumbnail etc) 408 // We need the instance because we have to remove some other stuffs (thumbnail etc)
399 return fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid) 409 return fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid)
400 .then(video => { 410 .then(video => {
401 logger.debug('Removing remote video %s.', video.uuid) 411 logger.debug('Removing remote video with uuid %s.', video.uuid)
402 return video.destroy() 412 return video.destroy()
403 }) 413 })
404 .catch(err => { 414 .catch(err => {
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 3a19fe989..7a9cd9d37 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -157,6 +157,7 @@ function addVideoRetryWrapper (req: express.Request, res: express.Response, next
157 157
158function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) { 158function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) {
159 const videoInfo: VideoCreate = req.body 159 const videoInfo: VideoCreate = req.body
160 let videoUUID = ''
160 161
161 return db.sequelize.transaction(t => { 162 return db.sequelize.transaction(t => {
162 const user = res.locals.oauth.token.User 163 const user = res.locals.oauth.token.User
@@ -241,6 +242,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
241 .then(videoCreated => { 242 .then(videoCreated => {
242 // Do not forget to add Author information to the created video 243 // Do not forget to add Author information to the created video
243 videoCreated.Author = author 244 videoCreated.Author = author
245 videoUUID = videoCreated.uuid
244 246
245 return { tagInstances, video: videoCreated, videoFile } 247 return { tagInstances, video: videoCreated, videoFile }
246 }) 248 })
@@ -274,7 +276,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
274 }) 276 })
275 }) 277 })
276 }) 278 })
277 .then(() => logger.info('Video with name %s created.', videoInfo.name)) 279 .then(() => logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoUUID))
278 .catch((err: Error) => { 280 .catch((err: Error) => {
279 logger.debug('Cannot insert the video.', err) 281 logger.debug('Cannot insert the video.', err)
280 throw err 282 throw err
@@ -342,7 +344,7 @@ function updateVideo (req: express.Request, res: express.Response) {
342 }) 344 })
343 }) 345 })
344 .then(() => { 346 .then(() => {
345 logger.info('Video with name %s updated.', videoInstance.name) 347 logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid)
346 }) 348 })
347 .catch(err => { 349 .catch(err => {
348 logger.debug('Cannot update the video.', err) 350 logger.debug('Cannot update the video.', err)
@@ -398,7 +400,10 @@ function removeVideo (req: express.Request, res: express.Response, next: express
398 const videoInstance = res.locals.video 400 const videoInstance = res.locals.video
399 401
400 videoInstance.destroy() 402 videoInstance.destroy()
401 .then(() => res.type('json').status(204).end()) 403 .then(() => {
404 logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid)
405 res.type('json').status(204).end()
406 })
402 .catch(err => { 407 .catch(err => {
403 logger.error('Errors when removed the video.', err) 408 logger.error('Errors when removed the video.', err)
404 return next(err) 409 return next(err)
diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts
index 249da668d..519e3d46c 100644
--- a/server/middlewares/validators/videos.ts
+++ b/server/middlewares/validators/videos.ts
@@ -109,8 +109,6 @@ function videosRemoveValidator (req: express.Request, res: express.Response, nex
109 109
110 checkErrors(req, res, () => { 110 checkErrors(req, res, () => {
111 checkVideoExists(req.params.id, res, () => { 111 checkVideoExists(req.params.id, res, () => {
112 // We need to make additional checks
113
114 // Check if the user who did the request is able to delete the video 112 // Check if the user who did the request is able to delete the video
115 checkUserCanDeleteVideo(res.locals.oauth.token.User.id, res, () => { 113 checkUserCanDeleteVideo(res.locals.oauth.token.User.id, res, () => {
116 next() 114 next()
@@ -205,17 +203,15 @@ function checkUserCanDeleteVideo (userId: number, res: express.Response, callbac
205 // Retrieve the user who did the request 203 // Retrieve the user who did the request
206 db.User.loadById(userId) 204 db.User.loadById(userId)
207 .then(user => { 205 .then(user => {
206 if (res.locals.video.isOwned() === false) {
207 return res.status(403).send('Cannot remove video of another pod, blacklist it')
208 }
209
208 // Check if the user can delete the video 210 // Check if the user can delete the video
209 // The user can delete it if s/he is an admin 211 // The user can delete it if s/he is an admin
210 // Or if s/he is the video's author 212 // Or if s/he is the video's author
211 if (user.isAdmin() === false) { 213 if (user.isAdmin() === false && res.locals.video.Author.userId !== res.locals.oauth.token.User.id) {
212 if (res.locals.video.isOwned() === false) { 214 return res.status(403).send('Cannot remove video of another user')
213 return res.status(403).send('Cannot remove video of another pod')
214 }
215
216 if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) {
217 return res.status(403).send('Cannot remove video of another user')
218 }
219 } 215 }
220 216
221 // If we reach this comment, we can delete the video 217 // If we reach this comment, we can delete the video
diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts
index a200bd02d..e225a1266 100644
--- a/server/tests/real-world/real-world.ts
+++ b/server/tests/real-world/real-world.ts
@@ -1,11 +1,10 @@
1import * as program from 'commander' 1import * as program from 'commander'
2import { isEqual, differenceWith } from 'lodash'
3 2
4// /!\ Before imports /!\ 3// /!\ Before imports /!\
5process.env.NODE_ENV = 'test' 4process.env.NODE_ENV = 'test'
6 5
7import { REQUESTS_INTERVAL } from '../../initializers/constants' 6import { REQUESTS_INTERVAL } from '../../initializers/constants'
8import { Video, VideoRateType } from '../../../shared' 7import { Video, VideoRateType, VideoFile } from '../../../shared'
9import { 8import {
10 ServerInfo as DefaultServerInfo, 9 ServerInfo as DefaultServerInfo,
11 flushAndRunMultipleServers, 10 flushAndRunMultipleServers,
@@ -121,12 +120,14 @@ async function start () {
121 checking = true 120 checking = true
122 121
123 const waitingInterval = setInterval(async () => { 122 const waitingInterval = setInterval(async () => {
124 const awaitingRequests = await isThereAwaitingRequests(servers) 123 const pendingRequests = await isTherePendingRequests(servers)
125 if (awaitingRequests === true) { 124 if (pendingRequests === true) {
126 console.log('A server has awaiting requests, waiting...') 125 console.log('A server has pending requests, waiting...')
127 return 126 return
128 } 127 }
129 128
129 // Even if there are no pending request, wait some potential processes
130 await wait(2000)
130 await checkIntegrity(servers) 131 await checkIntegrity(servers)
131 132
132 initializeRequestsPerServer(servers) 133 initializeRequestsPerServer(servers)
@@ -212,7 +213,7 @@ async function update (servers: ServerInfo[], numServer: number) {
212 213
213async function remove (servers: ServerInfo[], numServer: number) { 214async function remove (servers: ServerInfo[], numServer: number) {
214 const res = await getVideosList(servers[numServer].url) 215 const res = await getVideosList(servers[numServer].url)
215 const videos = res.body.data 216 const videos = res.body.data.filter(video => video.isLocal === true)
216 if (videos.length === 0) return undefined 217 if (videos.length === 0) return undefined
217 218
218 const toRemove = videos[getRandomInt(0, videos.length)].id 219 const toRemove = videos[getRandomInt(0, videos.length)].id
@@ -259,19 +260,7 @@ async function checkIntegrity (servers: ServerInfo[]) {
259 260
260 // Fetch all videos and remove some fields that can differ between pods 261 // Fetch all videos and remove some fields that can differ between pods
261 for (const server of servers) { 262 for (const server of servers) {
262 const p = getAllVideosListBy(server.url).then(res => { 263 const p = getAllVideosListBy(server.url).then(res => videos.push(res.body.data))
263 const serverVideos = res.body.data
264 for (const serverVideo of serverVideos) {
265 delete serverVideo.id
266 delete serverVideo.isLocal
267 delete serverVideo.thumbnailPath
268 delete serverVideo.updatedAt
269 delete serverVideo.views
270 }
271
272 videos.push(serverVideos)
273 })
274
275 tasks.push(p) 264 tasks.push(p)
276 } 265 }
277 266
@@ -279,12 +268,12 @@ async function checkIntegrity (servers: ServerInfo[]) {
279 268
280 let i = 0 269 let i = 0
281 for (const video of videos) { 270 for (const video of videos) {
282 if (!isEqual(video, videos[0])) { 271 const differences = areDifferences(video, videos[0])
272 if (differences !== undefined) {
283 console.error('Integrity not ok with server %d!', i + 1) 273 console.error('Integrity not ok with server %d!', i + 1)
284 274
285 if (displayDiffOnFail) { 275 if (displayDiffOnFail) {
286 console.log(differenceWith(videos[0], video, isEqual)) 276 console.log(differences)
287 console.log(differenceWith(video, videos[0], isEqual))
288 } 277 }
289 278
290 process.exit(-1) 279 process.exit(-1)
@@ -296,15 +285,74 @@ async function checkIntegrity (servers: ServerInfo[]) {
296 console.log('Integrity ok.') 285 console.log('Integrity ok.')
297} 286}
298 287
288function areDifferences (videos1: Video[], videos2: Video[]) {
289 // Remove some keys we don't want to compare
290 videos1.concat(videos2).forEach(video => {
291 delete video.id
292 delete video.isLocal
293 delete video.thumbnailPath
294 delete video.updatedAt
295 delete video.views
296 })
297
298 if (videos1.length !== videos2.length) {
299 return `Videos length are different (${videos1.length}/${videos2.length}).`
300 }
301
302 for (const video1 of videos1) {
303 const video2 = videos2.find(video => video.uuid === video1.uuid)
304
305 if (!video2) return 'Video ' + video1.uuid + ' is missing.'
306
307 for (const videoKey of Object.keys(video1)) {
308 const attribute1 = video1[videoKey]
309 const attribute2 = video2[videoKey]
310
311 if (videoKey === 'tags') {
312 if (attribute1.length !== attribute2.length) {
313 return 'Tags are different.'
314 }
315
316 attribute1.forEach(tag1 => {
317 if (attribute2.indexOf(tag1) === -1) {
318 return 'Tag ' + tag1 + ' is missing.'
319 }
320 })
321 } else if (videoKey === 'files') {
322 if (attribute1.length !== attribute2.length) {
323 return 'Video files are different.'
324 }
325
326 attribute1.forEach((videoFile1: VideoFile) => {
327 const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri)
328 if (!videoFile2) {
329 return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.`
330 }
331
332 if (videoFile1.size !== videoFile2.size || videoFile1.resolutionLabel !== videoFile2.resolutionLabel) {
333 return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.`
334 }
335 })
336 } else {
337 if (attribute1 !== attribute2) {
338 return `Video ${video1.uuid} has different value for attribute ${videoKey}.`
339 }
340 }
341 }
342 }
343
344 return undefined
345}
346
299function goodbye () { 347function goodbye () {
300 return process.exit(-1) 348 return process.exit(-1)
301} 349}
302 350
303async function isThereAwaitingRequests (servers: ServerInfo[]) { 351async function isTherePendingRequests (servers: ServerInfo[]) {
304 const tasks: Promise<any>[] = [] 352 const tasks: Promise<any>[] = []
305 let awaitingRequests = false 353 let pendingRequests = false
306 354
307 // Check if each server has awaiting request 355 // Check if each server has pending request
308 for (const server of servers) { 356 for (const server of servers) {
309 const p = getRequestsStats(server).then(res => { 357 const p = getRequestsStats(server).then(res => {
310 const stats = res.body 358 const stats = res.body
@@ -314,7 +362,7 @@ async function isThereAwaitingRequests (servers: ServerInfo[]) {
314 stats.requestVideoEventScheduler.totalRequests !== 0 || 362 stats.requestVideoEventScheduler.totalRequests !== 0 ||
315 stats.requestVideoQaduScheduler.totalRequests !== 0 363 stats.requestVideoQaduScheduler.totalRequests !== 0
316 ) { 364 ) {
317 awaitingRequests = true 365 pendingRequests = true
318 } 366 }
319 }) 367 })
320 368
@@ -323,5 +371,5 @@ async function isThereAwaitingRequests (servers: ServerInfo[]) {
323 371
324 await Promise.all(tasks) 372 await Promise.all(tasks)
325 373
326 return awaitingRequests 374 return pendingRequests
327} 375}