From 6d33593a0829a7f041127d50d4c455456550a47f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 12 Sep 2017 12:53:55 +0200 Subject: [PATCH] Improve real world script --- client/src/app/videos/shared/video.model.ts | 1 + server/controllers/api/remote/videos.ts | 30 ++++-- server/controllers/api/videos/index.ts | 11 ++- server/middlewares/validators/videos.ts | 16 ++- server/tests/real-world/real-world.ts | 102 ++++++++++++++------ shared/models/videos/video.model.ts | 1 + 6 files changed, 111 insertions(+), 50 deletions(-) diff --git a/client/src/app/videos/shared/video.model.ts b/client/src/app/videos/shared/video.model.ts index 438791368..1a413db9d 100644 --- a/client/src/app/videos/shared/video.model.ts +++ b/client/src/app/videos/shared/video.model.ts @@ -5,6 +5,7 @@ export class Video implements VideoServerModel { author: string by: string createdAt: Date + updatedAt: Date categoryLabel: string category: number licenceLabel: string diff --git a/server/controllers/api/remote/videos.ts b/server/controllers/api/remote/videos.ts index 9dd4afdb5..23023211f 100644 --- a/server/controllers/api/remote/videos.ts +++ b/server/controllers/api/remote/videos.ts @@ -17,7 +17,7 @@ import { } from '../../../middlewares' import { logger, retryTransactionWrapper } from '../../../helpers' import { quickAndDirtyUpdatesVideoToFriends } from '../../../lib' -import { PodInstance } from '../../../models' +import { PodInstance, VideoFileInstance } from '../../../models' import { RemoteVideoRequest, RemoteVideoCreateData, @@ -81,7 +81,7 @@ function remoteVideos (req: express.Request, res: express.Response, next: expres // Get the function we need to call in order to process the request const fun = functionsHash[request.type] if (fun === undefined) { - logger.error('Unkown remote request type %s.', request.type) + logger.error('Unknown remote request type %s.', request.type) return } @@ -176,7 +176,7 @@ function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInsta return quickAndDirtyUpdatesVideoToFriends(qadusParams, t) }) }) - .then(() => logger.info('Remote video event processed for video %s.', eventData.uuid)) + .then(() => logger.info('Remote video event processed for video with uuid %s.', eventData.uuid)) .catch(err => { logger.debug('Cannot process a video event.', err) throw err @@ -193,14 +193,14 @@ function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, f } function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) { - let videoName + let videoUUID = '' return db.sequelize.transaction(t => { return fetchVideoByHostAndUUID(fromPod.host, videoData.uuid) .then(videoInstance => { const options = { transaction: t } - videoName = videoInstance.name + videoUUID = videoInstance.uuid if (videoData.views) { videoInstance.set('views', videoData.views) @@ -217,7 +217,7 @@ function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodI return videoInstance.save(options) }) }) - .then(() => logger.info('Remote video %s quick and dirty updated', videoName)) + .then(() => logger.info('Remote video with uuid %s quick and dirty updated', videoUUID)) .catch(err => logger.debug('Cannot quick and dirty update the remote video.', err)) } @@ -315,7 +315,7 @@ function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodI return videoCreated.setTags(tagInstances, options) }) }) - .then(() => logger.info('Remote video %s inserted.', videoToCreateData.name)) + .then(() => logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid)) .catch(err => { logger.debug('Cannot insert the remote video.', err) throw err @@ -361,7 +361,17 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from return videoInstance.save(options).then(() => ({ videoInstance, tagInstances })) }) .then(({ tagInstances, videoInstance }) => { - const tasks = [] + const tasks: Promise[] = [] + + // Remove old video files + videoInstance.VideoFiles.forEach(videoFile => { + tasks.push(videoFile.destroy()) + }) + + return Promise.all(tasks).then(() => ({ tagInstances, videoInstance })) + }) + .then(({ tagInstances, videoInstance }) => { + const tasks: Promise[] = [] const options = { transaction: t } @@ -386,7 +396,7 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from return videoInstance.setTags(tagInstances, options) }) }) - .then(() => logger.info('Remote video %s updated', videoAttributesToUpdate.name)) + .then(() => logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid)) .catch(err => { // This is just a debug because we will retry the insert logger.debug('Cannot update the remote video.', err) @@ -398,7 +408,7 @@ function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: P // We need the instance because we have to remove some other stuffs (thumbnail etc) return fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid) .then(video => { - logger.debug('Removing remote video %s.', video.uuid) + logger.debug('Removing remote video with uuid %s.', video.uuid) return video.destroy() }) .catch(err => { diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 3a19fe989..7a9cd9d37 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -157,6 +157,7 @@ function addVideoRetryWrapper (req: express.Request, res: express.Response, next function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) { const videoInfo: VideoCreate = req.body + let videoUUID = '' return db.sequelize.transaction(t => { const user = res.locals.oauth.token.User @@ -241,6 +242,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil .then(videoCreated => { // Do not forget to add Author information to the created video videoCreated.Author = author + videoUUID = videoCreated.uuid return { tagInstances, video: videoCreated, videoFile } }) @@ -274,7 +276,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil }) }) }) - .then(() => logger.info('Video with name %s created.', videoInfo.name)) + .then(() => logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoUUID)) .catch((err: Error) => { logger.debug('Cannot insert the video.', err) throw err @@ -342,7 +344,7 @@ function updateVideo (req: express.Request, res: express.Response) { }) }) .then(() => { - logger.info('Video with name %s updated.', videoInstance.name) + logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid) }) .catch(err => { logger.debug('Cannot update the video.', err) @@ -398,7 +400,10 @@ function removeVideo (req: express.Request, res: express.Response, next: express const videoInstance = res.locals.video videoInstance.destroy() - .then(() => res.type('json').status(204).end()) + .then(() => { + logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) + res.type('json').status(204).end() + }) .catch(err => { logger.error('Errors when removed the video.', err) return next(err) diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts index 249da668d..519e3d46c 100644 --- a/server/middlewares/validators/videos.ts +++ b/server/middlewares/validators/videos.ts @@ -109,8 +109,6 @@ function videosRemoveValidator (req: express.Request, res: express.Response, nex checkErrors(req, res, () => { checkVideoExists(req.params.id, res, () => { - // We need to make additional checks - // Check if the user who did the request is able to delete the video checkUserCanDeleteVideo(res.locals.oauth.token.User.id, res, () => { next() @@ -205,17 +203,15 @@ function checkUserCanDeleteVideo (userId: number, res: express.Response, callbac // Retrieve the user who did the request db.User.loadById(userId) .then(user => { + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot remove video of another pod, blacklist it') + } + // Check if the user can delete the video // The user can delete it if s/he is an admin // Or if s/he is the video's author - if (user.isAdmin() === false) { - if (res.locals.video.isOwned() === false) { - return res.status(403).send('Cannot remove video of another pod') - } - - if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { - return res.status(403).send('Cannot remove video of another user') - } + if (user.isAdmin() === false && res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot remove video of another user') } // If we reach this comment, we can delete the video diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts index a200bd02d..e225a1266 100644 --- a/server/tests/real-world/real-world.ts +++ b/server/tests/real-world/real-world.ts @@ -1,11 +1,10 @@ import * as program from 'commander' -import { isEqual, differenceWith } from 'lodash' // /!\ Before imports /!\ process.env.NODE_ENV = 'test' import { REQUESTS_INTERVAL } from '../../initializers/constants' -import { Video, VideoRateType } from '../../../shared' +import { Video, VideoRateType, VideoFile } from '../../../shared' import { ServerInfo as DefaultServerInfo, flushAndRunMultipleServers, @@ -121,12 +120,14 @@ async function start () { checking = true const waitingInterval = setInterval(async () => { - const awaitingRequests = await isThereAwaitingRequests(servers) - if (awaitingRequests === true) { - console.log('A server has awaiting requests, waiting...') + const pendingRequests = await isTherePendingRequests(servers) + if (pendingRequests === true) { + console.log('A server has pending requests, waiting...') return } + // Even if there are no pending request, wait some potential processes + await wait(2000) await checkIntegrity(servers) initializeRequestsPerServer(servers) @@ -212,7 +213,7 @@ async function update (servers: ServerInfo[], numServer: number) { async function remove (servers: ServerInfo[], numServer: number) { const res = await getVideosList(servers[numServer].url) - const videos = res.body.data + const videos = res.body.data.filter(video => video.isLocal === true) if (videos.length === 0) return undefined const toRemove = videos[getRandomInt(0, videos.length)].id @@ -259,19 +260,7 @@ async function checkIntegrity (servers: ServerInfo[]) { // Fetch all videos and remove some fields that can differ between pods for (const server of servers) { - const p = getAllVideosListBy(server.url).then(res => { - const serverVideos = res.body.data - for (const serverVideo of serverVideos) { - delete serverVideo.id - delete serverVideo.isLocal - delete serverVideo.thumbnailPath - delete serverVideo.updatedAt - delete serverVideo.views - } - - videos.push(serverVideos) - }) - + const p = getAllVideosListBy(server.url).then(res => videos.push(res.body.data)) tasks.push(p) } @@ -279,12 +268,12 @@ async function checkIntegrity (servers: ServerInfo[]) { let i = 0 for (const video of videos) { - if (!isEqual(video, videos[0])) { + const differences = areDifferences(video, videos[0]) + if (differences !== undefined) { console.error('Integrity not ok with server %d!', i + 1) if (displayDiffOnFail) { - console.log(differenceWith(videos[0], video, isEqual)) - console.log(differenceWith(video, videos[0], isEqual)) + console.log(differences) } process.exit(-1) @@ -296,15 +285,74 @@ async function checkIntegrity (servers: ServerInfo[]) { console.log('Integrity ok.') } +function areDifferences (videos1: Video[], videos2: Video[]) { + // Remove some keys we don't want to compare + videos1.concat(videos2).forEach(video => { + delete video.id + delete video.isLocal + delete video.thumbnailPath + delete video.updatedAt + delete video.views + }) + + if (videos1.length !== videos2.length) { + return `Videos length are different (${videos1.length}/${videos2.length}).` + } + + for (const video1 of videos1) { + const video2 = videos2.find(video => video.uuid === video1.uuid) + + if (!video2) return 'Video ' + video1.uuid + ' is missing.' + + for (const videoKey of Object.keys(video1)) { + const attribute1 = video1[videoKey] + const attribute2 = video2[videoKey] + + if (videoKey === 'tags') { + if (attribute1.length !== attribute2.length) { + return 'Tags are different.' + } + + attribute1.forEach(tag1 => { + if (attribute2.indexOf(tag1) === -1) { + return 'Tag ' + tag1 + ' is missing.' + } + }) + } else if (videoKey === 'files') { + if (attribute1.length !== attribute2.length) { + return 'Video files are different.' + } + + attribute1.forEach((videoFile1: VideoFile) => { + const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri) + if (!videoFile2) { + return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.` + } + + if (videoFile1.size !== videoFile2.size || videoFile1.resolutionLabel !== videoFile2.resolutionLabel) { + return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.` + } + }) + } else { + if (attribute1 !== attribute2) { + return `Video ${video1.uuid} has different value for attribute ${videoKey}.` + } + } + } + } + + return undefined +} + function goodbye () { return process.exit(-1) } -async function isThereAwaitingRequests (servers: ServerInfo[]) { +async function isTherePendingRequests (servers: ServerInfo[]) { const tasks: Promise[] = [] - let awaitingRequests = false + let pendingRequests = false - // Check if each server has awaiting request + // Check if each server has pending request for (const server of servers) { const p = getRequestsStats(server).then(res => { const stats = res.body @@ -314,7 +362,7 @@ async function isThereAwaitingRequests (servers: ServerInfo[]) { stats.requestVideoEventScheduler.totalRequests !== 0 || stats.requestVideoQaduScheduler.totalRequests !== 0 ) { - awaitingRequests = true + pendingRequests = true } }) @@ -323,5 +371,5 @@ async function isThereAwaitingRequests (servers: ServerInfo[]) { await Promise.all(tasks) - return awaitingRequests + return pendingRequests } diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index 82c8763d0..6277dbe59 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts @@ -10,6 +10,7 @@ export interface Video { uuid: string author: string createdAt: Date + updatedAt: Date categoryLabel: string category: number licenceLabel: string -- 2.41.0