Optimise transaction for video upload

diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index f427a25c02639009e8aedc706830dda170316c4e..0f71a7f7fc7dba942d409f480b20b4a5b4515821 100644 (file)
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -164,52 +164,54 @@ async function addVideoRetryWrapper (req: express.Request, res: express.Response
   }).end()
 }
 
-function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) {
+async function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) {
   const videoInfo: VideoCreate = req.body
 
-  return db.sequelize.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
+  // Prepare data so we don't block the transaction
+  const videoData = {
+    name: videoInfo.name,
+    remote: false,
+    extname: extname(videoPhysicalFile.filename),
+    category: videoInfo.category,
+    licence: videoInfo.licence,
+    language: videoInfo.language,
+    nsfw: videoInfo.nsfw,
+    description: videoInfo.description,
+    privacy: videoInfo.privacy,
+    duration: videoPhysicalFile['duration'], // duration was added by a previous middleware
+    channelId: res.locals.videoChannel.id
+  }
+  const video = db.Video.build(videoData)
+  video.url = getVideoActivityPubUrl(video)
 
-    const videoData = {
-      name: videoInfo.name,
-      remote: false,
-      extname: extname(videoPhysicalFile.filename),
-      category: videoInfo.category,
-      licence: videoInfo.licence,
-      language: videoInfo.language,
-      nsfw: videoInfo.nsfw,
-      description: videoInfo.description,
-      privacy: videoInfo.privacy,
-      duration: videoPhysicalFile['duration'], // duration was added by a previous middleware
-      channelId: res.locals.videoChannel.id
-    }
-    const video = db.Video.build(videoData)
-    video.url = getVideoActivityPubUrl(video)
+  const videoFilePath = join(CONFIG.STORAGE.VIDEOS_DIR, videoPhysicalFile.filename)
+  const videoFileHeight = await getVideoFileHeight(videoFilePath)
 
-    const videoFilePath = join(CONFIG.STORAGE.VIDEOS_DIR, videoPhysicalFile.filename)
-    const videoFileHeight = await getVideoFileHeight(videoFilePath)
+  const videoFileData = {
+    extname: extname(videoPhysicalFile.filename),
+    resolution: videoFileHeight,
+    size: videoPhysicalFile.size
+  }
+  const videoFile = db.VideoFile.build(videoFileData)
+  const videoDir = CONFIG.STORAGE.VIDEOS_DIR
+  const source = join(videoDir, videoPhysicalFile.filename)
+  const destination = join(videoDir, video.getVideoFilename(videoFile))
 
-    const videoFileData = {
-      extname: extname(videoPhysicalFile.filename),
-      resolution: videoFileHeight,
-      size: videoPhysicalFile.size
-    }
-    const videoFile = db.VideoFile.build(videoFileData)
-    const videoDir = CONFIG.STORAGE.VIDEOS_DIR
-    const source = join(videoDir, videoPhysicalFile.filename)
-    const destination = join(videoDir, video.getVideoFilename(videoFile))
+  await renamePromise(source, destination)
+  // This is important in case if there is another attempt in the retry process
+  videoPhysicalFile.filename = video.getVideoFilename(videoFile)
 
-    await renamePromise(source, destination)
-    // This is important in case if there is another attempt in the retry process
-    videoPhysicalFile.filename = video.getVideoFilename(videoFile)
+  const tasks = []
 
-    const tasks = []
+  tasks.push(
+    video.createTorrentAndSetInfoHash(videoFile),
+    video.createThumbnail(videoFile),
+    video.createPreview(videoFile)
+  )
+  await Promise.all(tasks)
 
-    tasks.push(
-      video.createTorrentAndSetInfoHash(videoFile),
-      video.createThumbnail(videoFile),
-      video.createPreview(videoFile)
-    )
+  return db.sequelize.transaction(async t => {
+    const sequelizeOptions = { transaction: t }
 
     if (CONFIG.TRANSCODING.ENABLED === true) {
       // Put uuid because we don't have id auto incremented for now
@@ -217,20 +219,17 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
         videoUUID: video.uuid
       }
 
-      tasks.push(
-        transcodingJobScheduler.createJob(t, 'videoFileOptimizer', dataInput)
-      )
+      await transcodingJobScheduler.createJob(t, 'videoFileOptimizer', dataInput)
     }
-    await Promise.all(tasks)
 
     const videoCreated = await video.save(sequelizeOptions)
     // Do not forget to add video channel information to the created video
     videoCreated.VideoChannel = res.locals.videoChannel
 
     videoFile.videoId = video.id
-
     await videoFile.save(sequelizeOptions)
-    video.VideoFiles = [videoFile]
+
+    video.VideoFiles = [ videoFile ]
 
     if (videoInfo.tags) {
       const tagInstances = await db.Tag.findOrCreateTags(videoInfo.tags, t)

diff --git a/server/tests/api/single-server.ts b/server/tests/api/single-server.ts
index ed79f9e1c61d5a5d87f2b356780d2f58ae0a4dd3..e99955ef4356551c782bedd3ab66b35e5fad0769 100644 (file)
--- a/server/tests/api/single-server.ts
+++ b/server/tests/api/single-server.ts
@@ -364,7 +364,7 @@ describe('Test a single server', function () {
       'video_short1.webm', 'video_short2.webm', 'video_short3.webm'
     ]
 
-    // const tasks: Promise<any>[] = []
+    const tasks: Promise<any>[] = []
     for (const video of videos) {
       const videoAttributes = {
         name: video + ' name',
@@ -378,13 +378,10 @@ describe('Test a single server', function () {
       }
 
       const p = uploadVideo(server.url, server.accessToken, videoAttributes)
-      await p
+      tasks.push(p)
     }
-    // FIXME: concurrent uploads does not work :(
-    //   tasks.push(p)
-    // }
-    //
-    // await Promise.all(tasks)
+
+    await Promise.all(tasks)
   })
 
   it('Should have the correct durations', async function () {
@@ -712,7 +709,7 @@ describe('Test a single server', function () {
     const filePath = join(__dirname, '..', 'api', 'fixtures', 'video_short.webm')
 
     await req.attach('videofile', filePath)
-      .expect(204)
+      .expect(200)
 
     const res = await getVideosList(server.url)
     const video = res.body.data.find(v => v.name === 'minimum parameters')

diff --git a/server/tests/utils/videos.ts b/server/tests/utils/videos.ts
index bdf3368ac87d460401348eaead2596c7287f6d2a..fb758cf29bf42081998b10603626602b5a16b447 100644 (file)
--- a/server/tests/utils/videos.ts
+++ b/server/tests/utils/videos.ts
@@ -201,7 +201,7 @@ async function testVideoImage (url: string, imageName: string, imagePath: string
   }
 }
 
-async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = 201) {
+async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = 200) {
   const path = '/api/v1/videos/upload'
   let defaultChannelId = '1'