Async torrent creation

5 files changed, 66 insertions, 11 deletions

diff --git a/client/src/assets/player/webtorrent/webtorrent-plugin.ts b/client/src/assets/player/webtorrent/webtorrent-plugin.ts
index 4bb438e57..e557fe722 100644
--- a/client/src/assets/player/webtorrent/webtorrent-plugin.ts
+++ b/client/src/assets/player/webtorrent/webtorrent-plugin.ts
@@ -249,6 +249,8 @@ class WebTorrentPlugin extends Plugin {
     options: PlayOptions,
     done: Function
   ) {
+    if (!magnetOrTorrentUrl) return this.fallbackToHttp(options, done)
+
     console.log('Adding ' + magnetOrTorrentUrl + '.')
 
     const oldTorrent = this.torrent
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index e89315930..58ab72370 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -9,12 +9,12 @@ import { LiveManager } from '@server/lib/live-manager'
 import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
 import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
 import { getServerActor } from '@server/models/application/application'
-import { MVideoFullLight } from '@server/types/models'
+import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
 import { VideoCreate, VideoState, VideoUpdate } from '../../../../shared'
 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
 import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
-import { resetSequelizeInstance } from '../../../helpers/database-utils'
+import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
 import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils'
 import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
 import { logger } from '../../../helpers/logger'
@@ -221,9 +221,6 @@ async function addVideo (req: express.Request, res: express.Response) {
     fallback: type => generateVideoMiniature({ video, videoFile, type })
   })
 
-  // Create the torrent file
-  await createTorrentAndSetInfoHash(video, videoFile)
-
   const { videoCreated } = await sequelizeTypescript.transaction(async t => {
     const sequelizeOptions = { transaction: t }
 
@@ -258,7 +255,6 @@ async function addVideo (req: express.Request, res: express.Response) {
       isNew: true,
       transaction: t
     })
-    await federateVideoIfNeeded(video, true, t)
 
     auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
     logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid)
@@ -266,7 +262,21 @@ async function addVideo (req: express.Request, res: express.Response) {
     return { videoCreated }
   })
 
-  Notifier.Instance.notifyOnNewVideoIfNeeded(videoCreated)
+  // Create the torrent file in async way because it could be long
+  createTorrentAndSetInfoHashAsync(video, videoFile)
+    .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err }))
+    .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id))
+    .then(refreshedVideo => {
+      if (!refreshedVideo) return
+
+      // Only federate and notify after the torrent creation
+      Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo)
+
+      return retryTransactionWrapper(() => {
+        return sequelizeTypescript.transaction(t => federateVideoIfNeeded(refreshedVideo, true, t))
+      })
+    })
+    .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err }))
 
   if (video.state === VideoState.TO_TRANSCODE) {
     await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User)
@@ -526,3 +536,17 @@ async function removeVideo (req: express.Request, res: express.Response) {
             .status(HttpStatusCode.NO_CONTENT_204)
             .end()
 }
+
+async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoFile) {
+  await createTorrentAndSetInfoHash(video, fileArg)
+
+  // Refresh videoFile because the createTorrentAndSetInfoHash could be long
+  const refreshedFile = await VideoFileModel.loadWithVideo(fileArg.id)
+  // File does not exist anymore, remove the generated torrent
+  if (!refreshedFile) return fileArg.removeTorrent()
+
+  refreshedFile.infoHash = fileArg.infoHash
+  refreshedFile.torrentFilename = fileArg.torrentFilename
+
+  return refreshedFile.save()
+}
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 4df2c20bc..1ad796104 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -457,18 +457,26 @@ export class VideoFileModel extends Model {
 
   // We proxify torrent requests so use a local URL
   getTorrentUrl () {
+    if (!this.torrentFilename) return null
+
     return WEBSERVER.URL + this.getTorrentStaticPath()
   }
 
   getTorrentStaticPath () {
+    if (!this.torrentFilename) return null
+
     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
   }
 
   getTorrentDownloadUrl () {
+    if (!this.torrentFilename) return null
+
     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
   }
 
   removeTorrent () {
+    if (!this.torrentFilename) return null
+
     const torrentPath = getTorrentFilePath(this)
     return remove(torrentPath)
       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index a6a1a4f0d..bcba90093 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -205,7 +205,7 @@ function videoFilesModelToFormattedJSON (
           label: videoFile.resolution + 'p'
         },
 
-        magnetUri: includeMagnet
+        magnetUri: includeMagnet && videoFile.torrentFilename
           ? generateMagnetUri(video, videoFile, trackerUrls)
           : undefined,
 
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts
index 929eb42ca..0b6a54046 100644
--- a/shared/extra-utils/videos/videos.ts
+++ b/shared/extra-utils/videos/videos.ts
@@ -1,6 +1,5 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
 
-import { HttpStatusCode } from '@shared/core-utils'
 import { expect } from 'chai'
 import { pathExists, readdir, readFile } from 'fs-extra'
 import * as parseTorrent from 'parse-torrent'
@@ -8,9 +7,18 @@ import { extname, join } from 'path'
 import * as request from 'supertest'
 import { v4 as uuidv4 } from 'uuid'
 import validator from 'validator'
+import { HttpStatusCode } from '@shared/core-utils'
 import { loadLanguages, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
 import { VideoDetails, VideoPrivacy } from '../../models/videos'
-import { buildAbsoluteFixturePath, buildServerDirectory, dateIsValid, immutableAssign, testImage, webtorrentAdd } from '../miscs/miscs'
+import {
+  buildAbsoluteFixturePath,
+  buildServerDirectory,
+  dateIsValid,
+  immutableAssign,
+  testImage,
+  wait,
+  webtorrentAdd
+} from '../miscs/miscs'
 import { makeGetRequest, makePutBodyRequest, makeRawRequest, makeUploadRequest } from '../requests/requests'
 import { waitJobs } from '../server/jobs'
 import { ServerInfo } from '../server/servers'
@@ -423,8 +431,21 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
     req.field('originallyPublishedAt', attributes.originallyPublishedAt)
   }
 
-  return req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture))
+  const res = await req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture))
             .expect(specialStatus)
+
+  // Wait torrent generation
+  if (specialStatus === HttpStatusCode.OK_200) {
+    let video: VideoDetails
+    do {
+      const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid)
+      video = resVideo.body
+
+      await wait(50)
+    } while (!video.files[0].torrentUrl)
+  }
+
+  return res
 }
 
 function updateVideo (