Import magnets with webtorrent

2 files changed, 83 insertions, 23 deletions

diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index cdfe412cc..c457b71fc 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -13,30 +13,99 @@ import { VideoState } from '../../../../shared'
 import { JobQueue } from '../index'
 import { federateVideoIfNeeded } from '../../activitypub'
 import { VideoModel } from '../../../models/video/video'
+import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
 
-export type VideoImportPayload = {
+type VideoImportYoutubeDLPayload = {
   type: 'youtube-dl'
   videoImportId: number
+
   thumbnailUrl: string
   downloadThumbnail: boolean
   downloadPreview: boolean
 }
 
+type VideoImportTorrentPayload = {
+  type: 'magnet-uri'
+  videoImportId: number
+}
+
+export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
+
 async function processVideoImport (job: Bull.Job) {
   const payload = job.data as VideoImportPayload
-  logger.info('Processing video import in job %d.', job.id)
 
-  const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
+  if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
+  if (payload.type === 'magnet-uri') return processTorrentImport(job, payload)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processVideoImport
+}
+
+// ---------------------------------------------------------------------------
+
+async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
+  logger.info('Processing torrent video import in job %d.', job.id)
+
+  const videoImport = await getVideoImportOrDie(payload.videoImportId)
+  const options = {
+    videoImportId: payload.videoImportId,
+
+    downloadThumbnail: false,
+    downloadPreview: false,
+
+    generateThumbnail: true,
+    generatePreview: true
+  }
+  return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options)
+}
+
+async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
+  logger.info('Processing youtubeDL video import in job %d.', job.id)
+
+  const videoImport = await getVideoImportOrDie(payload.videoImportId)
+  const options = {
+    videoImportId: videoImport.id,
+
+    downloadThumbnail: payload.downloadThumbnail,
+    downloadPreview: payload.downloadPreview,
+    thumbnailUrl: payload.thumbnailUrl,
+
+    generateThumbnail: false,
+    generatePreview: false
+  }
+
+  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options)
+}
+
+async function getVideoImportOrDie (videoImportId: number) {
+  const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
   if (!videoImport || !videoImport.Video) {
     throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
   }
 
+  return videoImport
+}
+
+type ProcessFileOptions = {
+  videoImportId: number
+
+  downloadThumbnail: boolean
+  downloadPreview: boolean
+  thumbnailUrl?: string
+
+  generateThumbnail: boolean
+  generatePreview: boolean
+}
+async function processFile (downloader: () => Promise<string>, videoImport: VideoImportModel, options: ProcessFileOptions) {
   let tempVideoPath: string
   let videoDestFile: string
   let videoFile: VideoFileModel
   try {
     // Download video from youtubeDL
-    tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl)
+    tempVideoPath = await downloader()
 
     // Get information about this video
     const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
@@ -62,23 +131,27 @@ async function processVideoImport (job: Bull.Job) {
     tempVideoPath = null // This path is not used anymore
 
     // Process thumbnail
-    if (payload.downloadThumbnail) {
-      if (payload.thumbnailUrl) {
+    if (options.downloadThumbnail) {
+      if (options.thumbnailUrl) {
         const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
-        await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath)
+        await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath)
       } else {
         await videoImport.Video.createThumbnail(videoFile)
       }
+    } else if (options.generateThumbnail) {
+      await videoImport.Video.createThumbnail(videoFile)
     }
 
     // Process preview
-    if (payload.downloadPreview) {
-      if (payload.thumbnailUrl) {
+    if (options.downloadPreview) {
+      if (options.thumbnailUrl) {
         const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
-        await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath)
+        await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath)
       } else {
         await videoImport.Video.createPreview(videoFile)
       }
+    } else if (options.generatePreview) {
+      await videoImport.Video.createPreview(videoFile)
     }
 
     // Create torrent
@@ -137,9 +210,3 @@ async function processVideoImport (job: Bull.Job) {
     throw err
   }
 }
-
-// ---------------------------------------------------------------------------
-
-export {
-  processVideoImport
-}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 8a24604e1..ddb357db5 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -32,13 +32,6 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
   'video-import': processVideoImport
 }
 
-const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
-  'activitypub-http-broadcast': true,
-  'activitypub-http-unicast': true,
-  'activitypub-http-fetcher': true,
-  'activitypub-follow': true
-}
-
 const jobTypes: JobType[] = [
   'activitypub-follow',
   'activitypub-http-broadcast',