Cleanup unavailable remote AP resource

[github/Chocobozzz/PeerTube.git] / server / lib / job-queue / handlers / video-import.ts

diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 0d00c1b9d114520061bfe76c07ae5a63f03dac93..2f74e9fbd6c9548142f24582088f2c7870194a14 100644 (file)
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -1,30 +1,34 @@
-import * as Bull from 'bull'
+import { Job } from 'bull'
 import { move, remove, stat } from 'fs-extra'
-import { extname } from 'path'
 import { retryTransactionWrapper } from '@server/helpers/database-utils'
+import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
 import { isPostImportVideoAccepted } from '@server/lib/moderation'
+import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import { Hooks } from '@server/lib/plugins/hooks'
+import { ServerConfigManager } from '@server/lib/server-config-manager'
 import { isAbleToUploadVideo } from '@server/lib/user'
-import { addOptimizeOrMergeAudioJob } from '@server/lib/video'
-import { getVideoFilePath } from '@server/lib/video-paths'
+import { addMoveToObjectStorageJob, addOptimizeOrMergeAudioJob } from '@server/lib/video'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { buildNextVideoState } from '@server/lib/video-state'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
 import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
+import { getLowercaseExtension } from '@shared/core-utils'
+import { isAudioFile } from '@shared/extra-utils'
 import {
+  ThumbnailType,
   VideoImportPayload,
+  VideoImportState,
   VideoImportTorrentPayload,
   VideoImportTorrentPayloadType,
   VideoImportYoutubeDLPayload,
   VideoImportYoutubeDLPayloadType,
+  VideoResolution,
   VideoState
-} from '../../../../shared'
-import { VideoImportState } from '../../../../shared/models/videos'
-import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
-import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
+} from '@shared/models'
+import { ffprobePromise, getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
 import { logger } from '../../../helpers/logger'
 import { getSecureTorrentName } from '../../../helpers/utils'
 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
-import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
-import { CONFIG } from '../../../initializers/config'
 import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
 import { sequelizeTypescript } from '../../../initializers/database'
 import { VideoModel } from '../../../models/video/video'
@@ -35,7 +39,7 @@ import { federateVideoIfNeeded } from '../../activitypub/videos'
 import { Notifier } from '../../notifier'
 import { generateVideoMiniature } from '../../thumbnail'
 
-async function processVideoImport (job: Bull.Job) {
+async function processVideoImport (job: Job) {
   const payload = job.data as VideoImportPayload
 
   if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
@@ -50,7 +54,7 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
+async function processTorrentImport (job: Job, payload: VideoImportTorrentPayload) {
   logger.info('Processing torrent video import in job %d.', job.id)
 
   const videoImport = await getVideoImportOrDie(payload.videoImportId)
@@ -61,12 +65,12 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
   }
   const target = {
     torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
-    magnetUri: videoImport.magnetUri
+    uri: videoImport.magnetUri
   }
   return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options)
 }
 
-async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
+async function processYoutubeDLImport (job: Job, payload: VideoImportYoutubeDLPayload) {
   logger.info('Processing youtubeDL video import in job %d.', job.id)
 
   const videoImport = await getVideoImportOrDie(payload.videoImportId)
@@ -75,8 +79,10 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
     videoImportId: videoImport.id
   }
 
+  const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
+
   return processFile(
-    () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT),
+    () => youtubeDL.downloadVideo(payload.fileExt, VIDEO_IMPORT_TIMEOUT),
     videoImport,
     options
   )
@@ -97,7 +103,6 @@ type ProcessFileOptions = {
 }
 async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
   let tempVideoPath: string
-  let videoDestFile: string
   let videoFile: VideoFileModel
 
   try {
@@ -111,15 +116,22 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
       throw new Error('The user video quota is exceeded with this video to import.')
     }
 
-    const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
-    const fps = await getVideoFileFPS(tempVideoPath)
-    const duration = await getDurationFromVideoFile(tempVideoPath)
+    const probe = await ffprobePromise(tempVideoPath)
+
+    const { resolution } = await isAudioFile(tempVideoPath, probe)
+      ? { resolution: VideoResolution.H_NOVIDEO }
+      : await getVideoFileResolution(tempVideoPath)
+
+    const fps = await getVideoFileFPS(tempVideoPath, probe)
+    const duration = await getDurationFromVideoFile(tempVideoPath, probe)
 
     // Prepare video file object for creation in database
+    const fileExt = getLowercaseExtension(tempVideoPath)
     const videoFileData = {
-      extname: extname(tempVideoPath),
-      resolution: videoFileResolution,
+      extname: fileExt,
+      resolution,
       size: stats.size,
+      filename: generateWebTorrentVideoFilename(resolution, fileExt),
       fps,
       videoId: videoImport.videoId
     }
@@ -154,7 +166,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
 
     // Move file
-    videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile)
+    const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
     await move(tempVideoPath, videoDestFile)
     tempVideoPath = null // This path is not used anymore
 
@@ -199,7 +211,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
 
         // Update video DB object
         video.duration = duration
-        video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
+        video.state = buildNextVideoState(video.state)
         await video.save({ transaction: t })
 
         if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
@@ -230,7 +242,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
       })
     })
 
-    Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true)
+    Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: videoImportUpdated, success: true })
 
     if (video.isBlacklisted()) {
       const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
@@ -240,6 +252,10 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
       Notifier.Instance.notifyOnNewVideoIfNeeded(video)
     }
 
+    if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
+      return addMoveToObjectStorageJob(videoImportUpdated.Video)
+    }
+
     // Create transcoding jobs?
     if (video.state === VideoState.TO_TRANSCODE) {
       await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User)
@@ -258,7 +274,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     }
     await videoImport.save()
 
-    Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false)
+    Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false })
 
     throw err
   }