Refactor video creation

6 files changed, 151 insertions, 132 deletions

diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 24a237304..5840cd063 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,30 +1,10 @@
+import * as Bluebird from 'bluebird'
 import * as express from 'express'
+import { move, readFile } from 'fs-extra'
 import * as magnetUtil from 'magnet-uri'
-import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
-import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
-import { MIMETYPES } from '../../../initializers/constants'
-import { getYoutubeDLInfo, YoutubeDLInfo, getYoutubeDLSubs } from '../../../helpers/youtube-dl'
-import { createReqFiles } from '../../../helpers/express-utils'
-import { logger } from '../../../helpers/logger'
-import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
-import { VideoModel } from '../../../models/video/video'
-import { VideoCaptionModel } from '../../../models/video/video-caption'
-import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
-import { getVideoActivityPubUrl } from '../../../lib/activitypub/url'
-import { TagModel } from '../../../models/video/tag'
-import { VideoImportModel } from '../../../models/video/video-import'
-import { JobQueue } from '../../../lib/job-queue/job-queue'
-import { join } from 'path'
-import { isArray } from '../../../helpers/custom-validators/misc'
-import * as Bluebird from 'bluebird'
 import * as parseTorrent from 'parse-torrent'
-import { getSecureTorrentName } from '../../../helpers/utils'
-import { move, readFile } from 'fs-extra'
-import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
-import { CONFIG } from '../../../initializers/config'
-import { sequelizeTypescript } from '../../../initializers/database'
-import { createVideoMiniatureFromExisting, createVideoMiniatureFromUrl } from '../../../lib/thumbnail'
-import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
+import { join } from 'path'
+import { setVideoTags } from '@server/lib/video'
 import {
   MChannelAccountDefault,
   MThumbnail,
@@ -36,6 +16,26 @@ import {
   MVideoWithBlacklistLight
 } from '@server/types/models'
 import { MVideoImport, MVideoImportFormattable } from '@server/types/models/video/video-import'
+import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
+import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
+import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
+import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
+import { isArray } from '../../../helpers/custom-validators/misc'
+import { createReqFiles } from '../../../helpers/express-utils'
+import { logger } from '../../../helpers/logger'
+import { getSecureTorrentName } from '../../../helpers/utils'
+import { getYoutubeDLInfo, getYoutubeDLSubs, YoutubeDLInfo } from '../../../helpers/youtube-dl'
+import { CONFIG } from '../../../initializers/config'
+import { MIMETYPES } from '../../../initializers/constants'
+import { sequelizeTypescript } from '../../../initializers/database'
+import { getVideoActivityPubUrl } from '../../../lib/activitypub/url'
+import { JobQueue } from '../../../lib/job-queue/job-queue'
+import { createVideoMiniatureFromExisting, createVideoMiniatureFromUrl } from '../../../lib/thumbnail'
+import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
+import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
+import { VideoModel } from '../../../models/video/video'
+import { VideoCaptionModel } from '../../../models/video/video-caption'
+import { VideoImportModel } from '../../../models/video/video-import'
 
 const auditLogger = auditLoggerFactory('video-imports')
 const videoImportsRouter = express.Router()
@@ -260,7 +260,12 @@ async function processThumbnail (req: express.Request, video: VideoModel) {
   if (thumbnailField) {
     const thumbnailPhysicalFile = thumbnailField[0]
 
-    return createVideoMiniatureFromExisting(thumbnailPhysicalFile.path, video, ThumbnailType.MINIATURE, false)
+    return createVideoMiniatureFromExisting({
+      inputPath: thumbnailPhysicalFile.path,
+      video,
+      type: ThumbnailType.MINIATURE,
+      automaticallyGenerated: false
+    })
   }
 
   return undefined
@@ -271,7 +276,12 @@ async function processPreview (req: express.Request, video: VideoModel) {
   if (previewField) {
     const previewPhysicalFile = previewField[0]
 
-    return createVideoMiniatureFromExisting(previewPhysicalFile.path, video, ThumbnailType.PREVIEW, false)
+    return createVideoMiniatureFromExisting({
+      inputPath: previewPhysicalFile.path,
+      video,
+      type: ThumbnailType.PREVIEW,
+      automaticallyGenerated: false
+    })
   }
 
   return undefined
@@ -325,15 +335,7 @@ function insertIntoDB (parameters: {
       transaction: t
     })
 
-    // Set tags to the video
-    if (tags) {
-      const tagInstances = await TagModel.findOrCreateTags(tags, t)
-
-      await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
-      videoCreated.Tags = tagInstances
-    } else {
-      videoCreated.Tags = []
-    }
+    await setVideoTags({ video: videoCreated, tags, transaction: t })
 
     // Create video import object in database
     const videoImport = await VideoImportModel.create(
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 94f0361ee..1539afc35 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -6,11 +6,11 @@ import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { changeVideoChannelShare } from '@server/lib/activitypub/share'
 import { getVideoActivityPubUrl } from '@server/lib/activitypub/url'
+import { buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
 import { getVideoFilePath } from '@server/lib/video-paths'
 import { getServerActor } from '@server/models/application/application'
 import { MVideoDetails, MVideoFullLight } from '@server/types/models'
-import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../shared'
-import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
+import { VideoCreate, VideoState, VideoUpdate } from '../../../../shared'
 import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
 import { resetSequelizeInstance } from '../../../helpers/database-utils'
@@ -34,7 +34,7 @@ import { JobQueue } from '../../../lib/job-queue'
 import { Notifier } from '../../../lib/notifier'
 import { Hooks } from '../../../lib/plugins/hooks'
 import { Redis } from '../../../lib/redis'
-import { createVideoMiniatureFromExisting, generateVideoMiniature } from '../../../lib/thumbnail'
+import { generateVideoMiniature } from '../../../lib/thumbnail'
 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
 import {
   asyncMiddleware,
@@ -186,25 +186,9 @@ async function addVideo (req: express.Request, res: express.Response) {
   const videoPhysicalFile = req.files['videofile'][0]
   const videoInfo: VideoCreate = req.body
 
-  // Prepare data so we don't block the transaction
-  const videoData = {
-    name: videoInfo.name,
-    remote: false,
-    category: videoInfo.category,
-    licence: videoInfo.licence,
-    language: videoInfo.language,
-    commentsEnabled: videoInfo.commentsEnabled !== false, // If the value is not "false", the default is "true"
-    downloadEnabled: videoInfo.downloadEnabled !== false,
-    waitTranscoding: videoInfo.waitTranscoding || false,
-    state: CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED,
-    nsfw: videoInfo.nsfw || false,
-    description: videoInfo.description,
-    support: videoInfo.support,
-    privacy: videoInfo.privacy || VideoPrivacy.PRIVATE,
-    duration: videoPhysicalFile['duration'], // duration was added by a previous middleware
-    channelId: res.locals.videoChannel.id,
-    originallyPublishedAt: videoInfo.originallyPublishedAt
-  }
+  const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id)
+  videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
+  videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware
 
   const video = new VideoModel(videoData) as MVideoDetails
   video.url = getVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
@@ -230,17 +214,11 @@ async function addVideo (req: express.Request, res: express.Response) {
   videoPhysicalFile.filename = getVideoFilePath(video, videoFile)
   videoPhysicalFile.path = destination
 
-  // Process thumbnail or create it from the video
-  const thumbnailField = req.files['thumbnailfile']
-  const thumbnailModel = thumbnailField
-    ? await createVideoMiniatureFromExisting(thumbnailField[0].path, video, ThumbnailType.MINIATURE, false)
-    : await generateVideoMiniature(video, videoFile, ThumbnailType.MINIATURE)
-
-  // Process preview or create it from the video
-  const previewField = req.files['previewfile']
-  const previewModel = previewField
-    ? await createVideoMiniatureFromExisting(previewField[0].path, video, ThumbnailType.PREVIEW, false)
-    : await generateVideoMiniature(video, videoFile, ThumbnailType.PREVIEW)
+  const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
+    video,
+    files: req.files,
+    fallback: type => generateVideoMiniature(video, videoFile, type)
+  })
 
   // Create the torrent file
   await createTorrentAndSetInfoHash(video, videoFile)
@@ -261,13 +239,7 @@ async function addVideo (req: express.Request, res: express.Response) {
 
     video.VideoFiles = [ videoFile ]
 
-    // Create tags
-    if (videoInfo.tags !== undefined) {
-      const tagInstances = await TagModel.findOrCreateTags(videoInfo.tags, t)
-
-      await video.$set('Tags', tagInstances, sequelizeOptions)
-      video.Tags = tagInstances
-    }
+    await setVideoTags({ video, tags: videoInfo.tags, transaction: t })
 
     // Schedule an update in the future?
     if (videoInfo.scheduleUpdate) {
@@ -318,14 +290,12 @@ async function updateVideo (req: express.Request, res: express.Response) {
   const wasConfidentialVideo = videoInstance.isConfidential()
   const hadPrivacyForFederation = videoInstance.hasPrivacyForFederation()
 
-  // Process thumbnail or create it from the video
-  const thumbnailModel = req.files?.['thumbnailfile']
-    ? await createVideoMiniatureFromExisting(req.files['thumbnailfile'][0].path, videoInstance, ThumbnailType.MINIATURE, false)
-    : undefined
-
-  const previewModel = req.files?.['previewfile']
-    ? await createVideoMiniatureFromExisting(req.files['previewfile'][0].path, videoInstance, ThumbnailType.PREVIEW, false)
-    : undefined
+  const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
+    video: videoInstance,
+    files: req.files,
+    fallback: () => Promise.resolve(undefined),
+    automaticallyGenerated: false
+  })
 
   try {
     const videoInstanceUpdated = await sequelizeTypescript.transaction(async t => {
@@ -366,12 +336,12 @@ async function updateVideo (req: express.Request, res: express.Response) {
       if (previewModel) await videoInstanceUpdated.addAndSaveThumbnail(previewModel, t)
 
       // Video tags update?
-      if (videoInfoToUpdate.tags !== undefined) {
-        const tagInstances = await TagModel.findOrCreateTags(videoInfoToUpdate.tags, t)
-
-        await videoInstanceUpdated.$set('Tags', tagInstances, sequelizeOptions)
-        videoInstanceUpdated.Tags = tagInstances
-      }
+      await setVideoTags({
+        video: videoInstanceUpdated,
+        tags: videoInfoToUpdate.tags,
+        transaction: t,
+        defaultValue: videoInstanceUpdated.Tags
+      })
 
       // Video channel update?
       if (res.locals.videoChannel && videoInstanceUpdated.channelId !== res.locals.videoChannel.id) {
diff --git a/server/controllers/api/videos/live.ts b/server/controllers/api/videos/live.ts
index d08ef9869..97b135f96 100644
--- a/server/controllers/api/videos/live.ts
+++ b/server/controllers/api/videos/live.ts
@@ -4,18 +4,16 @@ import { createReqFiles } from '@server/helpers/express-utils'
 import { CONFIG } from '@server/initializers/config'
 import { ASSETS_PATH, MIMETYPES } from '@server/initializers/constants'
 import { getVideoActivityPubUrl } from '@server/lib/activitypub/url'
+import { buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
 import { videoLiveAddValidator, videoLiveGetValidator } from '@server/middlewares/validators/videos/video-live'
 import { VideoLiveModel } from '@server/models/video/video-live'
 import { MVideoDetails, MVideoFullLight } from '@server/types/models'
-import { VideoCreate, VideoPrivacy, VideoState } from '../../../../shared'
-import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
+import { VideoCreate, VideoState } from '../../../../shared'
 import { logger } from '../../../helpers/logger'
 import { sequelizeTypescript } from '../../../initializers/database'
 import { createVideoMiniatureFromExisting } from '../../../lib/thumbnail'
 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate } from '../../../middlewares'
-import { TagModel } from '../../../models/video/tag'
 import { VideoModel } from '../../../models/video/video'
-import { buildLocalVideoFromCreate } from '@server/lib/video'
 
 const liveRouter = express.Router()
 
@@ -59,26 +57,24 @@ async function addLiveVideo (req: express.Request, res: express.Response) {
   const videoInfo: VideoCreate = req.body
 
   // Prepare data so we don't block the transaction
-  const videoData = buildLocalVideoFromCreate(videoInfo, res.locals.videoChannel.id)
+  const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id)
   videoData.isLive = true
-
-  const videoLive = new VideoLiveModel()
-  videoLive.streamKey = uuidv4()
+  videoData.state = VideoState.WAITING_FOR_LIVE
+  videoData.duration = 0
 
   const video = new VideoModel(videoData) as MVideoDetails
   video.url = getVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
 
-  // Process thumbnail or create it from the video
-  const thumbnailField = req.files ? req.files['thumbnailfile'] : null
-  const thumbnailModel = thumbnailField
-    ? await createVideoMiniatureFromExisting(thumbnailField[0].path, video, ThumbnailType.MINIATURE, false)
-    : await createVideoMiniatureFromExisting(ASSETS_PATH.DEFAULT_LIVE_BACKGROUND, video, ThumbnailType.MINIATURE, true)
+  const videoLive = new VideoLiveModel()
+  videoLive.streamKey = uuidv4()
 
-  // Process preview or create it from the video
-  const previewField = req.files ? req.files['previewfile'] : null
-  const previewModel = previewField
-    ? await createVideoMiniatureFromExisting(previewField[0].path, video, ThumbnailType.PREVIEW, false)
-    : await createVideoMiniatureFromExisting(ASSETS_PATH.DEFAULT_LIVE_BACKGROUND, video, ThumbnailType.PREVIEW, true)
+  const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
+    video,
+    files: req.files,
+    fallback: type => {
+      return createVideoMiniatureFromExisting({ inputPath: ASSETS_PATH.DEFAULT_LIVE_BACKGROUND, video, type, automaticallyGenerated: true })
+    }
+  })
 
   const { videoCreated } = await sequelizeTypescript.transaction(async t => {
     const sequelizeOptions = { transaction: t }
@@ -94,13 +90,7 @@ async function addLiveVideo (req: express.Request, res: express.Response) {
     videoLive.videoId = videoCreated.id
     await videoLive.save(sequelizeOptions)
 
-    // Create tags
-    if (videoInfo.tags !== undefined) {
-      const tagInstances = await TagModel.findOrCreateTags(videoInfo.tags, t)
-
-      await video.$set('Tags', tagInstances, sequelizeOptions)
-      video.Tags = tagInstances
-    }
+    await setVideoTags({ video, tags: videoInfo.tags, transaction: t })
 
     logger.info('Video live %s with uuid %s created.', videoInfo.name, videoCreated.uuid)
 
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index cbbf23be1..096884776 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -68,6 +68,7 @@ import { ActorFollowScoreCache } from '../files-cache'
 import { JobQueue } from '../job-queue'
 import { Notifier } from '../notifier'
 import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
+import { setVideoTags } from '../video'
 import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
 import { getOrCreateActorAndServerAndModel } from './actor'
 import { crawlCollectionPage } from './crawl'
@@ -409,8 +410,7 @@ async function updateVideoFromAP (options: {
         const tags = videoObject.tag
                                 .filter(isAPHashTagObject)
                                 .map(tag => tag.name)
-        const tagInstances = await TagModel.findOrCreateTags(tags, t)
-        await videoUpdated.$set('Tags', tagInstances, sequelizeOptions)
+        await setVideoTags({ video: videoUpdated, tags, transaction: t, defaultValue: videoUpdated.Tags })
       }
 
       {
@@ -594,8 +594,7 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
     const tags = videoObject.tag
                             .filter(isAPHashTagObject)
                             .map(t => t.name)
-    const tagInstances = await TagModel.findOrCreateTags(tags, t)
-    await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
+    await setVideoTags({ video: videoCreated, tags, transaction: t })
 
     // Process captions
     const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
@@ -604,7 +603,6 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
     await Promise.all(videoCaptionsPromises)
 
     videoCreated.VideoFiles = videoFiles
-    videoCreated.Tags = tagInstances
 
     const autoBlacklisted = await autoBlacklistVideoIfNeeded({
       video: videoCreated,
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index 78d2f69e3..dc86423f8 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -42,15 +42,18 @@ function createVideoMiniatureFromUrl (fileUrl: string, video: MVideoThumbnail, t
   return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
 }
 
-function createVideoMiniatureFromExisting (
-  inputPath: string,
-  video: MVideoThumbnail,
-  type: ThumbnailType,
-  automaticallyGenerated: boolean,
+function createVideoMiniatureFromExisting (options: {
+  inputPath: string
+  video: MVideoThumbnail
+  type: ThumbnailType
+  automaticallyGenerated: boolean
   size?: ImageSize
-) {
+  keepOriginal?: boolean
+}) {
+  const { inputPath, video, type, automaticallyGenerated, size, keepOriginal } = options
+
   const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
-  const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height })
+  const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal)
 
   return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, automaticallyGenerated, existingThumbnail })
 }
diff --git a/server/lib/video.ts b/server/lib/video.ts
index a28f31529..6df41e6cd 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -1,9 +1,12 @@
-
+import { Transaction } from 'sequelize/types'
+import { TagModel } from '@server/models/video/tag'
 import { VideoModel } from '@server/models/video/video'
 import { FilteredModelAttributes } from '@server/types'
-import { VideoCreate, VideoPrivacy, VideoState } from '@shared/models'
+import { MTag, MThumbnail, MVideoTag, MVideoThumbnail } from '@server/types/models'
+import { ThumbnailType, VideoCreate, VideoPrivacy } from '@shared/models'
+import { createVideoMiniatureFromExisting } from './thumbnail'
 
-function buildLocalVideoFromCreate (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
+function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
   return {
     name: videoInfo.name,
     remote: false,
@@ -13,19 +16,72 @@ function buildLocalVideoFromCreate (videoInfo: VideoCreate, channelId: number):
     commentsEnabled: videoInfo.commentsEnabled !== false, // If the value is not "false", the default is "true"
     downloadEnabled: videoInfo.downloadEnabled !== false,
     waitTranscoding: videoInfo.waitTranscoding || false,
-    state: VideoState.WAITING_FOR_LIVE,
     nsfw: videoInfo.nsfw || false,
     description: videoInfo.description,
     support: videoInfo.support,
     privacy: videoInfo.privacy || VideoPrivacy.PRIVATE,
-    duration: 0,
     channelId: channelId,
     originallyPublishedAt: videoInfo.originallyPublishedAt
   }
 }
 
+async function buildVideoThumbnailsFromReq (options: {
+  video: MVideoThumbnail
+  files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[]
+  fallback: (type: ThumbnailType) => Promise<MThumbnail>
+  automaticallyGenerated?: boolean
+}) {
+  const { video, files, fallback, automaticallyGenerated } = options
+
+  const promises = [
+    {
+      type: ThumbnailType.MINIATURE,
+      fieldName: 'thumbnailfile'
+    },
+    {
+      type: ThumbnailType.PREVIEW,
+      fieldName: 'previewfile'
+    }
+  ].map(p => {
+    const fields = files?.[p.fieldName]
+
+    if (fields) {
+      return createVideoMiniatureFromExisting({
+        inputPath: fields[0].path,
+        video,
+        type: p.type,
+        automaticallyGenerated: automaticallyGenerated || false
+      })
+    }
+
+    return fallback(p.type)
+  })
+
+  return Promise.all(promises)
+}
+
+async function setVideoTags (options: {
+  video: MVideoTag
+  tags: string[]
+  transaction?: Transaction
+  defaultValue?: MTag[]
+}) {
+  const { video, tags, transaction, defaultValue } = options
+  // Set tags to the video
+  if (tags) {
+    const tagInstances = await TagModel.findOrCreateTags(tags, transaction)
+
+    await video.$set('Tags', tagInstances, { transaction })
+    video.Tags = tagInstances
+  } else {
+    video.Tags = defaultValue || []
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 export {
-  buildLocalVideoFromCreate
+  buildLocalVideoFromReq,
+  buildVideoThumbnailsFromReq,
+  setVideoTags
 }