Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

1 files changed, 51 insertions, 267 deletions

diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 5a2e1006a..9d7b0260b 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,49 +1,20 @@
 import express from 'express'
-import { move, readFile, remove } from 'fs-extra'
+import { move, readFile } from 'fs-extra'
 import { decode } from 'magnet-uri'
 import parseTorrent, { Instance } from 'parse-torrent'
 import { join } from 'path'
-import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
-import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
-import { isResolvingToUnicastOnly } from '@server/helpers/dns'
-import { Hooks } from '@server/lib/plugins/hooks'
-import { ServerConfigManager } from '@server/lib/server-config-manager'
-import { setVideoTags } from '@server/lib/video'
-import { FilteredModelAttributes } from '@server/types'
-import {
-  MChannelAccountDefault,
-  MThumbnail,
-  MUser,
-  MVideoAccountDefault,
-  MVideoCaption,
-  MVideoTag,
-  MVideoThumbnail,
-  MVideoWithBlacklistLight
-} from '@server/types/models'
-import { MVideoImportFormattable } from '@server/types/models/video/video-import'
-import {
-  HttpStatusCode,
-  ServerErrorCode,
-  ThumbnailType,
-  VideoImportCreate,
-  VideoImportState,
-  VideoPrivacy,
-  VideoState
-} from '@shared/models'
+import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
+import { MThumbnail, MVideoThumbnail } from '@server/types/models'
+import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
-import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
 import { isArray } from '../../../helpers/custom-validators/misc'
 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
 import { logger } from '../../../helpers/logger'
 import { getSecureTorrentName } from '../../../helpers/utils'
-import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
 import { CONFIG } from '../../../initializers/config'
 import { MIMETYPES } from '../../../initializers/constants'
-import { sequelizeTypescript } from '../../../initializers/database'
-import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
 import { JobQueue } from '../../../lib/job-queue/job-queue'
-import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
-import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
+import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
 import {
   asyncMiddleware,
   asyncRetryTransactionMiddleware,
@@ -52,9 +23,6 @@ import {
   videoImportCancelValidator,
   videoImportDeleteValidator
 } from '../../../middlewares'
-import { VideoModel } from '../../../models/video/video'
-import { VideoCaptionModel } from '../../../models/video/video-caption'
-import { VideoImportModel } from '../../../models/video/video-import'
 
 const auditLogger = auditLoggerFactory('video-imports')
 const videoImportsRouter = express.Router()
@@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
   authenticate,
   reqVideoFileImport,
   asyncMiddleware(videoImportAddValidator),
-  asyncRetryTransactionMiddleware(addVideoImport)
+  asyncRetryTransactionMiddleware(handleVideoImport)
 )
 
 videoImportsRouter.post('/imports/:id/cancel',
@@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
-function addVideoImport (req: express.Request, res: express.Response) {
-  if (req.body.targetUrl) return addYoutubeDLImport(req, res)
+function handleVideoImport (req: express.Request, res: express.Response) {
+  if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
 
   const file = req.files?.['torrentfile']?.[0]
-  if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
+  if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
 }
 
-async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
+async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
   const body: VideoImportCreate = req.body
   const user = res.locals.oauth.token.User
 
@@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
     videoName = result.name
   }
 
-  const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
+  const video = await buildVideoFromImport({
+    channelId: res.locals.videoChannel.id,
+    importData: { name: videoName },
+    importDataOverride: body,
+    importType: 'torrent'
+  })
 
   const thumbnailModel = await processThumbnail(req, video)
   const previewModel = await processPreview(req, video)
 
-  const videoImport = await insertIntoDB({
+  const videoImport = await insertFromImportIntoDB({
     video,
     thumbnailModel,
     previewModel,
@@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
     }
   })
 
-  // Create job to import the video
-  const payload = {
+  const payload: VideoImportPayload = {
     type: torrentfile
-      ? 'torrent-file' as 'torrent-file'
-      : 'magnet-uri' as 'magnet-uri',
+      ? 'torrent-file'
+      : 'magnet-uri',
     videoImportId: videoImport.id,
-    magnetUri
+    preventException: false
   }
   await JobQueue.Instance.createJob({ type: 'video-import', payload })
 
@@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
   return res.json(videoImport.toFormattedJSON()).end()
 }
 
-async function addYoutubeDLImport (req: express.Request, res: express.Response) {
+function statusFromYtDlImportError (err: YoutubeDlImportError): number {
+  switch (err.code) {
+    case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
+      return HttpStatusCode.FORBIDDEN_403
+
+    case YoutubeDlImportError.CODE.FETCH_ERROR:
+      return HttpStatusCode.BAD_REQUEST_400
+
+    default:
+      return HttpStatusCode.INTERNAL_SERVER_ERROR_500
+  }
+}
+
+async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
   const body: VideoImportCreate = req.body
   const targetUrl = body.targetUrl
   const user = res.locals.oauth.token.User
 
-  const youtubeDL = new YoutubeDLWrapper(
-    targetUrl,
-    ServerConfigManager.Instance.getEnabledResolutions('vod'),
-    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
-  )
-
-  // Get video infos
-  let youtubeDLInfo: YoutubeDLInfo
   try {
-    youtubeDLInfo = await youtubeDL.getInfoForDownload()
+    const { job, videoImport } = await buildYoutubeDLImport({
+      targetUrl,
+      channel: res.locals.videoChannel,
+      importDataOverride: body,
+      thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
+      previewFilePath: req.files?.['previewfile']?.[0].path,
+      user
+    })
+    await JobQueue.Instance.createJob(job)
+
+    auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
+
+    return res.json(videoImport.toFormattedJSON()).end()
   } catch (err) {
-    logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
+    logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
 
     return res.fail({
-      message: 'Cannot fetch remote information of this URL.',
+      message: err.message,
+      status: statusFromYtDlImportError(err),
       data: {
         targetUrl
       }
     })
   }
-
-  if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
-    return res.fail({
-      status: HttpStatusCode.FORBIDDEN_403,
-      message: 'Cannot use non unicast IP as targetUrl.'
-    })
-  }
-
-  const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
-
-  // Process video thumbnail from request.files
-  let thumbnailModel = await processThumbnail(req, video)
-
-  // Process video thumbnail from url if processing from request.files failed
-  if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
-    try {
-      thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
-    } catch (err) {
-      logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
-    }
-  }
-
-  // Process video preview from request.files
-  let previewModel = await processPreview(req, video)
-
-  // Process video preview from url if processing from request.files failed
-  if (!previewModel && youtubeDLInfo.thumbnailUrl) {
-    try {
-      previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
-    } catch (err) {
-      logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
-    }
-  }
-
-  const videoImport = await insertIntoDB({
-    video,
-    thumbnailModel,
-    previewModel,
-    videoChannel: res.locals.videoChannel,
-    tags: body.tags || youtubeDLInfo.tags,
-    user,
-    videoImportAttributes: {
-      targetUrl,
-      state: VideoImportState.PENDING,
-      userId: user.id
-    }
-  })
-
-  // Get video subtitles
-  await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
-
-  let fileExt = `.${youtubeDLInfo.ext}`
-  if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
-
-  // Create job to import the video
-  const payload = {
-    type: 'youtube-dl' as 'youtube-dl',
-    videoImportId: videoImport.id,
-    fileExt
-  }
-  await JobQueue.Instance.createJob({ type: 'video-import', payload })
-
-  auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
-
-  return res.json(videoImport.toFormattedJSON()).end()
-}
-
-async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
-  let videoData = {
-    name: body.name || importData.name || 'Unknown name',
-    remote: false,
-    category: body.category || importData.category,
-    licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
-    language: body.language || importData.language,
-    commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
-    downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
-    waitTranscoding: body.waitTranscoding || false,
-    state: VideoState.TO_IMPORT,
-    nsfw: body.nsfw || importData.nsfw || false,
-    description: body.description || importData.description,
-    support: body.support || null,
-    privacy: body.privacy || VideoPrivacy.PRIVATE,
-    duration: 0, // duration will be set by the import job
-    channelId,
-    originallyPublishedAt: body.originallyPublishedAt
-      ? new Date(body.originallyPublishedAt)
-      : importData.originallyPublishedAt
-  }
-
-  videoData = await Hooks.wrapObject(
-    videoData,
-    body.targetUrl
-      ? 'filter:api.video.import-url.video-attribute.result'
-      : 'filter:api.video.import-torrent.video-attribute.result'
-  )
-
-  const video = new VideoModel(videoData)
-  video.url = getLocalVideoActivityPubUrl(video)
-
-  return video
 }
 
 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
@@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
   return undefined
 }
 
-async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
-  try {
-    return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
-  } catch (err) {
-    logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
-    return undefined
-  }
-}
-
-async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
-  try {
-    return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
-  } catch (err) {
-    logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
-    return undefined
-  }
-}
-
-async function insertIntoDB (parameters: {
-  video: MVideoThumbnail
-  thumbnailModel: MThumbnail
-  previewModel: MThumbnail
-  videoChannel: MChannelAccountDefault
-  tags: string[]
-  videoImportAttributes: FilteredModelAttributes<VideoImportModel>
-  user: MUser
-}): Promise<MVideoImportFormattable> {
-  const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
-
-  const videoImport = await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
-
-    // Save video object in database
-    const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
-    videoCreated.VideoChannel = videoChannel
-
-    if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
-    if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
-
-    await autoBlacklistVideoIfNeeded({
-      video: videoCreated,
-      user,
-      notify: false,
-      isRemote: false,
-      isNew: true,
-      transaction: t
-    })
-
-    await setVideoTags({ video: videoCreated, tags, transaction: t })
-
-    // Create video import object in database
-    const videoImport = await VideoImportModel.create(
-      Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
-      sequelizeOptions
-    ) as MVideoImportFormattable
-    videoImport.Video = videoCreated
-
-    return videoImport
-  })
-
-  return videoImport
-}
-
 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
   const torrentName = torrentfile.originalname
 
@@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
 function extractNameFromArray (name: string | string[]) {
   return isArray(name) ? name[0] : name
 }
-
-async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
-  try {
-    const subtitles = await youtubeDL.getSubtitles()
-
-    logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
-
-    for (const subtitle of subtitles) {
-      if (!await isVTTFileValid(subtitle.path)) {
-        await remove(subtitle.path)
-        continue
-      }
-
-      const videoCaption = new VideoCaptionModel({
-        videoId,
-        language: subtitle.language,
-        filename: VideoCaptionModel.generateCaptionName(subtitle.language)
-      }) as MVideoCaption
-
-      // Move physical file
-      await moveAndProcessCaptionFile(subtitle, videoCaption)
-
-      await sequelizeTypescript.transaction(async t => {
-        await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
-      })
-    }
-  } catch (err) {
-    logger.warn('Cannot get video subtitles.', { err })
-  }
-}
-
-async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
-  const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
-  const uniqHosts = new Set(hosts)
-
-  for (const h of uniqHosts) {
-    if (await isResolvingToUnicastOnly(h) !== true) {
-      return false
-    }
-  }
-
-  return true
-}