Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

8 files changed, 556 insertions, 9 deletions

diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts
new file mode 100644
index 000000000..ffdd8c5b5
--- /dev/null
+++ b/server/lib/job-queue/handlers/after-video-channel-import.ts
@@ -0,0 +1,37 @@
+import { Job } from 'bullmq'
+import { logger } from '@server/helpers/logger'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
+
+export async function processAfterVideoChannelImport (job: Job) {
+  const payload = job.data as AfterVideoChannelImportPayload
+  if (!payload.channelSyncId) return
+
+  logger.info('Processing after video channel import in job %s.', job.id)
+
+  const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
+  if (!sync) {
+    logger.error('Unknown sync id %d.', payload.channelSyncId)
+    return
+  }
+
+  const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
+
+  let errors = 0
+  let successes = 0
+
+  for (const value of Object.values(childrenValues)) {
+    if (value.resultType === 'success') successes++
+    else if (value.resultType === 'error') errors++
+  }
+
+  if (errors > 0) {
+    sync.state = VideoChannelSyncState.FAILED
+    logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
+  } else {
+    sync.state = VideoChannelSyncState.SYNCED
+    logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
+  }
+
+  await sync.save()
+}
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
new file mode 100644
index 000000000..9bdb2d269
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -0,0 +1,36 @@
+import { Job } from 'bullmq'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { synchronizeChannel } from '@server/lib/sync-channel'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelImportPayload } from '@shared/models'
+
+export async function processVideoChannelImport (job: Job) {
+  const payload = job.data as VideoChannelImportPayload
+
+  logger.info('Processing video channel import in job %s.', job.id)
+
+  // Channel import requires only http upload to be allowed
+  if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
+    logger.error('Cannot import channel as the HTTP upload is disabled')
+    return
+  }
+
+  if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+    logger.error('Cannot import channel as the synchronization is disabled')
+    return
+  }
+
+  const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
+
+  try {
+    logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
+
+    await synchronizeChannel({
+      channel: videoChannel,
+      externalChannelUrl: payload.externalChannelUrl
+    })
+  } catch (err) {
+    logger.error(`Failed to import channel ${videoChannel.name}`, { err })
+  }
+}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index f4629159c..9901b878c 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import { Hooks } from '@server/lib/plugins/hooks'
 import { ServerConfigManager } from '@server/lib/server-config-manager'
 import { isAbleToUploadVideo } from '@server/lib/user'
-import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video'
+import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
@@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils'
 import {
   ThumbnailType,
   VideoImportPayload,
+  VideoImportPreventExceptionResult,
   VideoImportState,
   VideoImportTorrentPayload,
   VideoImportTorrentPayloadType,
@@ -41,20 +42,29 @@ import { Notifier } from '../../notifier'
 import { generateVideoMiniature } from '../../thumbnail'
 import { JobQueue } from '../job-queue'
 
-async function processVideoImport (job: Job) {
+async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
   const payload = job.data as VideoImportPayload
 
   const videoImport = await getVideoImportOrDie(payload)
   if (videoImport.state === VideoImportState.CANCELLED) {
     logger.info('Do not process import since it has been cancelled', { payload })
-    return
+    return { resultType: 'success' }
   }
 
   videoImport.state = VideoImportState.PROCESSING
   await videoImport.save()
 
-  if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload)
-  if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload)
+  try {
+    if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
+    if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
+
+    return { resultType: 'success' }
+  } catch (err) {
+    if (!payload.preventException) throw err
+
+    logger.warn('Catch error in video import to send value to parent job.', { payload, err })
+    return { resultType: 'error' }
+  }
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 281e2e51a..3970d48b7 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -22,6 +22,7 @@ import {
   ActivitypubHttpFetcherPayload,
   ActivitypubHttpUnicastPayload,
   ActorKeysPayload,
+  AfterVideoChannelImportPayload,
   DeleteResumableUploadMetaFilePayload,
   EmailPayload,
   FederateVideoPayload,
@@ -31,6 +32,7 @@ import {
   MoveObjectStoragePayload,
   NotifyPayload,
   RefreshPayload,
+  VideoChannelImportPayload,
   VideoFileImportPayload,
   VideoImportPayload,
   VideoLiveEndingPayload,
@@ -53,12 +55,14 @@ import { processFederateVideo } from './handlers/federate-video'
 import { processManageVideoTorrent } from './handlers/manage-video-torrent'
 import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
 import { processNotify } from './handlers/notify'
+import { processVideoChannelImport } from './handlers/video-channel-import'
 import { processVideoFileImport } from './handlers/video-file-import'
 import { processVideoImport } from './handlers/video-import'
 import { processVideoLiveEnding } from './handlers/video-live-ending'
 import { processVideoStudioEdition } from './handlers/video-studio-edition'
 import { processVideoTranscoding } from './handlers/video-transcoding'
 import { processVideosViewsStats } from './handlers/video-views-stats'
+import { processAfterVideoChannelImport } from './handlers/after-video-channel-import'
 
 export type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -79,6 +83,9 @@ export type CreateJobArgument =
   { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
   { type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
   { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
+  { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
+  { type: 'video-channel-import', payload: VideoChannelImportPayload } |
+  { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
   { type: 'notify', payload: NotifyPayload } |
   { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
   { type: 'federate-video', payload: FederateVideoPayload }
@@ -106,8 +113,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
   'video-redundancy': processVideoRedundancy,
   'move-to-object-storage': processMoveToObjectStorage,
   'manage-video-torrent': processManageVideoTorrent,
-  'notify': processNotify,
   'video-studio-edition': processVideoStudioEdition,
+  'video-channel-import': processVideoChannelImport,
+  'after-video-channel-import': processAfterVideoChannelImport,
+  'notify': processNotify,
   'federate-video': processFederateVideo
 }
 
@@ -134,6 +143,8 @@ const jobTypes: JobType[] = [
   'move-to-object-storage',
   'manage-video-torrent',
   'video-studio-edition',
+  'video-channel-import',
+  'after-video-channel-import',
   'notify',
   'federate-video'
 ]
@@ -306,7 +317,7 @@ class JobQueue {
         .catch(err => logger.error('Cannot create job.', { err, options }))
   }
 
-  async createJob (options: CreateJobArgument & CreateJobOptions) {
+  createJob (options: CreateJobArgument & CreateJobOptions) {
     const queue: Queue = this.queues[options.type]
     if (queue === undefined) {
       logger.error('Unknown queue %s: cannot create job.', options.type)
@@ -318,7 +329,7 @@ class JobQueue {
     return queue.add('job', options.payload, jobOptions)
   }
 
-  async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
+  createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
     let lastJob: FlowJob
 
     for (const job of jobs) {
@@ -336,7 +347,7 @@ class JobQueue {
     return this.flowProducer.add(lastJob)
   }
 
-  async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
+  createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
     return this.flowProducer.add({
       ...this.buildJobFlowOption(parent),
 
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
new file mode 100644
index 000000000..fd9a35299
--- /dev/null
+++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -0,0 +1,61 @@
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { VideoChannelSyncState } from '@shared/models'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { synchronizeChannel } from '../sync-channel'
+import { AbstractScheduler } from './abstract-scheduler'
+
+export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
+
+  private constructor () {
+    super()
+  }
+
+  protected async internalExecute () {
+    logger.debug('Running %s.%s', this.constructor.name, this.internalExecute.name)
+
+    if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+      logger.info('Discard channels synchronization as the feature is disabled')
+      return
+    }
+
+    const channelSyncs = await VideoChannelSyncModel.listSyncs()
+
+    for (const sync of channelSyncs) {
+      const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
+
+      try {
+        logger.info(
+          'Creating video import jobs for "%s" sync with external channel "%s"',
+          channel.Actor.preferredUsername, sync.externalChannelUrl
+        )
+
+        const onlyAfter = sync.lastSyncAt || sync.createdAt
+
+        sync.state = VideoChannelSyncState.PROCESSING
+        sync.lastSyncAt = new Date()
+        await sync.save()
+
+        await synchronizeChannel({
+          channel,
+          externalChannelUrl: sync.externalChannelUrl,
+          videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
+          channelSync: sync,
+          onlyAfter
+        })
+      } catch (err) {
+        logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
+        sync.state = VideoChannelSyncState.FAILED
+        await sync.save()
+      }
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts
index a3312fa20..78a9546ae 100644
--- a/server/lib/server-config-manager.ts
+++ b/server/lib/server-config-manager.ts
@@ -170,6 +170,9 @@ class ServerConfigManager {
           torrent: {
             enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
           }
+        },
+        videoChannelSynchronization: {
+          enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED
         }
       },
       autoBlacklist: {
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
new file mode 100644
index 000000000..50f80e6f9
--- /dev/null
+++ b/server/lib/sync-channel.ts
@@ -0,0 +1,81 @@
+import { logger } from '@server/helpers/logger'
+import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
+import { buildYoutubeDLImport } from '@server/lib/video-import'
+import { UserModel } from '@server/models/user/user'
+import { VideoImportModel } from '@server/models/video/video-import'
+import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
+import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
+import { CreateJobArgument, JobQueue } from './job-queue'
+import { ServerConfigManager } from './server-config-manager'
+
+export async function synchronizeChannel (options: {
+  channel: MChannelAccountDefault
+  externalChannelUrl: string
+  channelSync?: MChannelSync
+  videosCountLimit?: number
+  onlyAfter?: Date
+}) {
+  const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
+
+  const user = await UserModel.loadByChannelActorId(channel.actorId)
+  const youtubeDL = new YoutubeDLWrapper(
+    externalChannelUrl,
+    ServerConfigManager.Instance.getEnabledResolutions('vod'),
+    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+  )
+
+  const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
+
+  const targetUrls = infoList
+    .filter(videoInfo => {
+      if (!onlyAfter) return true
+
+      return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
+    })
+    .map(videoInfo => videoInfo.webpageUrl)
+
+  logger.info(
+    'Fetched %d candidate URLs for sync channel %s.',
+    targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
+  )
+
+  if (targetUrls.length === 0) {
+    if (channelSync) {
+      channelSync.state = VideoChannelSyncState.SYNCED
+      await channelSync.save()
+    }
+
+    return
+  }
+
+  const children: CreateJobArgument[] = []
+
+  for (const targetUrl of targetUrls) {
+    if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
+      logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
+      continue
+    }
+
+    const { job } = await buildYoutubeDLImport({
+      user,
+      channel,
+      targetUrl,
+      channelSync,
+      importDataOverride: {
+        privacy: VideoPrivacy.PUBLIC
+      }
+    })
+
+    children.push(job)
+  }
+
+  const parent: CreateJobArgument = {
+    type: 'after-video-channel-import',
+    payload: {
+      channelSyncId: channelSync?.id
+    }
+  }
+
+  await JobQueue.Instance.createJobWithChildren(parent, children)
+}
diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts
new file mode 100644
index 000000000..fb9306967
--- /dev/null
+++ b/server/lib/video-import.ts
@@ -0,0 +1,308 @@
+import { remove } from 'fs-extra'
+import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
+import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
+import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
+import { isResolvingToUnicastOnly } from '@server/helpers/dns'
+import { logger } from '@server/helpers/logger'
+import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { ServerConfigManager } from '@server/lib/server-config-manager'
+import { setVideoTags } from '@server/lib/video'
+import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
+import { VideoModel } from '@server/models/video/video'
+import { VideoCaptionModel } from '@server/models/video/video-caption'
+import { VideoImportModel } from '@server/models/video/video-import'
+import { FilteredModelAttributes } from '@server/types'
+import {
+  MChannelAccountDefault,
+  MChannelSync,
+  MThumbnail,
+  MUser,
+  MVideoAccountDefault,
+  MVideoCaption,
+  MVideoImportFormattable,
+  MVideoTag,
+  MVideoThumbnail,
+  MVideoWithBlacklistLight
+} from '@server/types/models'
+import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
+import { getLocalVideoActivityPubUrl } from './activitypub/url'
+import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
+
+class YoutubeDlImportError extends Error {
+  code: YoutubeDlImportError.CODE
+  cause?: Error // Property to remove once ES2022 is used
+  constructor ({ message, code }) {
+    super(message)
+    this.code = code
+  }
+
+  static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
+    const ytDlErr = new this({ message: message ?? err.message, code })
+    ytDlErr.cause = err
+    ytDlErr.stack = err.stack // Useless once ES2022 is used
+    return ytDlErr
+  }
+}
+
+namespace YoutubeDlImportError {
+  export enum CODE {
+    FETCH_ERROR,
+    NOT_ONLY_UNICAST_URL
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+async function insertFromImportIntoDB (parameters: {
+  video: MVideoThumbnail
+  thumbnailModel: MThumbnail
+  previewModel: MThumbnail
+  videoChannel: MChannelAccountDefault
+  tags: string[]
+  videoImportAttributes: FilteredModelAttributes<VideoImportModel>
+  user: MUser
+}): Promise<MVideoImportFormattable> {
+  const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
+
+  const videoImport = await sequelizeTypescript.transaction(async t => {
+    const sequelizeOptions = { transaction: t }
+
+    // Save video object in database
+    const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
+    videoCreated.VideoChannel = videoChannel
+
+    if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
+    if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
+
+    await autoBlacklistVideoIfNeeded({
+      video: videoCreated,
+      user,
+      notify: false,
+      isRemote: false,
+      isNew: true,
+      transaction: t
+    })
+
+    await setVideoTags({ video: videoCreated, tags, transaction: t })
+
+    // Create video import object in database
+    const videoImport = await VideoImportModel.create(
+      Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
+      sequelizeOptions
+    ) as MVideoImportFormattable
+    videoImport.Video = videoCreated
+
+    return videoImport
+  })
+
+  return videoImport
+}
+
+async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
+  channelId: number
+  importData: YoutubeDLInfo
+  importDataOverride?: Partial<VideoImportCreate>
+  importType: 'url' | 'torrent'
+}): Promise<MVideoThumbnail> {
+  let videoData = {
+    name: importDataOverride?.name || importData.name || 'Unknown name',
+    remote: false,
+    category: importDataOverride?.category || importData.category,
+    licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
+    language: importDataOverride?.language || importData.language,
+    commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
+    downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
+    waitTranscoding: importDataOverride?.waitTranscoding || false,
+    state: VideoState.TO_IMPORT,
+    nsfw: importDataOverride?.nsfw || importData.nsfw || false,
+    description: importDataOverride?.description || importData.description,
+    support: importDataOverride?.support || null,
+    privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
+    duration: 0, // duration will be set by the import job
+    channelId,
+    originallyPublishedAt: importDataOverride?.originallyPublishedAt
+      ? new Date(importDataOverride?.originallyPublishedAt)
+      : importData.originallyPublishedAt
+  }
+
+  videoData = await Hooks.wrapObject(
+    videoData,
+    importType === 'url'
+      ? 'filter:api.video.import-url.video-attribute.result'
+      : 'filter:api.video.import-torrent.video-attribute.result'
+  )
+
+  const video = new VideoModel(videoData)
+  video.url = getLocalVideoActivityPubUrl(video)
+
+  return video
+}
+
+async function buildYoutubeDLImport (options: {
+  targetUrl: string
+  channel: MChannelAccountDefault
+  user: MUser
+  channelSync?: MChannelSync
+  importDataOverride?: Partial<VideoImportCreate>
+  thumbnailFilePath?: string
+  previewFilePath?: string
+}) {
+  const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
+
+  const youtubeDL = new YoutubeDLWrapper(
+    targetUrl,
+    ServerConfigManager.Instance.getEnabledResolutions('vod'),
+    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+  )
+
+  // Get video infos
+  let youtubeDLInfo: YoutubeDLInfo
+  try {
+    youtubeDLInfo = await youtubeDL.getInfoForDownload()
+  } catch (err) {
+    throw YoutubeDlImportError.fromError(
+      err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
+    )
+  }
+
+  if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
+    throw new YoutubeDlImportError({
+      message: 'Cannot use non unicast IP as targetUrl.',
+      code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
+    })
+  }
+
+  const video = await buildVideoFromImport({
+    channelId: channel.id,
+    importData: youtubeDLInfo,
+    importDataOverride,
+    importType: 'url'
+  })
+
+  const thumbnailModel = await forgeThumbnail({
+    inputPath: thumbnailFilePath,
+    downloadUrl: youtubeDLInfo.thumbnailUrl,
+    video,
+    type: ThumbnailType.MINIATURE
+  })
+
+  const previewModel = await forgeThumbnail({
+    inputPath: previewFilePath,
+    downloadUrl: youtubeDLInfo.thumbnailUrl,
+    video,
+    type: ThumbnailType.PREVIEW
+  })
+
+  const videoImport = await insertFromImportIntoDB({
+    video,
+    thumbnailModel,
+    previewModel,
+    videoChannel: channel,
+    tags: importDataOverride?.tags || youtubeDLInfo.tags,
+    user,
+    videoImportAttributes: {
+      targetUrl,
+      state: VideoImportState.PENDING,
+      userId: user.id
+    }
+  })
+
+  // Get video subtitles
+  await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
+
+  let fileExt = `.${youtubeDLInfo.ext}`
+  if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
+
+  const payload: VideoImportPayload = {
+    type: 'youtube-dl' as 'youtube-dl',
+    videoImportId: videoImport.id,
+    fileExt,
+    // If part of a sync process, there is a parent job that will aggregate children results
+    preventException: !!channelSync
+  }
+
+  return {
+    videoImport,
+    job: { type: 'video-import' as 'video-import', payload }
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  buildYoutubeDLImport,
+  YoutubeDlImportError,
+  insertFromImportIntoDB,
+  buildVideoFromImport
+}
+
+// ---------------------------------------------------------------------------
+
+async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
+  inputPath?: string
+  downloadUrl?: string
+  video: MVideoThumbnail
+  type: ThumbnailType
+}): Promise<MThumbnail> {
+  if (inputPath) {
+    return updateVideoMiniatureFromExisting({
+      inputPath,
+      video,
+      type,
+      automaticallyGenerated: false
+    })
+  } else if (downloadUrl) {
+    try {
+      return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
+    } catch (err) {
+      logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
+    }
+  }
+  return null
+}
+
+async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
+  try {
+    const subtitles = await youtubeDL.getSubtitles()
+
+    logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
+
+    for (const subtitle of subtitles) {
+      if (!await isVTTFileValid(subtitle.path)) {
+        await remove(subtitle.path)
+        continue
+      }
+
+      const videoCaption = new VideoCaptionModel({
+        videoId,
+        language: subtitle.language,
+        filename: VideoCaptionModel.generateCaptionName(subtitle.language)
+      }) as MVideoCaption
+
+      // Move physical file
+      await moveAndProcessCaptionFile(subtitle, videoCaption)
+
+      await sequelizeTypescript.transaction(async t => {
+        await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
+      })
+    }
+  } catch (err) {
+    logger.warn('Cannot get video subtitles.', { err })
+  }
+}
+
+async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
+  const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
+  const uniqHosts = new Set(hosts)
+
+  for (const h of uniqHosts) {
+    if (await isResolvingToUnicastOnly(h) !== true) {
+      return false
+    }
+  }
+
+  return true
+}