Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

7 files changed, 192 insertions, 268 deletions

diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts
index 66cdaab82..7a530cde5 100644
--- a/server/controllers/api/accounts.ts
+++ b/server/controllers/api/accounts.ts
@@ -25,8 +25,10 @@ import {
   accountsFollowersSortValidator,
   accountsSortValidator,
   ensureAuthUserOwnsAccountValidator,
+  ensureCanManageUser,
   videoChannelsSortValidator,
   videoChannelStatsValidator,
+  videoChannelSyncsSortValidator,
   videosSortValidator
 } from '../../middlewares/validators'
 import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
@@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate'
 import { VideoModel } from '../../models/video/video'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { VideoPlaylistModel } from '../../models/video/video-playlist'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 
 const accountsRouter = express.Router()
 
@@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels',
   asyncMiddleware(listAccountChannels)
 )
 
+accountsRouter.get('/:accountName/video-channel-syncs',
+  authenticate,
+  asyncMiddleware(accountNameWithHostGetValidator),
+  ensureCanManageUser,
+  paginationValidator,
+  videoChannelSyncsSortValidator,
+  setDefaultSort,
+  setDefaultPagination,
+  asyncMiddleware(listAccountChannelsSync)
+)
+
 accountsRouter.get('/:accountName/video-playlists',
   optionalAuthenticate,
   asyncMiddleware(accountNameWithHostGetValidator),
@@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response)
   return res.json(getFormattedObjects(resultList.data, resultList.total))
 }
 
+async function listAccountChannelsSync (req: express.Request, res: express.Response) {
+  const options = {
+    accountId: res.locals.account.id,
+    start: req.query.start,
+    count: req.query.count,
+    sort: req.query.sort,
+    search: req.query.search
+  }
+
+  const resultList = await VideoChannelSyncModel.listByAccountForAPI(options)
+
+  return res.json(getFormattedObjects(resultList.data, resultList.total))
+}
+
 async function listAccountPlaylists (req: express.Request, res: express.Response) {
   const serverActor = await getServerActor()
 
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index ff2fa9d86..f0fb43071 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -273,6 +273,10 @@ function customConfig (): CustomConfig {
         torrent: {
           enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
         }
+      },
+      videoChannelSynchronization: {
+        enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
+        maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
       }
     },
     trending: {
diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts
index d1d4ef765..8c8ebd061 100644
--- a/server/controllers/api/index.ts
+++ b/server/controllers/api/index.ts
@@ -20,6 +20,7 @@ import { usersRouter } from './users'
 import { videoChannelRouter } from './video-channel'
 import { videoPlaylistRouter } from './video-playlist'
 import { videosRouter } from './videos'
+import { videoChannelSyncRouter } from './video-channel-sync'
 
 const apiRouter = express.Router()
 
@@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter)
 apiRouter.use('/users', usersRouter)
 apiRouter.use('/accounts', accountsRouter)
 apiRouter.use('/video-channels', videoChannelRouter)
+apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
 apiRouter.use('/video-playlists', videoPlaylistRouter)
 apiRouter.use('/videos', videosRouter)
 apiRouter.use('/jobs', jobsRouter)
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index e09510dc3..4e5333782 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models'
 import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
 import { UserRight } from '../../../../shared/models/users'
 import { authenticate, ensureUserHasRight } from '../../../middlewares'
+import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
 
 const debugRouter = express.Router()
 
@@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) {
   const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
     'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
     'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
-    'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats()
+    'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
+    'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
   }
 
   await processors[body.command]()
diff --git a/server/controllers/api/video-channel-sync.ts b/server/controllers/api/video-channel-sync.ts
new file mode 100644
index 000000000..c2770b8e4
--- /dev/null
+++ b/server/controllers/api/video-channel-sync.ts
@@ -0,0 +1,76 @@
+import express from 'express'
+import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger'
+import { logger } from '@server/helpers/logger'
+import {
+  asyncMiddleware,
+  asyncRetryTransactionMiddleware,
+  authenticate,
+  ensureCanManageChannel as ensureCanManageSyncedChannel,
+  ensureSyncExists,
+  ensureSyncIsEnabled,
+  videoChannelSyncValidator
+} from '@server/middlewares'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { MChannelSyncFormattable } from '@server/types/models'
+import { HttpStatusCode, VideoChannelSyncState } from '@shared/models'
+
+const videoChannelSyncRouter = express.Router()
+const auditLogger = auditLoggerFactory('channel-syncs')
+
+videoChannelSyncRouter.post('/',
+  authenticate,
+  ensureSyncIsEnabled,
+  asyncMiddleware(videoChannelSyncValidator),
+  ensureCanManageSyncedChannel,
+  asyncRetryTransactionMiddleware(createVideoChannelSync)
+)
+
+videoChannelSyncRouter.delete('/:id',
+  authenticate,
+  asyncMiddleware(ensureSyncExists),
+  ensureCanManageSyncedChannel,
+  asyncRetryTransactionMiddleware(removeVideoChannelSync)
+)
+
+export { videoChannelSyncRouter }
+
+// ---------------------------------------------------------------------------
+
+async function createVideoChannelSync (req: express.Request, res: express.Response) {
+  const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({
+    externalChannelUrl: req.body.externalChannelUrl,
+    videoChannelId: req.body.videoChannelId,
+    state: VideoChannelSyncState.WAITING_FIRST_RUN
+  })
+
+  await syncCreated.save()
+  syncCreated.VideoChannel = res.locals.videoChannel
+
+  auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON()))
+
+  logger.info(
+    'Video synchronization for channel "%s" with external channel "%s" created.',
+    syncCreated.VideoChannel.name,
+    syncCreated.externalChannelUrl
+  )
+
+  return res.json({
+    videoChannelSync: syncCreated.toFormattedJSON()
+  })
+}
+
+async function removeVideoChannelSync (req: express.Request, res: express.Response) {
+  const syncInstance = res.locals.videoChannelSync
+
+  await syncInstance.destroy()
+
+  auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON()))
+
+  logger.info(
+    'Video synchronization for channel "%s" with external channel "%s" deleted.',
+    syncInstance.VideoChannel.name,
+    syncInstance.externalChannelUrl
+  )
+
+  return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
+}
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts
index 6b33e894d..89c7181bd 100644
--- a/server/controllers/api/video-channel.ts
+++ b/server/controllers/api/video-channel.ts
@@ -36,7 +36,9 @@ import {
   videoPlaylistsSortValidator
 } from '../../middlewares'
 import {
+  ensureChannelOwnerCanUpload,
   ensureIsLocalChannel,
+  videoChannelImportVideosValidator,
   videoChannelsFollowersSortValidator,
   videoChannelsListValidator,
   videoChannelsNameWithHostValidator,
@@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers',
   asyncMiddleware(listVideoChannelFollowers)
 )
 
+videoChannelRouter.post('/:nameWithHost/import-videos',
+  authenticate,
+  asyncMiddleware(videoChannelsNameWithHostValidator),
+  videoChannelImportVideosValidator,
+  ensureIsLocalChannel,
+  ensureCanManageChannel,
+  asyncMiddleware(ensureChannelOwnerCanUpload),
+  asyncMiddleware(importVideosInChannel)
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res
 
   return res.json(getFormattedObjects(resultList.data, resultList.total))
 }
+
+async function importVideosInChannel (req: express.Request, res: express.Response) {
+  const { externalChannelUrl } = req.body
+
+  await JobQueue.Instance.createJob({
+    type: 'video-channel-import',
+    payload: {
+      externalChannelUrl,
+      videoChannelId: res.locals.videoChannel.id
+    }
+  })
+
+  logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl)
+
+  return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
+}
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 5a2e1006a..9d7b0260b 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,49 +1,20 @@
 import express from 'express'
-import { move, readFile, remove } from 'fs-extra'
+import { move, readFile } from 'fs-extra'
 import { decode } from 'magnet-uri'
 import parseTorrent, { Instance } from 'parse-torrent'
 import { join } from 'path'
-import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
-import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
-import { isResolvingToUnicastOnly } from '@server/helpers/dns'
-import { Hooks } from '@server/lib/plugins/hooks'
-import { ServerConfigManager } from '@server/lib/server-config-manager'
-import { setVideoTags } from '@server/lib/video'
-import { FilteredModelAttributes } from '@server/types'
-import {
-  MChannelAccountDefault,
-  MThumbnail,
-  MUser,
-  MVideoAccountDefault,
-  MVideoCaption,
-  MVideoTag,
-  MVideoThumbnail,
-  MVideoWithBlacklistLight
-} from '@server/types/models'
-import { MVideoImportFormattable } from '@server/types/models/video/video-import'
-import {
-  HttpStatusCode,
-  ServerErrorCode,
-  ThumbnailType,
-  VideoImportCreate,
-  VideoImportState,
-  VideoPrivacy,
-  VideoState
-} from '@shared/models'
+import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
+import { MThumbnail, MVideoThumbnail } from '@server/types/models'
+import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
-import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
 import { isArray } from '../../../helpers/custom-validators/misc'
 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
 import { logger } from '../../../helpers/logger'
 import { getSecureTorrentName } from '../../../helpers/utils'
-import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
 import { CONFIG } from '../../../initializers/config'
 import { MIMETYPES } from '../../../initializers/constants'
-import { sequelizeTypescript } from '../../../initializers/database'
-import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
 import { JobQueue } from '../../../lib/job-queue/job-queue'
-import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
-import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
+import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
 import {
   asyncMiddleware,
   asyncRetryTransactionMiddleware,
@@ -52,9 +23,6 @@ import {
   videoImportCancelValidator,
   videoImportDeleteValidator
 } from '../../../middlewares'
-import { VideoModel } from '../../../models/video/video'
-import { VideoCaptionModel } from '../../../models/video/video-caption'
-import { VideoImportModel } from '../../../models/video/video-import'
 
 const auditLogger = auditLoggerFactory('video-imports')
 const videoImportsRouter = express.Router()
@@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
   authenticate,
   reqVideoFileImport,
   asyncMiddleware(videoImportAddValidator),
-  asyncRetryTransactionMiddleware(addVideoImport)
+  asyncRetryTransactionMiddleware(handleVideoImport)
 )
 
 videoImportsRouter.post('/imports/:id/cancel',
@@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
-function addVideoImport (req: express.Request, res: express.Response) {
-  if (req.body.targetUrl) return addYoutubeDLImport(req, res)
+function handleVideoImport (req: express.Request, res: express.Response) {
+  if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
 
   const file = req.files?.['torrentfile']?.[0]
-  if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
+  if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
 }
 
-async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
+async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
   const body: VideoImportCreate = req.body
   const user = res.locals.oauth.token.User
 
@@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
     videoName = result.name
   }
 
-  const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
+  const video = await buildVideoFromImport({
+    channelId: res.locals.videoChannel.id,
+    importData: { name: videoName },
+    importDataOverride: body,
+    importType: 'torrent'
+  })
 
   const thumbnailModel = await processThumbnail(req, video)
   const previewModel = await processPreview(req, video)
 
-  const videoImport = await insertIntoDB({
+  const videoImport = await insertFromImportIntoDB({
     video,
     thumbnailModel,
     previewModel,
@@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
     }
   })
 
-  // Create job to import the video
-  const payload = {
+  const payload: VideoImportPayload = {
     type: torrentfile
-      ? 'torrent-file' as 'torrent-file'
-      : 'magnet-uri' as 'magnet-uri',
+      ? 'torrent-file'
+      : 'magnet-uri',
     videoImportId: videoImport.id,
-    magnetUri
+    preventException: false
   }
   await JobQueue.Instance.createJob({ type: 'video-import', payload })
 
@@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
   return res.json(videoImport.toFormattedJSON()).end()
 }
 
-async function addYoutubeDLImport (req: express.Request, res: express.Response) {
+function statusFromYtDlImportError (err: YoutubeDlImportError): number {
+  switch (err.code) {
+    case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
+      return HttpStatusCode.FORBIDDEN_403
+
+    case YoutubeDlImportError.CODE.FETCH_ERROR:
+      return HttpStatusCode.BAD_REQUEST_400
+
+    default:
+      return HttpStatusCode.INTERNAL_SERVER_ERROR_500
+  }
+}
+
+async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
   const body: VideoImportCreate = req.body
   const targetUrl = body.targetUrl
   const user = res.locals.oauth.token.User
 
-  const youtubeDL = new YoutubeDLWrapper(
-    targetUrl,
-    ServerConfigManager.Instance.getEnabledResolutions('vod'),
-    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
-  )
-
-  // Get video infos
-  let youtubeDLInfo: YoutubeDLInfo
   try {
-    youtubeDLInfo = await youtubeDL.getInfoForDownload()
+    const { job, videoImport } = await buildYoutubeDLImport({
+      targetUrl,
+      channel: res.locals.videoChannel,
+      importDataOverride: body,
+      thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
+      previewFilePath: req.files?.['previewfile']?.[0].path,
+      user
+    })
+    await JobQueue.Instance.createJob(job)
+
+    auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
+
+    return res.json(videoImport.toFormattedJSON()).end()
   } catch (err) {
-    logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
+    logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
 
     return res.fail({
-      message: 'Cannot fetch remote information of this URL.',
+      message: err.message,
+      status: statusFromYtDlImportError(err),
       data: {
         targetUrl
       }
     })
   }
-
-  if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
-    return res.fail({
-      status: HttpStatusCode.FORBIDDEN_403,
-      message: 'Cannot use non unicast IP as targetUrl.'
-    })
-  }
-
-  const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
-
-  // Process video thumbnail from request.files
-  let thumbnailModel = await processThumbnail(req, video)
-
-  // Process video thumbnail from url if processing from request.files failed
-  if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
-    try {
-      thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
-    } catch (err) {
-      logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
-    }
-  }
-
-  // Process video preview from request.files
-  let previewModel = await processPreview(req, video)
-
-  // Process video preview from url if processing from request.files failed
-  if (!previewModel && youtubeDLInfo.thumbnailUrl) {
-    try {
-      previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
-    } catch (err) {
-      logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
-    }
-  }
-
-  const videoImport = await insertIntoDB({
-    video,
-    thumbnailModel,
-    previewModel,
-    videoChannel: res.locals.videoChannel,
-    tags: body.tags || youtubeDLInfo.tags,
-    user,
-    videoImportAttributes: {
-      targetUrl,
-      state: VideoImportState.PENDING,
-      userId: user.id
-    }
-  })
-
-  // Get video subtitles
-  await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
-
-  let fileExt = `.${youtubeDLInfo.ext}`
-  if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
-
-  // Create job to import the video
-  const payload = {
-    type: 'youtube-dl' as 'youtube-dl',
-    videoImportId: videoImport.id,
-    fileExt
-  }
-  await JobQueue.Instance.createJob({ type: 'video-import', payload })
-
-  auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
-
-  return res.json(videoImport.toFormattedJSON()).end()
-}
-
-async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
-  let videoData = {
-    name: body.name || importData.name || 'Unknown name',
-    remote: false,
-    category: body.category || importData.category,
-    licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
-    language: body.language || importData.language,
-    commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
-    downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
-    waitTranscoding: body.waitTranscoding || false,
-    state: VideoState.TO_IMPORT,
-    nsfw: body.nsfw || importData.nsfw || false,
-    description: body.description || importData.description,
-    support: body.support || null,
-    privacy: body.privacy || VideoPrivacy.PRIVATE,
-    duration: 0, // duration will be set by the import job
-    channelId,
-    originallyPublishedAt: body.originallyPublishedAt
-      ? new Date(body.originallyPublishedAt)
-      : importData.originallyPublishedAt
-  }
-
-  videoData = await Hooks.wrapObject(
-    videoData,
-    body.targetUrl
-      ? 'filter:api.video.import-url.video-attribute.result'
-      : 'filter:api.video.import-torrent.video-attribute.result'
-  )
-
-  const video = new VideoModel(videoData)
-  video.url = getLocalVideoActivityPubUrl(video)
-
-  return video
 }
 
 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
@@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
   return undefined
 }
 
-async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
-  try {
-    return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
-  } catch (err) {
-    logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
-    return undefined
-  }
-}
-
-async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
-  try {
-    return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
-  } catch (err) {
-    logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
-    return undefined
-  }
-}
-
-async function insertIntoDB (parameters: {
-  video: MVideoThumbnail
-  thumbnailModel: MThumbnail
-  previewModel: MThumbnail
-  videoChannel: MChannelAccountDefault
-  tags: string[]
-  videoImportAttributes: FilteredModelAttributes<VideoImportModel>
-  user: MUser
-}): Promise<MVideoImportFormattable> {
-  const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
-
-  const videoImport = await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
-
-    // Save video object in database
-    const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
-    videoCreated.VideoChannel = videoChannel
-
-    if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
-    if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
-
-    await autoBlacklistVideoIfNeeded({
-      video: videoCreated,
-      user,
-      notify: false,
-      isRemote: false,
-      isNew: true,
-      transaction: t
-    })
-
-    await setVideoTags({ video: videoCreated, tags, transaction: t })
-
-    // Create video import object in database
-    const videoImport = await VideoImportModel.create(
-      Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
-      sequelizeOptions
-    ) as MVideoImportFormattable
-    videoImport.Video = videoCreated
-
-    return videoImport
-  })
-
-  return videoImport
-}
-
 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
   const torrentName = torrentfile.originalname
 
@@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
 function extractNameFromArray (name: string | string[]) {
   return isArray(name) ? name[0] : name
 }
-
-async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
-  try {
-    const subtitles = await youtubeDL.getSubtitles()
-
-    logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
-
-    for (const subtitle of subtitles) {
-      if (!await isVTTFileValid(subtitle.path)) {
-        await remove(subtitle.path)
-        continue
-      }
-
-      const videoCaption = new VideoCaptionModel({
-        videoId,
-        language: subtitle.language,
-        filename: VideoCaptionModel.generateCaptionName(subtitle.language)
-      }) as MVideoCaption
-
-      // Move physical file
-      await moveAndProcessCaptionFile(subtitle, videoCaption)
-
-      await sequelizeTypescript.transaction(async t => {
-        await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
-      })
-    }
-  } catch (err) {
-    logger.warn('Cannot get video subtitles.', { err })
-  }
-}
-
-async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
-  const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
-  const uniqHosts = new Set(hosts)
-
-  for (const h of uniqHosts) {
-    if (await isResolvingToUnicastOnly(h) !== true) {
-      return false
-    }
-  }
-
-  return true
-}