Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

1 files changed, 308 insertions, 0 deletions

diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts
new file mode 100644
index 000000000..fb9306967
--- /dev/null
+++ b/server/lib/video-import.ts
@@ -0,0 +1,308 @@
+import { remove } from 'fs-extra'
+import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
+import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
+import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
+import { isResolvingToUnicastOnly } from '@server/helpers/dns'
+import { logger } from '@server/helpers/logger'
+import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { ServerConfigManager } from '@server/lib/server-config-manager'
+import { setVideoTags } from '@server/lib/video'
+import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
+import { VideoModel } from '@server/models/video/video'
+import { VideoCaptionModel } from '@server/models/video/video-caption'
+import { VideoImportModel } from '@server/models/video/video-import'
+import { FilteredModelAttributes } from '@server/types'
+import {
+  MChannelAccountDefault,
+  MChannelSync,
+  MThumbnail,
+  MUser,
+  MVideoAccountDefault,
+  MVideoCaption,
+  MVideoImportFormattable,
+  MVideoTag,
+  MVideoThumbnail,
+  MVideoWithBlacklistLight
+} from '@server/types/models'
+import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
+import { getLocalVideoActivityPubUrl } from './activitypub/url'
+import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
+
+class YoutubeDlImportError extends Error {
+  code: YoutubeDlImportError.CODE
+  cause?: Error // Property to remove once ES2022 is used
+  constructor ({ message, code }) {
+    super(message)
+    this.code = code
+  }
+
+  static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
+    const ytDlErr = new this({ message: message ?? err.message, code })
+    ytDlErr.cause = err
+    ytDlErr.stack = err.stack // Useless once ES2022 is used
+    return ytDlErr
+  }
+}
+
+namespace YoutubeDlImportError {
+  export enum CODE {
+    FETCH_ERROR,
+    NOT_ONLY_UNICAST_URL
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+async function insertFromImportIntoDB (parameters: {
+  video: MVideoThumbnail
+  thumbnailModel: MThumbnail
+  previewModel: MThumbnail
+  videoChannel: MChannelAccountDefault
+  tags: string[]
+  videoImportAttributes: FilteredModelAttributes<VideoImportModel>
+  user: MUser
+}): Promise<MVideoImportFormattable> {
+  const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
+
+  const videoImport = await sequelizeTypescript.transaction(async t => {
+    const sequelizeOptions = { transaction: t }
+
+    // Save video object in database
+    const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
+    videoCreated.VideoChannel = videoChannel
+
+    if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
+    if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
+
+    await autoBlacklistVideoIfNeeded({
+      video: videoCreated,
+      user,
+      notify: false,
+      isRemote: false,
+      isNew: true,
+      transaction: t
+    })
+
+    await setVideoTags({ video: videoCreated, tags, transaction: t })
+
+    // Create video import object in database
+    const videoImport = await VideoImportModel.create(
+      Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
+      sequelizeOptions
+    ) as MVideoImportFormattable
+    videoImport.Video = videoCreated
+
+    return videoImport
+  })
+
+  return videoImport
+}
+
+async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
+  channelId: number
+  importData: YoutubeDLInfo
+  importDataOverride?: Partial<VideoImportCreate>
+  importType: 'url' | 'torrent'
+}): Promise<MVideoThumbnail> {
+  let videoData = {
+    name: importDataOverride?.name || importData.name || 'Unknown name',
+    remote: false,
+    category: importDataOverride?.category || importData.category,
+    licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
+    language: importDataOverride?.language || importData.language,
+    commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
+    downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
+    waitTranscoding: importDataOverride?.waitTranscoding || false,
+    state: VideoState.TO_IMPORT,
+    nsfw: importDataOverride?.nsfw || importData.nsfw || false,
+    description: importDataOverride?.description || importData.description,
+    support: importDataOverride?.support || null,
+    privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
+    duration: 0, // duration will be set by the import job
+    channelId,
+    originallyPublishedAt: importDataOverride?.originallyPublishedAt
+      ? new Date(importDataOverride?.originallyPublishedAt)
+      : importData.originallyPublishedAt
+  }
+
+  videoData = await Hooks.wrapObject(
+    videoData,
+    importType === 'url'
+      ? 'filter:api.video.import-url.video-attribute.result'
+      : 'filter:api.video.import-torrent.video-attribute.result'
+  )
+
+  const video = new VideoModel(videoData)
+  video.url = getLocalVideoActivityPubUrl(video)
+
+  return video
+}
+
+async function buildYoutubeDLImport (options: {
+  targetUrl: string
+  channel: MChannelAccountDefault
+  user: MUser
+  channelSync?: MChannelSync
+  importDataOverride?: Partial<VideoImportCreate>
+  thumbnailFilePath?: string
+  previewFilePath?: string
+}) {
+  const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
+
+  const youtubeDL = new YoutubeDLWrapper(
+    targetUrl,
+    ServerConfigManager.Instance.getEnabledResolutions('vod'),
+    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+  )
+
+  // Get video infos
+  let youtubeDLInfo: YoutubeDLInfo
+  try {
+    youtubeDLInfo = await youtubeDL.getInfoForDownload()
+  } catch (err) {
+    throw YoutubeDlImportError.fromError(
+      err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
+    )
+  }
+
+  if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
+    throw new YoutubeDlImportError({
+      message: 'Cannot use non unicast IP as targetUrl.',
+      code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
+    })
+  }
+
+  const video = await buildVideoFromImport({
+    channelId: channel.id,
+    importData: youtubeDLInfo,
+    importDataOverride,
+    importType: 'url'
+  })
+
+  const thumbnailModel = await forgeThumbnail({
+    inputPath: thumbnailFilePath,
+    downloadUrl: youtubeDLInfo.thumbnailUrl,
+    video,
+    type: ThumbnailType.MINIATURE
+  })
+
+  const previewModel = await forgeThumbnail({
+    inputPath: previewFilePath,
+    downloadUrl: youtubeDLInfo.thumbnailUrl,
+    video,
+    type: ThumbnailType.PREVIEW
+  })
+
+  const videoImport = await insertFromImportIntoDB({
+    video,
+    thumbnailModel,
+    previewModel,
+    videoChannel: channel,
+    tags: importDataOverride?.tags || youtubeDLInfo.tags,
+    user,
+    videoImportAttributes: {
+      targetUrl,
+      state: VideoImportState.PENDING,
+      userId: user.id
+    }
+  })
+
+  // Get video subtitles
+  await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
+
+  let fileExt = `.${youtubeDLInfo.ext}`
+  if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
+
+  const payload: VideoImportPayload = {
+    type: 'youtube-dl' as 'youtube-dl',
+    videoImportId: videoImport.id,
+    fileExt,
+    // If part of a sync process, there is a parent job that will aggregate children results
+    preventException: !!channelSync
+  }
+
+  return {
+    videoImport,
+    job: { type: 'video-import' as 'video-import', payload }
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  buildYoutubeDLImport,
+  YoutubeDlImportError,
+  insertFromImportIntoDB,
+  buildVideoFromImport
+}
+
+// ---------------------------------------------------------------------------
+
+async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
+  inputPath?: string
+  downloadUrl?: string
+  video: MVideoThumbnail
+  type: ThumbnailType
+}): Promise<MThumbnail> {
+  if (inputPath) {
+    return updateVideoMiniatureFromExisting({
+      inputPath,
+      video,
+      type,
+      automaticallyGenerated: false
+    })
+  } else if (downloadUrl) {
+    try {
+      return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
+    } catch (err) {
+      logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
+    }
+  }
+  return null
+}
+
+async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
+  try {
+    const subtitles = await youtubeDL.getSubtitles()
+
+    logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
+
+    for (const subtitle of subtitles) {
+      if (!await isVTTFileValid(subtitle.path)) {
+        await remove(subtitle.path)
+        continue
+      }
+
+      const videoCaption = new VideoCaptionModel({
+        videoId,
+        language: subtitle.language,
+        filename: VideoCaptionModel.generateCaptionName(subtitle.language)
+      }) as MVideoCaption
+
+      // Move physical file
+      await moveAndProcessCaptionFile(subtitle, videoCaption)
+
+      await sequelizeTypescript.transaction(async t => {
+        await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
+      })
+    }
+  } catch (err) {
+    logger.warn('Cannot get video subtitles.', { err })
+  }
+}
+
+async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
+  const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
+  const uniqHosts = new Set(hosts)
+
+  for (const h of uniqHosts) {
+    if (await isResolvingToUnicastOnly(h) !== true) {
+      return false
+    }
+  }
+
+  return true
+}