server/lib/sync-channel.ts

   1 import { logger } from '@server/helpers/logger'
   2 import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
   3 import { CONFIG } from '@server/initializers/config'
   4 import { buildYoutubeDLImport } from '@server/lib/video-pre-import'
   5 import { UserModel } from '@server/models/user/user'
   6 import { VideoImportModel } from '@server/models/video/video-import'
   7 import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
   8 import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
   9 import { CreateJobArgument, JobQueue } from './job-queue'
  10 import { ServerConfigManager } from './server-config-manager'
  11
  12 export async function synchronizeChannel (options: {
  13   channel: MChannelAccountDefault
  14   externalChannelUrl: string
  15   videosCountLimit: number
  16   channelSync?: MChannelSync
  17   onlyAfter?: Date
  18 }) {
  19   const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
  20
  21   if (channelSync) {
  22     channelSync.state = VideoChannelSyncState.PROCESSING
  23     channelSync.lastSyncAt = new Date()
  24     await channelSync.save()
  25   }
  26
  27   try {
  28     const user = await UserModel.loadByChannelActorId(channel.actorId)
  29     const youtubeDL = new YoutubeDLWrapper(
  30       externalChannelUrl,
  31       ServerConfigManager.Instance.getEnabledResolutions('vod'),
  32       CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
  33     )
  34
  35     const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
  36
  37     logger.info(
  38       'Fetched %d candidate URLs for sync channel %s.',
  39       targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
  40     )
  41
  42     if (targetUrls.length === 0) {
  43       if (channelSync) {
  44         channelSync.state = VideoChannelSyncState.SYNCED
  45         await channelSync.save()
  46       }
  47
  48       return
  49     }
  50
  51     const children: CreateJobArgument[] = []
  52
  53     for (const targetUrl of targetUrls) {
  54       if (await skipImport(channel, targetUrl, onlyAfter)) continue
  55
  56       const { job } = await buildYoutubeDLImport({
  57         user,
  58         channel,
  59         targetUrl,
  60         channelSync,
  61         importDataOverride: {
  62           privacy: VideoPrivacy.PUBLIC
  63         }
  64       })
  65
  66       children.push(job)
  67     }
  68
  69     // Will update the channel sync status
  70     const parent: CreateJobArgument = {
  71       type: 'after-video-channel-import',
  72       payload: {
  73         channelSyncId: channelSync?.id
  74       }
  75     }
  76
  77     await JobQueue.Instance.createJobWithChildren(parent, children)
  78   } catch (err) {
  79     logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err })
  80     channelSync.state = VideoChannelSyncState.FAILED
  81     await channelSync.save()
  82   }
  83 }
  84
  85 // ---------------------------------------------------------------------------
  86
  87 async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
  88   if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
  89     logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name)
  90     return true
  91   }
  92
  93   if (onlyAfter) {
  94     const youtubeDL = new YoutubeDLWrapper(
  95       targetUrl,
  96       ServerConfigManager.Instance.getEnabledResolutions('vod'),
  97       CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
  98     )
  99
 100     const videoInfo = await youtubeDL.getInfoForDownload()
 101
 102     const onlyAfterWithoutTime = new Date(onlyAfter)
 103     onlyAfterWithoutTime.setHours(0, 0, 0, 0)
 104
 105     if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) {
 106       return true
 107     }
 108   }
 109
 110   return false
 111 }