server/lib/sync-channel.ts

   1 import { logger } from '@server/helpers/logger'
   2 import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
   3 import { CONFIG } from '@server/initializers/config'
   4 import { buildYoutubeDLImport } from '@server/lib/video-import'
   5 import { UserModel } from '@server/models/user/user'
   6 import { VideoImportModel } from '@server/models/video/video-import'
   7 import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
   8 import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
   9 import { CreateJobArgument, JobQueue } from './job-queue'
  10 import { ServerConfigManager } from './server-config-manager'
  11
  12 export async function synchronizeChannel (options: {
  13   channel: MChannelAccountDefault
  14   externalChannelUrl: string
  15   channelSync?: MChannelSync
  16   videosCountLimit?: number
  17   onlyAfter?: Date
  18 }) {
  19   const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
  20
  21   if (channelSync) {
  22     channelSync.state = VideoChannelSyncState.PROCESSING
  23     channelSync.lastSyncAt = new Date()
  24     await channelSync.save()
  25   }
  26
  27   const user = await UserModel.loadByChannelActorId(channel.actorId)
  28   const youtubeDL = new YoutubeDLWrapper(
  29     externalChannelUrl,
  30     ServerConfigManager.Instance.getEnabledResolutions('vod'),
  31     CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
  32   )
  33
  34   const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
  35
  36   logger.info(
  37     'Fetched %d candidate URLs for sync channel %s.',
  38     targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
  39   )
  40
  41   if (targetUrls.length === 0) {
  42     if (channelSync) {
  43       channelSync.state = VideoChannelSyncState.SYNCED
  44       await channelSync.save()
  45     }
  46
  47     return
  48   }
  49
  50   const children: CreateJobArgument[] = []
  51
  52   for (const targetUrl of targetUrls) {
  53     if (await skipImport(channel, targetUrl, onlyAfter)) continue
  54
  55     const { job } = await buildYoutubeDLImport({
  56       user,
  57       channel,
  58       targetUrl,
  59       channelSync,
  60       importDataOverride: {
  61         privacy: VideoPrivacy.PUBLIC
  62       }
  63     })
  64
  65     children.push(job)
  66   }
  67
  68   // Will update the channel sync status
  69   const parent: CreateJobArgument = {
  70     type: 'after-video-channel-import',
  71     payload: {
  72       channelSyncId: channelSync?.id
  73     }
  74   }
  75
  76   await JobQueue.Instance.createJobWithChildren(parent, children)
  77 }
  78
  79 // ---------------------------------------------------------------------------
  80
  81 async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
  82   if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
  83     logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name)
  84     return true
  85   }
  86
  87   if (onlyAfter) {
  88     const youtubeDL = new YoutubeDLWrapper(
  89       targetUrl,
  90       ServerConfigManager.Instance.getEnabledResolutions('vod'),
  91       CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
  92     )
  93
  94     const videoInfo = await youtubeDL.getInfoForDownload()
  95
  96     const onlyAfterWithoutTime = new Date(onlyAfter)
  97     onlyAfterWithoutTime.setHours(0, 0, 0, 0)
  98
  99     if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) {
 100       return true
 101     }
 102   }
 103
 104   return false
 105 }