Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

1 files changed, 81 insertions, 0 deletions

diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
new file mode 100644
index 000000000..50f80e6f9
--- /dev/null
+++ b/server/lib/sync-channel.ts
@@ -0,0 +1,81 @@
+import { logger } from '@server/helpers/logger'
+import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
+import { buildYoutubeDLImport } from '@server/lib/video-import'
+import { UserModel } from '@server/models/user/user'
+import { VideoImportModel } from '@server/models/video/video-import'
+import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
+import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
+import { CreateJobArgument, JobQueue } from './job-queue'
+import { ServerConfigManager } from './server-config-manager'
+
+export async function synchronizeChannel (options: {
+  channel: MChannelAccountDefault
+  externalChannelUrl: string
+  channelSync?: MChannelSync
+  videosCountLimit?: number
+  onlyAfter?: Date
+}) {
+  const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
+
+  const user = await UserModel.loadByChannelActorId(channel.actorId)
+  const youtubeDL = new YoutubeDLWrapper(
+    externalChannelUrl,
+    ServerConfigManager.Instance.getEnabledResolutions('vod'),
+    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+  )
+
+  const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
+
+  const targetUrls = infoList
+    .filter(videoInfo => {
+      if (!onlyAfter) return true
+
+      return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
+    })
+    .map(videoInfo => videoInfo.webpageUrl)
+
+  logger.info(
+    'Fetched %d candidate URLs for sync channel %s.',
+    targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
+  )
+
+  if (targetUrls.length === 0) {
+    if (channelSync) {
+      channelSync.state = VideoChannelSyncState.SYNCED
+      await channelSync.save()
+    }
+
+    return
+  }
+
+  const children: CreateJobArgument[] = []
+
+  for (const targetUrl of targetUrls) {
+    if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
+      logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
+      continue
+    }
+
+    const { job } = await buildYoutubeDLImport({
+      user,
+      channel,
+      targetUrl,
+      channelSync,
+      importDataOverride: {
+        privacy: VideoPrivacy.PUBLIC
+      }
+    })
+
+    children.push(job)
+  }
+
+  const parent: CreateJobArgument = {
+    type: 'after-video-channel-import',
+    payload: {
+      channelSyncId: channelSync?.id
+    }
+  }
+
+  await JobQueue.Instance.createJobWithChildren(parent, children)
+}