Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

5 files changed, 68 insertions, 6 deletions

diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 076b7f11d..7e8a03e8f 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -5,7 +5,7 @@ import { chain } from 'lodash'
 import { join } from 'path'
 import { addColors, config, createLogger, format, transports } from 'winston'
 import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
-import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models'
+import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
 import { CONFIG } from '../initializers/config'
 import { jsonLoggerFormat, labelFormatter } from './logger'
 
@@ -260,6 +260,18 @@ class CustomConfigAuditView extends EntityAuditView {
   }
 }
 
+const channelSyncKeysToKeep = [
+  'id',
+  'externalChannelUrl',
+  'channel-id',
+  'channel-name'
+]
+class VideoChannelSyncAuditView extends EntityAuditView {
+  constructor (channelSync: VideoChannelSync) {
+    super(channelSyncKeysToKeep, 'channelSync', channelSync)
+  }
+}
+
 export {
   getAuditIdFromRes,
 
@@ -270,5 +282,6 @@ export {
   UserAuditView,
   VideoAuditView,
   AbuseAuditView,
-  CustomConfigAuditView
+  CustomConfigAuditView,
+  VideoChannelSyncAuditView
 }
diff --git a/server/helpers/custom-validators/video-channel-syncs.ts b/server/helpers/custom-validators/video-channel-syncs.ts
new file mode 100644
index 000000000..c5a9afa96
--- /dev/null
+++ b/server/helpers/custom-validators/video-channel-syncs.ts
@@ -0,0 +1,6 @@
+import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
+import { exists } from './misc'
+
+export function isVideoChannelSyncStateValid (value: any) {
+  return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
+}
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 13c990a1e..5a87b99b4 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -87,6 +87,7 @@ export class YoutubeDLCLI {
     return result.concat([
       'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
       'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
+      'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
       'best' // Ultimate fallback
     ]).join('/')
   }
@@ -103,11 +104,14 @@ export class YoutubeDLCLI {
     timeout?: number
     additionalYoutubeDLArgs?: string[]
   }) {
+    let args = options.additionalYoutubeDLArgs || []
+    args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
+
     return this.run({
       url: options.url,
       processOptions: options.processOptions,
       timeout: options.timeout,
-      args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
+      args
     })
   }
 
@@ -129,6 +133,25 @@ export class YoutubeDLCLI {
       : info
   }
 
+  getListInfo (options: {
+    url: string
+    latestVideosCount?: number
+    processOptions: execa.NodeOptions
+  }): Promise<{ upload_date: string, webpage_url: string }[]> {
+    const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
+
+    if (options.latestVideosCount !== undefined) {
+      additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
+    }
+
+    return this.getInfo({
+      url: options.url,
+      format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
+      processOptions: options.processOptions,
+      additionalYoutubeDLArgs
+    })
+  }
+
   async getSubs (options: {
     url: string
     format: 'vtt'
@@ -175,7 +198,7 @@ export class YoutubeDLCLI {
 
     const output = await subProcess
 
-    logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
+    logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
 
     return output.stdout
       ? output.stdout.trim().split(/\r?\n/)
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
index 71572f292..303e4051f 100644
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
@@ -13,6 +13,7 @@ type YoutubeDLInfo = {
   thumbnailUrl?: string
   ext?: string
   originallyPublishedAt?: Date
+  webpageUrl?: string
 
   urls?: string[]
 }
@@ -81,7 +82,8 @@ class YoutubeDLInfoBuilder {
       thumbnailUrl: obj.thumbnail || undefined,
       urls: this.buildAvailableUrl(obj),
       originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
-      ext: obj.ext
+      ext: obj.ext,
+      webpageUrl: obj.webpage_url
     }
   }
 
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index 176cf3b69..7cd5e3310 100644
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -46,6 +46,24 @@ class YoutubeDLWrapper {
     return infoBuilder.getInfo()
   }
 
+  async getInfoForListImport (options: {
+    latestVideosCount?: number
+  }) {
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const list = await youtubeDL.getListInfo({
+      url: this.url,
+      latestVideosCount: options.latestVideosCount,
+      processOptions
+    })
+
+    return list.map(info => {
+      const infoBuilder = new YoutubeDLInfoBuilder(info)
+
+      return infoBuilder.getInfo()
+    })
+  }
+
   async getSubtitles (): Promise<YoutubeDLSubs> {
     const cwd = CONFIG.STORAGE.TMP_DIR
 
@@ -103,7 +121,7 @@ class YoutubeDLWrapper {
 
           return remove(path)
         })
-        .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() }))
+        .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
 
       throw err
     }