Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

3 files changed, 210 insertions, 1 deletions

diff --git a/server/models/utils.ts b/server/models/utils.ts
index c468f748d..1e168d419 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A
   return getSort(value, lastSort)
 }
 
+function getChannelSyncSort (value: string): OrderItem[] {
+  const { direction, field } = buildDirectionAndField(value)
+  if (field.toLowerCase() === 'videochannel') {
+    return [
+      [ literal('"VideoChannel.name"'), direction ]
+    ]
+  }
+  return [ [ field, direction ] ]
+}
+
 function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
   if (!model.createdAt || !model.updatedAt) {
     throw new Error('Miss createdAt & updatedAt attributes to model')
@@ -280,6 +290,7 @@ export {
   getAdminUsersSort,
   getVideoSort,
   getBlacklistSort,
+  getChannelSyncSort,
   createSimilarityAttribute,
   throwIfNotValid,
   buildServerIdsFollowedBy,
diff --git a/server/models/video/video-channel-sync.ts b/server/models/video/video-channel-sync.ts
new file mode 100644
index 000000000..6e49cde10
--- /dev/null
+++ b/server/models/video/video-channel-sync.ts
@@ -0,0 +1,176 @@
+import { Op } from 'sequelize'
+import {
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  Default,
+  DefaultScope,
+  ForeignKey,
+  Is,
+  Model,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs'
+import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
+import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models'
+import { VideoChannelSync, VideoChannelSyncState } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { AccountModel } from '../account/account'
+import { UserModel } from '../user/user'
+import { getChannelSyncSort, throwIfNotValid } from '../utils'
+import { VideoChannelModel } from './video-channel'
+
+@DefaultScope(() => ({
+  include: [
+    {
+      model: VideoChannelModel, // Default scope includes avatar and server
+      required: true
+    }
+  ]
+}))
+@Table({
+  tableName: 'videoChannelSync',
+  indexes: [
+    {
+      fields: [ 'videoChannelId' ]
+    }
+  ]
+})
+export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> {
+
+  @AllowNull(false)
+  @Default(null)
+  @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max))
+  externalChannelUrl: string
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @ForeignKey(() => VideoChannelModel)
+  @Column
+  videoChannelId: number
+
+  @BelongsTo(() => VideoChannelModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  VideoChannel: VideoChannelModel
+
+  @AllowNull(false)
+  @Default(VideoChannelSyncState.WAITING_FIRST_RUN)
+  @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state'))
+  @Column
+  state: VideoChannelSyncState
+
+  @AllowNull(true)
+  @Column(DataType.DATE)
+  lastSyncAt: Date
+
+  static listByAccountForAPI (options: {
+    accountId: number
+    start: number
+    count: number
+    sort: string
+  }) {
+    const getQuery = (forCount: boolean) => {
+      const videoChannelModel = forCount
+        ? VideoChannelModel.unscoped()
+        : VideoChannelModel
+
+      return {
+        offset: options.start,
+        limit: options.count,
+        order: getChannelSyncSort(options.sort),
+        include: [
+          {
+            model: videoChannelModel,
+            required: true,
+            where: {
+              accountId: options.accountId
+            }
+          }
+        ]
+      }
+    }
+
+    return Promise.all([
+      VideoChannelSyncModel.unscoped().count(getQuery(true)),
+      VideoChannelSyncModel.unscoped().findAll(getQuery(false))
+    ]).then(([ total, data ]) => ({ total, data }))
+  }
+
+  static countByAccount (accountId: number) {
+    const query = {
+      include: [
+        {
+          model: VideoChannelModel.unscoped(),
+          required: true,
+          where: {
+            accountId
+          }
+        }
+      ]
+    }
+
+    return VideoChannelSyncModel.unscoped().count(query)
+  }
+
+  static loadWithChannel (id: number): Promise<MChannelSyncChannel> {
+    return VideoChannelSyncModel.findByPk(id)
+  }
+
+  static async listSyncs (): Promise<MChannelSync[]> {
+    const query = {
+      include: [
+        {
+          model: VideoChannelModel.unscoped(),
+          required: true,
+          include: [
+            {
+              model: AccountModel.unscoped(),
+              required: true,
+              include: [ {
+                attributes: [],
+                model: UserModel.unscoped(),
+                required: true,
+                where: {
+                  videoQuota: {
+                    [Op.ne]: 0
+                  },
+                  videoQuotaDaily: {
+                    [Op.ne]: 0
+                  }
+                }
+              } ]
+            }
+          ]
+        }
+      ]
+    }
+    return VideoChannelSyncModel.unscoped().findAll(query)
+  }
+
+  toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync {
+    return {
+      id: this.id,
+      state: {
+        id: this.state,
+        label: VIDEO_CHANNEL_SYNC_STATE[this.state]
+      },
+      externalChannelUrl: this.externalChannelUrl,
+      createdAt: this.createdAt.toISOString(),
+      channel: this.VideoChannel.toFormattedSummaryJSON(),
+      lastSyncAt: this.lastSyncAt?.toISOString()
+    }
+  }
+}
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index 1d8296060..b8e941623 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -1,4 +1,4 @@
-import { WhereOptions } from 'sequelize'
+import { Op, WhereOptions } from 'sequelize'
 import {
   AfterUpdate,
   AllowNull,
@@ -161,6 +161,28 @@ export class VideoImportModel extends Model<Partial<AttributesOnly<VideoImportMo
     ]).then(([ total, data ]) => ({ total, data }))
   }
 
+  static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
+    const element = await VideoImportModel.unscoped().findOne({
+      where: {
+        targetUrl,
+        state: {
+          [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
+        }
+      },
+      include: [
+        {
+          model: VideoModel,
+          required: true,
+          where: {
+            channelId
+          }
+        }
+      ]
+    })
+
+    return !!element
+  }
+
   getTargetIdentifier () {
     return this.targetUrl || this.magnetUri || this.torrentName
   }