From 2a491182e483b97afb1b65c908b23cb48d591807 Mon Sep 17 00:00:00 2001
From: Florent <florent.git@zeteo.me>
Date: Wed, 10 Aug 2022 09:53:39 +0200
Subject: Channel sync (#5135)
* Add external channel URL for channel update / creation (#754)
* Disallow synchronisation if user has no video quota (#754)
* More constraints serverside (#754)
* Disable sync if server configuration does not allow HTTP import (#754)
* Working version synchronizing videos with a job (#754)
TODO: refactoring, too much code duplication
* More logs and try/catch (#754)
* Fix eslint error (#754)
* WIP: support synchronization time change (#754)
* New frontend #754
* WIP: Create sync front (#754)
* Enhance UI, sync creation form (#754)
* Warning message when HTTP upload is disallowed
* More consistent names (#754)
* Binding Front with API (#754)
* Add a /me API (#754)
* Improve list UI (#754)
* Implement creation and deletion routes (#754)
* Lint (#754)
* Lint again (#754)
* WIP: UI for triggering import existing videos (#754)
* Implement jobs for syncing and importing channels
* Don't sync videos before sync creation + avoid concurrency issue (#754)
* Cleanup (#754)
* Cleanup: OpenAPI + API rework (#754)
* Remove dead code (#754)
* Eslint (#754)
* Revert the mess with whitespaces in constants.ts (#754)
* Some fixes after rebase (#754)
* Several fixes after PR remarks (#754)
* Front + API: Rename video-channels-sync to video-channel-syncs (#754)
* Allow enabling channel sync through UI (#754)
* getChannelInfo (#754)
* Minor fixes: openapi + model + sql (#754)
* Simplified API validators (#754)
* Rename MChannelSync to MChannelSyncChannel (#754)
* Add command for VideoChannelSync (#754)
* Use synchronization.enabled config (#754)
* Check parameters test + some fixes (#754)
* Fix conflict mistake (#754)
* Restrict access to video channel sync list API (#754)
* Start adding unit test for synchronization (#754)
* Continue testing (#754)
* Tests finished + convertion of job to scheduler (#754)
* Add lastSyncAt field (#754)
* Fix externalRemoteUrl sort + creation date not well formatted (#754)
* Small fix (#754)
* Factorize addYoutubeDLImport and buildVideo (#754)
* Check duplicates on channel not on users (#754)
* factorize thumbnail generation (#754)
* Fetch error should return status 400 (#754)
* Separate video-channel-import and video-channel-sync-latest (#754)
* Bump DB migration version after rebase (#754)
* Prettier states in UI table (#754)
* Add DefaultScope in VideoChannelSyncModel (#754)
* Fix audit logs (#754)
* Ensure user can upload when importing channel + minor fixes (#754)
* Mark synchronization as failed on exception + typos (#754)
* Change REST API for importing videos into channel (#754)
* Add option for fully synchronize a chnanel (#754)
* Return a whole sync object on creation to avoid tricks in Front (#754)
* Various remarks (#754)
* Single quotes by default (#754)
* Rename synchronization to video_channel_synchronization
* Add check.latest_videos_count and max_per_user options (#754)
* Better channel rendering in list #754
* Allow sorting with channel name and state (#754)
* Add missing tests for channel imports (#754)
* Prefer using a parent job for channel sync
* Styling
* Client styling
Co-authored-by: Chocobozzz <me@florianbigard.com>
---
server/models/utils.ts | 11 ++
server/models/video/video-channel-sync.ts | 176 ++++++++++++++++++++++++++++++
server/models/video/video-import.ts | 24 +++-
3 files changed, 210 insertions(+), 1 deletion(-)
create mode 100644 server/models/video/video-channel-sync.ts
(limited to 'server/models')
diff --git a/server/models/utils.ts b/server/models/utils.ts
index c468f748d..1e168d419 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A
return getSort(value, lastSort)
}
+function getChannelSyncSort (value: string): OrderItem[] {
+ const { direction, field } = buildDirectionAndField(value)
+ if (field.toLowerCase() === 'videochannel') {
+ return [
+ [ literal('"VideoChannel.name"'), direction ]
+ ]
+ }
+ return [ [ field, direction ] ]
+}
+
function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
if (!model.createdAt || !model.updatedAt) {
throw new Error('Miss createdAt & updatedAt attributes to model')
@@ -280,6 +290,7 @@ export {
getAdminUsersSort,
getVideoSort,
getBlacklistSort,
+ getChannelSyncSort,
createSimilarityAttribute,
throwIfNotValid,
buildServerIdsFollowedBy,
diff --git a/server/models/video/video-channel-sync.ts b/server/models/video/video-channel-sync.ts
new file mode 100644
index 000000000..6e49cde10
--- /dev/null
+++ b/server/models/video/video-channel-sync.ts
@@ -0,0 +1,176 @@
+import { Op } from 'sequelize'
+import {
+ AllowNull,
+ BelongsTo,
+ Column,
+ CreatedAt,
+ DataType,
+ Default,
+ DefaultScope,
+ ForeignKey,
+ Is,
+ Model,
+ Table,
+ UpdatedAt
+} from 'sequelize-typescript'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs'
+import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
+import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models'
+import { VideoChannelSync, VideoChannelSyncState } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { AccountModel } from '../account/account'
+import { UserModel } from '../user/user'
+import { getChannelSyncSort, throwIfNotValid } from '../utils'
+import { VideoChannelModel } from './video-channel'
+
+@DefaultScope(() => ({
+ include: [
+ {
+ model: VideoChannelModel, // Default scope includes avatar and server
+ required: true
+ }
+ ]
+}))
+@Table({
+ tableName: 'videoChannelSync',
+ indexes: [
+ {
+ fields: [ 'videoChannelId' ]
+ }
+ ]
+})
+export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> {
+
+ @AllowNull(false)
+ @Default(null)
+ @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true))
+ @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max))
+ externalChannelUrl: string
+
+ @CreatedAt
+ createdAt: Date
+
+ @UpdatedAt
+ updatedAt: Date
+
+ @ForeignKey(() => VideoChannelModel)
+ @Column
+ videoChannelId: number
+
+ @BelongsTo(() => VideoChannelModel, {
+ foreignKey: {
+ allowNull: false
+ },
+ onDelete: 'cascade'
+ })
+ VideoChannel: VideoChannelModel
+
+ @AllowNull(false)
+ @Default(VideoChannelSyncState.WAITING_FIRST_RUN)
+ @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state'))
+ @Column
+ state: VideoChannelSyncState
+
+ @AllowNull(true)
+ @Column(DataType.DATE)
+ lastSyncAt: Date
+
+ static listByAccountForAPI (options: {
+ accountId: number
+ start: number
+ count: number
+ sort: string
+ }) {
+ const getQuery = (forCount: boolean) => {
+ const videoChannelModel = forCount
+ ? VideoChannelModel.unscoped()
+ : VideoChannelModel
+
+ return {
+ offset: options.start,
+ limit: options.count,
+ order: getChannelSyncSort(options.sort),
+ include: [
+ {
+ model: videoChannelModel,
+ required: true,
+ where: {
+ accountId: options.accountId
+ }
+ }
+ ]
+ }
+ }
+
+ return Promise.all([
+ VideoChannelSyncModel.unscoped().count(getQuery(true)),
+ VideoChannelSyncModel.unscoped().findAll(getQuery(false))
+ ]).then(([ total, data ]) => ({ total, data }))
+ }
+
+ static countByAccount (accountId: number) {
+ const query = {
+ include: [
+ {
+ model: VideoChannelModel.unscoped(),
+ required: true,
+ where: {
+ accountId
+ }
+ }
+ ]
+ }
+
+ return VideoChannelSyncModel.unscoped().count(query)
+ }
+
+ static loadWithChannel (id: number): Promise<MChannelSyncChannel> {
+ return VideoChannelSyncModel.findByPk(id)
+ }
+
+ static async listSyncs (): Promise<MChannelSync[]> {
+ const query = {
+ include: [
+ {
+ model: VideoChannelModel.unscoped(),
+ required: true,
+ include: [
+ {
+ model: AccountModel.unscoped(),
+ required: true,
+ include: [ {
+ attributes: [],
+ model: UserModel.unscoped(),
+ required: true,
+ where: {
+ videoQuota: {
+ [Op.ne]: 0
+ },
+ videoQuotaDaily: {
+ [Op.ne]: 0
+ }
+ }
+ } ]
+ }
+ ]
+ }
+ ]
+ }
+ return VideoChannelSyncModel.unscoped().findAll(query)
+ }
+
+ toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync {
+ return {
+ id: this.id,
+ state: {
+ id: this.state,
+ label: VIDEO_CHANNEL_SYNC_STATE[this.state]
+ },
+ externalChannelUrl: this.externalChannelUrl,
+ createdAt: this.createdAt.toISOString(),
+ channel: this.VideoChannel.toFormattedSummaryJSON(),
+ lastSyncAt: this.lastSyncAt?.toISOString()
+ }
+ }
+}
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index 1d8296060..b8e941623 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -1,4 +1,4 @@
-import { WhereOptions } from 'sequelize'
+import { Op, WhereOptions } from 'sequelize'
import {
AfterUpdate,
AllowNull,
@@ -161,6 +161,28 @@ export class VideoImportModel extends Model<Partial<AttributesOnly<VideoImportMo
]).then(([ total, data ]) => ({ total, data }))
}
+ static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
+ const element = await VideoImportModel.unscoped().findOne({
+ where: {
+ targetUrl,
+ state: {
+ [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
+ }
+ },
+ include: [
+ {
+ model: VideoModel,
+ required: true,
+ where: {
+ channelId
+ }
+ }
+ ]
+ })
+
+ return !!element
+ }
+
getTargetIdentifier () {
return this.targetUrl || this.magnetUri || this.torrentName
}
--
cgit v1.2.3