From 2a491182e483b97afb1b65c908b23cb48d591807 Mon Sep 17 00:00:00 2001 From: Florent Date: Wed, 10 Aug 2022 09:53:39 +0200 Subject: Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz --- server/models/utils.ts | 11 ++ server/models/video/video-channel-sync.ts | 176 ++++++++++++++++++++++++++++++ server/models/video/video-import.ts | 24 +++- 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 server/models/video/video-channel-sync.ts (limited to 'server/models') diff --git a/server/models/utils.ts b/server/models/utils.ts index c468f748d..1e168d419 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts @@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A return getSort(value, lastSort) } +function getChannelSyncSort (value: string): OrderItem[] { + const { direction, field } = buildDirectionAndField(value) + if (field.toLowerCase() === 'videochannel') { + return [ + [ literal('"VideoChannel.name"'), direction ] + ] + } + return [ [ field, direction ] ] +} + function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { if (!model.createdAt || !model.updatedAt) { throw new Error('Miss createdAt & updatedAt attributes to model') @@ -280,6 +290,7 @@ export { getAdminUsersSort, getVideoSort, getBlacklistSort, + getChannelSyncSort, createSimilarityAttribute, throwIfNotValid, buildServerIdsFollowedBy, diff --git a/server/models/video/video-channel-sync.ts b/server/models/video/video-channel-sync.ts new file mode 100644 index 000000000..6e49cde10 --- /dev/null +++ b/server/models/video/video-channel-sync.ts @@ -0,0 +1,176 @@ +import { Op } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + ForeignKey, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' +import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs' +import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants' +import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models' +import { VideoChannelSync, VideoChannelSyncState } from '@shared/models' +import { AttributesOnly } from '@shared/typescript-utils' +import { AccountModel } from '../account/account' +import { UserModel } from '../user/user' +import { getChannelSyncSort, throwIfNotValid } from '../utils' +import { VideoChannelModel } from './video-channel' + +@DefaultScope(() => ({ + include: [ + { + model: VideoChannelModel, // Default scope includes avatar and server + required: true + } + ] +})) +@Table({ + tableName: 'videoChannelSync', + indexes: [ + { + fields: [ 'videoChannelId' ] + } + ] +}) +export class VideoChannelSyncModel extends Model>> { + + @AllowNull(false) + @Default(null) + @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max)) + externalChannelUrl: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoChannelModel) + @Column + videoChannelId: number + + @BelongsTo(() => VideoChannelModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + VideoChannel: VideoChannelModel + + @AllowNull(false) + @Default(VideoChannelSyncState.WAITING_FIRST_RUN) + @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state')) + @Column + state: VideoChannelSyncState + + @AllowNull(true) + @Column(DataType.DATE) + lastSyncAt: Date + + static listByAccountForAPI (options: { + accountId: number + start: number + count: number + sort: string + }) { + const getQuery = (forCount: boolean) => { + const videoChannelModel = forCount + ? VideoChannelModel.unscoped() + : VideoChannelModel + + return { + offset: options.start, + limit: options.count, + order: getChannelSyncSort(options.sort), + include: [ + { + model: videoChannelModel, + required: true, + where: { + accountId: options.accountId + } + } + ] + } + } + + return Promise.all([ + VideoChannelSyncModel.unscoped().count(getQuery(true)), + VideoChannelSyncModel.unscoped().findAll(getQuery(false)) + ]).then(([ total, data ]) => ({ total, data })) + } + + static countByAccount (accountId: number) { + const query = { + include: [ + { + model: VideoChannelModel.unscoped(), + required: true, + where: { + accountId + } + } + ] + } + + return VideoChannelSyncModel.unscoped().count(query) + } + + static loadWithChannel (id: number): Promise { + return VideoChannelSyncModel.findByPk(id) + } + + static async listSyncs (): Promise { + const query = { + include: [ + { + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + model: AccountModel.unscoped(), + required: true, + include: [ { + attributes: [], + model: UserModel.unscoped(), + required: true, + where: { + videoQuota: { + [Op.ne]: 0 + }, + videoQuotaDaily: { + [Op.ne]: 0 + } + } + } ] + } + ] + } + ] + } + return VideoChannelSyncModel.unscoped().findAll(query) + } + + toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync { + return { + id: this.id, + state: { + id: this.state, + label: VIDEO_CHANNEL_SYNC_STATE[this.state] + }, + externalChannelUrl: this.externalChannelUrl, + createdAt: this.createdAt.toISOString(), + channel: this.VideoChannel.toFormattedSummaryJSON(), + lastSyncAt: this.lastSyncAt?.toISOString() + } + } +} diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index 1d8296060..b8e941623 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts @@ -1,4 +1,4 @@ -import { WhereOptions } from 'sequelize' +import { Op, WhereOptions } from 'sequelize' import { AfterUpdate, AllowNull, @@ -161,6 +161,28 @@ export class VideoImportModel extends Model ({ total, data })) } + static async urlAlreadyImported (channelId: number, targetUrl: string): Promise { + const element = await VideoImportModel.unscoped().findOne({ + where: { + targetUrl, + state: { + [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ] + } + }, + include: [ + { + model: VideoModel, + required: true, + where: { + channelId + } + } + ] + }) + + return !!element + } + getTargetIdentifier () { return this.targetUrl || this.magnetUri || this.torrentName } -- cgit v1.2.3