From 2a491182e483b97afb1b65c908b23cb48d591807 Mon Sep 17 00:00:00 2001 From: Florent Date: Wed, 10 Aug 2022 09:53:39 +0200 Subject: Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz --- shared/models/server/custom-config.model.ts | 4 ++ shared/models/server/debug.model.ts | 5 +- shared/models/server/job.model.ts | 31 ++++++++++-- shared/models/server/server-config.model.ts | 3 ++ shared/models/videos/channel-sync/index.ts | 3 ++ .../video-channel-sync-create.model.ts | 4 ++ .../channel-sync/video-channel-sync-state.enum.ts | 6 +++ .../channel-sync/video-channel-sync.model.ts | 14 ++++++ shared/models/videos/index.ts | 1 + shared/server-commands/server/config-command.ts | 36 +++++++++++++- shared/server-commands/server/server.ts | 3 ++ shared/server-commands/server/servers.ts | 21 ++++++++- .../videos/channel-syncs-command.ts | 55 ++++++++++++++++++++++ shared/server-commands/videos/channels-command.ts | 18 +++++++ shared/server-commands/videos/index.ts | 1 + 15 files changed, 198 insertions(+), 7 deletions(-) create mode 100644 shared/models/videos/channel-sync/index.ts create mode 100644 shared/models/videos/channel-sync/video-channel-sync-create.model.ts create mode 100644 shared/models/videos/channel-sync/video-channel-sync-state.enum.ts create mode 100644 shared/models/videos/channel-sync/video-channel-sync.model.ts create mode 100644 shared/server-commands/videos/channel-syncs-command.ts (limited to 'shared') diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index bb9c7cef1..7d9d570b1 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -165,6 +165,10 @@ export interface CustomConfig { enabled: boolean } } + videoChannelSynchronization: { + enabled: boolean + maxPerUser: number + } } trending: { diff --git a/shared/models/server/debug.model.ts b/shared/models/server/debug.model.ts index 223d23362..1c4597b8b 100644 --- a/shared/models/server/debug.model.ts +++ b/shared/models/server/debug.model.ts @@ -4,5 +4,8 @@ export interface Debug { } export interface SendDebugCommand { - command: 'remove-dandling-resumable-uploads' | 'process-video-views-buffer' | 'process-video-viewers' + command: 'remove-dandling-resumable-uploads' + | 'process-video-views-buffer' + | 'process-video-viewers' + | 'process-video-channel-sync-latest' } diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 8c8f64de9..137da367c 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -25,6 +25,8 @@ export type JobType = | 'manage-video-torrent' | 'move-to-object-storage' | 'video-studio-edition' + | 'video-channel-import' + | 'after-video-channel-import' | 'notify' | 'federate-video' @@ -82,20 +84,32 @@ export type VideoFileImportPayload = { filePath: string } +// --------------------------------------------------------------------------- + export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file' export type VideoImportYoutubeDLPayloadType = 'youtube-dl' -export type VideoImportYoutubeDLPayload = { +export interface VideoImportYoutubeDLPayload { type: VideoImportYoutubeDLPayloadType videoImportId: number fileExt?: string } -export type VideoImportTorrentPayload = { + +export interface VideoImportTorrentPayload { type: VideoImportTorrentPayloadType videoImportId: number } -export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload + +export type VideoImportPayload = (VideoImportYoutubeDLPayload | VideoImportTorrentPayload) & { + preventException: boolean +} + +export interface VideoImportPreventExceptionResult { + resultType: 'success' | 'error' +} + +// --------------------------------------------------------------------------- export type VideoRedundancyPayload = { videoId: number @@ -219,6 +233,17 @@ export interface VideoStudioEditionPayload { // --------------------------------------------------------------------------- +export interface VideoChannelImportPayload { + externalChannelUrl: string + videoChannelId: number +} + +export interface AfterVideoChannelImportPayload { + channelSyncId: number +} + +// --------------------------------------------------------------------------- + export type NotifyPayload = { action: 'new-video' diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index 67ad809f7..3b6d0597c 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -188,6 +188,9 @@ export interface ServerConfig { enabled: boolean } } + videoChannelSynchronization: { + enabled: boolean + } } autoBlacklist: { diff --git a/shared/models/videos/channel-sync/index.ts b/shared/models/videos/channel-sync/index.ts new file mode 100644 index 000000000..7d25aaac3 --- /dev/null +++ b/shared/models/videos/channel-sync/index.ts @@ -0,0 +1,3 @@ +export * from './video-channel-sync-state.enum' +export * from './video-channel-sync.model' +export * from './video-channel-sync-create.model' diff --git a/shared/models/videos/channel-sync/video-channel-sync-create.model.ts b/shared/models/videos/channel-sync/video-channel-sync-create.model.ts new file mode 100644 index 000000000..753a8ee4c --- /dev/null +++ b/shared/models/videos/channel-sync/video-channel-sync-create.model.ts @@ -0,0 +1,4 @@ +export interface VideoChannelSyncCreate { + externalChannelUrl: string + videoChannelId: number +} diff --git a/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts b/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts new file mode 100644 index 000000000..3e9f5ddc2 --- /dev/null +++ b/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts @@ -0,0 +1,6 @@ +export const enum VideoChannelSyncState { + WAITING_FIRST_RUN = 1, + PROCESSING = 2, + SYNCED = 3, + FAILED = 4 +} diff --git a/shared/models/videos/channel-sync/video-channel-sync.model.ts b/shared/models/videos/channel-sync/video-channel-sync.model.ts new file mode 100644 index 000000000..73ac0615b --- /dev/null +++ b/shared/models/videos/channel-sync/video-channel-sync.model.ts @@ -0,0 +1,14 @@ +import { VideoChannelSummary } from '../channel/video-channel.model' +import { VideoConstant } from '../video-constant.model' +import { VideoChannelSyncState } from './video-channel-sync-state.enum' + +export interface VideoChannelSync { + id: number + + externalChannelUrl: string + + createdAt: string + channel: VideoChannelSummary + state: VideoConstant + lastSyncAt: string +} diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index 05497bda1..f8e6976d3 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts @@ -11,6 +11,7 @@ export * from './playlist' export * from './rate' export * from './stats' export * from './transcoding' +export * from './channel-sync' export * from './nsfw-policy.type' diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts index 8ab750983..1c2315ed1 100644 --- a/shared/server-commands/server/config-command.ts +++ b/shared/server-commands/server/config-command.ts @@ -18,17 +18,25 @@ export class ConfigCommand extends AbstractCommand { } } + disableImports () { + return this.setImportsEnabled(false) + } + enableImports () { + return this.setImportsEnabled(true) + } + + private setImportsEnabled (enabled: boolean) { return this.updateExistingSubConfig({ newConfig: { import: { videos: { http: { - enabled: true + enabled }, torrent: { - enabled: true + enabled } } } @@ -36,6 +44,26 @@ export class ConfigCommand extends AbstractCommand { }) } + private setChannelSyncEnabled (enabled: boolean) { + return this.updateExistingSubConfig({ + newConfig: { + import: { + videoChannelSynchronization: { + enabled + } + } + } + }) + } + + enableChannelSync () { + return this.setChannelSyncEnabled(true) + } + + disableChannelSync () { + return this.setChannelSyncEnabled(false) + } + enableLive (options: { allowReplay?: boolean transcoding?: boolean @@ -356,6 +384,10 @@ export class ConfigCommand extends AbstractCommand { torrent: { enabled: false } + }, + videoChannelSynchronization: { + enabled: false, + maxPerUser: 10 } }, trending: { diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts index 0ad818a11..7acbc978f 100644 --- a/shared/server-commands/server/server.ts +++ b/shared/server-commands/server/server.ts @@ -19,6 +19,7 @@ import { CaptionsCommand, ChangeOwnershipCommand, ChannelsCommand, + ChannelSyncsCommand, HistoryCommand, ImportsCommand, LiveCommand, @@ -118,6 +119,7 @@ export class PeerTubeServer { playlists?: PlaylistsCommand history?: HistoryCommand imports?: ImportsCommand + channelSyncs?: ChannelSyncsCommand streamingPlaylists?: StreamingPlaylistsCommand channels?: ChannelsCommand comments?: CommentsCommand @@ -390,6 +392,7 @@ export class PeerTubeServer { this.playlists = new PlaylistsCommand(this) this.history = new HistoryCommand(this) this.imports = new ImportsCommand(this) + this.channelSyncs = new ChannelSyncsCommand(this) this.streamingPlaylists = new StreamingPlaylistsCommand(this) this.channels = new ChannelsCommand(this) this.comments = new CommentsCommand(this) diff --git a/shared/server-commands/server/servers.ts b/shared/server-commands/server/servers.ts index 0faee3a8d..29f01774d 100644 --- a/shared/server-commands/server/servers.ts +++ b/shared/server-commands/server/servers.ts @@ -39,11 +39,30 @@ async function cleanupTests (servers: PeerTubeServer[]) { return Promise.all(p) } +function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') { + return { + import: { + videos: { + http: { + youtube_dl_release: { + url: mode === 'youtube-dl' + ? 'https://yt-dl.org/downloads/latest/youtube-dl' + : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases', + + name: mode + } + } + } + } + } +} + // --------------------------------------------------------------------------- export { createSingleServer, createMultipleServers, cleanupTests, - killallServers + killallServers, + getServerImportConfig } diff --git a/shared/server-commands/videos/channel-syncs-command.ts b/shared/server-commands/videos/channel-syncs-command.ts new file mode 100644 index 000000000..de4a160ec --- /dev/null +++ b/shared/server-commands/videos/channel-syncs-command.ts @@ -0,0 +1,55 @@ +import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@shared/models' +import { pick } from '@shared/core-utils' +import { unwrapBody } from '../requests' +import { AbstractCommand, OverrideCommandOptions } from '../shared' + +export class ChannelSyncsCommand extends AbstractCommand { + private static readonly API_PATH = '/api/v1/video-channel-syncs' + + listByAccount (options: OverrideCommandOptions & { + accountName: string + start?: number + count?: number + sort?: string + }) { + const { accountName, sort = 'createdAt' } = options + + const path = `/api/v1/accounts/${accountName}/video-channel-syncs` + + return this.getRequestBody>({ + ...options, + + path, + query: { sort, ...pick(options, [ 'start', 'count' ]) }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async create (options: OverrideCommandOptions & { + attributes: VideoChannelSyncCreate + }) { + return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({ + ...options, + + path: ChannelSyncsCommand.API_PATH, + fields: options.attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + delete (options: OverrideCommandOptions & { + channelSyncId: number + }) { + const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}` + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/shared/server-commands/videos/channels-command.ts b/shared/server-commands/videos/channels-command.ts index 8ab124658..a688a120f 100644 --- a/shared/server-commands/videos/channels-command.ts +++ b/shared/server-commands/videos/channels-command.ts @@ -181,4 +181,22 @@ export class ChannelsCommand extends AbstractCommand { defaultExpectedStatus: HttpStatusCode.OK_200 }) } + + importVideos (options: OverrideCommandOptions & { + channelName: string + externalChannelUrl: string + }) { + const { channelName, externalChannelUrl } = options + + const path = `/api/v1/video-channels/${channelName}/import-videos` + + return this.postBodyRequest({ + ...options, + + path, + fields: { externalChannelUrl }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } } diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts index b861731fb..b4d6fa37b 100644 --- a/shared/server-commands/videos/index.ts +++ b/shared/server-commands/videos/index.ts @@ -3,6 +3,7 @@ export * from './captions-command' export * from './change-ownership-command' export * from './channels' export * from './channels-command' +export * from './channel-syncs-command' export * from './comments-command' export * from './history-command' export * from './imports-command' -- cgit v1.2.3