From 2a491182e483b97afb1b65c908b23cb48d591807 Mon Sep 17 00:00:00 2001 From: Florent Date: Wed, 10 Aug 2022 09:53:39 +0200 Subject: Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz --- .../my-video-channels.component.html | 15 ++- .../my-video-channels.component.scss | 17 ++- .../app/+my-library/my-library-routing.module.ts | 22 ++++ client/src/app/+my-library/my-library.module.ts | 4 + .../my-video-channel-syncs.component.html | 83 +++++++++++++ .../my-video-channel-syncs.component.scss | 14 +++ .../my-video-channel-syncs.component.ts | 129 +++++++++++++++++++++ .../video-channel-sync-edit.component.html | 64 ++++++++++ .../video-channel-sync-edit.component.scss | 17 +++ .../video-channel-sync-edit.component.ts | 76 ++++++++++++ 10 files changed, 435 insertions(+), 6 deletions(-) create mode 100644 client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html create mode 100644 client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss create mode 100644 client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts create mode 100644 client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html create mode 100644 client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss create mode 100644 client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts (limited to 'client/src/app/+my-library') diff --git a/client/src/app/+my-library/+my-video-channels/my-video-channels.component.html b/client/src/app/+my-library/+my-video-channels/my-video-channels.component.html index e942e002b..a48731e7c 100644 --- a/client/src/app/+my-library/+my-video-channels/my-video-channels.component.html +++ b/client/src/app/+my-library/+my-video-channels/my-video-channels.component.html @@ -1,7 +1,16 @@

- - My channels - {{ totalItems }} + + + My channels + {{ totalItems }} + + +
+ + + My synchronizations + +

diff --git a/client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss b/client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss index ab80f3d01..6c5be9240 100644 --- a/client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss +++ b/client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss @@ -1,9 +1,20 @@ @use '_variables' as *; @use '_mixins' as *; -h1 my-global-icon { - position: relative; - top: -2px; +h1 { + display: flex; + justify-content: space-between; + + my-global-icon { + position: relative; + top: -2px; + } + + .button-link { + @include peertube-button-link; + @include grey-button; + @include button-with-icon(18px, 3px, -1px); + } } .create-button { diff --git a/client/src/app/+my-library/my-library-routing.module.ts b/client/src/app/+my-library/my-library-routing.module.ts index 73858fb82..de3ef4d96 100644 --- a/client/src/app/+my-library/my-library-routing.module.ts +++ b/client/src/app/+my-library/my-library-routing.module.ts @@ -6,6 +6,8 @@ import { MySubscriptionsComponent } from './my-follows/my-subscriptions.componen import { MyHistoryComponent } from './my-history/my-history.component' import { MyLibraryComponent } from './my-library.component' import { MyOwnershipComponent } from './my-ownership/my-ownership.component' +import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component' +import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component' import { MyVideoImportsComponent } from './my-video-imports/my-video-imports.component' import { MyVideoPlaylistCreateComponent } from './my-video-playlists/my-video-playlist-create.component' import { MyVideoPlaylistElementsComponent } from './my-video-playlists/my-video-playlist-elements.component' @@ -131,6 +133,26 @@ const myLibraryRoutes: Routes = [ key: 'my-videos-history-list' } } + }, + + { + path: 'video-channel-syncs', + component: MyVideoChannelSyncsComponent, + data: { + meta: { + title: $localize`My synchronizations` + } + } + }, + + { + path: 'video-channel-syncs/create', + component: VideoChannelSyncEditComponent, + data: { + meta: { + title: $localize`Create new synchronization` + } + } } ] } diff --git a/client/src/app/+my-library/my-library.module.ts b/client/src/app/+my-library/my-library.module.ts index bfafcb3e4..4acb3b75e 100644 --- a/client/src/app/+my-library/my-library.module.ts +++ b/client/src/app/+my-library/my-library.module.ts @@ -29,6 +29,8 @@ import { MyVideoPlaylistUpdateComponent } from './my-video-playlists/my-video-pl import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlists.component' import { VideoChangeOwnershipComponent } from './my-videos/modals/video-change-ownership.component' import { MyVideosComponent } from './my-videos/my-videos.component' +import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component' +import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component' @NgModule({ imports: [ @@ -63,6 +65,8 @@ import { MyVideosComponent } from './my-videos/my-videos.component' MyOwnershipComponent, MyAcceptOwnershipComponent, MyVideoImportsComponent, + MyVideoChannelSyncsComponent, + VideoChannelSyncEditComponent, MySubscriptionsComponent, MyFollowersComponent, MyHistoryComponent, diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html new file mode 100644 index 000000000..5141607b1 --- /dev/null +++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html @@ -0,0 +1,83 @@ +
{{ error }}
+ +

+ + My synchronizations +

+ +
+

⚠️ The instance doesn't allow channel synchronization

+
+ + + +
+
+ + + Add synchronization + +
+
+ + + + + + External Channel + Channel + State + Created + Last synchronization at + + + + + + + + + + + {{ videoChannelSync.externalChannelUrl }} + + + +
+ + +
+ +
{{ videoChannelSync.channel.displayName }}
+
{{ videoChannelSync.channel.name }}
+ +
+
+ + + + + {{ videoChannelSync.state.label }} + + + + {{ videoChannelSync.createdAt | date: 'short' }} + {{ videoChannelSync.lastSyncAt | date: 'short' }} + + + diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss new file mode 100644 index 000000000..88738e54d --- /dev/null +++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss @@ -0,0 +1,14 @@ +@use '_mixins' as *; +@use '_variables' as *; +@use '_actor' as *; + +.add-sync { + @include create-button; +} + +.actor { + @include actor-row($min-height: auto, $separator: true); + margin-bottom: 0; + padding-bottom: 0; + border: 0; +} diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts new file mode 100644 index 000000000..81bdaf9f2 --- /dev/null +++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts @@ -0,0 +1,129 @@ +import { Component, OnInit } from '@angular/core' +import { AuthService, Notifier, RestPagination, RestTable, ServerService } from '@app/core' +import { DropdownAction, VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main' +import { HTMLServerConfig } from '@shared/models/server' +import { VideoChannelSync, VideoChannelSyncState } from '@shared/models/videos' +import { SortMeta } from 'primeng/api' +import { mergeMap } from 'rxjs' + +@Component({ + templateUrl: './my-video-channel-syncs.component.html', + styleUrls: [ './my-video-channel-syncs.component.scss' ] +}) +export class MyVideoChannelSyncsComponent extends RestTable implements OnInit { + error: string + + channelSyncs: VideoChannelSync[] = [] + totalRecords = 0 + + videoChannelSyncActions: DropdownAction[][] = [] + sort: SortMeta = { field: 'createdAt', order: 1 } + pagination: RestPagination = { count: this.rowsPerPage, start: 0 } + + private static STATE_CLASS_BY_ID = { + [VideoChannelSyncState.FAILED]: 'badge-red', + [VideoChannelSyncState.PROCESSING]: 'badge-blue', + [VideoChannelSyncState.SYNCED]: 'badge-green', + [VideoChannelSyncState.WAITING_FIRST_RUN]: 'badge-yellow' + } + + private serverConfig: HTMLServerConfig + + constructor ( + private videoChannelsSyncService: VideoChannelSyncService, + private serverService: ServerService, + private notifier: Notifier, + private authService: AuthService, + private videoChannelService: VideoChannelService + ) { + super() + } + + ngOnInit () { + this.serverConfig = this.serverService.getHTMLConfig() + this.initialize() + + this.videoChannelSyncActions = [ + [ + { + label: $localize`Delete`, + iconName: 'delete', + handler: videoChannelSync => this.deleteSync(videoChannelSync) + }, + { + label: $localize`Fully synchronize the channel`, + description: $localize`This fetches any missing videos on the local channel`, + iconName: 'refresh', + handler: videoChannelSync => this.fullySynchronize(videoChannelSync) + } + ] + ] + } + + protected reloadData () { + this.error = undefined + + this.authService.userInformationLoaded + .pipe(mergeMap(() => { + const user = this.authService.getUser() + return this.videoChannelsSyncService.listAccountVideoChannelsSyncs({ + sort: this.sort, + account: user.account, + pagination: this.pagination + }) + })) + .subscribe({ + next: res => { + this.channelSyncs = res.data + }, + error: err => { + this.error = err.message + } + }) + } + + syncEnabled () { + return this.serverConfig.import.videoChannelSynchronization.enabled + } + + deleteSync (videoChannelSync: VideoChannelSync) { + this.videoChannelsSyncService.deleteSync(videoChannelSync.id) + .subscribe({ + next: () => { + this.notifier.success($localize`Synchronization removed successfully for ${videoChannelSync.channel.displayName}.`) + this.reloadData() + }, + error: err => { + this.error = err.message + } + }) + } + + fullySynchronize (videoChannelSync: VideoChannelSync) { + this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl) + .subscribe({ + next: () => { + this.notifier.success($localize`Full synchronization requested successfully for ${videoChannelSync.channel.displayName}.`) + }, + error: err => { + this.error = err.message + } + }) + } + + getSyncCreateLink () { + return '/my-library/video-channel-syncs/create' + } + + getSyncStateClass (stateId: number) { + return [ 'pt-badge', MyVideoChannelSyncsComponent.STATE_CLASS_BY_ID[stateId] ] + } + + getIdentifier () { + return 'MyVideoChannelsSyncComponent' + } + + getChannelUrl (name: string) { + return '/c/' + name + } +} diff --git a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html new file mode 100644 index 000000000..611146c1a --- /dev/null +++ b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html @@ -0,0 +1,64 @@ +
{{ error }}
+ +
+
+ +
+
+
NEW SYNCHRONIZATION
+
+ +
+
+ + +
+ +
+ +
+ {{ formErrors['externalChannelUrl'] }} +
+
+ +
+ + + +
+ {{ formErrors['videoChannel'] }} +
+
+ +
+ + +
+ + +
+ +
+ + +
+
+
+
+ +
+
+
+ +
+
+
+
diff --git a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss new file mode 100644 index 000000000..d0d8c2a68 --- /dev/null +++ b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss @@ -0,0 +1,17 @@ +@use '_variables' as *; +@use '_mixins' as *; + +$form-base-input-width: 480px; + +input[type=text] { + @include peertube-input-text($form-base-input-width); +} + +.video-channel-sync-title { + @include settings-big-title; +} + +my-select-channel { + display: block; + max-width: $form-base-input-width; +} diff --git a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts new file mode 100644 index 000000000..836582609 --- /dev/null +++ b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts @@ -0,0 +1,76 @@ +import { mergeMap } from 'rxjs' +import { SelectChannelItem } from 'src/types' +import { Component, OnInit } from '@angular/core' +import { Router } from '@angular/router' +import { AuthService, Notifier } from '@app/core' +import { listUserChannelsForSelect } from '@app/helpers' +import { VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR } from '@app/shared/form-validators/video-channel-validators' +import { FormReactive, FormValidatorService } from '@app/shared/shared-forms' +import { VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main' +import { VideoChannelSyncCreate } from '@shared/models/videos' + +@Component({ + selector: 'my-video-channel-sync-edit', + templateUrl: './video-channel-sync-edit.component.html', + styleUrls: [ './video-channel-sync-edit.component.scss' ] +}) +export class VideoChannelSyncEditComponent extends FormReactive implements OnInit { + error: string + userVideoChannels: SelectChannelItem[] = [] + existingVideosStrategy: string + + constructor ( + protected formValidatorService: FormValidatorService, + private authService: AuthService, + private router: Router, + private notifier: Notifier, + private videoChannelSyncService: VideoChannelSyncService, + private videoChannelService: VideoChannelService + ) { + super() + } + + ngOnInit () { + this.buildForm({ + externalChannelUrl: VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR, + videoChannel: null, + existingVideoStrategy: null + }) + + listUserChannelsForSelect(this.authService) + .subscribe(channels => this.userVideoChannels = channels) + } + + getFormButtonTitle () { + return $localize`Create` + } + + formValidated () { + this.error = undefined + + const body = this.form.value + const videoChannelSyncCreate: VideoChannelSyncCreate = { + externalChannelUrl: body.externalChannelUrl, + videoChannelId: body.videoChannel + } + + const importExistingVideos = body['existingVideoStrategy'] === 'import' + + this.videoChannelSyncService.createSync(videoChannelSyncCreate) + .pipe(mergeMap(({ videoChannelSync }) => { + return importExistingVideos + ? this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl) + : Promise.resolve(null) + })) + .subscribe({ + next: () => { + this.notifier.success($localize`Synchronization created successfully.`) + this.router.navigate([ '/my-library', 'video-channel-syncs' ]) + }, + + error: err => { + this.error = err.message + } + }) + } +} -- cgit v1.2.3