aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorFlorent <florent.git@zeteo.me>2022-08-10 09:53:39 +0200
committerGitHub <noreply@github.com>2022-08-10 09:53:39 +0200
commit2a491182e483b97afb1b65c908b23cb48d591807 (patch)
treeec13503216ad72a3ea8f1ce3b659899f8167fb47
parent06ac128958c489efe1008eeca1df683819bd2f18 (diff)
downloadPeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.gz
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.zst
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.zip
Channel sync (#5135)
* Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html24
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts26
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts3
-rw-r--r--client/src/app/+admin/system/jobs/jobs.component.ts3
-rw-r--r--client/src/app/+manage/video-channel-edit/video-channel-edit.component.html2
-rw-r--r--client/src/app/+my-library/+my-video-channels/my-video-channels.component.html15
-rw-r--r--client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss17
-rw-r--r--client/src/app/+my-library/my-library-routing.module.ts22
-rw-r--r--client/src/app/+my-library/my-library.module.ts4
-rw-r--r--client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html83
-rw-r--r--client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss14
-rw-r--r--client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts129
-rw-r--r--client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html64
-rw-r--r--client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss17
-rw-r--r--client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts76
-rw-r--r--client/src/app/shared/form-validators/video-channel-validators.ts13
-rw-r--r--client/src/app/shared/shared-instance/instance-features-table.component.html7
-rw-r--r--client/src/app/shared/shared-main/index.ts1
-rw-r--r--client/src/app/shared/shared-main/video-channel-sync/index.ts1
-rw-r--r--client/src/app/shared/shared-main/video-channel-sync/video-channel-sync.service.ts50
-rw-r--r--client/src/app/shared/shared-main/video-channel/video-channel.service.ts6
-rw-r--r--config/default.yaml11
-rw-r--r--config/dev.yaml5
-rw-r--r--config/production.yaml.example11
-rw-r--r--server.ts2
-rw-r--r--server/controllers/api/accounts.ts28
-rw-r--r--server/controllers/api/config.ts4
-rw-r--r--server/controllers/api/index.ts2
-rw-r--r--server/controllers/api/server/debug.ts4
-rw-r--r--server/controllers/api/video-channel-sync.ts76
-rw-r--r--server/controllers/api/video-channel.ts28
-rw-r--r--server/controllers/api/videos/import.ts318
-rw-r--r--server/helpers/audit-logger.ts17
-rw-r--r--server/helpers/custom-validators/video-channel-syncs.ts6
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts27
-rw-r--r--server/helpers/youtube-dl/youtube-dl-info-builder.ts4
-rw-r--r--server/helpers/youtube-dl/youtube-dl-wrapper.ts20
-rw-r--r--server/initializers/checker-after-init.ts7
-rw-r--r--server/initializers/checker-before-init.ts2
-rw-r--r--server/initializers/config.ts9
-rw-r--r--server/initializers/constants.ts29
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/initializers/migrations/0730-video-channel-sync.ts36
-rw-r--r--server/lib/job-queue/handlers/after-video-channel-import.ts37
-rw-r--r--server/lib/job-queue/handlers/video-channel-import.ts36
-rw-r--r--server/lib/job-queue/handlers/video-import.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts19
-rw-r--r--server/lib/schedulers/video-channel-sync-latest-scheduler.ts61
-rw-r--r--server/lib/server-config-manager.ts3
-rw-r--r--server/lib/sync-channel.ts81
-rw-r--r--server/lib/video-import.ts308
-rw-r--r--server/middlewares/validators/config.ts11
-rw-r--r--server/middlewares/validators/sort.ts2
-rw-r--r--server/middlewares/validators/videos/index.ts1
-rw-r--r--server/middlewares/validators/videos/video-channel-sync.ts66
-rw-r--r--server/middlewares/validators/videos/video-channels.ts55
-rw-r--r--server/models/utils.ts11
-rw-r--r--server/models/video/video-channel-sync.ts176
-rw-r--r--server/models/video/video-import.ts24
-rw-r--r--server/tests/api/check-params/config.ts29
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/upload-quota.ts2
-rw-r--r--server/tests/api/check-params/video-channel-syncs.ts318
-rw-r--r--server/tests/api/check-params/video-channels.ts134
-rw-r--r--server/tests/api/check-params/video-imports.ts8
-rw-r--r--server/tests/api/server/config.ts4
-rw-r--r--server/tests/api/videos/channel-import-videos.ts50
-rw-r--r--server/tests/api/videos/index.ts2
-rw-r--r--server/tests/api/videos/video-channel-syncs.ts226
-rw-r--r--server/tests/api/videos/video-imports.ts22
-rw-r--r--server/tests/shared/tests.ts2
-rw-r--r--server/types/express.d.ts3
-rw-r--r--server/types/models/video/index.ts1
-rw-r--r--server/types/models/video/video-channel-sync.ts17
-rw-r--r--shared/models/server/custom-config.model.ts4
-rw-r--r--shared/models/server/debug.model.ts5
-rw-r--r--shared/models/server/job.model.ts31
-rw-r--r--shared/models/server/server-config.model.ts3
-rw-r--r--shared/models/videos/channel-sync/index.ts3
-rw-r--r--shared/models/videos/channel-sync/video-channel-sync-create.model.ts4
-rw-r--r--shared/models/videos/channel-sync/video-channel-sync-state.enum.ts6
-rw-r--r--shared/models/videos/channel-sync/video-channel-sync.model.ts14
-rw-r--r--shared/models/videos/index.ts1
-rw-r--r--shared/server-commands/server/config-command.ts36
-rw-r--r--shared/server-commands/server/server.ts3
-rw-r--r--shared/server-commands/server/servers.ts21
-rw-r--r--shared/server-commands/videos/channel-syncs-command.ts55
-rw-r--r--shared/server-commands/videos/channels-command.ts18
-rw-r--r--shared/server-commands/videos/index.ts1
-rw-r--r--support/doc/api/openapi.yaml143
90 files changed, 2945 insertions, 360 deletions
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html
index 7dfe5f5f9..929ea3a90 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html
+++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html
@@ -267,10 +267,10 @@
267 inputName="importVideosHttpEnabled" formControlName="enabled" 267 inputName="importVideosHttpEnabled" formControlName="enabled"
268 i18n-labelText labelText="Allow import with HTTP URL (e.g. YouTube)" 268 i18n-labelText labelText="Allow import with HTTP URL (e.g. YouTube)"
269 > 269 >
270 <ng-container ngProjectAs="description"> 270 <ng-container ngProjectAs="description">
271 <span i18n>⚠️ If enabled, we recommend to use <a class="link-orange" href="https://docs.joinpeertube.org/maintain-configuration?id=security">a HTTP proxy</a> to prevent private URL access from your PeerTube server</span> 271 <span i18n>⚠️ If enabled, we recommend to use <a class="link-orange" href="https://docs.joinpeertube.org/maintain-configuration?id=security">a HTTP proxy</a> to prevent private URL access from your PeerTube server</span>
272 </ng-container> 272 </ng-container>
273 </my-peertube-checkbox> 273 </my-peertube-checkbox>
274 </div> 274 </div>
275 275
276 <div class="form-group" formGroupName="torrent"> 276 <div class="form-group" formGroupName="torrent">
@@ -285,6 +285,22 @@
285 </div> 285 </div>
286 286
287 </ng-container> 287 </ng-container>
288
289 <ng-container formGroupName="videoChannelSynchronization">
290 <div class="form-group">
291 <my-peertube-checkbox
292 inputName="importSynchronizationEnabled" formControlName="enabled"
293 i18n-labelText labelText="Allow channel synchronization with channel of other platforms like YouTube (requires allowing import with HTTP URL)"
294 >
295 <ng-container ngProjectAs="description">
296 <span i18n [hidden]="isImportVideosHttpEnabled()">
297 ⛔ You need to allow import with HTTP URL to be able to activate this feature.
298 </span>
299 </ng-container>
300 </my-peertube-checkbox>
301 </div>
302 </ng-container>
303
288 </ng-container> 304 </ng-container>
289 305
290 <ng-container formGroupName="autoBlacklist"> 306 <ng-container formGroupName="autoBlacklist">
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts
index 29910369a..90ed58c99 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts
+++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts
@@ -25,11 +25,12 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
25 private configService: ConfigService, 25 private configService: ConfigService,
26 private menuService: MenuService, 26 private menuService: MenuService,
27 private themeService: ThemeService 27 private themeService: ThemeService
28 ) { } 28 ) {}
29 29
30 ngOnInit () { 30 ngOnInit () {
31 this.buildLandingPageOptions() 31 this.buildLandingPageOptions()
32 this.checkSignupField() 32 this.checkSignupField()
33 this.checkImportSyncField()
33 34
34 this.availableThemes = this.themeService.buildAvailableThemes() 35 this.availableThemes = this.themeService.buildAvailableThemes()
35 } 36 }
@@ -67,6 +68,14 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
67 return { 'disabled-checkbox-extra': !this.isSignupEnabled() } 68 return { 'disabled-checkbox-extra': !this.isSignupEnabled() }
68 } 69 }
69 70
71 isImportVideosHttpEnabled (): boolean {
72 return this.form.value['import']['videos']['http']['enabled'] === true
73 }
74
75 importSynchronizationChecked () {
76 return this.isImportVideosHttpEnabled() && this.form.value['import']['videoChannelSynchronization']['enabled']
77 }
78
70 hasUnlimitedSignup () { 79 hasUnlimitedSignup () {
71 return this.form.value['signup']['limit'] === -1 80 return this.form.value['signup']['limit'] === -1
72 } 81 }
@@ -97,6 +106,21 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
97 return this.themeService.getDefaultThemeLabel() 106 return this.themeService.getDefaultThemeLabel()
98 } 107 }
99 108
109 private checkImportSyncField () {
110 const importSyncControl = this.form.get('import.videoChannelSynchronization.enabled')
111 const importVideosHttpControl = this.form.get('import.videos.http.enabled')
112
113 importVideosHttpControl.valueChanges
114 .subscribe((httpImportEnabled) => {
115 importSyncControl.setValue(httpImportEnabled && importSyncControl.value)
116 if (httpImportEnabled) {
117 importSyncControl.enable()
118 } else {
119 importSyncControl.disable()
120 }
121 })
122 }
123
100 private checkSignupField () { 124 private checkSignupField () {
101 const signupControl = this.form.get('signup.enabled') 125 const signupControl = this.form.get('signup.enabled')
102 126
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
index ce01f8b59..5cab9e9df 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
+++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
@@ -144,6 +144,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
144 torrent: { 144 torrent: {
145 enabled: null 145 enabled: null
146 } 146 }
147 },
148 videoChannelSynchronization: {
149 enabled: null
147 } 150 }
148 }, 151 },
149 trending: { 152 trending: {
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts
index 42f503be6..4cda63272 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.ts
+++ b/client/src/app/+admin/system/jobs/jobs.component.ts
@@ -38,7 +38,8 @@ export class JobsComponent extends RestTable implements OnInit {
38 'video-redundancy', 38 'video-redundancy',
39 'video-transcoding', 39 'video-transcoding',
40 'videos-views-stats', 40 'videos-views-stats',
41 'move-to-object-storage' 41 'move-to-object-storage',
42 'video-channel-import'
42 ] 43 ]
43 44
44 jobs: Job[] = [] 45 jobs: Job[] = []
diff --git a/client/src/app/+manage/video-channel-edit/video-channel-edit.component.html b/client/src/app/+manage/video-channel-edit/video-channel-edit.component.html
index b557fb011..b93dc2b12 100644
--- a/client/src/app/+manage/video-channel-edit/video-channel-edit.component.html
+++ b/client/src/app/+manage/video-channel-edit/video-channel-edit.component.html
@@ -61,7 +61,7 @@
61 </div> 61 </div>
62 62
63 <div class="form-group"> 63 <div class="form-group">
64 <label for="support">Support</label> 64 <label i18n for="support">Support</label>
65 <my-help 65 <my-help
66 helpType="markdownEnhanced" i18n-preHtml preHtml="Short text to tell people how they can support the channel (membership platform...).<br /><br /> 66 helpType="markdownEnhanced" i18n-preHtml preHtml="Short text to tell people how they can support the channel (membership platform...).<br /><br />
67 When a video is uploaded in this channel, the video support field will be automatically filled by this text." 67 When a video is uploaded in this channel, the video support field will be automatically filled by this text."
diff --git a/client/src/app/+my-library/+my-video-channels/my-video-channels.component.html b/client/src/app/+my-library/+my-video-channels/my-video-channels.component.html
index e942e002b..a48731e7c 100644
--- a/client/src/app/+my-library/+my-video-channels/my-video-channels.component.html
+++ b/client/src/app/+my-library/+my-video-channels/my-video-channels.component.html
@@ -1,7 +1,16 @@
1<h1> 1<h1>
2 <my-global-icon iconName="channel" aria-hidden="true"></my-global-icon> 2 <span>
3 <ng-container i18n>My channels</ng-container> 3 <my-global-icon iconName="channel" aria-hidden="true"></my-global-icon>
4 <span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span> 4 <ng-container i18n>My channels</ng-container>
5 <span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span>
6 </span>
7
8 <div>
9 <a routerLink="/my-library/video-channel-syncs" class="button-link">
10 <my-global-icon iconName="repeat" aria-hidden="true"></my-global-icon>
11 <ng-container i18n>My synchronizations</ng-container>
12 </a>
13 </div>
5</h1> 14</h1>
6 15
7<my-channels-setup-message [hideLink]="true"></my-channels-setup-message> 16<my-channels-setup-message [hideLink]="true"></my-channels-setup-message>
diff --git a/client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss b/client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss
index ab80f3d01..6c5be9240 100644
--- a/client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss
+++ b/client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss
@@ -1,9 +1,20 @@
1@use '_variables' as *; 1@use '_variables' as *;
2@use '_mixins' as *; 2@use '_mixins' as *;
3 3
4h1 my-global-icon { 4h1 {
5 position: relative; 5 display: flex;
6 top: -2px; 6 justify-content: space-between;
7
8 my-global-icon {
9 position: relative;
10 top: -2px;
11 }
12
13 .button-link {
14 @include peertube-button-link;
15 @include grey-button;
16 @include button-with-icon(18px, 3px, -1px);
17 }
7} 18}
8 19
9.create-button { 20.create-button {
diff --git a/client/src/app/+my-library/my-library-routing.module.ts b/client/src/app/+my-library/my-library-routing.module.ts
index 73858fb82..de3ef4d96 100644
--- a/client/src/app/+my-library/my-library-routing.module.ts
+++ b/client/src/app/+my-library/my-library-routing.module.ts
@@ -6,6 +6,8 @@ import { MySubscriptionsComponent } from './my-follows/my-subscriptions.componen
6import { MyHistoryComponent } from './my-history/my-history.component' 6import { MyHistoryComponent } from './my-history/my-history.component'
7import { MyLibraryComponent } from './my-library.component' 7import { MyLibraryComponent } from './my-library.component'
8import { MyOwnershipComponent } from './my-ownership/my-ownership.component' 8import { MyOwnershipComponent } from './my-ownership/my-ownership.component'
9import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
10import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
9import { MyVideoImportsComponent } from './my-video-imports/my-video-imports.component' 11import { MyVideoImportsComponent } from './my-video-imports/my-video-imports.component'
10import { MyVideoPlaylistCreateComponent } from './my-video-playlists/my-video-playlist-create.component' 12import { MyVideoPlaylistCreateComponent } from './my-video-playlists/my-video-playlist-create.component'
11import { MyVideoPlaylistElementsComponent } from './my-video-playlists/my-video-playlist-elements.component' 13import { MyVideoPlaylistElementsComponent } from './my-video-playlists/my-video-playlist-elements.component'
@@ -131,6 +133,26 @@ const myLibraryRoutes: Routes = [
131 key: 'my-videos-history-list' 133 key: 'my-videos-history-list'
132 } 134 }
133 } 135 }
136 },
137
138 {
139 path: 'video-channel-syncs',
140 component: MyVideoChannelSyncsComponent,
141 data: {
142 meta: {
143 title: $localize`My synchronizations`
144 }
145 }
146 },
147
148 {
149 path: 'video-channel-syncs/create',
150 component: VideoChannelSyncEditComponent,
151 data: {
152 meta: {
153 title: $localize`Create new synchronization`
154 }
155 }
134 } 156 }
135 ] 157 ]
136 } 158 }
diff --git a/client/src/app/+my-library/my-library.module.ts b/client/src/app/+my-library/my-library.module.ts
index bfafcb3e4..4acb3b75e 100644
--- a/client/src/app/+my-library/my-library.module.ts
+++ b/client/src/app/+my-library/my-library.module.ts
@@ -29,6 +29,8 @@ import { MyVideoPlaylistUpdateComponent } from './my-video-playlists/my-video-pl
29import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlists.component' 29import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlists.component'
30import { VideoChangeOwnershipComponent } from './my-videos/modals/video-change-ownership.component' 30import { VideoChangeOwnershipComponent } from './my-videos/modals/video-change-ownership.component'
31import { MyVideosComponent } from './my-videos/my-videos.component' 31import { MyVideosComponent } from './my-videos/my-videos.component'
32import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
33import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
32 34
33@NgModule({ 35@NgModule({
34 imports: [ 36 imports: [
@@ -63,6 +65,8 @@ import { MyVideosComponent } from './my-videos/my-videos.component'
63 MyOwnershipComponent, 65 MyOwnershipComponent,
64 MyAcceptOwnershipComponent, 66 MyAcceptOwnershipComponent,
65 MyVideoImportsComponent, 67 MyVideoImportsComponent,
68 MyVideoChannelSyncsComponent,
69 VideoChannelSyncEditComponent,
66 MySubscriptionsComponent, 70 MySubscriptionsComponent,
67 MyFollowersComponent, 71 MyFollowersComponent,
68 MyHistoryComponent, 72 MyHistoryComponent,
diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html
new file mode 100644
index 000000000..5141607b1
--- /dev/null
+++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html
@@ -0,0 +1,83 @@
1<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
2
3<h1>
4 <my-global-icon iconName="refresh" aria-hidden="true"></my-global-icon>
5 <ng-container i18n>My synchronizations</ng-container>
6</h1>
7
8<div *ngIf="!syncEnabled()">
9 <p class="muted" i18n>⚠️ The instance doesn't allow channel synchronization</p>
10</div>
11
12<p-table
13 *ngIf="syncEnabled()" [value]="channelSyncs" [lazy]="true"
14 [paginator]="totalRecords > 0" [totalRecords]="totalRecords" [rows]="rowsPerPage" [rowsPerPageOptions]="rowsPerPageOptions"
15 [sortField]="sort.field" [sortOrder]="sort.order" (onLazyLoad)="loadLazy($event)" dataKey="id"
16 [showCurrentPageReport]="true" i18n-currentPageReportTemplate
17 currentPageReportTemplate="Showing {{'{first}'}} to {{'{last}'}} of {{'{totalRecords}'}} synchronizations"
18 [expandedRowKeys]="expandedRows"
19>
20 <ng-template pTemplate="caption">
21 <div class="caption">
22 <div class="left-buttons">
23 <a class="add-sync" routerLink="{{ getSyncCreateLink() }}">
24 <my-global-icon iconName="add" aria-hidden="true"></my-global-icon>
25 <ng-container i18n>Add synchronization</ng-container>
26 </a>
27 </div>
28 </div>
29 </ng-template>
30
31 <ng-template pTemplate="header">
32 <tr>
33 <th style="width: 10%"><my-global-icon iconName="columns"></my-global-icon></th>
34 <th style="width: 25%" i18n pSortableColumn="externalChannelUrl">External Channel <p-sortIcon field="externalChannelUrl"></p-sortIcon></th>
35 <th style="width: 25%" i18n pSortableColumn="videoChannel">Channel <p-sortIcon field="videoChannel"></p-sortIcon></th>
36 <th style="width: 10%" i18n pSortableColumn="state">State <p-sortIcon field="state"></p-sortIcon></th>
37 <th style="width: 10%" i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
38 <th style="width: 10%" i18n pSortableColumn="lastSyncAt">Last synchronization at <p-sortIcon field="lastSyncAt"></p-sortIcon></th>
39 </tr>
40 </ng-template>
41
42 <ng-template pTemplate="body" let-expanded="expanded" let-videoChannelSync>
43 <tr>
44 <td class="action-cell">
45 <my-action-dropdown
46 container="body"
47 [actions]="videoChannelSyncActions" [entry]="videoChannelSync"
48 ></my-action-dropdown>
49 </td>
50
51 <td>
52 <a [href]="videoChannelSync.externalChannelUrl" target="_blank" rel="noopener noreferrer">{{ videoChannelSync.externalChannelUrl }}</a>
53 </td>
54
55 <td>
56 <div class="actor">
57 <my-actor-avatar
58 class="channel"
59 [actor]="videoChannelSync.channel" actorType="channel"
60 [internalHref]="[ '/c', videoChannelSync.channel.name ]"
61 size="25"
62 ></my-actor-avatar>
63
64 <div class="actor-info">
65 <a [routerLink]="[ '/c', videoChannelSync.channel.name ]" class="actor-names" i18n-title title="Channel page">
66 <div class="actor-display-name">{{ videoChannelSync.channel.displayName }}</div>
67 <div class="actor-name">{{ videoChannelSync.channel.name }}</div>
68 </a>
69 </div>
70 </div>
71 </td>
72
73 <td>
74 <span [ngClass]="getSyncStateClass(videoChannelSync.state.id)">
75 {{ videoChannelSync.state.label }}
76 </span>
77 </td>
78
79 <td>{{ videoChannelSync.createdAt | date: 'short' }}</td>
80 <td>{{ videoChannelSync.lastSyncAt | date: 'short' }}</td>
81 </tr>
82 </ng-template>
83</p-table>
diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss
new file mode 100644
index 000000000..88738e54d
--- /dev/null
+++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss
@@ -0,0 +1,14 @@
1@use '_mixins' as *;
2@use '_variables' as *;
3@use '_actor' as *;
4
5.add-sync {
6 @include create-button;
7}
8
9.actor {
10 @include actor-row($min-height: auto, $separator: true);
11 margin-bottom: 0;
12 padding-bottom: 0;
13 border: 0;
14}
diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts
new file mode 100644
index 000000000..81bdaf9f2
--- /dev/null
+++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts
@@ -0,0 +1,129 @@
1import { Component, OnInit } from '@angular/core'
2import { AuthService, Notifier, RestPagination, RestTable, ServerService } from '@app/core'
3import { DropdownAction, VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
4import { HTMLServerConfig } from '@shared/models/server'
5import { VideoChannelSync, VideoChannelSyncState } from '@shared/models/videos'
6import { SortMeta } from 'primeng/api'
7import { mergeMap } from 'rxjs'
8
9@Component({
10 templateUrl: './my-video-channel-syncs.component.html',
11 styleUrls: [ './my-video-channel-syncs.component.scss' ]
12})
13export class MyVideoChannelSyncsComponent extends RestTable implements OnInit {
14 error: string
15
16 channelSyncs: VideoChannelSync[] = []
17 totalRecords = 0
18
19 videoChannelSyncActions: DropdownAction<VideoChannelSync>[][] = []
20 sort: SortMeta = { field: 'createdAt', order: 1 }
21 pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
22
23 private static STATE_CLASS_BY_ID = {
24 [VideoChannelSyncState.FAILED]: 'badge-red',
25 [VideoChannelSyncState.PROCESSING]: 'badge-blue',
26 [VideoChannelSyncState.SYNCED]: 'badge-green',
27 [VideoChannelSyncState.WAITING_FIRST_RUN]: 'badge-yellow'
28 }
29
30 private serverConfig: HTMLServerConfig
31
32 constructor (
33 private videoChannelsSyncService: VideoChannelSyncService,
34 private serverService: ServerService,
35 private notifier: Notifier,
36 private authService: AuthService,
37 private videoChannelService: VideoChannelService
38 ) {
39 super()
40 }
41
42 ngOnInit () {
43 this.serverConfig = this.serverService.getHTMLConfig()
44 this.initialize()
45
46 this.videoChannelSyncActions = [
47 [
48 {
49 label: $localize`Delete`,
50 iconName: 'delete',
51 handler: videoChannelSync => this.deleteSync(videoChannelSync)
52 },
53 {
54 label: $localize`Fully synchronize the channel`,
55 description: $localize`This fetches any missing videos on the local channel`,
56 iconName: 'refresh',
57 handler: videoChannelSync => this.fullySynchronize(videoChannelSync)
58 }
59 ]
60 ]
61 }
62
63 protected reloadData () {
64 this.error = undefined
65
66 this.authService.userInformationLoaded
67 .pipe(mergeMap(() => {
68 const user = this.authService.getUser()
69 return this.videoChannelsSyncService.listAccountVideoChannelsSyncs({
70 sort: this.sort,
71 account: user.account,
72 pagination: this.pagination
73 })
74 }))
75 .subscribe({
76 next: res => {
77 this.channelSyncs = res.data
78 },
79 error: err => {
80 this.error = err.message
81 }
82 })
83 }
84
85 syncEnabled () {
86 return this.serverConfig.import.videoChannelSynchronization.enabled
87 }
88
89 deleteSync (videoChannelSync: VideoChannelSync) {
90 this.videoChannelsSyncService.deleteSync(videoChannelSync.id)
91 .subscribe({
92 next: () => {
93 this.notifier.success($localize`Synchronization removed successfully for ${videoChannelSync.channel.displayName}.`)
94 this.reloadData()
95 },
96 error: err => {
97 this.error = err.message
98 }
99 })
100 }
101
102 fullySynchronize (videoChannelSync: VideoChannelSync) {
103 this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
104 .subscribe({
105 next: () => {
106 this.notifier.success($localize`Full synchronization requested successfully for ${videoChannelSync.channel.displayName}.`)
107 },
108 error: err => {
109 this.error = err.message
110 }
111 })
112 }
113
114 getSyncCreateLink () {
115 return '/my-library/video-channel-syncs/create'
116 }
117
118 getSyncStateClass (stateId: number) {
119 return [ 'pt-badge', MyVideoChannelSyncsComponent.STATE_CLASS_BY_ID[stateId] ]
120 }
121
122 getIdentifier () {
123 return 'MyVideoChannelsSyncComponent'
124 }
125
126 getChannelUrl (name: string) {
127 return '/c/' + name
128 }
129}
diff --git a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html
new file mode 100644
index 000000000..611146c1a
--- /dev/null
+++ b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html
@@ -0,0 +1,64 @@
1<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
2
3<div class="margin-content">
4 <form role="form" (ngSubmit)="formValidated()" [formGroup]="form">
5
6 <div class="row">
7 <div class="col-12 col-lg-4 col-xl-3">
8 <div class="video-channel-sync-title" i18n>NEW SYNCHRONIZATION</div>
9 </div>
10
11 <div class="col-12 col-lg-8 col-xl-9">
12 <div class="form-group">
13 <label i18n for="externalChannelUrl">Remote channel URL</label>
14
15 <div class="input-group">
16 <input
17 type="text"
18 id="externalChannelUrl"
19 i18n-placeholder
20 placeholder="Example: https://youtube.com/channel/UC_fancy_channel"
21 formControlName="externalChannelUrl"
22 [ngClass]="{ 'input-error': formErrors['externalChannelUrl'] }"
23 class="form-control"
24 >
25 </div>
26
27 <div *ngIf="formErrors['externalChannelUrl']" class="form-error">
28 {{ formErrors['externalChannelUrl'] }}
29 </div>
30 </div>
31
32 <div class="form-group">
33 <label i18n for="videoChannel">Video Channel</label>
34 <my-select-channel required [items]="userVideoChannels" formControlName="videoChannel"></my-select-channel>
35
36 <div *ngIf="formErrors['videoChannel']" class="form-error">
37 {{ formErrors['videoChannel'] }}
38 </div>
39 </div>
40
41 <div class="form-group">
42 <label for="existingVideoStrategy" i18n>Options for existing videos on remote channel:</label>
43
44 <div class="peertube-radio-container">
45 <input type="radio" name="existingVideoStrategy" id="import" value="import" formControlName="existingVideoStrategy" required />
46 <label for="import" i18n>Import all and watch for new publications</label>
47 </div>
48
49 <div class="peertube-radio-container">
50 <input type="radio" name="existingVideoStrategy" id="doNothing" value="nothing" formControlName="existingVideoStrategy" required />
51 <label for="doNothing" i18n>Only watch for new publications</label>
52 </div>
53 </div>
54 </div>
55 </div>
56
57 <div class="row"> <!-- submit placement block -->
58 <div class="col-md-7 col-xl-5"></div>
59 <div class="col-md-5 col-xl-5 d-inline-flex">
60 <input type="submit" class="peertube-button orange-button ms-auto" value="{{ getFormButtonTitle() }}" [disabled]="!form.valid">
61 </div>
62 </div>
63 </form>
64</div>
diff --git a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss
new file mode 100644
index 000000000..d0d8c2a68
--- /dev/null
+++ b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss
@@ -0,0 +1,17 @@
1@use '_variables' as *;
2@use '_mixins' as *;
3
4$form-base-input-width: 480px;
5
6input[type=text] {
7 @include peertube-input-text($form-base-input-width);
8}
9
10.video-channel-sync-title {
11 @include settings-big-title;
12}
13
14my-select-channel {
15 display: block;
16 max-width: $form-base-input-width;
17}
diff --git a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts
new file mode 100644
index 000000000..836582609
--- /dev/null
+++ b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts
@@ -0,0 +1,76 @@
1import { mergeMap } from 'rxjs'
2import { SelectChannelItem } from 'src/types'
3import { Component, OnInit } from '@angular/core'
4import { Router } from '@angular/router'
5import { AuthService, Notifier } from '@app/core'
6import { listUserChannelsForSelect } from '@app/helpers'
7import { VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR } from '@app/shared/form-validators/video-channel-validators'
8import { FormReactive, FormValidatorService } from '@app/shared/shared-forms'
9import { VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
10import { VideoChannelSyncCreate } from '@shared/models/videos'
11
12@Component({
13 selector: 'my-video-channel-sync-edit',
14 templateUrl: './video-channel-sync-edit.component.html',
15 styleUrls: [ './video-channel-sync-edit.component.scss' ]
16})
17export class VideoChannelSyncEditComponent extends FormReactive implements OnInit {
18 error: string
19 userVideoChannels: SelectChannelItem[] = []
20 existingVideosStrategy: string
21
22 constructor (
23 protected formValidatorService: FormValidatorService,
24 private authService: AuthService,
25 private router: Router,
26 private notifier: Notifier,
27 private videoChannelSyncService: VideoChannelSyncService,
28 private videoChannelService: VideoChannelService
29 ) {
30 super()
31 }
32
33 ngOnInit () {
34 this.buildForm({
35 externalChannelUrl: VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR,
36 videoChannel: null,
37 existingVideoStrategy: null
38 })
39
40 listUserChannelsForSelect(this.authService)
41 .subscribe(channels => this.userVideoChannels = channels)
42 }
43
44 getFormButtonTitle () {
45 return $localize`Create`
46 }
47
48 formValidated () {
49 this.error = undefined
50
51 const body = this.form.value
52 const videoChannelSyncCreate: VideoChannelSyncCreate = {
53 externalChannelUrl: body.externalChannelUrl,
54 videoChannelId: body.videoChannel
55 }
56
57 const importExistingVideos = body['existingVideoStrategy'] === 'import'
58
59 this.videoChannelSyncService.createSync(videoChannelSyncCreate)
60 .pipe(mergeMap(({ videoChannelSync }) => {
61 return importExistingVideos
62 ? this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
63 : Promise.resolve(null)
64 }))
65 .subscribe({
66 next: () => {
67 this.notifier.success($localize`Synchronization created successfully.`)
68 this.router.navigate([ '/my-library', 'video-channel-syncs' ])
69 },
70
71 error: err => {
72 this.error = err.message
73 }
74 })
75 }
76}
diff --git a/client/src/app/shared/form-validators/video-channel-validators.ts b/client/src/app/shared/form-validators/video-channel-validators.ts
index 163faf270..b12b3caaf 100644
--- a/client/src/app/shared/form-validators/video-channel-validators.ts
+++ b/client/src/app/shared/form-validators/video-channel-validators.ts
@@ -48,3 +48,16 @@ export const VIDEO_CHANNEL_SUPPORT_VALIDATOR: BuildFormValidator = {
48 maxlength: $localize`Support text cannot be more than 1000 characters long.` 48 maxlength: $localize`Support text cannot be more than 1000 characters long.`
49 } 49 }
50} 50}
51
52export const VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR: BuildFormValidator = {
53 VALIDATORS: [
54 Validators.required,
55 Validators.pattern(/^https?:\/\//),
56 Validators.maxLength(1000)
57 ],
58 MESSAGES: {
59 required: $localize`Remote channel url is required.`,
60 pattern: $localize`External channel URL must begin with "https://" or "http://"`,
61 maxlength: $localize`External channel URL cannot be more than 1000 characters long`
62 }
63}
diff --git a/client/src/app/shared/shared-instance/instance-features-table.component.html b/client/src/app/shared/shared-instance/instance-features-table.component.html
index 761243bfe..6c05764df 100644
--- a/client/src/app/shared/shared-instance/instance-features-table.component.html
+++ b/client/src/app/shared/shared-instance/instance-features-table.component.html
@@ -107,6 +107,13 @@
107 </tr> 107 </tr>
108 108
109 <tr> 109 <tr>
110 <th i18n class="sub-label" scope="row">Channel synchronization with other platforms (YouTube, Vimeo, ...)</th>
111 <td>
112 <my-feature-boolean [value]="serverConfig.import.videoChannelSynchronization.enabled"></my-feature-boolean>
113 </td>
114 </tr>
115
116 <tr>
110 <th i18n class="label" colspan="2">Search</th> 117 <th i18n class="label" colspan="2">Search</th>
111 </tr> 118 </tr>
112 119
diff --git a/client/src/app/shared/shared-main/index.ts b/client/src/app/shared/shared-main/index.ts
index 3a7fd4c34..9faa28e32 100644
--- a/client/src/app/shared/shared-main/index.ts
+++ b/client/src/app/shared/shared-main/index.ts
@@ -13,3 +13,4 @@ export * from './video'
13export * from './video-caption' 13export * from './video-caption'
14export * from './video-channel' 14export * from './video-channel'
15export * from './shared-main.module' 15export * from './shared-main.module'
16export * from './video-channel-sync'
diff --git a/client/src/app/shared/shared-main/video-channel-sync/index.ts b/client/src/app/shared/shared-main/video-channel-sync/index.ts
new file mode 100644
index 000000000..7134bcd18
--- /dev/null
+++ b/client/src/app/shared/shared-main/video-channel-sync/index.ts
@@ -0,0 +1 @@
export * from './video-channel-sync.service'
diff --git a/client/src/app/shared/shared-main/video-channel-sync/video-channel-sync.service.ts b/client/src/app/shared/shared-main/video-channel-sync/video-channel-sync.service.ts
new file mode 100644
index 000000000..a4e216869
--- /dev/null
+++ b/client/src/app/shared/shared-main/video-channel-sync/video-channel-sync.service.ts
@@ -0,0 +1,50 @@
1import { SortMeta } from 'primeng/api'
2import { catchError, Observable } from 'rxjs'
3import { environment } from 'src/environments/environment'
4import { HttpClient, HttpParams } from '@angular/common/http'
5import { Injectable } from '@angular/core'
6import { RestExtractor, RestPagination, RestService } from '@app/core'
7import { ResultList } from '@shared/models/common'
8import { VideoChannelSync, VideoChannelSyncCreate } from '@shared/models/videos'
9import { Account, AccountService } from '../account'
10
11@Injectable({
12 providedIn: 'root'
13})
14export class VideoChannelSyncService {
15 static BASE_VIDEO_CHANNEL_URL = environment.apiUrl + '/api/v1/video-channel-syncs'
16
17 constructor (
18 private authHttp: HttpClient,
19 private restExtractor: RestExtractor,
20 private restService: RestService
21 ) { }
22
23 listAccountVideoChannelsSyncs (parameters: {
24 sort: SortMeta
25 pagination: RestPagination
26 account: Account
27 }): Observable<ResultList<VideoChannelSync>> {
28 const { pagination, sort, account } = parameters
29
30 let params = new HttpParams()
31 params = this.restService.addRestGetParams(params, pagination, sort)
32
33 const url = AccountService.BASE_ACCOUNT_URL + account.nameWithHost + '/video-channel-syncs'
34
35 return this.authHttp.get<ResultList<VideoChannelSync>>(url, { params })
36 .pipe(catchError(err => this.restExtractor.handleError(err)))
37 }
38
39 createSync (body: VideoChannelSyncCreate) {
40 return this.authHttp.post<{ videoChannelSync: VideoChannelSync }>(VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL, body)
41 .pipe(catchError(err => this.restExtractor.handleError(err)))
42 }
43
44 deleteSync (videoChannelsSyncId: number) {
45 const url = `${VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL}/${videoChannelsSyncId}`
46
47 return this.authHttp.delete(url)
48 .pipe(catchError(err => this.restExtractor.handleError(err)))
49 }
50}
diff --git a/client/src/app/shared/shared-main/video-channel/video-channel.service.ts b/client/src/app/shared/shared-main/video-channel/video-channel.service.ts
index 480d250fb..fa97025ac 100644
--- a/client/src/app/shared/shared-main/video-channel/video-channel.service.ts
+++ b/client/src/app/shared/shared-main/video-channel/video-channel.service.ts
@@ -95,4 +95,10 @@ export class VideoChannelService {
95 return this.authHttp.delete(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannel.nameWithHost) 95 return this.authHttp.delete(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannel.nameWithHost)
96 .pipe(catchError(err => this.restExtractor.handleError(err))) 96 .pipe(catchError(err => this.restExtractor.handleError(err)))
97 } 97 }
98
99 importVideos (videoChannelName: string, externalChannelUrl: string) {
100 const path = VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannelName + '/import-videos'
101 return this.authHttp.post(path, { externalChannelUrl })
102 .pipe(catchError(err => this.restExtractor.handleError(err)))
103 }
98} 104}
diff --git a/config/default.yaml b/config/default.yaml
index 3a577d31d..9bf1ca284 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -546,6 +546,17 @@ import:
546 # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information 546 # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
547 enabled: false 547 enabled: false
548 548
549 # Add ability for your users to synchronize their channels with external channels, playlists, etc
550 video_channel_synchronization:
551 enabled: false
552
553 max_per_user: 10
554
555 check_interval: 1 hour
556
557 # Number of latest published videos to check and to potentially import when syncing a channel
558 videos_limit_per_synchronization: 10
559
549auto_blacklist: 560auto_blacklist:
550 # New videos automatically blacklisted so moderators can review before publishing 561 # New videos automatically blacklisted so moderators can review before publishing
551 videos: 562 videos:
diff --git a/config/dev.yaml b/config/dev.yaml
index 15e239b29..ca93874d2 100644
--- a/config/dev.yaml
+++ b/config/dev.yaml
@@ -81,6 +81,11 @@ import:
81 enabled: true 81 enabled: true
82 torrent: 82 torrent:
83 enabled: true 83 enabled: true
84 video_channel_synchronization:
85 enabled: true
86 max_per_user: 10
87 check_interval: 5 minutes
88 videos_limit_per_synchronization: 3
84 89
85instance: 90instance:
86 default_nsfw_policy: 'display' 91 default_nsfw_policy: 'display'
diff --git a/config/production.yaml.example b/config/production.yaml.example
index b5ea7fec5..f6dc6ccdb 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -556,6 +556,17 @@ import:
556 # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information 556 # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
557 enabled: false 557 enabled: false
558 558
559 # Add ability for your users to synchronize their channels with external channels, playlists, etc.
560 video_channel_synchronization:
561 enabled: false
562
563 max_per_user: 10
564
565 check_interval: 1 hour
566
567 # Number of latest published videos to check and to potentially import when syncing a channel
568 videos_limit_per_synchronization: 10
569
559auto_blacklist: 570auto_blacklist:
560 # New videos automatically blacklisted so moderators can review before publishing 571 # New videos automatically blacklisted so moderators can review before publishing
561 videos: 572 videos:
diff --git a/server.ts b/server.ts
index 3b9353e2f..6073d2ea4 100644
--- a/server.ts
+++ b/server.ts
@@ -139,6 +139,7 @@ import { VideoViewsManager } from '@server/lib/views/video-views-manager'
139import { isTestOrDevInstance } from './server/helpers/core-utils' 139import { isTestOrDevInstance } from './server/helpers/core-utils'
140import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics' 140import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
141import { ApplicationModel } from '@server/models/application/application' 141import { ApplicationModel } from '@server/models/application/application'
142import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
142 143
143// ----------- Command line ----------- 144// ----------- Command line -----------
144 145
@@ -314,6 +315,7 @@ async function startApplication () {
314 PeerTubeVersionCheckScheduler.Instance.enable() 315 PeerTubeVersionCheckScheduler.Instance.enable()
315 AutoFollowIndexInstances.Instance.enable() 316 AutoFollowIndexInstances.Instance.enable()
316 RemoveDanglingResumableUploadsScheduler.Instance.enable() 317 RemoveDanglingResumableUploadsScheduler.Instance.enable()
318 VideoChannelSyncLatestScheduler.Instance.enable()
317 VideoViewsBufferScheduler.Instance.enable() 319 VideoViewsBufferScheduler.Instance.enable()
318 GeoIPUpdateScheduler.Instance.enable() 320 GeoIPUpdateScheduler.Instance.enable()
319 OpenTelemetryMetrics.Instance.registerMetrics() 321 OpenTelemetryMetrics.Instance.registerMetrics()
diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts
index 66cdaab82..7a530cde5 100644
--- a/server/controllers/api/accounts.ts
+++ b/server/controllers/api/accounts.ts
@@ -25,8 +25,10 @@ import {
25 accountsFollowersSortValidator, 25 accountsFollowersSortValidator,
26 accountsSortValidator, 26 accountsSortValidator,
27 ensureAuthUserOwnsAccountValidator, 27 ensureAuthUserOwnsAccountValidator,
28 ensureCanManageUser,
28 videoChannelsSortValidator, 29 videoChannelsSortValidator,
29 videoChannelStatsValidator, 30 videoChannelStatsValidator,
31 videoChannelSyncsSortValidator,
30 videosSortValidator 32 videosSortValidator
31} from '../../middlewares/validators' 33} from '../../middlewares/validators'
32import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists' 34import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
@@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate'
35import { VideoModel } from '../../models/video/video' 37import { VideoModel } from '../../models/video/video'
36import { VideoChannelModel } from '../../models/video/video-channel' 38import { VideoChannelModel } from '../../models/video/video-channel'
37import { VideoPlaylistModel } from '../../models/video/video-playlist' 39import { VideoPlaylistModel } from '../../models/video/video-playlist'
40import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
38 41
39const accountsRouter = express.Router() 42const accountsRouter = express.Router()
40 43
@@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels',
72 asyncMiddleware(listAccountChannels) 75 asyncMiddleware(listAccountChannels)
73) 76)
74 77
78accountsRouter.get('/:accountName/video-channel-syncs',
79 authenticate,
80 asyncMiddleware(accountNameWithHostGetValidator),
81 ensureCanManageUser,
82 paginationValidator,
83 videoChannelSyncsSortValidator,
84 setDefaultSort,
85 setDefaultPagination,
86 asyncMiddleware(listAccountChannelsSync)
87)
88
75accountsRouter.get('/:accountName/video-playlists', 89accountsRouter.get('/:accountName/video-playlists',
76 optionalAuthenticate, 90 optionalAuthenticate,
77 asyncMiddleware(accountNameWithHostGetValidator), 91 asyncMiddleware(accountNameWithHostGetValidator),
@@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response)
146 return res.json(getFormattedObjects(resultList.data, resultList.total)) 160 return res.json(getFormattedObjects(resultList.data, resultList.total))
147} 161}
148 162
163async function listAccountChannelsSync (req: express.Request, res: express.Response) {
164 const options = {
165 accountId: res.locals.account.id,
166 start: req.query.start,
167 count: req.query.count,
168 sort: req.query.sort,
169 search: req.query.search
170 }
171
172 const resultList = await VideoChannelSyncModel.listByAccountForAPI(options)
173
174 return res.json(getFormattedObjects(resultList.data, resultList.total))
175}
176
149async function listAccountPlaylists (req: express.Request, res: express.Response) { 177async function listAccountPlaylists (req: express.Request, res: express.Response) {
150 const serverActor = await getServerActor() 178 const serverActor = await getServerActor()
151 179
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index ff2fa9d86..f0fb43071 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -273,6 +273,10 @@ function customConfig (): CustomConfig {
273 torrent: { 273 torrent: {
274 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED 274 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
275 } 275 }
276 },
277 videoChannelSynchronization: {
278 enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
279 maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
276 } 280 }
277 }, 281 },
278 trending: { 282 trending: {
diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts
index d1d4ef765..8c8ebd061 100644
--- a/server/controllers/api/index.ts
+++ b/server/controllers/api/index.ts
@@ -20,6 +20,7 @@ import { usersRouter } from './users'
20import { videoChannelRouter } from './video-channel' 20import { videoChannelRouter } from './video-channel'
21import { videoPlaylistRouter } from './video-playlist' 21import { videoPlaylistRouter } from './video-playlist'
22import { videosRouter } from './videos' 22import { videosRouter } from './videos'
23import { videoChannelSyncRouter } from './video-channel-sync'
23 24
24const apiRouter = express.Router() 25const apiRouter = express.Router()
25 26
@@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter)
43apiRouter.use('/users', usersRouter) 44apiRouter.use('/users', usersRouter)
44apiRouter.use('/accounts', accountsRouter) 45apiRouter.use('/accounts', accountsRouter)
45apiRouter.use('/video-channels', videoChannelRouter) 46apiRouter.use('/video-channels', videoChannelRouter)
47apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
46apiRouter.use('/video-playlists', videoPlaylistRouter) 48apiRouter.use('/video-playlists', videoPlaylistRouter)
47apiRouter.use('/videos', videosRouter) 49apiRouter.use('/videos', videosRouter)
48apiRouter.use('/jobs', jobsRouter) 50apiRouter.use('/jobs', jobsRouter)
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index e09510dc3..4e5333782 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models'
7import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' 7import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
8import { UserRight } from '../../../../shared/models/users' 8import { UserRight } from '../../../../shared/models/users'
9import { authenticate, ensureUserHasRight } from '../../../middlewares' 9import { authenticate, ensureUserHasRight } from '../../../middlewares'
10import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
10 11
11const debugRouter = express.Router() 12const debugRouter = express.Router()
12 13
@@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) {
43 const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = { 44 const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
44 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), 45 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
45 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), 46 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
46 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats() 47 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
48 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
47 } 49 }
48 50
49 await processors[body.command]() 51 await processors[body.command]()
diff --git a/server/controllers/api/video-channel-sync.ts b/server/controllers/api/video-channel-sync.ts
new file mode 100644
index 000000000..c2770b8e4
--- /dev/null
+++ b/server/controllers/api/video-channel-sync.ts
@@ -0,0 +1,76 @@
1import express from 'express'
2import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger'
3import { logger } from '@server/helpers/logger'
4import {
5 asyncMiddleware,
6 asyncRetryTransactionMiddleware,
7 authenticate,
8 ensureCanManageChannel as ensureCanManageSyncedChannel,
9 ensureSyncExists,
10 ensureSyncIsEnabled,
11 videoChannelSyncValidator
12} from '@server/middlewares'
13import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
14import { MChannelSyncFormattable } from '@server/types/models'
15import { HttpStatusCode, VideoChannelSyncState } from '@shared/models'
16
17const videoChannelSyncRouter = express.Router()
18const auditLogger = auditLoggerFactory('channel-syncs')
19
20videoChannelSyncRouter.post('/',
21 authenticate,
22 ensureSyncIsEnabled,
23 asyncMiddleware(videoChannelSyncValidator),
24 ensureCanManageSyncedChannel,
25 asyncRetryTransactionMiddleware(createVideoChannelSync)
26)
27
28videoChannelSyncRouter.delete('/:id',
29 authenticate,
30 asyncMiddleware(ensureSyncExists),
31 ensureCanManageSyncedChannel,
32 asyncRetryTransactionMiddleware(removeVideoChannelSync)
33)
34
35export { videoChannelSyncRouter }
36
37// ---------------------------------------------------------------------------
38
39async function createVideoChannelSync (req: express.Request, res: express.Response) {
40 const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({
41 externalChannelUrl: req.body.externalChannelUrl,
42 videoChannelId: req.body.videoChannelId,
43 state: VideoChannelSyncState.WAITING_FIRST_RUN
44 })
45
46 await syncCreated.save()
47 syncCreated.VideoChannel = res.locals.videoChannel
48
49 auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON()))
50
51 logger.info(
52 'Video synchronization for channel "%s" with external channel "%s" created.',
53 syncCreated.VideoChannel.name,
54 syncCreated.externalChannelUrl
55 )
56
57 return res.json({
58 videoChannelSync: syncCreated.toFormattedJSON()
59 })
60}
61
62async function removeVideoChannelSync (req: express.Request, res: express.Response) {
63 const syncInstance = res.locals.videoChannelSync
64
65 await syncInstance.destroy()
66
67 auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON()))
68
69 logger.info(
70 'Video synchronization for channel "%s" with external channel "%s" deleted.',
71 syncInstance.VideoChannel.name,
72 syncInstance.externalChannelUrl
73 )
74
75 return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
76}
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts
index 6b33e894d..89c7181bd 100644
--- a/server/controllers/api/video-channel.ts
+++ b/server/controllers/api/video-channel.ts
@@ -36,7 +36,9 @@ import {
36 videoPlaylistsSortValidator 36 videoPlaylistsSortValidator
37} from '../../middlewares' 37} from '../../middlewares'
38import { 38import {
39 ensureChannelOwnerCanUpload,
39 ensureIsLocalChannel, 40 ensureIsLocalChannel,
41 videoChannelImportVideosValidator,
40 videoChannelsFollowersSortValidator, 42 videoChannelsFollowersSortValidator,
41 videoChannelsListValidator, 43 videoChannelsListValidator,
42 videoChannelsNameWithHostValidator, 44 videoChannelsNameWithHostValidator,
@@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers',
161 asyncMiddleware(listVideoChannelFollowers) 163 asyncMiddleware(listVideoChannelFollowers)
162) 164)
163 165
166videoChannelRouter.post('/:nameWithHost/import-videos',
167 authenticate,
168 asyncMiddleware(videoChannelsNameWithHostValidator),
169 videoChannelImportVideosValidator,
170 ensureIsLocalChannel,
171 ensureCanManageChannel,
172 asyncMiddleware(ensureChannelOwnerCanUpload),
173 asyncMiddleware(importVideosInChannel)
174)
175
164// --------------------------------------------------------------------------- 176// ---------------------------------------------------------------------------
165 177
166export { 178export {
@@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res
404 416
405 return res.json(getFormattedObjects(resultList.data, resultList.total)) 417 return res.json(getFormattedObjects(resultList.data, resultList.total))
406} 418}
419
420async function importVideosInChannel (req: express.Request, res: express.Response) {
421 const { externalChannelUrl } = req.body
422
423 await JobQueue.Instance.createJob({
424 type: 'video-channel-import',
425 payload: {
426 externalChannelUrl,
427 videoChannelId: res.locals.videoChannel.id
428 }
429 })
430
431 logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl)
432
433 return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
434}
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 5a2e1006a..9d7b0260b 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,49 +1,20 @@
1import express from 'express' 1import express from 'express'
2import { move, readFile, remove } from 'fs-extra' 2import { move, readFile } from 'fs-extra'
3import { decode } from 'magnet-uri' 3import { decode } from 'magnet-uri'
4import parseTorrent, { Instance } from 'parse-torrent' 4import parseTorrent, { Instance } from 'parse-torrent'
5import { join } from 'path' 5import { join } from 'path'
6import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' 6import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
7import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' 7import { MThumbnail, MVideoThumbnail } from '@server/types/models'
8import { isResolvingToUnicastOnly } from '@server/helpers/dns' 8import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
9import { Hooks } from '@server/lib/plugins/hooks'
10import { ServerConfigManager } from '@server/lib/server-config-manager'
11import { setVideoTags } from '@server/lib/video'
12import { FilteredModelAttributes } from '@server/types'
13import {
14 MChannelAccountDefault,
15 MThumbnail,
16 MUser,
17 MVideoAccountDefault,
18 MVideoCaption,
19 MVideoTag,
20 MVideoThumbnail,
21 MVideoWithBlacklistLight
22} from '@server/types/models'
23import { MVideoImportFormattable } from '@server/types/models/video/video-import'
24import {
25 HttpStatusCode,
26 ServerErrorCode,
27 ThumbnailType,
28 VideoImportCreate,
29 VideoImportState,
30 VideoPrivacy,
31 VideoState
32} from '@shared/models'
33import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' 9import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
34import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
35import { isArray } from '../../../helpers/custom-validators/misc' 10import { isArray } from '../../../helpers/custom-validators/misc'
36import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' 11import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
37import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
38import { getSecureTorrentName } from '../../../helpers/utils' 13import { getSecureTorrentName } from '../../../helpers/utils'
39import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
40import { CONFIG } from '../../../initializers/config' 14import { CONFIG } from '../../../initializers/config'
41import { MIMETYPES } from '../../../initializers/constants' 15import { MIMETYPES } from '../../../initializers/constants'
42import { sequelizeTypescript } from '../../../initializers/database'
43import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
44import { JobQueue } from '../../../lib/job-queue/job-queue' 16import { JobQueue } from '../../../lib/job-queue/job-queue'
45import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail' 17import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
46import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
47import { 18import {
48 asyncMiddleware, 19 asyncMiddleware,
49 asyncRetryTransactionMiddleware, 20 asyncRetryTransactionMiddleware,
@@ -52,9 +23,6 @@ import {
52 videoImportCancelValidator, 23 videoImportCancelValidator,
53 videoImportDeleteValidator 24 videoImportDeleteValidator
54} from '../../../middlewares' 25} from '../../../middlewares'
55import { VideoModel } from '../../../models/video/video'
56import { VideoCaptionModel } from '../../../models/video/video-caption'
57import { VideoImportModel } from '../../../models/video/video-import'
58 26
59const auditLogger = auditLoggerFactory('video-imports') 27const auditLogger = auditLoggerFactory('video-imports')
60const videoImportsRouter = express.Router() 28const videoImportsRouter = express.Router()
@@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
68 authenticate, 36 authenticate,
69 reqVideoFileImport, 37 reqVideoFileImport,
70 asyncMiddleware(videoImportAddValidator), 38 asyncMiddleware(videoImportAddValidator),
71 asyncRetryTransactionMiddleware(addVideoImport) 39 asyncRetryTransactionMiddleware(handleVideoImport)
72) 40)
73 41
74videoImportsRouter.post('/imports/:id/cancel', 42videoImportsRouter.post('/imports/:id/cancel',
@@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
108 return res.sendStatus(HttpStatusCode.NO_CONTENT_204) 76 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
109} 77}
110 78
111function addVideoImport (req: express.Request, res: express.Response) { 79function handleVideoImport (req: express.Request, res: express.Response) {
112 if (req.body.targetUrl) return addYoutubeDLImport(req, res) 80 if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
113 81
114 const file = req.files?.['torrentfile']?.[0] 82 const file = req.files?.['torrentfile']?.[0]
115 if (req.body.magnetUri || file) return addTorrentImport(req, res, file) 83 if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
116} 84}
117 85
118async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { 86async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
119 const body: VideoImportCreate = req.body 87 const body: VideoImportCreate = req.body
120 const user = res.locals.oauth.token.User 88 const user = res.locals.oauth.token.User
121 89
@@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
135 videoName = result.name 103 videoName = result.name
136 } 104 }
137 105
138 const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName }) 106 const video = await buildVideoFromImport({
107 channelId: res.locals.videoChannel.id,
108 importData: { name: videoName },
109 importDataOverride: body,
110 importType: 'torrent'
111 })
139 112
140 const thumbnailModel = await processThumbnail(req, video) 113 const thumbnailModel = await processThumbnail(req, video)
141 const previewModel = await processPreview(req, video) 114 const previewModel = await processPreview(req, video)
142 115
143 const videoImport = await insertIntoDB({ 116 const videoImport = await insertFromImportIntoDB({
144 video, 117 video,
145 thumbnailModel, 118 thumbnailModel,
146 previewModel, 119 previewModel,
@@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
155 } 128 }
156 }) 129 })
157 130
158 // Create job to import the video 131 const payload: VideoImportPayload = {
159 const payload = {
160 type: torrentfile 132 type: torrentfile
161 ? 'torrent-file' as 'torrent-file' 133 ? 'torrent-file'
162 : 'magnet-uri' as 'magnet-uri', 134 : 'magnet-uri',
163 videoImportId: videoImport.id, 135 videoImportId: videoImport.id,
164 magnetUri 136 preventException: false
165 } 137 }
166 await JobQueue.Instance.createJob({ type: 'video-import', payload }) 138 await JobQueue.Instance.createJob({ type: 'video-import', payload })
167 139
@@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
170 return res.json(videoImport.toFormattedJSON()).end() 142 return res.json(videoImport.toFormattedJSON()).end()
171} 143}
172 144
173async function addYoutubeDLImport (req: express.Request, res: express.Response) { 145function statusFromYtDlImportError (err: YoutubeDlImportError): number {
146 switch (err.code) {
147 case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
148 return HttpStatusCode.FORBIDDEN_403
149
150 case YoutubeDlImportError.CODE.FETCH_ERROR:
151 return HttpStatusCode.BAD_REQUEST_400
152
153 default:
154 return HttpStatusCode.INTERNAL_SERVER_ERROR_500
155 }
156}
157
158async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
174 const body: VideoImportCreate = req.body 159 const body: VideoImportCreate = req.body
175 const targetUrl = body.targetUrl 160 const targetUrl = body.targetUrl
176 const user = res.locals.oauth.token.User 161 const user = res.locals.oauth.token.User
177 162
178 const youtubeDL = new YoutubeDLWrapper(
179 targetUrl,
180 ServerConfigManager.Instance.getEnabledResolutions('vod'),
181 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
182 )
183
184 // Get video infos
185 let youtubeDLInfo: YoutubeDLInfo
186 try { 163 try {
187 youtubeDLInfo = await youtubeDL.getInfoForDownload() 164 const { job, videoImport } = await buildYoutubeDLImport({
165 targetUrl,
166 channel: res.locals.videoChannel,
167 importDataOverride: body,
168 thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
169 previewFilePath: req.files?.['previewfile']?.[0].path,
170 user
171 })
172 await JobQueue.Instance.createJob(job)
173
174 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
175
176 return res.json(videoImport.toFormattedJSON()).end()
188 } catch (err) { 177 } catch (err) {
189 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) 178 logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
190 179
191 return res.fail({ 180 return res.fail({
192 message: 'Cannot fetch remote information of this URL.', 181 message: err.message,
182 status: statusFromYtDlImportError(err),
193 data: { 183 data: {
194 targetUrl 184 targetUrl
195 } 185 }
196 }) 186 })
197 } 187 }
198
199 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
200 return res.fail({
201 status: HttpStatusCode.FORBIDDEN_403,
202 message: 'Cannot use non unicast IP as targetUrl.'
203 })
204 }
205
206 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
207
208 // Process video thumbnail from request.files
209 let thumbnailModel = await processThumbnail(req, video)
210
211 // Process video thumbnail from url if processing from request.files failed
212 if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
213 try {
214 thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
215 } catch (err) {
216 logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
217 }
218 }
219
220 // Process video preview from request.files
221 let previewModel = await processPreview(req, video)
222
223 // Process video preview from url if processing from request.files failed
224 if (!previewModel && youtubeDLInfo.thumbnailUrl) {
225 try {
226 previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
227 } catch (err) {
228 logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
229 }
230 }
231
232 const videoImport = await insertIntoDB({
233 video,
234 thumbnailModel,
235 previewModel,
236 videoChannel: res.locals.videoChannel,
237 tags: body.tags || youtubeDLInfo.tags,
238 user,
239 videoImportAttributes: {
240 targetUrl,
241 state: VideoImportState.PENDING,
242 userId: user.id
243 }
244 })
245
246 // Get video subtitles
247 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
248
249 let fileExt = `.${youtubeDLInfo.ext}`
250 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
251
252 // Create job to import the video
253 const payload = {
254 type: 'youtube-dl' as 'youtube-dl',
255 videoImportId: videoImport.id,
256 fileExt
257 }
258 await JobQueue.Instance.createJob({ type: 'video-import', payload })
259
260 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
261
262 return res.json(videoImport.toFormattedJSON()).end()
263}
264
265async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
266 let videoData = {
267 name: body.name || importData.name || 'Unknown name',
268 remote: false,
269 category: body.category || importData.category,
270 licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
271 language: body.language || importData.language,
272 commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
273 downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
274 waitTranscoding: body.waitTranscoding || false,
275 state: VideoState.TO_IMPORT,
276 nsfw: body.nsfw || importData.nsfw || false,
277 description: body.description || importData.description,
278 support: body.support || null,
279 privacy: body.privacy || VideoPrivacy.PRIVATE,
280 duration: 0, // duration will be set by the import job
281 channelId,
282 originallyPublishedAt: body.originallyPublishedAt
283 ? new Date(body.originallyPublishedAt)
284 : importData.originallyPublishedAt
285 }
286
287 videoData = await Hooks.wrapObject(
288 videoData,
289 body.targetUrl
290 ? 'filter:api.video.import-url.video-attribute.result'
291 : 'filter:api.video.import-torrent.video-attribute.result'
292 )
293
294 const video = new VideoModel(videoData)
295 video.url = getLocalVideoActivityPubUrl(video)
296
297 return video
298} 188}
299 189
300async function processThumbnail (req: express.Request, video: MVideoThumbnail) { 190async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
@@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
329 return undefined 219 return undefined
330} 220}
331 221
332async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
333 try {
334 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
335 } catch (err) {
336 logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
337 return undefined
338 }
339}
340
341async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
342 try {
343 return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
344 } catch (err) {
345 logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
346 return undefined
347 }
348}
349
350async function insertIntoDB (parameters: {
351 video: MVideoThumbnail
352 thumbnailModel: MThumbnail
353 previewModel: MThumbnail
354 videoChannel: MChannelAccountDefault
355 tags: string[]
356 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
357 user: MUser
358}): Promise<MVideoImportFormattable> {
359 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
360
361 const videoImport = await sequelizeTypescript.transaction(async t => {
362 const sequelizeOptions = { transaction: t }
363
364 // Save video object in database
365 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
366 videoCreated.VideoChannel = videoChannel
367
368 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
369 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
370
371 await autoBlacklistVideoIfNeeded({
372 video: videoCreated,
373 user,
374 notify: false,
375 isRemote: false,
376 isNew: true,
377 transaction: t
378 })
379
380 await setVideoTags({ video: videoCreated, tags, transaction: t })
381
382 // Create video import object in database
383 const videoImport = await VideoImportModel.create(
384 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
385 sequelizeOptions
386 ) as MVideoImportFormattable
387 videoImport.Video = videoCreated
388
389 return videoImport
390 })
391
392 return videoImport
393}
394
395async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { 222async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
396 const torrentName = torrentfile.originalname 223 const torrentName = torrentfile.originalname
397 224
@@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
432function extractNameFromArray (name: string | string[]) { 259function extractNameFromArray (name: string | string[]) {
433 return isArray(name) ? name[0] : name 260 return isArray(name) ? name[0] : name
434} 261}
435
436async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
437 try {
438 const subtitles = await youtubeDL.getSubtitles()
439
440 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
441
442 for (const subtitle of subtitles) {
443 if (!await isVTTFileValid(subtitle.path)) {
444 await remove(subtitle.path)
445 continue
446 }
447
448 const videoCaption = new VideoCaptionModel({
449 videoId,
450 language: subtitle.language,
451 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
452 }) as MVideoCaption
453
454 // Move physical file
455 await moveAndProcessCaptionFile(subtitle, videoCaption)
456
457 await sequelizeTypescript.transaction(async t => {
458 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
459 })
460 }
461 } catch (err) {
462 logger.warn('Cannot get video subtitles.', { err })
463 }
464}
465
466async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
467 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
468 const uniqHosts = new Set(hosts)
469
470 for (const h of uniqHosts) {
471 if (await isResolvingToUnicastOnly(h) !== true) {
472 return false
473 }
474 }
475
476 return true
477}
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 076b7f11d..7e8a03e8f 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -5,7 +5,7 @@ import { chain } from 'lodash'
5import { join } from 'path' 5import { join } from 'path'
6import { addColors, config, createLogger, format, transports } from 'winston' 6import { addColors, config, createLogger, format, transports } from 'winston'
7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' 7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
8import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models' 8import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
9import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
10import { jsonLoggerFormat, labelFormatter } from './logger' 10import { jsonLoggerFormat, labelFormatter } from './logger'
11 11
@@ -260,6 +260,18 @@ class CustomConfigAuditView extends EntityAuditView {
260 } 260 }
261} 261}
262 262
263const channelSyncKeysToKeep = [
264 'id',
265 'externalChannelUrl',
266 'channel-id',
267 'channel-name'
268]
269class VideoChannelSyncAuditView extends EntityAuditView {
270 constructor (channelSync: VideoChannelSync) {
271 super(channelSyncKeysToKeep, 'channelSync', channelSync)
272 }
273}
274
263export { 275export {
264 getAuditIdFromRes, 276 getAuditIdFromRes,
265 277
@@ -270,5 +282,6 @@ export {
270 UserAuditView, 282 UserAuditView,
271 VideoAuditView, 283 VideoAuditView,
272 AbuseAuditView, 284 AbuseAuditView,
273 CustomConfigAuditView 285 CustomConfigAuditView,
286 VideoChannelSyncAuditView
274} 287}
diff --git a/server/helpers/custom-validators/video-channel-syncs.ts b/server/helpers/custom-validators/video-channel-syncs.ts
new file mode 100644
index 000000000..c5a9afa96
--- /dev/null
+++ b/server/helpers/custom-validators/video-channel-syncs.ts
@@ -0,0 +1,6 @@
1import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
2import { exists } from './misc'
3
4export function isVideoChannelSyncStateValid (value: any) {
5 return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
6}
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 13c990a1e..5a87b99b4 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -87,6 +87,7 @@ export class YoutubeDLCLI {
87 return result.concat([ 87 return result.concat([
88 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', 88 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
89 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats 89 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
90 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
90 'best' // Ultimate fallback 91 'best' // Ultimate fallback
91 ]).join('/') 92 ]).join('/')
92 } 93 }
@@ -103,11 +104,14 @@ export class YoutubeDLCLI {
103 timeout?: number 104 timeout?: number
104 additionalYoutubeDLArgs?: string[] 105 additionalYoutubeDLArgs?: string[]
105 }) { 106 }) {
107 let args = options.additionalYoutubeDLArgs || []
108 args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
109
106 return this.run({ 110 return this.run({
107 url: options.url, 111 url: options.url,
108 processOptions: options.processOptions, 112 processOptions: options.processOptions,
109 timeout: options.timeout, 113 timeout: options.timeout,
110 args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ]) 114 args
111 }) 115 })
112 } 116 }
113 117
@@ -129,6 +133,25 @@ export class YoutubeDLCLI {
129 : info 133 : info
130 } 134 }
131 135
136 getListInfo (options: {
137 url: string
138 latestVideosCount?: number
139 processOptions: execa.NodeOptions
140 }): Promise<{ upload_date: string, webpage_url: string }[]> {
141 const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
142
143 if (options.latestVideosCount !== undefined) {
144 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
145 }
146
147 return this.getInfo({
148 url: options.url,
149 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
150 processOptions: options.processOptions,
151 additionalYoutubeDLArgs
152 })
153 }
154
132 async getSubs (options: { 155 async getSubs (options: {
133 url: string 156 url: string
134 format: 'vtt' 157 format: 'vtt'
@@ -175,7 +198,7 @@ export class YoutubeDLCLI {
175 198
176 const output = await subProcess 199 const output = await subProcess
177 200
178 logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() }) 201 logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
179 202
180 return output.stdout 203 return output.stdout
181 ? output.stdout.trim().split(/\r?\n/) 204 ? output.stdout.trim().split(/\r?\n/)
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
index 71572f292..303e4051f 100644
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
@@ -13,6 +13,7 @@ type YoutubeDLInfo = {
13 thumbnailUrl?: string 13 thumbnailUrl?: string
14 ext?: string 14 ext?: string
15 originallyPublishedAt?: Date 15 originallyPublishedAt?: Date
16 webpageUrl?: string
16 17
17 urls?: string[] 18 urls?: string[]
18} 19}
@@ -81,7 +82,8 @@ class YoutubeDLInfoBuilder {
81 thumbnailUrl: obj.thumbnail || undefined, 82 thumbnailUrl: obj.thumbnail || undefined,
82 urls: this.buildAvailableUrl(obj), 83 urls: this.buildAvailableUrl(obj),
83 originallyPublishedAt: this.buildOriginallyPublishedAt(obj), 84 originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
84 ext: obj.ext 85 ext: obj.ext,
86 webpageUrl: obj.webpage_url
85 } 87 }
86 } 88 }
87 89
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index 176cf3b69..7cd5e3310 100644
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -46,6 +46,24 @@ class YoutubeDLWrapper {
46 return infoBuilder.getInfo() 46 return infoBuilder.getInfo()
47 } 47 }
48 48
49 async getInfoForListImport (options: {
50 latestVideosCount?: number
51 }) {
52 const youtubeDL = await YoutubeDLCLI.safeGet()
53
54 const list = await youtubeDL.getListInfo({
55 url: this.url,
56 latestVideosCount: options.latestVideosCount,
57 processOptions
58 })
59
60 return list.map(info => {
61 const infoBuilder = new YoutubeDLInfoBuilder(info)
62
63 return infoBuilder.getInfo()
64 })
65 }
66
49 async getSubtitles (): Promise<YoutubeDLSubs> { 67 async getSubtitles (): Promise<YoutubeDLSubs> {
50 const cwd = CONFIG.STORAGE.TMP_DIR 68 const cwd = CONFIG.STORAGE.TMP_DIR
51 69
@@ -103,7 +121,7 @@ class YoutubeDLWrapper {
103 121
104 return remove(path) 122 return remove(path)
105 }) 123 })
106 .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) 124 .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
107 125
108 throw err 126 throw err
109 } 127 }
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index f0f16d9bd..74c82541e 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -48,6 +48,7 @@ function checkConfig () {
48 checkRemoteRedundancyConfig() 48 checkRemoteRedundancyConfig()
49 checkStorageConfig() 49 checkStorageConfig()
50 checkTranscodingConfig() 50 checkTranscodingConfig()
51 checkImportConfig()
51 checkBroadcastMessageConfig() 52 checkBroadcastMessageConfig()
52 checkSearchConfig() 53 checkSearchConfig()
53 checkLiveConfig() 54 checkLiveConfig()
@@ -200,6 +201,12 @@ function checkTranscodingConfig () {
200 } 201 }
201} 202}
202 203
204function checkImportConfig () {
205 if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) {
206 throw new Error('You need to enable HTTP import to allow synchronization')
207 }
208}
209
203function checkBroadcastMessageConfig () { 210function checkBroadcastMessageConfig () {
204 if (CONFIG.BROADCAST_MESSAGE.ENABLED) { 211 if (CONFIG.BROADCAST_MESSAGE.ENABLED) {
205 const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL 212 const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index f4057b81b..3188903be 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -32,6 +32,8 @@ function checkMissedConfig () {
32 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', 32 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
33 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled', 33 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
34 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', 34 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
35 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
36 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
35 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', 37 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
36 'client.videos.miniature.display_author_avatar', 38 'client.videos.miniature.display_author_avatar',
37 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', 39 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 1a0b8942c..2c92bea22 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -398,6 +398,14 @@ const CONFIG = {
398 TORRENT: { 398 TORRENT: {
399 get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') } 399 get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
400 } 400 }
401 },
402 VIDEO_CHANNEL_SYNCHRONIZATION: {
403 get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') },
404 get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') },
405 get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
406 get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
407 return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
408 }
401 } 409 }
402 }, 410 },
403 AUTO_BLACKLIST: { 411 AUTO_BLACKLIST: {
@@ -499,6 +507,7 @@ const CONFIG = {
499 get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') } 507 get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
500 } 508 }
501 } 509 }
510
502} 511}
503 512
504function registerConfigChangedHandler (fun: Function) { 513function registerConfigChangedHandler (fun: Function) {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5a5f2d666..697a64d42 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
6import { 6import {
7 AbuseState, 7 AbuseState,
8 JobType, 8 JobType,
9 VideoChannelSyncState,
9 VideoImportState, 10 VideoImportState,
10 VideoPrivacy, 11 VideoPrivacy,
11 VideoRateType, 12 VideoRateType,
@@ -24,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
24 25
25// --------------------------------------------------------------------------- 26// ---------------------------------------------------------------------------
26 27
27const LAST_MIGRATION_VERSION = 725 28const LAST_MIGRATION_VERSION = 730
28 29
29// --------------------------------------------------------------------------- 30// ---------------------------------------------------------------------------
30 31
@@ -64,6 +65,7 @@ const SORTABLE_COLUMNS = {
64 JOBS: [ 'createdAt' ], 65 JOBS: [ 'createdAt' ],
65 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], 66 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
66 VIDEO_IMPORTS: [ 'createdAt' ], 67 VIDEO_IMPORTS: [ 'createdAt' ],
68 VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ],
67 69
68 VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ], 70 VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
69 VIDEO_COMMENTS: [ 'createdAt' ], 71 VIDEO_COMMENTS: [ 'createdAt' ],
@@ -156,6 +158,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
156 'video-live-ending': 1, 158 'video-live-ending': 1,
157 'video-studio-edition': 1, 159 'video-studio-edition': 1,
158 'manage-video-torrent': 1, 160 'manage-video-torrent': 1,
161 'video-channel-import': 1,
162 'after-video-channel-import': 1,
159 'move-to-object-storage': 3, 163 'move-to-object-storage': 3,
160 'notify': 1, 164 'notify': 1,
161 'federate-video': 1 165 'federate-video': 1
@@ -178,6 +182,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
178 'video-studio-edition': 1, 182 'video-studio-edition': 1,
179 'manage-video-torrent': 1, 183 'manage-video-torrent': 1,
180 'move-to-object-storage': 1, 184 'move-to-object-storage': 1,
185 'video-channel-import': 1,
186 'after-video-channel-import': 1,
181 'notify': 5, 187 'notify': 5,
182 'federate-video': 3 188 'federate-video': 3
183} 189}
@@ -199,9 +205,11 @@ const JOB_TTL: { [id in JobType]: number } = {
199 'video-redundancy': 1000 * 3600 * 3, // 3 hours 205 'video-redundancy': 1000 * 3600 * 3, // 3 hours
200 'video-live-ending': 1000 * 60 * 10, // 10 minutes 206 'video-live-ending': 1000 * 60 * 10, // 10 minutes
201 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours 207 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
208 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
209 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
210 'after-video-channel-import': 60000 * 5, // 5 minutes
202 'notify': 60000 * 5, // 5 minutes 211 'notify': 60000 * 5, // 5 minutes
203 'federate-video': 60000 * 5, // 5 minutes 212 'federate-video': 60000 * 5 // 5 minutes
204 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
205} 213}
206const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = { 214const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
207 'videos-views-stats': { 215 'videos-views-stats': {
@@ -246,7 +254,8 @@ const SCHEDULER_INTERVALS_MS = {
246 REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day 254 REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
247 REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day 255 REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
248 UPDATE_INBOX_STATS: 1000 * 60, // 1 minute 256 UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
249 REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour 257 REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
258 CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
250} 259}
251 260
252// --------------------------------------------------------------------------- 261// ---------------------------------------------------------------------------
@@ -276,8 +285,12 @@ const CONSTRAINTS_FIELDS = {
276 NAME: { min: 1, max: 120 }, // Length 285 NAME: { min: 1, max: 120 }, // Length
277 DESCRIPTION: { min: 3, max: 1000 }, // Length 286 DESCRIPTION: { min: 3, max: 1000 }, // Length
278 SUPPORT: { min: 3, max: 1000 }, // Length 287 SUPPORT: { min: 3, max: 1000 }, // Length
288 EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length
279 URL: { min: 3, max: 2000 } // Length 289 URL: { min: 3, max: 2000 } // Length
280 }, 290 },
291 VIDEO_CHANNEL_SYNCS: {
292 EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length
293 },
281 VIDEO_CAPTIONS: { 294 VIDEO_CAPTIONS: {
282 CAPTION_FILE: { 295 CAPTION_FILE: {
283 EXTNAME: [ '.vtt', '.srt' ], 296 EXTNAME: [ '.vtt', '.srt' ],
@@ -478,6 +491,13 @@ const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
478 [VideoImportState.PROCESSING]: 'Processing' 491 [VideoImportState.PROCESSING]: 'Processing'
479} 492}
480 493
494const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = {
495 [VideoChannelSyncState.FAILED]: 'Failed',
496 [VideoChannelSyncState.SYNCED]: 'Synchronized',
497 [VideoChannelSyncState.PROCESSING]: 'Processing',
498 [VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run'
499}
500
481const ABUSE_STATES: { [ id in AbuseState ]: string } = { 501const ABUSE_STATES: { [ id in AbuseState ]: string } = {
482 [AbuseState.PENDING]: 'Pending', 502 [AbuseState.PENDING]: 'Pending',
483 [AbuseState.REJECTED]: 'Rejected', 503 [AbuseState.REJECTED]: 'Rejected',
@@ -1005,6 +1025,7 @@ export {
1005 JOB_COMPLETED_LIFETIME, 1025 JOB_COMPLETED_LIFETIME,
1006 HTTP_SIGNATURE, 1026 HTTP_SIGNATURE,
1007 VIDEO_IMPORT_STATES, 1027 VIDEO_IMPORT_STATES,
1028 VIDEO_CHANNEL_SYNC_STATE,
1008 VIEW_LIFETIME, 1029 VIEW_LIFETIME,
1009 CONTACT_FORM_LIFETIME, 1030 CONTACT_FORM_LIFETIME,
1010 VIDEO_PLAYLIST_PRIVACIES, 1031 VIDEO_PLAYLIST_PRIVACIES,
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 91286241b..f55f40df0 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -50,6 +50,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
50import { VideoTagModel } from '../models/video/video-tag' 50import { VideoTagModel } from '../models/video/video-tag'
51import { VideoViewModel } from '../models/view/video-view' 51import { VideoViewModel } from '../models/view/video-view'
52import { CONFIG } from './config' 52import { CONFIG } from './config'
53import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
53 54
54require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 55require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
55 56
@@ -153,7 +154,8 @@ async function initDatabaseModels (silent: boolean) {
153 VideoTrackerModel, 154 VideoTrackerModel,
154 PluginModel, 155 PluginModel,
155 ActorCustomPageModel, 156 ActorCustomPageModel,
156 VideoJobInfoModel 157 VideoJobInfoModel,
158 VideoChannelSyncModel
157 ]) 159 ])
158 160
159 // Check extensions exist in the database 161 // Check extensions exist in the database
diff --git a/server/initializers/migrations/0730-video-channel-sync.ts b/server/initializers/migrations/0730-video-channel-sync.ts
new file mode 100644
index 000000000..a2fe8211f
--- /dev/null
+++ b/server/initializers/migrations/0730-video-channel-sync.ts
@@ -0,0 +1,36 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction
5 queryInterface: Sequelize.QueryInterface
6 sequelize: Sequelize.Sequelize
7 db: any
8}): Promise<void> {
9 const query = `
10 CREATE TABLE IF NOT EXISTS "videoChannelSync" (
11 "id" SERIAL,
12 "externalChannelUrl" VARCHAR(2000) NOT NULL DEFAULT NULL,
13 "videoChannelId" INTEGER NOT NULL REFERENCES "videoChannel" ("id")
14 ON DELETE CASCADE
15 ON UPDATE CASCADE,
16 "state" INTEGER NOT NULL DEFAULT 1,
17 "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
18 "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
19 "lastSyncAt" TIMESTAMP WITH TIME ZONE,
20 PRIMARY KEY ("id")
21 );
22 `
23 await utils.sequelize.query(query, { transaction: utils.transaction })
24}
25
26async function down (utils: {
27 queryInterface: Sequelize.QueryInterface
28 transaction: Sequelize.Transaction
29}) {
30 await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction })
31}
32
33export {
34 up,
35 down
36}
diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts
new file mode 100644
index 000000000..ffdd8c5b5
--- /dev/null
+++ b/server/lib/job-queue/handlers/after-video-channel-import.ts
@@ -0,0 +1,37 @@
1import { Job } from 'bullmq'
2import { logger } from '@server/helpers/logger'
3import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
4import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
5
6export async function processAfterVideoChannelImport (job: Job) {
7 const payload = job.data as AfterVideoChannelImportPayload
8 if (!payload.channelSyncId) return
9
10 logger.info('Processing after video channel import in job %s.', job.id)
11
12 const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
13 if (!sync) {
14 logger.error('Unknown sync id %d.', payload.channelSyncId)
15 return
16 }
17
18 const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
19
20 let errors = 0
21 let successes = 0
22
23 for (const value of Object.values(childrenValues)) {
24 if (value.resultType === 'success') successes++
25 else if (value.resultType === 'error') errors++
26 }
27
28 if (errors > 0) {
29 sync.state = VideoChannelSyncState.FAILED
30 logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
31 } else {
32 sync.state = VideoChannelSyncState.SYNCED
33 logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
34 }
35
36 await sync.save()
37}
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
new file mode 100644
index 000000000..9bdb2d269
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -0,0 +1,36 @@
1import { Job } from 'bullmq'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { synchronizeChannel } from '@server/lib/sync-channel'
5import { VideoChannelModel } from '@server/models/video/video-channel'
6import { VideoChannelImportPayload } from '@shared/models'
7
8export async function processVideoChannelImport (job: Job) {
9 const payload = job.data as VideoChannelImportPayload
10
11 logger.info('Processing video channel import in job %s.', job.id)
12
13 // Channel import requires only http upload to be allowed
14 if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
15 logger.error('Cannot import channel as the HTTP upload is disabled')
16 return
17 }
18
19 if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
20 logger.error('Cannot import channel as the synchronization is disabled')
21 return
22 }
23
24 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
25
26 try {
27 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
28
29 await synchronizeChannel({
30 channel: videoChannel,
31 externalChannelUrl: payload.externalChannelUrl
32 })
33 } catch (err) {
34 logger.error(`Failed to import channel ${videoChannel.name}`, { err })
35 }
36}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index f4629159c..9901b878c 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
8import { Hooks } from '@server/lib/plugins/hooks' 8import { Hooks } from '@server/lib/plugins/hooks'
9import { ServerConfigManager } from '@server/lib/server-config-manager' 9import { ServerConfigManager } from '@server/lib/server-config-manager'
10import { isAbleToUploadVideo } from '@server/lib/user' 10import { isAbleToUploadVideo } from '@server/lib/user'
11import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video' 11import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
12import { VideoPathManager } from '@server/lib/video-path-manager' 12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { buildNextVideoState } from '@server/lib/video-state' 13import { buildNextVideoState } from '@server/lib/video-state'
14import { ThumbnailModel } from '@server/models/video/thumbnail' 14import { ThumbnailModel } from '@server/models/video/thumbnail'
@@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils'
18import { 18import {
19 ThumbnailType, 19 ThumbnailType,
20 VideoImportPayload, 20 VideoImportPayload,
21 VideoImportPreventExceptionResult,
21 VideoImportState, 22 VideoImportState,
22 VideoImportTorrentPayload, 23 VideoImportTorrentPayload,
23 VideoImportTorrentPayloadType, 24 VideoImportTorrentPayloadType,
@@ -41,20 +42,29 @@ import { Notifier } from '../../notifier'
41import { generateVideoMiniature } from '../../thumbnail' 42import { generateVideoMiniature } from '../../thumbnail'
42import { JobQueue } from '../job-queue' 43import { JobQueue } from '../job-queue'
43 44
44async function processVideoImport (job: Job) { 45async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
45 const payload = job.data as VideoImportPayload 46 const payload = job.data as VideoImportPayload
46 47
47 const videoImport = await getVideoImportOrDie(payload) 48 const videoImport = await getVideoImportOrDie(payload)
48 if (videoImport.state === VideoImportState.CANCELLED) { 49 if (videoImport.state === VideoImportState.CANCELLED) {
49 logger.info('Do not process import since it has been cancelled', { payload }) 50 logger.info('Do not process import since it has been cancelled', { payload })
50 return 51 return { resultType: 'success' }
51 } 52 }
52 53
53 videoImport.state = VideoImportState.PROCESSING 54 videoImport.state = VideoImportState.PROCESSING
54 await videoImport.save() 55 await videoImport.save()
55 56
56 if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload) 57 try {
57 if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload) 58 if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
59 if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
60
61 return { resultType: 'success' }
62 } catch (err) {
63 if (!payload.preventException) throw err
64
65 logger.warn('Catch error in video import to send value to parent job.', { payload, err })
66 return { resultType: 'error' }
67 }
58} 68}
59 69
60// --------------------------------------------------------------------------- 70// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 281e2e51a..3970d48b7 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -22,6 +22,7 @@ import {
22 ActivitypubHttpFetcherPayload, 22 ActivitypubHttpFetcherPayload,
23 ActivitypubHttpUnicastPayload, 23 ActivitypubHttpUnicastPayload,
24 ActorKeysPayload, 24 ActorKeysPayload,
25 AfterVideoChannelImportPayload,
25 DeleteResumableUploadMetaFilePayload, 26 DeleteResumableUploadMetaFilePayload,
26 EmailPayload, 27 EmailPayload,
27 FederateVideoPayload, 28 FederateVideoPayload,
@@ -31,6 +32,7 @@ import {
31 MoveObjectStoragePayload, 32 MoveObjectStoragePayload,
32 NotifyPayload, 33 NotifyPayload,
33 RefreshPayload, 34 RefreshPayload,
35 VideoChannelImportPayload,
34 VideoFileImportPayload, 36 VideoFileImportPayload,
35 VideoImportPayload, 37 VideoImportPayload,
36 VideoLiveEndingPayload, 38 VideoLiveEndingPayload,
@@ -53,12 +55,14 @@ import { processFederateVideo } from './handlers/federate-video'
53import { processManageVideoTorrent } from './handlers/manage-video-torrent' 55import { processManageVideoTorrent } from './handlers/manage-video-torrent'
54import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' 56import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
55import { processNotify } from './handlers/notify' 57import { processNotify } from './handlers/notify'
58import { processVideoChannelImport } from './handlers/video-channel-import'
56import { processVideoFileImport } from './handlers/video-file-import' 59import { processVideoFileImport } from './handlers/video-file-import'
57import { processVideoImport } from './handlers/video-import' 60import { processVideoImport } from './handlers/video-import'
58import { processVideoLiveEnding } from './handlers/video-live-ending' 61import { processVideoLiveEnding } from './handlers/video-live-ending'
59import { processVideoStudioEdition } from './handlers/video-studio-edition' 62import { processVideoStudioEdition } from './handlers/video-studio-edition'
60import { processVideoTranscoding } from './handlers/video-transcoding' 63import { processVideoTranscoding } from './handlers/video-transcoding'
61import { processVideosViewsStats } from './handlers/video-views-stats' 64import { processVideosViewsStats } from './handlers/video-views-stats'
65import { processAfterVideoChannelImport } from './handlers/after-video-channel-import'
62 66
63export type CreateJobArgument = 67export type CreateJobArgument =
64 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 68 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -79,6 +83,9 @@ export type CreateJobArgument =
79 { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | 83 { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
80 { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | 84 { type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
81 { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | 85 { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
86 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
87 { type: 'video-channel-import', payload: VideoChannelImportPayload } |
88 { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
82 { type: 'notify', payload: NotifyPayload } | 89 { type: 'notify', payload: NotifyPayload } |
83 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | 90 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
84 { type: 'federate-video', payload: FederateVideoPayload } 91 { type: 'federate-video', payload: FederateVideoPayload }
@@ -106,8 +113,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
106 'video-redundancy': processVideoRedundancy, 113 'video-redundancy': processVideoRedundancy,
107 'move-to-object-storage': processMoveToObjectStorage, 114 'move-to-object-storage': processMoveToObjectStorage,
108 'manage-video-torrent': processManageVideoTorrent, 115 'manage-video-torrent': processManageVideoTorrent,
109 'notify': processNotify,
110 'video-studio-edition': processVideoStudioEdition, 116 'video-studio-edition': processVideoStudioEdition,
117 'video-channel-import': processVideoChannelImport,
118 'after-video-channel-import': processAfterVideoChannelImport,
119 'notify': processNotify,
111 'federate-video': processFederateVideo 120 'federate-video': processFederateVideo
112} 121}
113 122
@@ -134,6 +143,8 @@ const jobTypes: JobType[] = [
134 'move-to-object-storage', 143 'move-to-object-storage',
135 'manage-video-torrent', 144 'manage-video-torrent',
136 'video-studio-edition', 145 'video-studio-edition',
146 'video-channel-import',
147 'after-video-channel-import',
137 'notify', 148 'notify',
138 'federate-video' 149 'federate-video'
139] 150]
@@ -306,7 +317,7 @@ class JobQueue {
306 .catch(err => logger.error('Cannot create job.', { err, options })) 317 .catch(err => logger.error('Cannot create job.', { err, options }))
307 } 318 }
308 319
309 async createJob (options: CreateJobArgument & CreateJobOptions) { 320 createJob (options: CreateJobArgument & CreateJobOptions) {
310 const queue: Queue = this.queues[options.type] 321 const queue: Queue = this.queues[options.type]
311 if (queue === undefined) { 322 if (queue === undefined) {
312 logger.error('Unknown queue %s: cannot create job.', options.type) 323 logger.error('Unknown queue %s: cannot create job.', options.type)
@@ -318,7 +329,7 @@ class JobQueue {
318 return queue.add('job', options.payload, jobOptions) 329 return queue.add('job', options.payload, jobOptions)
319 } 330 }
320 331
321 async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { 332 createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
322 let lastJob: FlowJob 333 let lastJob: FlowJob
323 334
324 for (const job of jobs) { 335 for (const job of jobs) {
@@ -336,7 +347,7 @@ class JobQueue {
336 return this.flowProducer.add(lastJob) 347 return this.flowProducer.add(lastJob)
337 } 348 }
338 349
339 async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { 350 createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
340 return this.flowProducer.add({ 351 return this.flowProducer.add({
341 ...this.buildJobFlowOption(parent), 352 ...this.buildJobFlowOption(parent),
342 353
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
new file mode 100644
index 000000000..fd9a35299
--- /dev/null
+++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -0,0 +1,61 @@
1import { logger } from '@server/helpers/logger'
2import { CONFIG } from '@server/initializers/config'
3import { VideoChannelModel } from '@server/models/video/video-channel'
4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
5import { VideoChannelSyncState } from '@shared/models'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { synchronizeChannel } from '../sync-channel'
8import { AbstractScheduler } from './abstract-scheduler'
9
10export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
11 private static instance: AbstractScheduler
12 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
13
14 private constructor () {
15 super()
16 }
17
18 protected async internalExecute () {
19 logger.debug('Running %s.%s', this.constructor.name, this.internalExecute.name)
20
21 if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
22 logger.info('Discard channels synchronization as the feature is disabled')
23 return
24 }
25
26 const channelSyncs = await VideoChannelSyncModel.listSyncs()
27
28 for (const sync of channelSyncs) {
29 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
30
31 try {
32 logger.info(
33 'Creating video import jobs for "%s" sync with external channel "%s"',
34 channel.Actor.preferredUsername, sync.externalChannelUrl
35 )
36
37 const onlyAfter = sync.lastSyncAt || sync.createdAt
38
39 sync.state = VideoChannelSyncState.PROCESSING
40 sync.lastSyncAt = new Date()
41 await sync.save()
42
43 await synchronizeChannel({
44 channel,
45 externalChannelUrl: sync.externalChannelUrl,
46 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
47 channelSync: sync,
48 onlyAfter
49 })
50 } catch (err) {
51 logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
52 sync.state = VideoChannelSyncState.FAILED
53 await sync.save()
54 }
55 }
56 }
57
58 static get Instance () {
59 return this.instance || (this.instance = new this())
60 }
61}
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts
index a3312fa20..78a9546ae 100644
--- a/server/lib/server-config-manager.ts
+++ b/server/lib/server-config-manager.ts
@@ -170,6 +170,9 @@ class ServerConfigManager {
170 torrent: { 170 torrent: {
171 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED 171 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
172 } 172 }
173 },
174 videoChannelSynchronization: {
175 enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED
173 } 176 }
174 }, 177 },
175 autoBlacklist: { 178 autoBlacklist: {
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
new file mode 100644
index 000000000..50f80e6f9
--- /dev/null
+++ b/server/lib/sync-channel.ts
@@ -0,0 +1,81 @@
1import { logger } from '@server/helpers/logger'
2import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
3import { CONFIG } from '@server/initializers/config'
4import { buildYoutubeDLImport } from '@server/lib/video-import'
5import { UserModel } from '@server/models/user/user'
6import { VideoImportModel } from '@server/models/video/video-import'
7import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
8import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
9import { CreateJobArgument, JobQueue } from './job-queue'
10import { ServerConfigManager } from './server-config-manager'
11
12export async function synchronizeChannel (options: {
13 channel: MChannelAccountDefault
14 externalChannelUrl: string
15 channelSync?: MChannelSync
16 videosCountLimit?: number
17 onlyAfter?: Date
18}) {
19 const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
20
21 const user = await UserModel.loadByChannelActorId(channel.actorId)
22 const youtubeDL = new YoutubeDLWrapper(
23 externalChannelUrl,
24 ServerConfigManager.Instance.getEnabledResolutions('vod'),
25 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
26 )
27
28 const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
29
30 const targetUrls = infoList
31 .filter(videoInfo => {
32 if (!onlyAfter) return true
33
34 return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
35 })
36 .map(videoInfo => videoInfo.webpageUrl)
37
38 logger.info(
39 'Fetched %d candidate URLs for sync channel %s.',
40 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
41 )
42
43 if (targetUrls.length === 0) {
44 if (channelSync) {
45 channelSync.state = VideoChannelSyncState.SYNCED
46 await channelSync.save()
47 }
48
49 return
50 }
51
52 const children: CreateJobArgument[] = []
53
54 for (const targetUrl of targetUrls) {
55 if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
56 logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
57 continue
58 }
59
60 const { job } = await buildYoutubeDLImport({
61 user,
62 channel,
63 targetUrl,
64 channelSync,
65 importDataOverride: {
66 privacy: VideoPrivacy.PUBLIC
67 }
68 })
69
70 children.push(job)
71 }
72
73 const parent: CreateJobArgument = {
74 type: 'after-video-channel-import',
75 payload: {
76 channelSyncId: channelSync?.id
77 }
78 }
79
80 await JobQueue.Instance.createJobWithChildren(parent, children)
81}
diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts
new file mode 100644
index 000000000..fb9306967
--- /dev/null
+++ b/server/lib/video-import.ts
@@ -0,0 +1,308 @@
1import { remove } from 'fs-extra'
2import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
3import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
4import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
5import { isResolvingToUnicastOnly } from '@server/helpers/dns'
6import { logger } from '@server/helpers/logger'
7import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
8import { CONFIG } from '@server/initializers/config'
9import { sequelizeTypescript } from '@server/initializers/database'
10import { Hooks } from '@server/lib/plugins/hooks'
11import { ServerConfigManager } from '@server/lib/server-config-manager'
12import { setVideoTags } from '@server/lib/video'
13import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
14import { VideoModel } from '@server/models/video/video'
15import { VideoCaptionModel } from '@server/models/video/video-caption'
16import { VideoImportModel } from '@server/models/video/video-import'
17import { FilteredModelAttributes } from '@server/types'
18import {
19 MChannelAccountDefault,
20 MChannelSync,
21 MThumbnail,
22 MUser,
23 MVideoAccountDefault,
24 MVideoCaption,
25 MVideoImportFormattable,
26 MVideoTag,
27 MVideoThumbnail,
28 MVideoWithBlacklistLight
29} from '@server/types/models'
30import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
31import { getLocalVideoActivityPubUrl } from './activitypub/url'
32import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
33
34class YoutubeDlImportError extends Error {
35 code: YoutubeDlImportError.CODE
36 cause?: Error // Property to remove once ES2022 is used
37 constructor ({ message, code }) {
38 super(message)
39 this.code = code
40 }
41
42 static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
43 const ytDlErr = new this({ message: message ?? err.message, code })
44 ytDlErr.cause = err
45 ytDlErr.stack = err.stack // Useless once ES2022 is used
46 return ytDlErr
47 }
48}
49
50namespace YoutubeDlImportError {
51 export enum CODE {
52 FETCH_ERROR,
53 NOT_ONLY_UNICAST_URL
54 }
55}
56
57// ---------------------------------------------------------------------------
58
59async function insertFromImportIntoDB (parameters: {
60 video: MVideoThumbnail
61 thumbnailModel: MThumbnail
62 previewModel: MThumbnail
63 videoChannel: MChannelAccountDefault
64 tags: string[]
65 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
66 user: MUser
67}): Promise<MVideoImportFormattable> {
68 const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
69
70 const videoImport = await sequelizeTypescript.transaction(async t => {
71 const sequelizeOptions = { transaction: t }
72
73 // Save video object in database
74 const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
75 videoCreated.VideoChannel = videoChannel
76
77 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
78 if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
79
80 await autoBlacklistVideoIfNeeded({
81 video: videoCreated,
82 user,
83 notify: false,
84 isRemote: false,
85 isNew: true,
86 transaction: t
87 })
88
89 await setVideoTags({ video: videoCreated, tags, transaction: t })
90
91 // Create video import object in database
92 const videoImport = await VideoImportModel.create(
93 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
94 sequelizeOptions
95 ) as MVideoImportFormattable
96 videoImport.Video = videoCreated
97
98 return videoImport
99 })
100
101 return videoImport
102}
103
104async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
105 channelId: number
106 importData: YoutubeDLInfo
107 importDataOverride?: Partial<VideoImportCreate>
108 importType: 'url' | 'torrent'
109}): Promise<MVideoThumbnail> {
110 let videoData = {
111 name: importDataOverride?.name || importData.name || 'Unknown name',
112 remote: false,
113 category: importDataOverride?.category || importData.category,
114 licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
115 language: importDataOverride?.language || importData.language,
116 commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
117 downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
118 waitTranscoding: importDataOverride?.waitTranscoding || false,
119 state: VideoState.TO_IMPORT,
120 nsfw: importDataOverride?.nsfw || importData.nsfw || false,
121 description: importDataOverride?.description || importData.description,
122 support: importDataOverride?.support || null,
123 privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
124 duration: 0, // duration will be set by the import job
125 channelId,
126 originallyPublishedAt: importDataOverride?.originallyPublishedAt
127 ? new Date(importDataOverride?.originallyPublishedAt)
128 : importData.originallyPublishedAt
129 }
130
131 videoData = await Hooks.wrapObject(
132 videoData,
133 importType === 'url'
134 ? 'filter:api.video.import-url.video-attribute.result'
135 : 'filter:api.video.import-torrent.video-attribute.result'
136 )
137
138 const video = new VideoModel(videoData)
139 video.url = getLocalVideoActivityPubUrl(video)
140
141 return video
142}
143
144async function buildYoutubeDLImport (options: {
145 targetUrl: string
146 channel: MChannelAccountDefault
147 user: MUser
148 channelSync?: MChannelSync
149 importDataOverride?: Partial<VideoImportCreate>
150 thumbnailFilePath?: string
151 previewFilePath?: string
152}) {
153 const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
154
155 const youtubeDL = new YoutubeDLWrapper(
156 targetUrl,
157 ServerConfigManager.Instance.getEnabledResolutions('vod'),
158 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
159 )
160
161 // Get video infos
162 let youtubeDLInfo: YoutubeDLInfo
163 try {
164 youtubeDLInfo = await youtubeDL.getInfoForDownload()
165 } catch (err) {
166 throw YoutubeDlImportError.fromError(
167 err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
168 )
169 }
170
171 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
172 throw new YoutubeDlImportError({
173 message: 'Cannot use non unicast IP as targetUrl.',
174 code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
175 })
176 }
177
178 const video = await buildVideoFromImport({
179 channelId: channel.id,
180 importData: youtubeDLInfo,
181 importDataOverride,
182 importType: 'url'
183 })
184
185 const thumbnailModel = await forgeThumbnail({
186 inputPath: thumbnailFilePath,
187 downloadUrl: youtubeDLInfo.thumbnailUrl,
188 video,
189 type: ThumbnailType.MINIATURE
190 })
191
192 const previewModel = await forgeThumbnail({
193 inputPath: previewFilePath,
194 downloadUrl: youtubeDLInfo.thumbnailUrl,
195 video,
196 type: ThumbnailType.PREVIEW
197 })
198
199 const videoImport = await insertFromImportIntoDB({
200 video,
201 thumbnailModel,
202 previewModel,
203 videoChannel: channel,
204 tags: importDataOverride?.tags || youtubeDLInfo.tags,
205 user,
206 videoImportAttributes: {
207 targetUrl,
208 state: VideoImportState.PENDING,
209 userId: user.id
210 }
211 })
212
213 // Get video subtitles
214 await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
215
216 let fileExt = `.${youtubeDLInfo.ext}`
217 if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
218
219 const payload: VideoImportPayload = {
220 type: 'youtube-dl' as 'youtube-dl',
221 videoImportId: videoImport.id,
222 fileExt,
223 // If part of a sync process, there is a parent job that will aggregate children results
224 preventException: !!channelSync
225 }
226
227 return {
228 videoImport,
229 job: { type: 'video-import' as 'video-import', payload }
230 }
231}
232
233// ---------------------------------------------------------------------------
234
235export {
236 buildYoutubeDLImport,
237 YoutubeDlImportError,
238 insertFromImportIntoDB,
239 buildVideoFromImport
240}
241
242// ---------------------------------------------------------------------------
243
244async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
245 inputPath?: string
246 downloadUrl?: string
247 video: MVideoThumbnail
248 type: ThumbnailType
249}): Promise<MThumbnail> {
250 if (inputPath) {
251 return updateVideoMiniatureFromExisting({
252 inputPath,
253 video,
254 type,
255 automaticallyGenerated: false
256 })
257 } else if (downloadUrl) {
258 try {
259 return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
260 } catch (err) {
261 logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
262 }
263 }
264 return null
265}
266
267async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
268 try {
269 const subtitles = await youtubeDL.getSubtitles()
270
271 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
272
273 for (const subtitle of subtitles) {
274 if (!await isVTTFileValid(subtitle.path)) {
275 await remove(subtitle.path)
276 continue
277 }
278
279 const videoCaption = new VideoCaptionModel({
280 videoId,
281 language: subtitle.language,
282 filename: VideoCaptionModel.generateCaptionName(subtitle.language)
283 }) as MVideoCaption
284
285 // Move physical file
286 await moveAndProcessCaptionFile(subtitle, videoCaption)
287
288 await sequelizeTypescript.transaction(async t => {
289 await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
290 })
291 }
292 } catch (err) {
293 logger.warn('Cannot get video subtitles.', { err })
294 }
295}
296
297async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
298 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
299 const uniqHosts = new Set(hosts)
300
301 for (const h of uniqHosts) {
302 if (await isResolvingToUnicastOnly(h) !== true) {
303 return false
304 }
305 }
306
307 return true
308}
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index 9ce47c5aa..f60103f48 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -66,6 +66,8 @@ const customConfigUpdateValidator = [
66 body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'), 66 body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
67 body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'), 67 body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
68 68
69 body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'),
70
69 body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'), 71 body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'),
70 body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'), 72 body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'),
71 73
@@ -110,6 +112,7 @@ const customConfigUpdateValidator = [
110 if (areValidationErrors(req, res)) return 112 if (areValidationErrors(req, res)) return
111 if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return 113 if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
112 if (!checkInvalidTranscodingConfig(req.body, res)) return 114 if (!checkInvalidTranscodingConfig(req.body, res)) return
115 if (!checkInvalidSynchronizationConfig(req.body, res)) return
113 if (!checkInvalidLiveConfig(req.body, res)) return 116 if (!checkInvalidLiveConfig(req.body, res)) return
114 if (!checkInvalidVideoStudioConfig(req.body, res)) return 117 if (!checkInvalidVideoStudioConfig(req.body, res)) return
115 118
@@ -157,6 +160,14 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express
157 return true 160 return true
158} 161}
159 162
163function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) {
164 if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) {
165 res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' })
166 return false
167 }
168 return true
169}
170
160function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { 171function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
161 if (customConfig.live.enabled === false) return true 172 if (customConfig.live.enabled === false) return true
162 173
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts
index c9978e3b4..0354e3fc6 100644
--- a/server/middlewares/validators/sort.ts
+++ b/server/middlewares/validators/sort.ts
@@ -52,6 +52,7 @@ const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAY
52const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS) 52const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS)
53const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) 53const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
54const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) 54const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
55const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS)
55 56
56const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS) 57const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
57const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS) 58const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)
@@ -84,5 +85,6 @@ export {
84 videoPlaylistsSearchSortValidator, 85 videoPlaylistsSearchSortValidator,
85 accountsFollowersSortValidator, 86 accountsFollowersSortValidator,
86 videoChannelsFollowersSortValidator, 87 videoChannelsFollowersSortValidator,
88 videoChannelSyncsSortValidator,
87 pluginsSortValidator 89 pluginsSortValidator
88} 90}
diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts
index 1dd7b5d2e..d225dfe45 100644
--- a/server/middlewares/validators/videos/index.ts
+++ b/server/middlewares/validators/videos/index.ts
@@ -14,3 +14,4 @@ export * from './video-stats'
14export * from './video-studio' 14export * from './video-studio'
15export * from './video-transcoding' 15export * from './video-transcoding'
16export * from './videos' 16export * from './videos'
17export * from './video-channel-sync'
diff --git a/server/middlewares/validators/videos/video-channel-sync.ts b/server/middlewares/validators/videos/video-channel-sync.ts
new file mode 100644
index 000000000..b18498243
--- /dev/null
+++ b/server/middlewares/validators/videos/video-channel-sync.ts
@@ -0,0 +1,66 @@
1import * as express from 'express'
2import { body, param } from 'express-validator'
3import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
4import { logger } from '@server/helpers/logger'
5import { CONFIG } from '@server/initializers/config'
6import { VideoChannelModel } from '@server/models/video/video-channel'
7import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
8import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
9import { areValidationErrors, doesVideoChannelIdExist } from '../shared'
10
11export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => {
12 if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
13 return res.fail({
14 status: HttpStatusCode.FORBIDDEN_403,
15 message: 'Synchronization is impossible as video channel synchronization is not enabled on the server'
16 })
17 }
18
19 return next()
20}
21
22export const videoChannelSyncValidator = [
23 body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
24 body('videoChannelId').isInt().withMessage('Should have a valid video channel id'),
25
26 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
27 logger.debug('Checking videoChannelSync parameters', { parameters: req.body })
28
29 if (areValidationErrors(req, res)) return
30
31 const body: VideoChannelSyncCreate = req.body
32 if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return
33
34 const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId)
35 if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) {
36 return res.fail({
37 message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations`
38 })
39 }
40
41 return next()
42 }
43]
44
45export const ensureSyncExists = [
46 param('id').exists().isInt().withMessage('Should have an sync id'),
47
48 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
49 if (areValidationErrors(req, res)) return
50
51 const syncId = parseInt(req.params.id, 10)
52 const sync = await VideoChannelSyncModel.loadWithChannel(syncId)
53
54 if (!sync) {
55 return res.fail({
56 status: HttpStatusCode.NOT_FOUND_404,
57 message: 'Synchronization not found'
58 })
59 }
60
61 res.locals.videoChannelSync = sync
62 res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
63
64 return next()
65 }
66]
diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts
index 3bfdebbb1..88f8b814d 100644
--- a/server/middlewares/validators/videos/video-channels.ts
+++ b/server/middlewares/validators/videos/video-channels.ts
@@ -1,5 +1,6 @@
1import express from 'express' 1import express from 'express'
2import { body, param, query } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
3import { CONFIG } from '@server/initializers/config' 4import { CONFIG } from '@server/initializers/config'
4import { MChannelAccountDefault } from '@server/types/models' 5import { MChannelAccountDefault } from '@server/types/models'
5import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' 6import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
@@ -13,9 +14,9 @@ import {
13import { logger } from '../../../helpers/logger' 14import { logger } from '../../../helpers/logger'
14import { ActorModel } from '../../../models/actor/actor' 15import { ActorModel } from '../../../models/actor/actor'
15import { VideoChannelModel } from '../../../models/video/video-channel' 16import { VideoChannelModel } from '../../../models/video/video-channel'
16import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared' 17import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared'
17 18
18const videoChannelsAddValidator = [ 19export const videoChannelsAddValidator = [
19 body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'), 20 body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'),
20 body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'), 21 body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'),
21 body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'), 22 body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'),
@@ -45,7 +46,7 @@ const videoChannelsAddValidator = [
45 } 46 }
46] 47]
47 48
48const videoChannelsUpdateValidator = [ 49export const videoChannelsUpdateValidator = [
49 param('nameWithHost').exists().withMessage('Should have an video channel name with host'), 50 param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
50 body('displayName') 51 body('displayName')
51 .optional() 52 .optional()
@@ -69,7 +70,7 @@ const videoChannelsUpdateValidator = [
69 } 70 }
70] 71]
71 72
72const videoChannelsRemoveValidator = [ 73export const videoChannelsRemoveValidator = [
73 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 74 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
74 logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params }) 75 logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params })
75 76
@@ -79,7 +80,7 @@ const videoChannelsRemoveValidator = [
79 } 80 }
80] 81]
81 82
82const videoChannelsNameWithHostValidator = [ 83export const videoChannelsNameWithHostValidator = [
83 param('nameWithHost').exists().withMessage('Should have an video channel name with host'), 84 param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
84 85
85 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 86 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -93,7 +94,7 @@ const videoChannelsNameWithHostValidator = [
93 } 94 }
94] 95]
95 96
96const ensureIsLocalChannel = [ 97export const ensureIsLocalChannel = [
97 (req: express.Request, res: express.Response, next: express.NextFunction) => { 98 (req: express.Request, res: express.Response, next: express.NextFunction) => {
98 if (res.locals.videoChannel.Actor.isOwned() === false) { 99 if (res.locals.videoChannel.Actor.isOwned() === false) {
99 return res.fail({ 100 return res.fail({
@@ -106,7 +107,18 @@ const ensureIsLocalChannel = [
106 } 107 }
107] 108]
108 109
109const videoChannelStatsValidator = [ 110export const ensureChannelOwnerCanUpload = [
111 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
112 const channel = res.locals.videoChannel
113 const user = { id: channel.Account.userId }
114
115 if (!await checkUserQuota(user, 1, res)) return
116
117 next()
118 }
119]
120
121export const videoChannelStatsValidator = [
110 query('withStats') 122 query('withStats')
111 .optional() 123 .optional()
112 .customSanitizer(toBooleanOrNull) 124 .customSanitizer(toBooleanOrNull)
@@ -118,7 +130,7 @@ const videoChannelStatsValidator = [
118 } 130 }
119] 131]
120 132
121const videoChannelsListValidator = [ 133export const videoChannelsListValidator = [
122 query('search').optional().not().isEmpty().withMessage('Should have a valid search'), 134 query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
123 135
124 (req: express.Request, res: express.Response, next: express.NextFunction) => { 136 (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -130,17 +142,24 @@ const videoChannelsListValidator = [
130 } 142 }
131] 143]
132 144
133// --------------------------------------------------------------------------- 145export const videoChannelImportVideosValidator = [
146 body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
134 147
135export { 148 (req: express.Request, res: express.Response, next: express.NextFunction) => {
136 videoChannelsAddValidator, 149 logger.debug('Checking videoChannelImport parameters', { parameters: req.body })
137 videoChannelsUpdateValidator, 150
138 videoChannelsRemoveValidator, 151 if (areValidationErrors(req, res)) return
139 videoChannelsNameWithHostValidator, 152
140 ensureIsLocalChannel, 153 if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
141 videoChannelsListValidator, 154 return res.fail({
142 videoChannelStatsValidator 155 status: HttpStatusCode.FORBIDDEN_403,
143} 156 message: 'Channel import is impossible as video upload via HTTP is not enabled on the server'
157 })
158 }
159
160 return next()
161 }
162]
144 163
145// --------------------------------------------------------------------------- 164// ---------------------------------------------------------------------------
146 165
diff --git a/server/models/utils.ts b/server/models/utils.ts
index c468f748d..1e168d419 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A
117 return getSort(value, lastSort) 117 return getSort(value, lastSort)
118} 118}
119 119
120function getChannelSyncSort (value: string): OrderItem[] {
121 const { direction, field } = buildDirectionAndField(value)
122 if (field.toLowerCase() === 'videochannel') {
123 return [
124 [ literal('"VideoChannel.name"'), direction ]
125 ]
126 }
127 return [ [ field, direction ] ]
128}
129
120function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { 130function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
121 if (!model.createdAt || !model.updatedAt) { 131 if (!model.createdAt || !model.updatedAt) {
122 throw new Error('Miss createdAt & updatedAt attributes to model') 132 throw new Error('Miss createdAt & updatedAt attributes to model')
@@ -280,6 +290,7 @@ export {
280 getAdminUsersSort, 290 getAdminUsersSort,
281 getVideoSort, 291 getVideoSort,
282 getBlacklistSort, 292 getBlacklistSort,
293 getChannelSyncSort,
283 createSimilarityAttribute, 294 createSimilarityAttribute,
284 throwIfNotValid, 295 throwIfNotValid,
285 buildServerIdsFollowedBy, 296 buildServerIdsFollowedBy,
diff --git a/server/models/video/video-channel-sync.ts b/server/models/video/video-channel-sync.ts
new file mode 100644
index 000000000..6e49cde10
--- /dev/null
+++ b/server/models/video/video-channel-sync.ts
@@ -0,0 +1,176 @@
1import { Op } from 'sequelize'
2import {
3 AllowNull,
4 BelongsTo,
5 Column,
6 CreatedAt,
7 DataType,
8 Default,
9 DefaultScope,
10 ForeignKey,
11 Is,
12 Model,
13 Table,
14 UpdatedAt
15} from 'sequelize-typescript'
16import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
17import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs'
18import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
19import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models'
20import { VideoChannelSync, VideoChannelSyncState } from '@shared/models'
21import { AttributesOnly } from '@shared/typescript-utils'
22import { AccountModel } from '../account/account'
23import { UserModel } from '../user/user'
24import { getChannelSyncSort, throwIfNotValid } from '../utils'
25import { VideoChannelModel } from './video-channel'
26
27@DefaultScope(() => ({
28 include: [
29 {
30 model: VideoChannelModel, // Default scope includes avatar and server
31 required: true
32 }
33 ]
34}))
35@Table({
36 tableName: 'videoChannelSync',
37 indexes: [
38 {
39 fields: [ 'videoChannelId' ]
40 }
41 ]
42})
43export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> {
44
45 @AllowNull(false)
46 @Default(null)
47 @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true))
48 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max))
49 externalChannelUrl: string
50
51 @CreatedAt
52 createdAt: Date
53
54 @UpdatedAt
55 updatedAt: Date
56
57 @ForeignKey(() => VideoChannelModel)
58 @Column
59 videoChannelId: number
60
61 @BelongsTo(() => VideoChannelModel, {
62 foreignKey: {
63 allowNull: false
64 },
65 onDelete: 'cascade'
66 })
67 VideoChannel: VideoChannelModel
68
69 @AllowNull(false)
70 @Default(VideoChannelSyncState.WAITING_FIRST_RUN)
71 @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state'))
72 @Column
73 state: VideoChannelSyncState
74
75 @AllowNull(true)
76 @Column(DataType.DATE)
77 lastSyncAt: Date
78
79 static listByAccountForAPI (options: {
80 accountId: number
81 start: number
82 count: number
83 sort: string
84 }) {
85 const getQuery = (forCount: boolean) => {
86 const videoChannelModel = forCount
87 ? VideoChannelModel.unscoped()
88 : VideoChannelModel
89
90 return {
91 offset: options.start,
92 limit: options.count,
93 order: getChannelSyncSort(options.sort),
94 include: [
95 {
96 model: videoChannelModel,
97 required: true,
98 where: {
99 accountId: options.accountId
100 }
101 }
102 ]
103 }
104 }
105
106 return Promise.all([
107 VideoChannelSyncModel.unscoped().count(getQuery(true)),
108 VideoChannelSyncModel.unscoped().findAll(getQuery(false))
109 ]).then(([ total, data ]) => ({ total, data }))
110 }
111
112 static countByAccount (accountId: number) {
113 const query = {
114 include: [
115 {
116 model: VideoChannelModel.unscoped(),
117 required: true,
118 where: {
119 accountId
120 }
121 }
122 ]
123 }
124
125 return VideoChannelSyncModel.unscoped().count(query)
126 }
127
128 static loadWithChannel (id: number): Promise<MChannelSyncChannel> {
129 return VideoChannelSyncModel.findByPk(id)
130 }
131
132 static async listSyncs (): Promise<MChannelSync[]> {
133 const query = {
134 include: [
135 {
136 model: VideoChannelModel.unscoped(),
137 required: true,
138 include: [
139 {
140 model: AccountModel.unscoped(),
141 required: true,
142 include: [ {
143 attributes: [],
144 model: UserModel.unscoped(),
145 required: true,
146 where: {
147 videoQuota: {
148 [Op.ne]: 0
149 },
150 videoQuotaDaily: {
151 [Op.ne]: 0
152 }
153 }
154 } ]
155 }
156 ]
157 }
158 ]
159 }
160 return VideoChannelSyncModel.unscoped().findAll(query)
161 }
162
163 toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync {
164 return {
165 id: this.id,
166 state: {
167 id: this.state,
168 label: VIDEO_CHANNEL_SYNC_STATE[this.state]
169 },
170 externalChannelUrl: this.externalChannelUrl,
171 createdAt: this.createdAt.toISOString(),
172 channel: this.VideoChannel.toFormattedSummaryJSON(),
173 lastSyncAt: this.lastSyncAt?.toISOString()
174 }
175 }
176}
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index 1d8296060..b8e941623 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -1,4 +1,4 @@
1import { WhereOptions } from 'sequelize' 1import { Op, WhereOptions } from 'sequelize'
2import { 2import {
3 AfterUpdate, 3 AfterUpdate,
4 AllowNull, 4 AllowNull,
@@ -161,6 +161,28 @@ export class VideoImportModel extends Model<Partial<AttributesOnly<VideoImportMo
161 ]).then(([ total, data ]) => ({ total, data })) 161 ]).then(([ total, data ]) => ({ total, data }))
162 } 162 }
163 163
164 static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
165 const element = await VideoImportModel.unscoped().findOne({
166 where: {
167 targetUrl,
168 state: {
169 [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
170 }
171 },
172 include: [
173 {
174 model: VideoModel,
175 required: true,
176 where: {
177 channelId
178 }
179 }
180 ]
181 })
182
183 return !!element
184 }
185
164 getTargetIdentifier () { 186 getTargetIdentifier () {
165 return this.targetUrl || this.magnetUri || this.torrentName 187 return this.targetUrl || this.magnetUri || this.torrentName
166 } 188 }
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 2f9f553ab..d67e51123 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -1,7 +1,8 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import 'mocha' 3import 'mocha'
4import { omit } from 'lodash' 4import { merge, omit } from 'lodash'
5import { CustomConfig, HttpStatusCode } from '@shared/models'
5import { 6import {
6 cleanupTests, 7 cleanupTests,
7 createSingleServer, 8 createSingleServer,
@@ -11,7 +12,6 @@ import {
11 PeerTubeServer, 12 PeerTubeServer,
12 setAccessTokensToServers 13 setAccessTokensToServers
13} from '@shared/server-commands' 14} from '@shared/server-commands'
14import { CustomConfig, HttpStatusCode } from '@shared/models'
15 15
16describe('Test config API validators', function () { 16describe('Test config API validators', function () {
17 const path = '/api/v1/config/custom' 17 const path = '/api/v1/config/custom'
@@ -162,6 +162,10 @@ describe('Test config API validators', function () {
162 torrent: { 162 torrent: {
163 enabled: false 163 enabled: false
164 } 164 }
165 },
166 videoChannelSynchronization: {
167 enabled: false,
168 maxPerUser: 10
165 } 169 }
166 }, 170 },
167 trending: { 171 trending: {
@@ -346,7 +350,26 @@ describe('Test config API validators', function () {
346 }) 350 })
347 }) 351 })
348 352
349 it('Should success with the correct parameters', async function () { 353 it('Should fail with a disabled http upload & enabled sync', async function () {
354 const newUpdateParams: CustomConfig = merge({}, updateParams, {
355 import: {
356 videos: {
357 http: { enabled: false }
358 },
359 videoChannelSynchronization: { enabled: true }
360 }
361 })
362
363 await makePutBodyRequest({
364 url: server.url,
365 path,
366 fields: newUpdateParams,
367 token: server.accessToken,
368 expectedStatus: HttpStatusCode.BAD_REQUEST_400
369 })
370 })
371
372 it('Should succeed with the correct parameters', async function () {
350 await makePutBodyRequest({ 373 await makePutBodyRequest({
351 url: server.url, 374 url: server.url,
352 path, 375 path,
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index a27bc8509..5f1168b53 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -27,6 +27,7 @@ import './video-channels'
27import './video-comments' 27import './video-comments'
28import './video-files' 28import './video-files'
29import './video-imports' 29import './video-imports'
30import './video-channel-syncs'
30import './video-playlists' 31import './video-playlists'
31import './video-source' 32import './video-source'
32import './video-studio' 33import './video-studio'
diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts
index deb4a7aa3..f64eafc18 100644
--- a/server/tests/api/check-params/upload-quota.ts
+++ b/server/tests/api/check-params/upload-quota.ts
@@ -70,7 +70,7 @@ describe('Test upload quota', function () {
70 }) 70 })
71 71
72 it('Should fail to import with HTTP/Torrent/magnet', async function () { 72 it('Should fail to import with HTTP/Torrent/magnet', async function () {
73 this.timeout(120000) 73 this.timeout(120_000)
74 74
75 const baseAttributes = { 75 const baseAttributes = {
76 channelId: server.store.channel.id, 76 channelId: server.store.channel.id,
diff --git a/server/tests/api/check-params/video-channel-syncs.ts b/server/tests/api/check-params/video-channel-syncs.ts
new file mode 100644
index 000000000..bcd8984df
--- /dev/null
+++ b/server/tests/api/check-params/video-channel-syncs.ts
@@ -0,0 +1,318 @@
1import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
2import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
3import {
4 ChannelSyncsCommand,
5 createSingleServer,
6 makePostBodyRequest,
7 PeerTubeServer,
8 setAccessTokensToServers,
9 setDefaultVideoChannel
10} from '@shared/server-commands'
11
12describe('Test video channel sync API validator', () => {
13 const path = '/api/v1/video-channel-syncs'
14 let server: PeerTubeServer
15 let command: ChannelSyncsCommand
16 let rootChannelId: number
17 let rootChannelSyncId: number
18 const userInfo = {
19 accessToken: '',
20 username: 'user1',
21 id: -1,
22 channelId: -1,
23 syncId: -1
24 }
25
26 async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> {
27 try {
28 await server.config.disableChannelSync()
29 await callback()
30 } finally {
31 await server.config.enableChannelSync()
32 }
33 }
34
35 async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> {
36 const origConfig = await server.config.getCustomConfig()
37
38 await server.config.updateExistingSubConfig({
39 newConfig: {
40 import: {
41 videoChannelSynchronization: {
42 maxPerUser: maxSync
43 }
44 }
45 }
46 })
47
48 try {
49 await callback()
50 } finally {
51 await server.config.updateCustomConfig({ newCustomConfig: origConfig })
52 }
53 }
54
55 before(async function () {
56 this.timeout(30_000)
57
58 server = await createSingleServer(1)
59
60 await setAccessTokensToServers([ server ])
61 await setDefaultVideoChannel([ server ])
62
63 command = server.channelSyncs
64
65 rootChannelId = server.store.channel.id
66
67 {
68 userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
69
70 const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken })
71 userInfo.id = userId
72 userInfo.channelId = videoChannels[0].id
73 }
74
75 await server.config.enableChannelSync()
76 })
77
78 describe('When creating a sync', function () {
79 let baseCorrectParams: VideoChannelSyncCreate
80
81 before(function () {
82 baseCorrectParams = {
83 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
84 videoChannelId: rootChannelId
85 }
86 })
87
88 it('Should fail when sync is disabled', async function () {
89 await withChannelSyncDisabled(async () => {
90 await command.create({
91 token: server.accessToken,
92 attributes: baseCorrectParams,
93 expectedStatus: HttpStatusCode.FORBIDDEN_403
94 })
95 })
96 })
97
98 it('Should fail with nothing', async function () {
99 const fields = {}
100 await makePostBodyRequest({
101 url: server.url,
102 path,
103 token: server.accessToken,
104 fields,
105 expectedStatus: HttpStatusCode.BAD_REQUEST_400
106 })
107 })
108
109 it('Should fail with no authentication', async function () {
110 await command.create({
111 token: null,
112 attributes: baseCorrectParams,
113 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
114 })
115 })
116
117 it('Should fail without a target url', async function () {
118 const attributes: VideoChannelSyncCreate = {
119 ...baseCorrectParams,
120 externalChannelUrl: null
121 }
122 await command.create({
123 token: server.accessToken,
124 attributes,
125 expectedStatus: HttpStatusCode.BAD_REQUEST_400
126 })
127 })
128
129 it('Should fail without a channelId', async function () {
130 const attributes: VideoChannelSyncCreate = {
131 ...baseCorrectParams,
132 videoChannelId: null
133 }
134 await command.create({
135 token: server.accessToken,
136 attributes,
137 expectedStatus: HttpStatusCode.BAD_REQUEST_400
138 })
139 })
140
141 it('Should fail with a channelId refering nothing', async function () {
142 const attributes: VideoChannelSyncCreate = {
143 ...baseCorrectParams,
144 videoChannelId: 42
145 }
146 await command.create({
147 token: server.accessToken,
148 attributes,
149 expectedStatus: HttpStatusCode.NOT_FOUND_404
150 })
151 })
152
153 it('Should fail to create a sync when the user does not own the channel', async function () {
154 await command.create({
155 token: userInfo.accessToken,
156 attributes: baseCorrectParams,
157 expectedStatus: HttpStatusCode.FORBIDDEN_403
158 })
159 })
160
161 it('Should succeed to create a sync with root and for another user\'s channel', async function () {
162 const { videoChannelSync } = await command.create({
163 token: server.accessToken,
164 attributes: {
165 ...baseCorrectParams,
166 videoChannelId: userInfo.channelId
167 },
168 expectedStatus: HttpStatusCode.OK_200
169 })
170 userInfo.syncId = videoChannelSync.id
171 })
172
173 it('Should succeed with the correct parameters', async function () {
174 const { videoChannelSync } = await command.create({
175 token: server.accessToken,
176 attributes: baseCorrectParams,
177 expectedStatus: HttpStatusCode.OK_200
178 })
179 rootChannelSyncId = videoChannelSync.id
180 })
181
182 it('Should fail when the user exceeds allowed number of synchronizations', async function () {
183 await withMaxSyncsPerUser(1, async () => {
184 await command.create({
185 token: server.accessToken,
186 attributes: {
187 ...baseCorrectParams,
188 videoChannelId: userInfo.channelId
189 },
190 expectedStatus: HttpStatusCode.BAD_REQUEST_400
191 })
192 })
193 })
194 })
195
196 describe('When listing my channel syncs', function () {
197 const myPath = '/api/v1/accounts/root/video-channel-syncs'
198
199 it('Should fail with a bad start pagination', async function () {
200 await checkBadStartPagination(server.url, myPath, server.accessToken)
201 })
202
203 it('Should fail with a bad count pagination', async function () {
204 await checkBadCountPagination(server.url, myPath, server.accessToken)
205 })
206
207 it('Should fail with an incorrect sort', async function () {
208 await checkBadSortPagination(server.url, myPath, server.accessToken)
209 })
210
211 it('Should succeed with the correct parameters', async function () {
212 await command.listByAccount({
213 accountName: 'root',
214 token: server.accessToken,
215 expectedStatus: HttpStatusCode.OK_200
216 })
217 })
218
219 it('Should fail with no authentication', async function () {
220 await command.listByAccount({
221 accountName: 'root',
222 token: null,
223 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
224 })
225 })
226
227 it('Should fail when a simple user lists another user\'s synchronizations', async function () {
228 await command.listByAccount({
229 accountName: 'root',
230 token: userInfo.accessToken,
231 expectedStatus: HttpStatusCode.FORBIDDEN_403
232 })
233 })
234
235 it('Should succeed when root lists another user\'s synchronizations', async function () {
236 await command.listByAccount({
237 accountName: userInfo.username,
238 token: server.accessToken,
239 expectedStatus: HttpStatusCode.OK_200
240 })
241 })
242
243 it('Should succeed even with synchronization disabled', async function () {
244 await withChannelSyncDisabled(async function () {
245 await command.listByAccount({
246 accountName: 'root',
247 token: server.accessToken,
248 expectedStatus: HttpStatusCode.OK_200
249 })
250 })
251 })
252 })
253
254 describe('When triggering deletion', function () {
255 it('should fail with no authentication', async function () {
256 await command.delete({
257 channelSyncId: userInfo.syncId,
258 token: null,
259 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
260 })
261 })
262
263 it('Should fail when channelSyncId does not refer to any sync', async function () {
264 await command.delete({
265 channelSyncId: 42,
266 token: server.accessToken,
267 expectedStatus: HttpStatusCode.NOT_FOUND_404
268 })
269 })
270
271 it('Should fail when sync is not owned by the user', async function () {
272 await command.delete({
273 channelSyncId: rootChannelSyncId,
274 token: userInfo.accessToken,
275 expectedStatus: HttpStatusCode.FORBIDDEN_403
276 })
277 })
278
279 it('Should succeed when root delete a sync they do not own', async function () {
280 await command.delete({
281 channelSyncId: userInfo.syncId,
282 token: server.accessToken,
283 expectedStatus: HttpStatusCode.NO_CONTENT_204
284 })
285 })
286
287 it('should succeed when user delete a sync they own', async function () {
288 const { videoChannelSync } = await command.create({
289 attributes: {
290 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
291 videoChannelId: userInfo.channelId
292 },
293 token: server.accessToken,
294 expectedStatus: HttpStatusCode.OK_200
295 })
296
297 await command.delete({
298 channelSyncId: videoChannelSync.id,
299 token: server.accessToken,
300 expectedStatus: HttpStatusCode.NO_CONTENT_204
301 })
302 })
303
304 it('Should succeed even when synchronization is disabled', async function () {
305 await withChannelSyncDisabled(async function () {
306 await command.delete({
307 channelSyncId: rootChannelSyncId,
308 token: server.accessToken,
309 expectedStatus: HttpStatusCode.NO_CONTENT_204
310 })
311 })
312 })
313 })
314
315 after(async function () {
316 await server?.kill()
317 })
318})
diff --git a/server/tests/api/check-params/video-channels.ts b/server/tests/api/check-params/video-channels.ts
index 5c2650fac..337ea1dd4 100644
--- a/server/tests/api/check-params/video-channels.ts
+++ b/server/tests/api/check-params/video-channels.ts
@@ -3,8 +3,8 @@
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { omit } from 'lodash' 5import { omit } from 'lodash'
6import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' 6import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
7import { buildAbsoluteFixturePath } from '@shared/core-utils' 7import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils'
8import { HttpStatusCode, VideoChannelUpdate } from '@shared/models' 8import { HttpStatusCode, VideoChannelUpdate } from '@shared/models'
9import { 9import {
10 ChannelsCommand, 10 ChannelsCommand,
@@ -23,7 +23,13 @@ const expect = chai.expect
23describe('Test video channels API validator', function () { 23describe('Test video channels API validator', function () {
24 const videoChannelPath = '/api/v1/video-channels' 24 const videoChannelPath = '/api/v1/video-channels'
25 let server: PeerTubeServer 25 let server: PeerTubeServer
26 let accessTokenUser: string 26 const userInfo = {
27 accessToken: '',
28 channelName: 'fake_channel',
29 id: -1,
30 videoQuota: -1,
31 videoQuotaDaily: -1
32 }
27 let command: ChannelsCommand 33 let command: ChannelsCommand
28 34
29 // --------------------------------------------------------------- 35 // ---------------------------------------------------------------
@@ -35,14 +41,15 @@ describe('Test video channels API validator', function () {
35 41
36 await setAccessTokensToServers([ server ]) 42 await setAccessTokensToServers([ server ])
37 43
38 const user = { 44 const userCreds = {
39 username: 'fake', 45 username: 'fake',
40 password: 'fake_password' 46 password: 'fake_password'
41 } 47 }
42 48
43 { 49 {
44 await server.users.create({ username: user.username, password: user.password }) 50 const user = await server.users.create({ username: userCreds.username, password: userCreds.password })
45 accessTokenUser = await server.login.getAccessToken(user) 51 userInfo.id = user.id
52 userInfo.accessToken = await server.login.getAccessToken(userCreds)
46 } 53 }
47 54
48 command = server.channels 55 command = server.channels
@@ -191,7 +198,7 @@ describe('Test video channels API validator', function () {
191 await makePutBodyRequest({ 198 await makePutBodyRequest({
192 url: server.url, 199 url: server.url,
193 path, 200 path,
194 token: accessTokenUser, 201 token: userInfo.accessToken,
195 fields: baseCorrectParams, 202 fields: baseCorrectParams,
196 expectedStatus: HttpStatusCode.FORBIDDEN_403 203 expectedStatus: HttpStatusCode.FORBIDDEN_403
197 }) 204 })
@@ -339,7 +346,7 @@ describe('Test video channels API validator', function () {
339 }) 346 })
340 347
341 it('Should fail with a another user', async function () { 348 it('Should fail with a another user', async function () {
342 await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) 349 await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
343 }) 350 })
344 351
345 it('Should succeed with the correct params', async function () { 352 it('Should succeed with the correct params', async function () {
@@ -347,13 +354,122 @@ describe('Test video channels API validator', function () {
347 }) 354 })
348 }) 355 })
349 356
357 describe('When triggering full synchronization', function () {
358
359 it('Should fail when HTTP upload is disabled', async function () {
360 await server.config.disableImports()
361
362 await command.importVideos({
363 channelName: 'super_channel',
364 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
365 token: server.accessToken,
366 expectedStatus: HttpStatusCode.FORBIDDEN_403
367 })
368
369 await server.config.enableImports()
370 })
371
372 it('Should fail when externalChannelUrl is not provided', async function () {
373 await command.importVideos({
374 channelName: 'super_channel',
375 externalChannelUrl: null,
376 token: server.accessToken,
377 expectedStatus: HttpStatusCode.BAD_REQUEST_400
378 })
379 })
380
381 it('Should fail when externalChannelUrl is malformed', async function () {
382 await command.importVideos({
383 channelName: 'super_channel',
384 externalChannelUrl: 'not-a-url',
385 token: server.accessToken,
386 expectedStatus: HttpStatusCode.BAD_REQUEST_400
387 })
388 })
389
390 it('Should fail with no authentication', async function () {
391 await command.importVideos({
392 channelName: 'super_channel',
393 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
394 token: null,
395 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
396 })
397 })
398
399 it('Should fail when sync is not owned by the user', async function () {
400 await command.importVideos({
401 channelName: 'super_channel',
402 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
403 token: userInfo.accessToken,
404 expectedStatus: HttpStatusCode.FORBIDDEN_403
405 })
406 })
407
408 it('Should fail when the user has no quota', async function () {
409 await server.users.update({
410 userId: userInfo.id,
411 videoQuota: 0
412 })
413
414 await command.importVideos({
415 channelName: 'fake_channel',
416 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
417 token: userInfo.accessToken,
418 expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
419 })
420
421 await server.users.update({
422 userId: userInfo.id,
423 videoQuota: userInfo.videoQuota
424 })
425 })
426
427 it('Should fail when the user has no daily quota', async function () {
428 await server.users.update({
429 userId: userInfo.id,
430 videoQuotaDaily: 0
431 })
432
433 await command.importVideos({
434 channelName: 'fake_channel',
435 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
436 token: userInfo.accessToken,
437 expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
438 })
439
440 await server.users.update({
441 userId: userInfo.id,
442 videoQuotaDaily: userInfo.videoQuotaDaily
443 })
444 })
445
446 it('Should succeed when sync is run by its owner', async function () {
447 if (!areHttpImportTestsDisabled()) return
448
449 await command.importVideos({
450 channelName: 'fake_channel',
451 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
452 token: userInfo.accessToken
453 })
454 })
455
456 it('Should succeed when sync is run with root and for another user\'s channel', async function () {
457 if (!areHttpImportTestsDisabled()) return
458
459 await command.importVideos({
460 channelName: 'fake_channel',
461 externalChannelUrl: FIXTURE_URLS.youtubeChannel
462 })
463 })
464 })
465
350 describe('When deleting a video channel', function () { 466 describe('When deleting a video channel', function () {
351 it('Should fail with a non authenticated user', async function () { 467 it('Should fail with a non authenticated user', async function () {
352 await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) 468 await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
353 }) 469 })
354 470
355 it('Should fail with another authenticated user', async function () { 471 it('Should fail with another authenticated user', async function () {
356 await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) 472 await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
357 }) 473 })
358 474
359 it('Should fail with an unknown video channel id', async function () { 475 it('Should fail with an unknown video channel id', async function () {
diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts
index 4439810e8..5cdd0d925 100644
--- a/server/tests/api/check-params/video-imports.ts
+++ b/server/tests/api/check-params/video-imports.ts
@@ -88,7 +88,13 @@ describe('Test video imports API validator', function () {
88 88
89 it('Should fail with nothing', async function () { 89 it('Should fail with nothing', async function () {
90 const fields = {} 90 const fields = {}
91 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) 91 await makePostBodyRequest({
92 url: server.url,
93 path,
94 token: server.accessToken,
95 fields,
96 expectedStatus: HttpStatusCode.BAD_REQUEST_400
97 })
92 }) 98 })
93 99
94 it('Should fail without a target url', async function () { 100 it('Should fail without a target url', async function () {
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index efc57b345..fc8711161 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -368,6 +368,10 @@ const newCustomConfig: CustomConfig = {
368 torrent: { 368 torrent: {
369 enabled: false 369 enabled: false
370 } 370 }
371 },
372 videoChannelSynchronization: {
373 enabled: false,
374 maxPerUser: 10
371 } 375 }
372 }, 376 },
373 trending: { 377 trending: {
diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts
new file mode 100644
index 000000000..f7540e1ba
--- /dev/null
+++ b/server/tests/api/videos/channel-import-videos.ts
@@ -0,0 +1,50 @@
1import { expect } from 'chai'
2import { FIXTURE_URLS } from '@server/tests/shared'
3import { areHttpImportTestsDisabled } from '@shared/core-utils'
4import {
5 createSingleServer,
6 getServerImportConfig,
7 PeerTubeServer,
8 setAccessTokensToServers,
9 setDefaultVideoChannel,
10 waitJobs
11} from '@shared/server-commands'
12
13describe('Test videos import in a channel', function () {
14 if (areHttpImportTestsDisabled()) return
15
16 function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
17
18 describe('Import using ' + mode, function () {
19 let server: PeerTubeServer
20
21 before(async function () {
22 this.timeout(120_000)
23
24 server = await createSingleServer(1, getServerImportConfig(mode))
25
26 await setAccessTokensToServers([ server ])
27 await setDefaultVideoChannel([ server ])
28
29 await server.config.enableChannelSync()
30 })
31
32 it('Should import a whole channel', async function () {
33 this.timeout(240_000)
34
35 await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel })
36 await waitJobs(server)
37
38 const videos = await server.videos.listByChannel({ handle: server.store.channel.name })
39 expect(videos.total).to.equal(2)
40 })
41
42 after(async function () {
43 await server?.kill()
44 })
45 })
46 }
47
48 runSuite('yt-dlp')
49 runSuite('youtube-dl')
50})
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index a0b6b01cf..266155297 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -4,6 +4,8 @@ import './single-server'
4import './video-captions' 4import './video-captions'
5import './video-change-ownership' 5import './video-change-ownership'
6import './video-channels' 6import './video-channels'
7import './channel-import-videos'
8import './video-channel-syncs'
7import './video-comments' 9import './video-comments'
8import './video-description' 10import './video-description'
9import './video-files' 11import './video-files'
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts
new file mode 100644
index 000000000..229c01f68
--- /dev/null
+++ b/server/tests/api/videos/video-channel-syncs.ts
@@ -0,0 +1,226 @@
1import 'mocha'
2import { expect } from 'chai'
3import { FIXTURE_URLS } from '@server/tests/shared'
4import { areHttpImportTestsDisabled } from '@shared/core-utils'
5import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
6import {
7 ChannelSyncsCommand,
8 createSingleServer,
9 getServerImportConfig,
10 PeerTubeServer,
11 setAccessTokensToServers,
12 setDefaultAccountAvatar,
13 setDefaultChannelAvatar,
14 setDefaultVideoChannel,
15 waitJobs
16} from '@shared/server-commands'
17
18describe('Test channel synchronizations', function () {
19 if (areHttpImportTestsDisabled()) return
20
21 function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
22
23 describe('Sync using ' + mode, function () {
24 let server: PeerTubeServer
25 let command: ChannelSyncsCommand
26 let startTestDate: Date
27 const userInfo = {
28 accessToken: '',
29 username: 'user1',
30 channelName: 'user1_channel',
31 channelId: -1,
32 syncId: -1
33 }
34
35 async function changeDateForSync (channelSyncId: number, newDate: string) {
36 await server.sql.updateQuery(
37 `UPDATE "videoChannelSync" ` +
38 `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
39 `WHERE id=${channelSyncId}`
40 )
41 }
42
43 before(async function () {
44 this.timeout(120_000)
45
46 startTestDate = new Date()
47
48 server = await createSingleServer(1, getServerImportConfig(mode))
49
50 await setAccessTokensToServers([ server ])
51 await setDefaultVideoChannel([ server ])
52 await setDefaultChannelAvatar([ server ])
53 await setDefaultAccountAvatar([ server ])
54
55 await server.config.enableChannelSync()
56
57 command = server.channelSyncs
58
59 {
60 userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
61
62 const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
63 userInfo.channelId = videoChannels[0].id
64 }
65 })
66
67 it('Should fetch the latest channel videos of a remote channel', async function () {
68 this.timeout(120_000)
69
70 {
71 const { video } = await server.imports.importVideo({
72 attributes: {
73 channelId: server.store.channel.id,
74 privacy: VideoPrivacy.PUBLIC,
75 targetUrl: FIXTURE_URLS.youtube
76 }
77 })
78
79 expect(video.name).to.equal('small video - youtube')
80
81 const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
82 expect(total).to.equal(1)
83 }
84
85 const { videoChannelSync } = await command.create({
86 attributes: {
87 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
88 videoChannelId: server.store.channel.id
89 },
90 token: server.accessToken,
91 expectedStatus: HttpStatusCode.OK_200
92 })
93
94 // Ensure any missing video not already fetched will be considered as new
95 await changeDateForSync(videoChannelSync.id, '1970-01-01')
96
97 await server.debug.sendCommand({
98 body: {
99 command: 'process-video-channel-sync-latest'
100 }
101 })
102
103 {
104 await waitJobs(server)
105
106 const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
107 expect(total).to.equal(2)
108 expect(data[0].name).to.equal('test')
109 }
110 })
111
112 it('Should add another synchronization', async function () {
113 const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
114
115 const { videoChannelSync } = await command.create({
116 attributes: {
117 externalChannelUrl,
118 videoChannelId: server.store.channel.id
119 },
120 token: server.accessToken,
121 expectedStatus: HttpStatusCode.OK_200
122 })
123
124 expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
125 expect(videoChannelSync.channel).to.include({
126 id: server.store.channel.id,
127 name: 'root_channel'
128 })
129 expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
130 expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
131 })
132
133 it('Should add a synchronization for another user', async function () {
134 const { videoChannelSync } = await command.create({
135 attributes: {
136 externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
137 videoChannelId: userInfo.channelId
138 },
139 token: userInfo.accessToken
140 })
141 userInfo.syncId = videoChannelSync.id
142 })
143
144 it('Should not import a channel if not asked', async function () {
145 await waitJobs(server)
146
147 const { data } = await command.listByAccount({ accountName: userInfo.username })
148
149 expect(data[0].state).to.contain({
150 id: VideoChannelSyncState.WAITING_FIRST_RUN,
151 label: 'Waiting first run'
152 })
153 })
154
155 it('Should only fetch the videos newer than the creation date', async function () {
156 this.timeout(120_000)
157
158 await changeDateForSync(userInfo.syncId, '2019-03-01')
159
160 await server.debug.sendCommand({
161 body: {
162 command: 'process-video-channel-sync-latest'
163 }
164 })
165
166 await waitJobs(server)
167
168 const { data, total } = await server.videos.listByChannel({
169 handle: userInfo.channelName,
170 include: VideoInclude.NOT_PUBLISHED_STATE
171 })
172
173 expect(total).to.equal(1)
174 expect(data[0].name).to.equal('test')
175 })
176
177 it('Should list channel synchronizations', async function () {
178 // Root
179 {
180 const { total, data } = await command.listByAccount({ accountName: 'root' })
181 expect(total).to.equal(2)
182
183 expect(data[0]).to.deep.contain({
184 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
185 state: {
186 id: VideoChannelSyncState.SYNCED,
187 label: 'Synchronized'
188 }
189 })
190
191 expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
192
193 expect(data[0].channel).to.contain({ id: server.store.channel.id })
194 expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
195 }
196
197 // User
198 {
199 const { total, data } = await command.listByAccount({ accountName: userInfo.username })
200 expect(total).to.equal(1)
201 expect(data[0]).to.deep.contain({
202 externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
203 state: {
204 id: VideoChannelSyncState.SYNCED,
205 label: 'Synchronized'
206 }
207 })
208 }
209 })
210
211 it('Should remove user\'s channel synchronizations', async function () {
212 await command.delete({ channelSyncId: userInfo.syncId })
213
214 const { total } = await command.listByAccount({ accountName: userInfo.username })
215 expect(total).to.equal(0)
216 })
217
218 after(async function () {
219 await server?.kill()
220 })
221 })
222 }
223
224 runSuite('youtube-dl')
225 runSuite('yt-dlp')
226})
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
index 603e2d234..a487062a2 100644
--- a/server/tests/api/videos/video-imports.ts
+++ b/server/tests/api/videos/video-imports.ts
@@ -12,6 +12,7 @@ import {
12 createMultipleServers, 12 createMultipleServers,
13 createSingleServer, 13 createSingleServer,
14 doubleFollow, 14 doubleFollow,
15 getServerImportConfig,
15 PeerTubeServer, 16 PeerTubeServer,
16 setAccessTokensToServers, 17 setAccessTokensToServers,
17 setDefaultVideoChannel, 18 setDefaultVideoChannel,
@@ -84,24 +85,9 @@ describe('Test video imports', function () {
84 let servers: PeerTubeServer[] = [] 85 let servers: PeerTubeServer[] = []
85 86
86 before(async function () { 87 before(async function () {
87 this.timeout(30_000) 88 this.timeout(60_000)
88 89
89 // Run servers 90 servers = await createMultipleServers(2, getServerImportConfig(mode))
90 servers = await createMultipleServers(2, {
91 import: {
92 videos: {
93 http: {
94 youtube_dl_release: {
95 url: mode === 'youtube-dl'
96 ? 'https://yt-dl.org/downloads/latest/youtube-dl'
97 : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
98
99 name: mode
100 }
101 }
102 }
103 }
104 })
105 91
106 await setAccessTokensToServers(servers) 92 await setAccessTokensToServers(servers)
107 await setDefaultVideoChannel(servers) 93 await setDefaultVideoChannel(servers)
diff --git a/server/tests/shared/tests.ts b/server/tests/shared/tests.ts
index 3abaf833d..e67a294dc 100644
--- a/server/tests/shared/tests.ts
+++ b/server/tests/shared/tests.ts
@@ -16,6 +16,8 @@ const FIXTURE_URLS = {
16 */ 16 */
17 youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4', 17 youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4',
18 18
19 youtubeChannel: 'https://youtube.com/channel/UCtnlZdXv3-xQzxiqfn6cjIA',
20
19 // eslint-disable-next-line max-len 21 // eslint-disable-next-line max-len
20 magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4', 22 magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4',
21 23
diff --git a/server/types/express.d.ts b/server/types/express.d.ts
index 8f8c65102..27d60da72 100644
--- a/server/types/express.d.ts
+++ b/server/types/express.d.ts
@@ -8,6 +8,7 @@ import {
8 MActorFollowActorsDefault, 8 MActorFollowActorsDefault,
9 MActorUrl, 9 MActorUrl,
10 MChannelBannerAccountDefault, 10 MChannelBannerAccountDefault,
11 MChannelSyncChannel,
11 MStreamingPlaylist, 12 MStreamingPlaylist,
12 MVideoChangeOwnershipFull, 13 MVideoChangeOwnershipFull,
13 MVideoFile, 14 MVideoFile,
@@ -145,6 +146,7 @@ declare module 'express' {
145 videoStreamingPlaylist?: MStreamingPlaylist 146 videoStreamingPlaylist?: MStreamingPlaylist
146 147
147 videoChannel?: MChannelBannerAccountDefault 148 videoChannel?: MChannelBannerAccountDefault
149 videoChannelSync?: MChannelSyncChannel
148 150
149 videoPlaylistFull?: MVideoPlaylistFull 151 videoPlaylistFull?: MVideoPlaylistFull
150 videoPlaylistSummary?: MVideoPlaylistFullSummary 152 videoPlaylistSummary?: MVideoPlaylistFullSummary
@@ -194,6 +196,7 @@ declare module 'express' {
194 plugin?: MPlugin 196 plugin?: MPlugin
195 197
196 localViewerFull?: MLocalVideoViewerWithWatchSections 198 localViewerFull?: MLocalVideoViewerWithWatchSections
199
197 } 200 }
198 } 201 }
199} 202}
diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts
index fdf8e1ddb..940f0ac0d 100644
--- a/server/types/models/video/index.ts
+++ b/server/types/models/video/index.ts
@@ -8,6 +8,7 @@ export * from './video'
8export * from './video-blacklist' 8export * from './video-blacklist'
9export * from './video-caption' 9export * from './video-caption'
10export * from './video-change-ownership' 10export * from './video-change-ownership'
11export * from './video-channel-sync'
11export * from './video-channels' 12export * from './video-channels'
12export * from './video-comment' 13export * from './video-comment'
13export * from './video-file' 14export * from './video-file'
diff --git a/server/types/models/video/video-channel-sync.ts b/server/types/models/video/video-channel-sync.ts
new file mode 100644
index 000000000..429ab70b0
--- /dev/null
+++ b/server/types/models/video/video-channel-sync.ts
@@ -0,0 +1,17 @@
1import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
2import { FunctionProperties, PickWith } from '@shared/typescript-utils'
3import { MChannelAccountDefault, MChannelFormattable } from './video-channels'
4
5type Use<K extends keyof VideoChannelSyncModel, M> = PickWith<VideoChannelSyncModel, K, M>
6
7export type MChannelSync = Omit<VideoChannelSyncModel, 'VideoChannel'>
8
9export type MChannelSyncChannel =
10 MChannelSync &
11 Use<'VideoChannel', MChannelAccountDefault> &
12 FunctionProperties<VideoChannelSyncModel>
13
14export type MChannelSyncFormattable =
15 FunctionProperties<MChannelSyncChannel> &
16 Use<'VideoChannel', MChannelFormattable> &
17 MChannelSync
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts
index bb9c7cef1..7d9d570b1 100644
--- a/shared/models/server/custom-config.model.ts
+++ b/shared/models/server/custom-config.model.ts
@@ -165,6 +165,10 @@ export interface CustomConfig {
165 enabled: boolean 165 enabled: boolean
166 } 166 }
167 } 167 }
168 videoChannelSynchronization: {
169 enabled: boolean
170 maxPerUser: number
171 }
168 } 172 }
169 173
170 trending: { 174 trending: {
diff --git a/shared/models/server/debug.model.ts b/shared/models/server/debug.model.ts
index 223d23362..1c4597b8b 100644
--- a/shared/models/server/debug.model.ts
+++ b/shared/models/server/debug.model.ts
@@ -4,5 +4,8 @@ export interface Debug {
4} 4}
5 5
6export interface SendDebugCommand { 6export interface SendDebugCommand {
7 command: 'remove-dandling-resumable-uploads' | 'process-video-views-buffer' | 'process-video-viewers' 7 command: 'remove-dandling-resumable-uploads'
8 | 'process-video-views-buffer'
9 | 'process-video-viewers'
10 | 'process-video-channel-sync-latest'
8} 11}
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 8c8f64de9..137da367c 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -25,6 +25,8 @@ export type JobType =
25 | 'manage-video-torrent' 25 | 'manage-video-torrent'
26 | 'move-to-object-storage' 26 | 'move-to-object-storage'
27 | 'video-studio-edition' 27 | 'video-studio-edition'
28 | 'video-channel-import'
29 | 'after-video-channel-import'
28 | 'notify' 30 | 'notify'
29 | 'federate-video' 31 | 'federate-video'
30 32
@@ -82,20 +84,32 @@ export type VideoFileImportPayload = {
82 filePath: string 84 filePath: string
83} 85}
84 86
87// ---------------------------------------------------------------------------
88
85export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file' 89export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file'
86export type VideoImportYoutubeDLPayloadType = 'youtube-dl' 90export type VideoImportYoutubeDLPayloadType = 'youtube-dl'
87 91
88export type VideoImportYoutubeDLPayload = { 92export interface VideoImportYoutubeDLPayload {
89 type: VideoImportYoutubeDLPayloadType 93 type: VideoImportYoutubeDLPayloadType
90 videoImportId: number 94 videoImportId: number
91 95
92 fileExt?: string 96 fileExt?: string
93} 97}
94export type VideoImportTorrentPayload = { 98
99export interface VideoImportTorrentPayload {
95 type: VideoImportTorrentPayloadType 100 type: VideoImportTorrentPayloadType
96 videoImportId: number 101 videoImportId: number
97} 102}
98export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload 103
104export type VideoImportPayload = (VideoImportYoutubeDLPayload | VideoImportTorrentPayload) & {
105 preventException: boolean
106}
107
108export interface VideoImportPreventExceptionResult {
109 resultType: 'success' | 'error'
110}
111
112// ---------------------------------------------------------------------------
99 113
100export type VideoRedundancyPayload = { 114export type VideoRedundancyPayload = {
101 videoId: number 115 videoId: number
@@ -219,6 +233,17 @@ export interface VideoStudioEditionPayload {
219 233
220// --------------------------------------------------------------------------- 234// ---------------------------------------------------------------------------
221 235
236export interface VideoChannelImportPayload {
237 externalChannelUrl: string
238 videoChannelId: number
239}
240
241export interface AfterVideoChannelImportPayload {
242 channelSyncId: number
243}
244
245// ---------------------------------------------------------------------------
246
222export type NotifyPayload = 247export type NotifyPayload =
223 { 248 {
224 action: 'new-video' 249 action: 'new-video'
diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts
index 67ad809f7..3b6d0597c 100644
--- a/shared/models/server/server-config.model.ts
+++ b/shared/models/server/server-config.model.ts
@@ -188,6 +188,9 @@ export interface ServerConfig {
188 enabled: boolean 188 enabled: boolean
189 } 189 }
190 } 190 }
191 videoChannelSynchronization: {
192 enabled: boolean
193 }
191 } 194 }
192 195
193 autoBlacklist: { 196 autoBlacklist: {
diff --git a/shared/models/videos/channel-sync/index.ts b/shared/models/videos/channel-sync/index.ts
new file mode 100644
index 000000000..7d25aaac3
--- /dev/null
+++ b/shared/models/videos/channel-sync/index.ts
@@ -0,0 +1,3 @@
1export * from './video-channel-sync-state.enum'
2export * from './video-channel-sync.model'
3export * from './video-channel-sync-create.model'
diff --git a/shared/models/videos/channel-sync/video-channel-sync-create.model.ts b/shared/models/videos/channel-sync/video-channel-sync-create.model.ts
new file mode 100644
index 000000000..753a8ee4c
--- /dev/null
+++ b/shared/models/videos/channel-sync/video-channel-sync-create.model.ts
@@ -0,0 +1,4 @@
1export interface VideoChannelSyncCreate {
2 externalChannelUrl: string
3 videoChannelId: number
4}
diff --git a/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts b/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts
new file mode 100644
index 000000000..3e9f5ddc2
--- /dev/null
+++ b/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts
@@ -0,0 +1,6 @@
1export const enum VideoChannelSyncState {
2 WAITING_FIRST_RUN = 1,
3 PROCESSING = 2,
4 SYNCED = 3,
5 FAILED = 4
6}
diff --git a/shared/models/videos/channel-sync/video-channel-sync.model.ts b/shared/models/videos/channel-sync/video-channel-sync.model.ts
new file mode 100644
index 000000000..73ac0615b
--- /dev/null
+++ b/shared/models/videos/channel-sync/video-channel-sync.model.ts
@@ -0,0 +1,14 @@
1import { VideoChannelSummary } from '../channel/video-channel.model'
2import { VideoConstant } from '../video-constant.model'
3import { VideoChannelSyncState } from './video-channel-sync-state.enum'
4
5export interface VideoChannelSync {
6 id: number
7
8 externalChannelUrl: string
9
10 createdAt: string
11 channel: VideoChannelSummary
12 state: VideoConstant<VideoChannelSyncState>
13 lastSyncAt: string
14}
diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts
index 05497bda1..f8e6976d3 100644
--- a/shared/models/videos/index.ts
+++ b/shared/models/videos/index.ts
@@ -11,6 +11,7 @@ export * from './playlist'
11export * from './rate' 11export * from './rate'
12export * from './stats' 12export * from './stats'
13export * from './transcoding' 13export * from './transcoding'
14export * from './channel-sync'
14 15
15export * from './nsfw-policy.type' 16export * from './nsfw-policy.type'
16 17
diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts
index 8ab750983..1c2315ed1 100644
--- a/shared/server-commands/server/config-command.ts
+++ b/shared/server-commands/server/config-command.ts
@@ -18,17 +18,25 @@ export class ConfigCommand extends AbstractCommand {
18 } 18 }
19 } 19 }
20 20
21 disableImports () {
22 return this.setImportsEnabled(false)
23 }
24
21 enableImports () { 25 enableImports () {
26 return this.setImportsEnabled(true)
27 }
28
29 private setImportsEnabled (enabled: boolean) {
22 return this.updateExistingSubConfig({ 30 return this.updateExistingSubConfig({
23 newConfig: { 31 newConfig: {
24 import: { 32 import: {
25 videos: { 33 videos: {
26 http: { 34 http: {
27 enabled: true 35 enabled
28 }, 36 },
29 37
30 torrent: { 38 torrent: {
31 enabled: true 39 enabled
32 } 40 }
33 } 41 }
34 } 42 }
@@ -36,6 +44,26 @@ export class ConfigCommand extends AbstractCommand {
36 }) 44 })
37 } 45 }
38 46
47 private setChannelSyncEnabled (enabled: boolean) {
48 return this.updateExistingSubConfig({
49 newConfig: {
50 import: {
51 videoChannelSynchronization: {
52 enabled
53 }
54 }
55 }
56 })
57 }
58
59 enableChannelSync () {
60 return this.setChannelSyncEnabled(true)
61 }
62
63 disableChannelSync () {
64 return this.setChannelSyncEnabled(false)
65 }
66
39 enableLive (options: { 67 enableLive (options: {
40 allowReplay?: boolean 68 allowReplay?: boolean
41 transcoding?: boolean 69 transcoding?: boolean
@@ -356,6 +384,10 @@ export class ConfigCommand extends AbstractCommand {
356 torrent: { 384 torrent: {
357 enabled: false 385 enabled: false
358 } 386 }
387 },
388 videoChannelSynchronization: {
389 enabled: false,
390 maxPerUser: 10
359 } 391 }
360 }, 392 },
361 trending: { 393 trending: {
diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index 0ad818a11..7acbc978f 100644
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -19,6 +19,7 @@ import {
19 CaptionsCommand, 19 CaptionsCommand,
20 ChangeOwnershipCommand, 20 ChangeOwnershipCommand,
21 ChannelsCommand, 21 ChannelsCommand,
22 ChannelSyncsCommand,
22 HistoryCommand, 23 HistoryCommand,
23 ImportsCommand, 24 ImportsCommand,
24 LiveCommand, 25 LiveCommand,
@@ -118,6 +119,7 @@ export class PeerTubeServer {
118 playlists?: PlaylistsCommand 119 playlists?: PlaylistsCommand
119 history?: HistoryCommand 120 history?: HistoryCommand
120 imports?: ImportsCommand 121 imports?: ImportsCommand
122 channelSyncs?: ChannelSyncsCommand
121 streamingPlaylists?: StreamingPlaylistsCommand 123 streamingPlaylists?: StreamingPlaylistsCommand
122 channels?: ChannelsCommand 124 channels?: ChannelsCommand
123 comments?: CommentsCommand 125 comments?: CommentsCommand
@@ -390,6 +392,7 @@ export class PeerTubeServer {
390 this.playlists = new PlaylistsCommand(this) 392 this.playlists = new PlaylistsCommand(this)
391 this.history = new HistoryCommand(this) 393 this.history = new HistoryCommand(this)
392 this.imports = new ImportsCommand(this) 394 this.imports = new ImportsCommand(this)
395 this.channelSyncs = new ChannelSyncsCommand(this)
393 this.streamingPlaylists = new StreamingPlaylistsCommand(this) 396 this.streamingPlaylists = new StreamingPlaylistsCommand(this)
394 this.channels = new ChannelsCommand(this) 397 this.channels = new ChannelsCommand(this)
395 this.comments = new CommentsCommand(this) 398 this.comments = new CommentsCommand(this)
diff --git a/shared/server-commands/server/servers.ts b/shared/server-commands/server/servers.ts
index 0faee3a8d..29f01774d 100644
--- a/shared/server-commands/server/servers.ts
+++ b/shared/server-commands/server/servers.ts
@@ -39,11 +39,30 @@ async function cleanupTests (servers: PeerTubeServer[]) {
39 return Promise.all(p) 39 return Promise.all(p)
40} 40}
41 41
42function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
43 return {
44 import: {
45 videos: {
46 http: {
47 youtube_dl_release: {
48 url: mode === 'youtube-dl'
49 ? 'https://yt-dl.org/downloads/latest/youtube-dl'
50 : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
51
52 name: mode
53 }
54 }
55 }
56 }
57 }
58}
59
42// --------------------------------------------------------------------------- 60// ---------------------------------------------------------------------------
43 61
44export { 62export {
45 createSingleServer, 63 createSingleServer,
46 createMultipleServers, 64 createMultipleServers,
47 cleanupTests, 65 cleanupTests,
48 killallServers 66 killallServers,
67 getServerImportConfig
49} 68}
diff --git a/shared/server-commands/videos/channel-syncs-command.ts b/shared/server-commands/videos/channel-syncs-command.ts
new file mode 100644
index 000000000..de4a160ec
--- /dev/null
+++ b/shared/server-commands/videos/channel-syncs-command.ts
@@ -0,0 +1,55 @@
1import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@shared/models'
2import { pick } from '@shared/core-utils'
3import { unwrapBody } from '../requests'
4import { AbstractCommand, OverrideCommandOptions } from '../shared'
5
6export class ChannelSyncsCommand extends AbstractCommand {
7 private static readonly API_PATH = '/api/v1/video-channel-syncs'
8
9 listByAccount (options: OverrideCommandOptions & {
10 accountName: string
11 start?: number
12 count?: number
13 sort?: string
14 }) {
15 const { accountName, sort = 'createdAt' } = options
16
17 const path = `/api/v1/accounts/${accountName}/video-channel-syncs`
18
19 return this.getRequestBody<ResultList<VideoChannelSync>>({
20 ...options,
21
22 path,
23 query: { sort, ...pick(options, [ 'start', 'count' ]) },
24 implicitToken: true,
25 defaultExpectedStatus: HttpStatusCode.OK_200
26 })
27 }
28
29 async create (options: OverrideCommandOptions & {
30 attributes: VideoChannelSyncCreate
31 }) {
32 return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({
33 ...options,
34
35 path: ChannelSyncsCommand.API_PATH,
36 fields: options.attributes,
37 implicitToken: true,
38 defaultExpectedStatus: HttpStatusCode.OK_200
39 }))
40 }
41
42 delete (options: OverrideCommandOptions & {
43 channelSyncId: number
44 }) {
45 const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}`
46
47 return this.deleteRequest({
48 ...options,
49
50 path,
51 implicitToken: true,
52 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
53 })
54 }
55}
diff --git a/shared/server-commands/videos/channels-command.ts b/shared/server-commands/videos/channels-command.ts
index 8ab124658..a688a120f 100644
--- a/shared/server-commands/videos/channels-command.ts
+++ b/shared/server-commands/videos/channels-command.ts
@@ -181,4 +181,22 @@ export class ChannelsCommand extends AbstractCommand {
181 defaultExpectedStatus: HttpStatusCode.OK_200 181 defaultExpectedStatus: HttpStatusCode.OK_200
182 }) 182 })
183 } 183 }
184
185 importVideos (options: OverrideCommandOptions & {
186 channelName: string
187 externalChannelUrl: string
188 }) {
189 const { channelName, externalChannelUrl } = options
190
191 const path = `/api/v1/video-channels/${channelName}/import-videos`
192
193 return this.postBodyRequest({
194 ...options,
195
196 path,
197 fields: { externalChannelUrl },
198 implicitToken: true,
199 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
200 })
201 }
184} 202}
diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts
index b861731fb..b4d6fa37b 100644
--- a/shared/server-commands/videos/index.ts
+++ b/shared/server-commands/videos/index.ts
@@ -3,6 +3,7 @@ export * from './captions-command'
3export * from './change-ownership-command' 3export * from './change-ownership-command'
4export * from './channels' 4export * from './channels'
5export * from './channels-command' 5export * from './channels-command'
6export * from './channel-syncs-command'
6export * from './comments-command' 7export * from './comments-command'
7export * from './history-command' 8export * from './history-command'
8export * from './imports-command' 9export * from './imports-command'
diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml
index 74963df14..ac8cde565 100644
--- a/support/doc/api/openapi.yaml
+++ b/support/doc/api/openapi.yaml
@@ -254,6 +254,8 @@ tags:
254 download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have. 254 download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have.
255 - name: Video Imports 255 - name: Video Imports
256 description: Operations dealing with listing, adding and removing video imports. 256 description: Operations dealing with listing, adding and removing video imports.
257 - name: Channels Sync
258 description: Operations dealing with synchronizing PeerTube user's channel with channels of other platforms
257 - name: Video Captions 259 - name: Video Captions
258 description: Operations dealing with listing, adding and removing closed captions of a video. 260 description: Operations dealing with listing, adding and removing closed captions of a video.
259 - name: Video Channels 261 - name: Video Channels
@@ -327,6 +329,7 @@ x-tagGroups:
327 - Video Transcoding 329 - Video Transcoding
328 - Live Videos 330 - Live Videos
329 - Feeds 331 - Feeds
332 - Channels Sync
330 - name: Search 333 - name: Search
331 tags: 334 tags:
332 - Search 335 - Search
@@ -3050,7 +3053,7 @@ paths:
3050 tags: 3053 tags:
3051 - Video Channels 3054 - Video Channels
3052 responses: 3055 responses:
3053 '204': 3056 '200':
3054 description: successful operation 3057 description: successful operation
3055 content: 3058 content:
3056 application/json: 3059 application/json:
@@ -3288,6 +3291,59 @@ paths:
3288 '204': 3291 '204':
3289 description: successful operation 3292 description: successful operation
3290 3293
3294 '/video-channel-syncs':
3295 post:
3296 summary: Create a synchronization for a video channel
3297 operationId: addVideoChannelSync
3298 security:
3299 - OAuth2: []
3300 tags:
3301 - Channels Sync
3302 requestBody:
3303 content:
3304 application/json:
3305 schema:
3306 $ref: '#/components/schemas/VideoChannelSyncCreate'
3307 responses:
3308 '200':
3309 description: successful operation
3310 content:
3311 application/json:
3312 schema:
3313 type: object
3314 properties:
3315 videoChannelSync:
3316 $ref: "#/components/schemas/VideoChannelSync"
3317
3318 '/video-channel-syncs/{channelSyncId}':
3319 delete:
3320 summary: Delete a video channel synchronization
3321 operationId: delVideoChannelSync
3322 security:
3323 - OAuth2: []
3324 tags:
3325 - Channels Sync
3326 parameters:
3327 - $ref: '#/components/parameters/channelSyncId'
3328 responses:
3329 '204':
3330 description: successful operation
3331
3332 '/video-channel-syncs/{channelSyncId}/sync':
3333 post:
3334 summary: Triggers the channel synchronization job, fetching all the videos from the remote channel
3335 operationId: triggerVideoChannelSync
3336 security:
3337 - OAuth2: []
3338 tags:
3339 - Channels Sync
3340 parameters:
3341 - $ref: '#/components/parameters/channelSyncId'
3342 responses:
3343 '204':
3344 description: successful operation
3345
3346
3291 /video-playlists/privacies: 3347 /video-playlists/privacies:
3292 get: 3348 get:
3293 summary: List available playlist privacy policies 3349 summary: List available playlist privacy policies
@@ -3659,6 +3715,26 @@ paths:
3659 schema: 3715 schema:
3660 $ref: '#/components/schemas/VideoChannelList' 3716 $ref: '#/components/schemas/VideoChannelList'
3661 3717
3718 '/accounts/{name}/video-channel-syncs':
3719 get:
3720 summary: List the synchronizations of video channels of an account
3721 tags:
3722 - Video Channels
3723 - Channels Sync
3724 - Accounts
3725 parameters:
3726 - $ref: '#/components/parameters/name'
3727 - $ref: '#/components/parameters/start'
3728 - $ref: '#/components/parameters/count'
3729 - $ref: '#/components/parameters/sort'
3730 responses:
3731 '200':
3732 description: successful operation
3733 content:
3734 application/json:
3735 schema:
3736 $ref: '#/components/schemas/VideoChannelSyncList'
3737
3662 '/accounts/{name}/ratings': 3738 '/accounts/{name}/ratings':
3663 get: 3739 get:
3664 summary: List ratings of an account 3740 summary: List ratings of an account
@@ -5141,6 +5217,13 @@ components:
5141 schema: 5217 schema:
5142 type: string 5218 type: string
5143 example: my_username | my_username@example.com 5219 example: my_username | my_username@example.com
5220 channelSyncId:
5221 name: channelSyncId
5222 in: path
5223 required: true
5224 description: Channel Sync id
5225 schema:
5226 $ref: '#/components/schemas/Abuse/properties/id'
5144 subscriptionHandle: 5227 subscriptionHandle:
5145 name: subscriptionHandle 5228 name: subscriptionHandle
5146 in: path 5229 in: path
@@ -5347,6 +5430,7 @@ components:
5347 - activitypub-refresher 5430 - activitypub-refresher
5348 - video-redundancy 5431 - video-redundancy
5349 - video-live-ending 5432 - video-live-ending
5433 - video-channel-import
5350 followState: 5434 followState:
5351 name: state 5435 name: state
5352 in: query 5436 in: query
@@ -6497,6 +6581,11 @@ components:
6497 properties: 6581 properties:
6498 enabled: 6582 enabled:
6499 type: boolean 6583 type: boolean
6584 videoChannelSynchronization:
6585 type: object
6586 properties:
6587 enabled:
6588 type: boolean
6500 autoBlacklist: 6589 autoBlacklist:
6501 type: object 6590 type: object
6502 properties: 6591 properties:
@@ -6861,6 +6950,11 @@ components:
6861 properties: 6950 properties:
6862 enabled: 6951 enabled:
6863 type: boolean 6952 type: boolean
6953 video_channel_synchronization:
6954 type: object
6955 properties:
6956 enabled:
6957 type: boolean
6864 autoBlacklist: 6958 autoBlacklist:
6865 type: object 6959 type: object
6866 properties: 6960 properties:
@@ -6953,6 +7047,7 @@ components:
6953 - videos-views-stats 7047 - videos-views-stats
6954 - activitypub-refresher 7048 - activitypub-refresher
6955 - video-redundancy 7049 - video-redundancy
7050 - video-channel-import
6956 data: 7051 data:
6957 type: object 7052 type: object
6958 additionalProperties: true 7053 additionalProperties: true
@@ -7473,6 +7568,7 @@ components:
7473 type: integer 7568 type: integer
7474 uuid: 7569 uuid:
7475 $ref: '#/components/schemas/UUIDv4' 7570 $ref: '#/components/schemas/UUIDv4'
7571
7476 VideoChannelCreate: 7572 VideoChannelCreate:
7477 allOf: 7573 allOf:
7478 - $ref: '#/components/schemas/VideoChannel' 7574 - $ref: '#/components/schemas/VideoChannel'
@@ -7503,6 +7599,51 @@ components:
7503 - $ref: '#/components/schemas/VideoChannel' 7599 - $ref: '#/components/schemas/VideoChannel'
7504 - $ref: '#/components/schemas/Actor' 7600 - $ref: '#/components/schemas/Actor'
7505 7601
7602 VideoChannelSync:
7603 type: object
7604 properties:
7605 id:
7606 $ref: '#/components/schemas/id'
7607 state:
7608 type: object
7609 properties:
7610 id:
7611 type: integer
7612 example: 2
7613 label:
7614 type: string
7615 example: PROCESSING
7616 externalChannelUrl:
7617 type: string
7618 example: 'https://youtube.com/c/UC_myfancychannel'
7619 createdAt:
7620 type: string
7621 format: date-time
7622 lastSyncAt:
7623 type: string
7624 format: date-time
7625 nullable: true
7626 channel:
7627 $ref: '#/components/schemas/VideoChannel'
7628 VideoChannelSyncList:
7629 type: object
7630 properties:
7631 total:
7632 type: integer
7633 example: 1
7634 data:
7635 type: array
7636 items:
7637 allOf:
7638 - $ref: '#/components/schemas/VideoChannelSync'
7639 VideoChannelSyncCreate:
7640 type: object
7641 properties:
7642 externalChannelUrl:
7643 type: string
7644 example: https://youtube.com/c/UC_myfancychannel
7645 videoChannelId:
7646 $ref: '#/components/schemas/id'
7506 MRSSPeerLink: 7647 MRSSPeerLink:
7507 type: object 7648 type: object
7508 xml: 7649 xml: