inputName="importVideosHttpEnabled" formControlName="enabled"
i18n-labelText labelText="Allow import with HTTP URL (e.g. YouTube)"
>
- <ng-container ngProjectAs="description">
- <span i18n>⚠️ If enabled, we recommend to use <a class="link-orange" href="https://docs.joinpeertube.org/maintain-configuration?id=security">a HTTP proxy</a> to prevent private URL access from your PeerTube server</span>
- </ng-container>
- </my-peertube-checkbox>
+ <ng-container ngProjectAs="description">
+
<span i18n>⚠️ If enabled, we recommend to use <a class="link-orange" href="https://docs.joinpeertube.org/maintain-configuration?id=security">a HTTP proxy</a> to prevent private URL access from your PeerTube server</span>
+ </ng-container>
+ </my-peertube-checkbox>
</div>
<div class="form-group" formGroupName="torrent">
</div>
</ng-container>
+
+ <ng-container formGroupName="videoChannelSynchronization">
+ <div class="form-group">
+ <my-peertube-checkbox
+ inputName="importSynchronizationEnabled" formControlName="enabled"
+ i18n-labelText labelText="Allow channel synchronization with channel of other platforms like YouTube (requires allowing import with HTTP URL)"
+ >
+ <ng-container ngProjectAs="description">
+ <span i18n [hidden]="isImportVideosHttpEnabled()">
+ ⛔ You need to allow import with HTTP URL to be able to activate this feature.
+ </span>
+ </ng-container>
+ </my-peertube-checkbox>
+ </div>
+ </ng-container>
+
</ng-container>
<ng-container formGroupName="autoBlacklist">
private configService: ConfigService,
private menuService: MenuService,
private themeService: ThemeService
- ) { }
+ ) {}
ngOnInit () {
this.buildLandingPageOptions()
this.checkSignupField()
+ this.checkImportSyncField()
this.availableThemes = this.themeService.buildAvailableThemes()
}
return { 'disabled-checkbox-extra': !this.isSignupEnabled() }
}
+ isImportVideosHttpEnabled (): boolean {
+ return this.form.value['import']['videos']['http']['enabled'] === true
+ }
+
+ importSynchronizationChecked () {
+ return this.isImportVideosHttpEnabled() && this.form.value['import']['videoChannelSynchronization']['enabled']
+ }
+
hasUnlimitedSignup () {
return this.form.value['signup']['limit'] === -1
}
return this.themeService.getDefaultThemeLabel()
}
+ private checkImportSyncField () {
+ const importSyncControl = this.form.get('import.videoChannelSynchronization.enabled')
+ const importVideosHttpControl = this.form.get('import.videos.http.enabled')
+
+ importVideosHttpControl.valueChanges
+ .subscribe((httpImportEnabled) => {
+ importSyncControl.setValue(httpImportEnabled && importSyncControl.value)
+ if (httpImportEnabled) {
+ importSyncControl.enable()
+ } else {
+ importSyncControl.disable()
+ }
+ })
+ }
+
private checkSignupField () {
const signupControl = this.form.get('signup.enabled')
torrent: {
enabled: null
}
+ },
+ videoChannelSynchronization: {
+ enabled: null
}
},
trending: {
'video-redundancy',
'video-transcoding',
'videos-views-stats',
- 'move-to-object-storage'
+ 'move-to-object-storage',
+ 'video-channel-import'
]
jobs: Job[] = []
</div>
<div class="form-group">
- <label for="support">Support</label>
+ <label i18n for="support">Support</label>
<my-help
helpType="markdownEnhanced" i18n-preHtml preHtml="Short text to tell people how they can support the channel (membership platform...).<br /><br />
When a video is uploaded in this channel, the video support field will be automatically filled by this text."
<h1>
- <my-global-icon iconName="channel" aria-hidden="true"></my-global-icon>
- <ng-container i18n>My channels</ng-container>
- <span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span>
+ <span>
+ <my-global-icon iconName="channel" aria-hidden="true"></my-global-icon>
+ <ng-container i18n>My channels</ng-container>
+ <span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span>
+ </span>
+
+ <div>
+ <a routerLink="/my-library/video-channel-syncs" class="button-link">
+ <my-global-icon iconName="repeat" aria-hidden="true"></my-global-icon>
+ <ng-container i18n>My synchronizations</ng-container>
+ </a>
+ </div>
</h1>
<my-channels-setup-message [hideLink]="true"></my-channels-setup-message>
@use '_variables' as *;
@use '_mixins' as *;
-h1 my-global-icon {
- position: relative;
- top: -2px;
+h1 {
+ display: flex;
+ justify-content: space-between;
+
+ my-global-icon {
+ position: relative;
+ top: -2px;
+ }
+
+ .button-link {
+ @include peertube-button-link;
+ @include grey-button;
+ @include button-with-icon(18px, 3px, -1px);
+ }
}
.create-button {
import { MyHistoryComponent } from './my-history/my-history.component'
import { MyLibraryComponent } from './my-library.component'
import { MyOwnershipComponent } from './my-ownership/my-ownership.component'
+import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
+import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
import { MyVideoImportsComponent } from './my-video-imports/my-video-imports.component'
import { MyVideoPlaylistCreateComponent } from './my-video-playlists/my-video-playlist-create.component'
import { MyVideoPlaylistElementsComponent } from './my-video-playlists/my-video-playlist-elements.component'
key: 'my-videos-history-list'
}
}
+ },
+
+ {
+ path: 'video-channel-syncs',
+ component: MyVideoChannelSyncsComponent,
+ data: {
+ meta: {
+ title: $localize`My synchronizations`
+ }
+ }
+ },
+
+ {
+ path: 'video-channel-syncs/create',
+ component: VideoChannelSyncEditComponent,
+ data: {
+ meta: {
+ title: $localize`Create new synchronization`
+ }
+ }
}
]
}
import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlists.component'
import { VideoChangeOwnershipComponent } from './my-videos/modals/video-change-ownership.component'
import { MyVideosComponent } from './my-videos/my-videos.component'
+import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
+import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
@NgModule({
imports: [
MyOwnershipComponent,
MyAcceptOwnershipComponent,
MyVideoImportsComponent,
+ MyVideoChannelSyncsComponent,
+ VideoChannelSyncEditComponent,
MySubscriptionsComponent,
MyFollowersComponent,
MyHistoryComponent,
--- /dev/null
+<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
+
+<h1>
+ <my-global-icon iconName="refresh" aria-hidden="true"></my-global-icon>
+ <ng-container i18n>My synchronizations</ng-container>
+</h1>
+
+<div *ngIf="!syncEnabled()">
+ <p class="muted" i18n>⚠️ The instance doesn't allow channel synchronization</p>
+</div>
+
+<p-table
+ *ngIf="syncEnabled()" [value]="channelSyncs" [lazy]="true"
+ [paginator]="totalRecords > 0" [totalRecords]="totalRecords" [rows]="rowsPerPage" [rowsPerPageOptions]="rowsPerPageOptions"
+ [sortField]="sort.field" [sortOrder]="sort.order" (onLazyLoad)="loadLazy($event)" dataKey="id"
+ [showCurrentPageReport]="true" i18n-currentPageReportTemplate
+ currentPageReportTemplate="Showing {{'{first}'}} to {{'{last}'}} of {{'{totalRecords}'}} synchronizations"
+ [expandedRowKeys]="expandedRows"
+>
+ <ng-template pTemplate="caption">
+ <div class="caption">
+ <div class="left-buttons">
+ <a class="add-sync" routerLink="{{ getSyncCreateLink() }}">
+ <my-global-icon iconName="add" aria-hidden="true"></my-global-icon>
+ <ng-container i18n>Add synchronization</ng-container>
+ </a>
+ </div>
+ </div>
+ </ng-template>
+
+ <ng-template pTemplate="header">
+ <tr>
+ <th style="width: 10%"><my-global-icon iconName="columns"></my-global-icon></th>
+ <th style="width: 25%" i18n pSortableColumn="externalChannelUrl">External Channel <p-sortIcon field="externalChannelUrl"></p-sortIcon></th>
+ <th style="width: 25%" i18n pSortableColumn="videoChannel">Channel <p-sortIcon field="videoChannel"></p-sortIcon></th>
+ <th style="width: 10%" i18n pSortableColumn="state">State <p-sortIcon field="state"></p-sortIcon></th>
+ <th style="width: 10%" i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
+ <th style="width: 10%" i18n pSortableColumn="lastSyncAt">Last synchronization at <p-sortIcon field="lastSyncAt"></p-sortIcon></th>
+ </tr>
+ </ng-template>
+
+ <ng-template pTemplate="body" let-expanded="expanded" let-videoChannelSync>
+ <tr>
+ <td class="action-cell">
+ <my-action-dropdown
+ container="body"
+ [actions]="videoChannelSyncActions" [entry]="videoChannelSync"
+ ></my-action-dropdown>
+ </td>
+
+ <td>
+ <a [href]="videoChannelSync.externalChannelUrl" target="_blank" rel="noopener noreferrer">{{ videoChannelSync.externalChannelUrl }}</a>
+ </td>
+
+ <td>
+ <div class="actor">
+ <my-actor-avatar
+ class="channel"
+ [actor]="videoChannelSync.channel" actorType="channel"
+ [internalHref]="[ '/c', videoChannelSync.channel.name ]"
+ size="25"
+ ></my-actor-avatar>
+
+ <div class="actor-info">
+ <a [routerLink]="[ '/c', videoChannelSync.channel.name ]" class="actor-names" i18n-title title="Channel page">
+ <div class="actor-display-name">{{ videoChannelSync.channel.displayName }}</div>
+ <div class="actor-name">{{ videoChannelSync.channel.name }}</div>
+ </a>
+ </div>
+ </div>
+ </td>
+
+ <td>
+ <span [ngClass]="getSyncStateClass(videoChannelSync.state.id)">
+ {{ videoChannelSync.state.label }}
+ </span>
+ </td>
+
+ <td>{{ videoChannelSync.createdAt | date: 'short' }}</td>
+ <td>{{ videoChannelSync.lastSyncAt | date: 'short' }}</td>
+ </tr>
+ </ng-template>
+</p-table>
--- /dev/null
+@use '_mixins' as *;
+@use '_variables' as *;
+@use '_actor' as *;
+
+.add-sync {
+ @include create-button;
+}
+
+.actor {
+ @include actor-row($min-height: auto, $separator: true);
+ margin-bottom: 0;
+ padding-bottom: 0;
+ border: 0;
+}
--- /dev/null
+import { Component, OnInit } from '@angular/core'
+import { AuthService, Notifier, RestPagination, RestTable, ServerService } from '@app/core'
+import { DropdownAction, VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
+import { HTMLServerConfig } from '@shared/models/server'
+import { VideoChannelSync, VideoChannelSyncState } from '@shared/models/videos'
+import { SortMeta } from 'primeng/api'
+import { mergeMap } from 'rxjs'
+
+@Component({
+ templateUrl: './my-video-channel-syncs.component.html',
+ styleUrls: [ './my-video-channel-syncs.component.scss' ]
+})
+export class MyVideoChannelSyncsComponent extends RestTable implements OnInit {
+ error: string
+
+ channelSyncs: VideoChannelSync[] = []
+ totalRecords = 0
+
+ videoChannelSyncActions: DropdownAction<VideoChannelSync>[][] = []
+ sort: SortMeta = { field: 'createdAt', order: 1 }
+ pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
+
+ private static STATE_CLASS_BY_ID = {
+ [VideoChannelSyncState.FAILED]: 'badge-red',
+ [VideoChannelSyncState.PROCESSING]: 'badge-blue',
+ [VideoChannelSyncState.SYNCED]: 'badge-green',
+ [VideoChannelSyncState.WAITING_FIRST_RUN]: 'badge-yellow'
+ }
+
+ private serverConfig: HTMLServerConfig
+
+ constructor (
+ private videoChannelsSyncService: VideoChannelSyncService,
+ private serverService: ServerService,
+ private notifier: Notifier,
+ private authService: AuthService,
+ private videoChannelService: VideoChannelService
+ ) {
+ super()
+ }
+
+ ngOnInit () {
+ this.serverConfig = this.serverService.getHTMLConfig()
+ this.initialize()
+
+ this.videoChannelSyncActions = [
+ [
+ {
+ label: $localize`Delete`,
+ iconName: 'delete',
+ handler: videoChannelSync => this.deleteSync(videoChannelSync)
+ },
+ {
+ label: $localize`Fully synchronize the channel`,
+ description: $localize`This fetches any missing videos on the local channel`,
+ iconName: 'refresh',
+ handler: videoChannelSync => this.fullySynchronize(videoChannelSync)
+ }
+ ]
+ ]
+ }
+
+ protected reloadData () {
+ this.error = undefined
+
+ this.authService.userInformationLoaded
+ .pipe(mergeMap(() => {
+ const user = this.authService.getUser()
+ return this.videoChannelsSyncService.listAccountVideoChannelsSyncs({
+ sort: this.sort,
+ account: user.account,
+ pagination: this.pagination
+ })
+ }))
+ .subscribe({
+ next: res => {
+ this.channelSyncs = res.data
+ },
+ error: err => {
+ this.error = err.message
+ }
+ })
+ }
+
+ syncEnabled () {
+ return this.serverConfig.import.videoChannelSynchronization.enabled
+ }
+
+ deleteSync (videoChannelSync: VideoChannelSync) {
+ this.videoChannelsSyncService.deleteSync(videoChannelSync.id)
+ .subscribe({
+ next: () => {
+ this.notifier.success($localize`Synchronization removed successfully for ${videoChannelSync.channel.displayName}.`)
+ this.reloadData()
+ },
+ error: err => {
+ this.error = err.message
+ }
+ })
+ }
+
+ fullySynchronize (videoChannelSync: VideoChannelSync) {
+ this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
+ .subscribe({
+ next: () => {
+ this.notifier.success($localize`Full synchronization requested successfully for ${videoChannelSync.channel.displayName}.`)
+ },
+ error: err => {
+ this.error = err.message
+ }
+ })
+ }
+
+ getSyncCreateLink () {
+ return '/my-library/video-channel-syncs/create'
+ }
+
+ getSyncStateClass (stateId: number) {
+ return [ 'pt-badge', MyVideoChannelSyncsComponent.STATE_CLASS_BY_ID[stateId] ]
+ }
+
+ getIdentifier () {
+ return 'MyVideoChannelsSyncComponent'
+ }
+
+ getChannelUrl (name: string) {
+ return '/c/' + name
+ }
+}
--- /dev/null
+<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
+
+<div class="margin-content">
+ <form role="form" (ngSubmit)="formValidated()" [formGroup]="form">
+
+ <div class="row">
+ <div class="col-12 col-lg-4 col-xl-3">
+ <div class="video-channel-sync-title" i18n>NEW SYNCHRONIZATION</div>
+ </div>
+
+ <div class="col-12 col-lg-8 col-xl-9">
+ <div class="form-group">
+ <label i18n for="externalChannelUrl">Remote channel URL</label>
+
+ <div class="input-group">
+ <input
+ type="text"
+ id="externalChannelUrl"
+ i18n-placeholder
+ placeholder="Example: https://youtube.com/channel/UC_fancy_channel"
+ formControlName="externalChannelUrl"
+ [ngClass]="{ 'input-error': formErrors['externalChannelUrl'] }"
+ class="form-control"
+ >
+ </div>
+
+ <div *ngIf="formErrors['externalChannelUrl']" class="form-error">
+ {{ formErrors['externalChannelUrl'] }}
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label i18n for="videoChannel">Video Channel</label>
+ <my-select-channel required [items]="userVideoChannels" formControlName="videoChannel"></my-select-channel>
+
+ <div *ngIf="formErrors['videoChannel']" class="form-error">
+ {{ formErrors['videoChannel'] }}
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="existingVideoStrategy" i18n>Options for existing videos on remote channel:</label>
+
+ <div class="peertube-radio-container">
+ <input type="radio" name="existingVideoStrategy" id="import" value="import" formControlName="existingVideoStrategy" required />
+ <label for="import" i18n>Import all and watch for new publications</label>
+ </div>
+
+ <div class="peertube-radio-container">
+ <input type="radio" name="existingVideoStrategy" id="doNothing" value="nothing" formControlName="existingVideoStrategy" required />
+ <label for="doNothing" i18n>Only watch for new publications</label>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="row"> <!-- submit placement block -->
+ <div class="col-md-7 col-xl-5"></div>
+ <div class="col-md-5 col-xl-5 d-inline-flex">
+ <input type="submit" class="peertube-button orange-button ms-auto" value="{{ getFormButtonTitle() }}" [disabled]="!form.valid">
+ </div>
+ </div>
+ </form>
+</div>
--- /dev/null
+@use '_variables' as *;
+@use '_mixins' as *;
+
+$form-base-input-width: 480px;
+
+input[type=text] {
+ @include peertube-input-text($form-base-input-width);
+}
+
+.video-channel-sync-title {
+ @include settings-big-title;
+}
+
+my-select-channel {
+ display: block;
+ max-width: $form-base-input-width;
+}
--- /dev/null
+import { mergeMap } from 'rxjs'
+import { SelectChannelItem } from 'src/types'
+import { Component, OnInit } from '@angular/core'
+import { Router } from '@angular/router'
+import { AuthService, Notifier } from '@app/core'
+import { listUserChannelsForSelect } from '@app/helpers'
+import { VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR } from '@app/shared/form-validators/video-channel-validators'
+import { FormReactive, FormValidatorService } from '@app/shared/shared-forms'
+import { VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
+import { VideoChannelSyncCreate } from '@shared/models/videos'
+
+@Component({
+ selector: 'my-video-channel-sync-edit',
+ templateUrl: './video-channel-sync-edit.component.html',
+ styleUrls: [ './video-channel-sync-edit.component.scss' ]
+})
+export class VideoChannelSyncEditComponent extends FormReactive implements OnInit {
+ error: string
+ userVideoChannels: SelectChannelItem[] = []
+ existingVideosStrategy: string
+
+ constructor (
+ protected formValidatorService: FormValidatorService,
+ private authService: AuthService,
+ private router: Router,
+ private notifier: Notifier,
+ private videoChannelSyncService: VideoChannelSyncService,
+ private videoChannelService: VideoChannelService
+ ) {
+ super()
+ }
+
+ ngOnInit () {
+ this.buildForm({
+ externalChannelUrl: VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR,
+ videoChannel: null,
+ existingVideoStrategy: null
+ })
+
+ listUserChannelsForSelect(this.authService)
+ .subscribe(channels => this.userVideoChannels = channels)
+ }
+
+ getFormButtonTitle () {
+ return $localize`Create`
+ }
+
+ formValidated () {
+ this.error = undefined
+
+ const body = this.form.value
+ const videoChannelSyncCreate: VideoChannelSyncCreate = {
+ externalChannelUrl: body.externalChannelUrl,
+ videoChannelId: body.videoChannel
+ }
+
+ const importExistingVideos = body['existingVideoStrategy'] === 'import'
+
+ this.videoChannelSyncService.createSync(videoChannelSyncCreate)
+ .pipe(mergeMap(({ videoChannelSync }) => {
+ return importExistingVideos
+ ? this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
+ : Promise.resolve(null)
+ }))
+ .subscribe({
+ next: () => {
+ this.notifier.success($localize`Synchronization created successfully.`)
+ this.router.navigate([ '/my-library', 'video-channel-syncs' ])
+ },
+
+ error: err => {
+ this.error = err.message
+ }
+ })
+ }
+}
maxlength: $localize`Support text cannot be more than 1000 characters long.`
}
}
+
+export const VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR: BuildFormValidator = {
+ VALIDATORS: [
+ Validators.required,
+ Validators.pattern(/^https?:\/\//),
+ Validators.maxLength(1000)
+ ],
+ MESSAGES: {
+ required: $localize`Remote channel url is required.`,
+ pattern: $localize`External channel URL must begin with "https://" or "http://"`,
+ maxlength: $localize`External channel URL cannot be more than 1000 characters long`
+ }
+}
</td>
</tr>
+ <tr>
+ <th i18n class="sub-label" scope="row">Channel synchronization with other platforms (YouTube, Vimeo, ...)</th>
+ <td>
+ <my-feature-boolean [value]="serverConfig.import.videoChannelSynchronization.enabled"></my-feature-boolean>
+ </td>
+ </tr>
+
<tr>
<th i18n class="label" colspan="2">Search</th>
</tr>
export * from './video-caption'
export * from './video-channel'
export * from './shared-main.module'
+export * from './video-channel-sync'
--- /dev/null
+export * from './video-channel-sync.service'
--- /dev/null
+import { SortMeta } from 'primeng/api'
+import { catchError, Observable } from 'rxjs'
+import { environment } from 'src/environments/environment'
+import { HttpClient, HttpParams } from '@angular/common/http'
+import { Injectable } from '@angular/core'
+import { RestExtractor, RestPagination, RestService } from '@app/core'
+import { ResultList } from '@shared/models/common'
+import { VideoChannelSync, VideoChannelSyncCreate } from '@shared/models/videos'
+import { Account, AccountService } from '../account'
+
+@Injectable({
+ providedIn: 'root'
+})
+export class VideoChannelSyncService {
+ static BASE_VIDEO_CHANNEL_URL = environment.apiUrl + '/api/v1/video-channel-syncs'
+
+ constructor (
+ private authHttp: HttpClient,
+ private restExtractor: RestExtractor,
+ private restService: RestService
+ ) { }
+
+ listAccountVideoChannelsSyncs (parameters: {
+ sort: SortMeta
+ pagination: RestPagination
+ account: Account
+ }): Observable<ResultList<VideoChannelSync>> {
+ const { pagination, sort, account } = parameters
+
+ let params = new HttpParams()
+ params = this.restService.addRestGetParams(params, pagination, sort)
+
+ const url = AccountService.BASE_ACCOUNT_URL + account.nameWithHost + '/video-channel-syncs'
+
+ return this.authHttp.get<ResultList<VideoChannelSync>>(url, { params })
+ .pipe(catchError(err => this.restExtractor.handleError(err)))
+ }
+
+ createSync (body: VideoChannelSyncCreate) {
+ return this.authHttp.post<{ videoChannelSync: VideoChannelSync }>(VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL, body)
+ .pipe(catchError(err => this.restExtractor.handleError(err)))
+ }
+
+ deleteSync (videoChannelsSyncId: number) {
+ const url = `${VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL}/${videoChannelsSyncId}`
+
+ return this.authHttp.delete(url)
+ .pipe(catchError(err => this.restExtractor.handleError(err)))
+ }
+}
return this.authHttp.delete(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannel.nameWithHost)
.pipe(catchError(err => this.restExtractor.handleError(err)))
}
+
+ importVideos (videoChannelName: string, externalChannelUrl: string) {
+ const path = VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannelName + '/import-videos'
+ return this.authHttp.post(path, { externalChannelUrl })
+ .pipe(catchError(err => this.restExtractor.handleError(err)))
+ }
}
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
enabled: false
+ # Add ability for your users to synchronize their channels with external channels, playlists, etc
+ video_channel_synchronization:
+ enabled: false
+
+ max_per_user: 10
+
+ check_interval: 1 hour
+
+ # Number of latest published videos to check and to potentially import when syncing a channel
+ videos_limit_per_synchronization: 10
+
auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:
enabled: true
torrent:
enabled: true
+ video_channel_synchronization:
+ enabled: true
+ max_per_user: 10
+ check_interval: 5 minutes
+ videos_limit_per_synchronization: 3
instance:
default_nsfw_policy: 'display'
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
enabled: false
+ # Add ability for your users to synchronize their channels with external channels, playlists, etc.
+ video_channel_synchronization:
+ enabled: false
+
+ max_per_user: 10
+
+ check_interval: 1 hour
+
+ # Number of latest published videos to check and to potentially import when syncing a channel
+ videos_limit_per_synchronization: 10
+
auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:
import { isTestOrDevInstance } from './server/helpers/core-utils'
import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
import { ApplicationModel } from '@server/models/application/application'
+import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
// ----------- Command line -----------
PeerTubeVersionCheckScheduler.Instance.enable()
AutoFollowIndexInstances.Instance.enable()
RemoveDanglingResumableUploadsScheduler.Instance.enable()
+ VideoChannelSyncLatestScheduler.Instance.enable()
VideoViewsBufferScheduler.Instance.enable()
GeoIPUpdateScheduler.Instance.enable()
OpenTelemetryMetrics.Instance.registerMetrics()
accountsFollowersSortValidator,
accountsSortValidator,
ensureAuthUserOwnsAccountValidator,
+ ensureCanManageUser,
videoChannelsSortValidator,
videoChannelStatsValidator,
+ videoChannelSyncsSortValidator,
videosSortValidator
} from '../../middlewares/validators'
import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
import { VideoModel } from '../../models/video/video'
import { VideoChannelModel } from '../../models/video/video-channel'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
const accountsRouter = express.Router()
asyncMiddleware(listAccountChannels)
)
+accountsRouter.get('/:accountName/video-channel-syncs',
+ authenticate,
+ asyncMiddleware(accountNameWithHostGetValidator),
+ ensureCanManageUser,
+ paginationValidator,
+ videoChannelSyncsSortValidator,
+ setDefaultSort,
+ setDefaultPagination,
+ asyncMiddleware(listAccountChannelsSync)
+)
+
accountsRouter.get('/:accountName/video-playlists',
optionalAuthenticate,
asyncMiddleware(accountNameWithHostGetValidator),
return res.json(getFormattedObjects(resultList.data, resultList.total))
}
+async function listAccountChannelsSync (req: express.Request, res: express.Response) {
+ const options = {
+ accountId: res.locals.account.id,
+ start: req.query.start,
+ count: req.query.count,
+ sort: req.query.sort,
+ search: req.query.search
+ }
+
+ const resultList = await VideoChannelSyncModel.listByAccountForAPI(options)
+
+ return res.json(getFormattedObjects(resultList.data, resultList.total))
+}
+
async function listAccountPlaylists (req: express.Request, res: express.Response) {
const serverActor = await getServerActor()
torrent: {
enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
}
+ },
+ videoChannelSynchronization: {
+ enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
+ maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
}
},
trending: {
import { videoChannelRouter } from './video-channel'
import { videoPlaylistRouter } from './video-playlist'
import { videosRouter } from './videos'
+import { videoChannelSyncRouter } from './video-channel-sync'
const apiRouter = express.Router()
apiRouter.use('/users', usersRouter)
apiRouter.use('/accounts', accountsRouter)
apiRouter.use('/video-channels', videoChannelRouter)
+apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
apiRouter.use('/video-playlists', videoPlaylistRouter)
apiRouter.use('/videos', videosRouter)
apiRouter.use('/jobs', jobsRouter)
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
import { UserRight } from '../../../../shared/models/users'
import { authenticate, ensureUserHasRight } from '../../../middlewares'
+import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
const debugRouter = express.Router()
const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
- 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats()
+ 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
+ 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
}
await processors[body.command]()
--- /dev/null
+import express from 'express'
+import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger'
+import { logger } from '@server/helpers/logger'
+import {
+ asyncMiddleware,
+ asyncRetryTransactionMiddleware,
+ authenticate,
+ ensureCanManageChannel as ensureCanManageSyncedChannel,
+ ensureSyncExists,
+ ensureSyncIsEnabled,
+ videoChannelSyncValidator
+} from '@server/middlewares'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { MChannelSyncFormattable } from '@server/types/models'
+import { HttpStatusCode, VideoChannelSyncState } from '@shared/models'
+
+const videoChannelSyncRouter = express.Router()
+const auditLogger = auditLoggerFactory('channel-syncs')
+
+videoChannelSyncRouter.post('/',
+ authenticate,
+ ensureSyncIsEnabled,
+ asyncMiddleware(videoChannelSyncValidator),
+ ensureCanManageSyncedChannel,
+ asyncRetryTransactionMiddleware(createVideoChannelSync)
+)
+
+videoChannelSyncRouter.delete('/:id',
+ authenticate,
+ asyncMiddleware(ensureSyncExists),
+ ensureCanManageSyncedChannel,
+ asyncRetryTransactionMiddleware(removeVideoChannelSync)
+)
+
+export { videoChannelSyncRouter }
+
+// ---------------------------------------------------------------------------
+
+async function createVideoChannelSync (req: express.Request, res: express.Response) {
+ const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({
+ externalChannelUrl: req.body.externalChannelUrl,
+ videoChannelId: req.body.videoChannelId,
+ state: VideoChannelSyncState.WAITING_FIRST_RUN
+ })
+
+ await syncCreated.save()
+ syncCreated.VideoChannel = res.locals.videoChannel
+
+ auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON()))
+
+ logger.info(
+ 'Video synchronization for channel "%s" with external channel "%s" created.',
+ syncCreated.VideoChannel.name,
+ syncCreated.externalChannelUrl
+ )
+
+ return res.json({
+ videoChannelSync: syncCreated.toFormattedJSON()
+ })
+}
+
+async function removeVideoChannelSync (req: express.Request, res: express.Response) {
+ const syncInstance = res.locals.videoChannelSync
+
+ await syncInstance.destroy()
+
+ auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON()))
+
+ logger.info(
+ 'Video synchronization for channel "%s" with external channel "%s" deleted.',
+ syncInstance.VideoChannel.name,
+ syncInstance.externalChannelUrl
+ )
+
+ return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
+}
videoPlaylistsSortValidator
} from '../../middlewares'
import {
+ ensureChannelOwnerCanUpload,
ensureIsLocalChannel,
+ videoChannelImportVideosValidator,
videoChannelsFollowersSortValidator,
videoChannelsListValidator,
videoChannelsNameWithHostValidator,
asyncMiddleware(listVideoChannelFollowers)
)
+videoChannelRouter.post('/:nameWithHost/import-videos',
+ authenticate,
+ asyncMiddleware(videoChannelsNameWithHostValidator),
+ videoChannelImportVideosValidator,
+ ensureIsLocalChannel,
+ ensureCanManageChannel,
+ asyncMiddleware(ensureChannelOwnerCanUpload),
+ asyncMiddleware(importVideosInChannel)
+)
+
// ---------------------------------------------------------------------------
export {
return res.json(getFormattedObjects(resultList.data, resultList.total))
}
+
+async function importVideosInChannel (req: express.Request, res: express.Response) {
+ const { externalChannelUrl } = req.body
+
+ await JobQueue.Instance.createJob({
+ type: 'video-channel-import',
+ payload: {
+ externalChannelUrl,
+ videoChannelId: res.locals.videoChannel.id
+ }
+ })
+
+ logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl)
+
+ return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
+}
import express from 'express'
-import { move, readFile, remove } from 'fs-extra'
+import { move, readFile } from 'fs-extra'
import { decode } from 'magnet-uri'
import parseTorrent, { Instance } from 'parse-torrent'
import { join } from 'path'
-import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
-import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
-import { isResolvingToUnicastOnly } from '@server/helpers/dns'
-import { Hooks } from '@server/lib/plugins/hooks'
-import { ServerConfigManager } from '@server/lib/server-config-manager'
-import { setVideoTags } from '@server/lib/video'
-import { FilteredModelAttributes } from '@server/types'
-import {
- MChannelAccountDefault,
- MThumbnail,
- MUser,
- MVideoAccountDefault,
- MVideoCaption,
- MVideoTag,
- MVideoThumbnail,
- MVideoWithBlacklistLight
-} from '@server/types/models'
-import { MVideoImportFormattable } from '@server/types/models/video/video-import'
-import {
- HttpStatusCode,
- ServerErrorCode,
- ThumbnailType,
- VideoImportCreate,
- VideoImportState,
- VideoPrivacy,
- VideoState
-} from '@shared/models'
+import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
+import { MThumbnail, MVideoThumbnail } from '@server/types/models'
+import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
-import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
import { isArray } from '../../../helpers/custom-validators/misc'
import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
import { logger } from '../../../helpers/logger'
import { getSecureTorrentName } from '../../../helpers/utils'
-import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
import { CONFIG } from '../../../initializers/config'
import { MIMETYPES } from '../../../initializers/constants'
-import { sequelizeTypescript } from '../../../initializers/database'
-import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
import { JobQueue } from '../../../lib/job-queue/job-queue'
-import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
-import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
+import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
videoImportCancelValidator,
videoImportDeleteValidator
} from '../../../middlewares'
-import { VideoModel } from '../../../models/video/video'
-import { VideoCaptionModel } from '../../../models/video/video-caption'
-import { VideoImportModel } from '../../../models/video/video-import'
const auditLogger = auditLoggerFactory('video-imports')
const videoImportsRouter = express.Router()
authenticate,
reqVideoFileImport,
asyncMiddleware(videoImportAddValidator),
- asyncRetryTransactionMiddleware(addVideoImport)
+ asyncRetryTransactionMiddleware(handleVideoImport)
)
videoImportsRouter.post('/imports/:id/cancel',
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
-function addVideoImport (req: express.Request, res: express.Response) {
- if (req.body.targetUrl) return addYoutubeDLImport(req, res)
+function handleVideoImport (req: express.Request, res: express.Response) {
+ if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
const file = req.files?.['torrentfile']?.[0]
- if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
+ if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
}
-async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
+async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
const body: VideoImportCreate = req.body
const user = res.locals.oauth.token.User
videoName = result.name
}
- const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
+ const video = await buildVideoFromImport({
+ channelId: res.locals.videoChannel.id,
+ importData: { name: videoName },
+ importDataOverride: body,
+ importType: 'torrent'
+ })
const thumbnailModel = await processThumbnail(req, video)
const previewModel = await processPreview(req, video)
- const videoImport = await insertIntoDB({
+ const videoImport = await insertFromImportIntoDB({
video,
thumbnailModel,
previewModel,
}
})
- // Create job to import the video
- const payload = {
+ const payload: VideoImportPayload = {
type: torrentfile
- ? 'torrent-file' as 'torrent-file'
- : 'magnet-uri' as 'magnet-uri',
+ ? 'torrent-file'
+ : 'magnet-uri',
videoImportId: videoImport.id,
- magnetUri
+ preventException: false
}
await JobQueue.Instance.createJob({ type: 'video-import', payload })
return res.json(videoImport.toFormattedJSON()).end()
}
-async function addYoutubeDLImport (req: express.Request, res: express.Response) {
+function statusFromYtDlImportError (err: YoutubeDlImportError): number {
+ switch (err.code) {
+ case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
+ return HttpStatusCode.FORBIDDEN_403
+
+ case YoutubeDlImportError.CODE.FETCH_ERROR:
+ return HttpStatusCode.BAD_REQUEST_400
+
+ default:
+ return HttpStatusCode.INTERNAL_SERVER_ERROR_500
+ }
+}
+
+async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
const body: VideoImportCreate = req.body
const targetUrl = body.targetUrl
const user = res.locals.oauth.token.User
- const youtubeDL = new YoutubeDLWrapper(
- targetUrl,
- ServerConfigManager.Instance.getEnabledResolutions('vod'),
- CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
- )
-
- // Get video infos
- let youtubeDLInfo: YoutubeDLInfo
try {
- youtubeDLInfo = await youtubeDL.getInfoForDownload()
+ const { job, videoImport } = await buildYoutubeDLImport({
+ targetUrl,
+ channel: res.locals.videoChannel,
+ importDataOverride: body,
+ thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
+ previewFilePath: req.files?.['previewfile']?.[0].path,
+ user
+ })
+ await JobQueue.Instance.createJob(job)
+
+ auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
+
+ return res.json(videoImport.toFormattedJSON()).end()
} catch (err) {
- logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
+ logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
return res.fail({
- message: 'Cannot fetch remote information of this URL.',
+ message: err.message,
+ status: statusFromYtDlImportError(err),
data: {
targetUrl
}
})
}
-
- if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
- return res.fail({
- status: HttpStatusCode.FORBIDDEN_403,
- message: 'Cannot use non unicast IP as targetUrl.'
- })
- }
-
- const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
-
- // Process video thumbnail from request.files
- let thumbnailModel = await processThumbnail(req, video)
-
- // Process video thumbnail from url if processing from request.files failed
- if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
- try {
- thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
- } catch (err) {
- logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
- }
- }
-
- // Process video preview from request.files
- let previewModel = await processPreview(req, video)
-
- // Process video preview from url if processing from request.files failed
- if (!previewModel && youtubeDLInfo.thumbnailUrl) {
- try {
- previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
- } catch (err) {
- logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
- }
- }
-
- const videoImport = await insertIntoDB({
- video,
- thumbnailModel,
- previewModel,
- videoChannel: res.locals.videoChannel,
- tags: body.tags || youtubeDLInfo.tags,
- user,
- videoImportAttributes: {
- targetUrl,
- state: VideoImportState.PENDING,
- userId: user.id
- }
- })
-
- // Get video subtitles
- await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
-
- let fileExt = `.${youtubeDLInfo.ext}`
- if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
-
- // Create job to import the video
- const payload = {
- type: 'youtube-dl' as 'youtube-dl',
- videoImportId: videoImport.id,
- fileExt
- }
- await JobQueue.Instance.createJob({ type: 'video-import', payload })
-
- auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
-
- return res.json(videoImport.toFormattedJSON()).end()
-}
-
-async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
- let videoData = {
- name: body.name || importData.name || 'Unknown name',
- remote: false,
- category: body.category || importData.category,
- licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
- language: body.language || importData.language,
- commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
- downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
- waitTranscoding: body.waitTranscoding || false,
- state: VideoState.TO_IMPORT,
- nsfw: body.nsfw || importData.nsfw || false,
- description: body.description || importData.description,
- support: body.support || null,
- privacy: body.privacy || VideoPrivacy.PRIVATE,
- duration: 0, // duration will be set by the import job
- channelId,
- originallyPublishedAt: body.originallyPublishedAt
- ? new Date(body.originallyPublishedAt)
- : importData.originallyPublishedAt
- }
-
- videoData = await Hooks.wrapObject(
- videoData,
- body.targetUrl
- ? 'filter:api.video.import-url.video-attribute.result'
- : 'filter:api.video.import-torrent.video-attribute.result'
- )
-
- const video = new VideoModel(videoData)
- video.url = getLocalVideoActivityPubUrl(video)
-
- return video
}
async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
return undefined
}
-async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
- try {
- return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
- } catch (err) {
- logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
- return undefined
- }
-}
-
-async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
- try {
- return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
- } catch (err) {
- logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
- return undefined
- }
-}
-
-async function insertIntoDB (parameters: {
- video: MVideoThumbnail
- thumbnailModel: MThumbnail
- previewModel: MThumbnail
- videoChannel: MChannelAccountDefault
- tags: string[]
- videoImportAttributes: FilteredModelAttributes<VideoImportModel>
- user: MUser
-}): Promise<MVideoImportFormattable> {
- const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
-
- const videoImport = await sequelizeTypescript.transaction(async t => {
- const sequelizeOptions = { transaction: t }
-
- // Save video object in database
- const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
- videoCreated.VideoChannel = videoChannel
-
- if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
- if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
-
- await autoBlacklistVideoIfNeeded({
- video: videoCreated,
- user,
- notify: false,
- isRemote: false,
- isNew: true,
- transaction: t
- })
-
- await setVideoTags({ video: videoCreated, tags, transaction: t })
-
- // Create video import object in database
- const videoImport = await VideoImportModel.create(
- Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
- sequelizeOptions
- ) as MVideoImportFormattable
- videoImport.Video = videoCreated
-
- return videoImport
- })
-
- return videoImport
-}
-
async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
const torrentName = torrentfile.originalname
function extractNameFromArray (name: string | string[]) {
return isArray(name) ? name[0] : name
}
-
-async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
- try {
- const subtitles = await youtubeDL.getSubtitles()
-
- logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
-
- for (const subtitle of subtitles) {
- if (!await isVTTFileValid(subtitle.path)) {
- await remove(subtitle.path)
- continue
- }
-
- const videoCaption = new VideoCaptionModel({
- videoId,
- language: subtitle.language,
- filename: VideoCaptionModel.generateCaptionName(subtitle.language)
- }) as MVideoCaption
-
- // Move physical file
- await moveAndProcessCaptionFile(subtitle, videoCaption)
-
- await sequelizeTypescript.transaction(async t => {
- await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
- })
- }
- } catch (err) {
- logger.warn('Cannot get video subtitles.', { err })
- }
-}
-
-async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
- const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
- const uniqHosts = new Set(hosts)
-
- for (const h of uniqHosts) {
- if (await isResolvingToUnicastOnly(h) !== true) {
- return false
- }
- }
-
- return true
-}
import { join } from 'path'
import { addColors, config, createLogger, format, transports } from 'winston'
import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
-import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models'
+import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
import { CONFIG } from '../initializers/config'
import { jsonLoggerFormat, labelFormatter } from './logger'
}
}
+const channelSyncKeysToKeep = [
+ 'id',
+ 'externalChannelUrl',
+ 'channel-id',
+ 'channel-name'
+]
+class VideoChannelSyncAuditView extends EntityAuditView {
+ constructor (channelSync: VideoChannelSync) {
+ super(channelSyncKeysToKeep, 'channelSync', channelSync)
+ }
+}
+
export {
getAuditIdFromRes,
UserAuditView,
VideoAuditView,
AbuseAuditView,
- CustomConfigAuditView
+ CustomConfigAuditView,
+ VideoChannelSyncAuditView
}
--- /dev/null
+import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
+import { exists } from './misc'
+
+export function isVideoChannelSyncStateValid (value: any) {
+ return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
+}
return result.concat([
'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
+ 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
'best' // Ultimate fallback
]).join('/')
}
timeout?: number
additionalYoutubeDLArgs?: string[]
}) {
+ let args = options.additionalYoutubeDLArgs || []
+ args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
+
return this.run({
url: options.url,
processOptions: options.processOptions,
timeout: options.timeout,
- args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
+ args
})
}
: info
}
+ getListInfo (options: {
+ url: string
+ latestVideosCount?: number
+ processOptions: execa.NodeOptions
+ }): Promise<{ upload_date: string, webpage_url: string }[]> {
+ const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
+
+ if (options.latestVideosCount !== undefined) {
+ additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
+ }
+
+ return this.getInfo({
+ url: options.url,
+ format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
+ processOptions: options.processOptions,
+ additionalYoutubeDLArgs
+ })
+ }
+
async getSubs (options: {
url: string
format: 'vtt'
const output = await subProcess
- logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
+ logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
return output.stdout
? output.stdout.trim().split(/\r?\n/)
thumbnailUrl?: string
ext?: string
originallyPublishedAt?: Date
+ webpageUrl?: string
urls?: string[]
}
thumbnailUrl: obj.thumbnail || undefined,
urls: this.buildAvailableUrl(obj),
originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
- ext: obj.ext
+ ext: obj.ext,
+ webpageUrl: obj.webpage_url
}
}
return infoBuilder.getInfo()
}
+ async getInfoForListImport (options: {
+ latestVideosCount?: number
+ }) {
+ const youtubeDL = await YoutubeDLCLI.safeGet()
+
+ const list = await youtubeDL.getListInfo({
+ url: this.url,
+ latestVideosCount: options.latestVideosCount,
+ processOptions
+ })
+
+ return list.map(info => {
+ const infoBuilder = new YoutubeDLInfoBuilder(info)
+
+ return infoBuilder.getInfo()
+ })
+ }
+
async getSubtitles (): Promise<YoutubeDLSubs> {
const cwd = CONFIG.STORAGE.TMP_DIR
return remove(path)
})
- .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() }))
+ .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
throw err
}
checkRemoteRedundancyConfig()
checkStorageConfig()
checkTranscodingConfig()
+ checkImportConfig()
checkBroadcastMessageConfig()
checkSearchConfig()
checkLiveConfig()
}
}
+function checkImportConfig () {
+ if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) {
+ throw new Error('You need to enable HTTP import to allow synchronization')
+ }
+}
+
function checkBroadcastMessageConfig () {
if (CONFIG.BROADCAST_MESSAGE.ENABLED) {
const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL
'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
+ 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
+ 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
'client.videos.miniature.display_author_avatar',
'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
TORRENT: {
get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
}
+ },
+ VIDEO_CHANNEL_SYNCHRONIZATION: {
+ get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') },
+ get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') },
+ get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
+ get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
+ return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
+ }
}
},
AUTO_BLACKLIST: {
get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
}
}
+
}
function registerConfigChangedHandler (fun: Function) {
import {
AbuseState,
JobType,
+ VideoChannelSyncState,
VideoImportState,
VideoPrivacy,
VideoRateType,
// ---------------------------------------------------------------------------
-const LAST_MIGRATION_VERSION = 725
+const LAST_MIGRATION_VERSION = 730
// ---------------------------------------------------------------------------
JOBS: [ 'createdAt' ],
VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
VIDEO_IMPORTS: [ 'createdAt' ],
+ VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ],
VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
VIDEO_COMMENTS: [ 'createdAt' ],
'video-live-ending': 1,
'video-studio-edition': 1,
'manage-video-torrent': 1,
+ 'video-channel-import': 1,
+ 'after-video-channel-import': 1,
'move-to-object-storage': 3,
'notify': 1,
'federate-video': 1
'video-studio-edition': 1,
'manage-video-torrent': 1,
'move-to-object-storage': 1,
+ 'video-channel-import': 1,
+ 'after-video-channel-import': 1,
'notify': 5,
'federate-video': 3
}
'video-redundancy': 1000 * 3600 * 3, // 3 hours
'video-live-ending': 1000 * 60 * 10, // 10 minutes
'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
+ 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
+ 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
+ 'after-video-channel-import': 60000 * 5, // 5 minutes
'notify': 60000 * 5, // 5 minutes
- 'federate-video': 60000 * 5, // 5 minutes
- 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
+ 'federate-video': 60000 * 5 // 5 minutes
}
const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
'videos-views-stats': {
REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
- REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour
+ REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
+ CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
}
// ---------------------------------------------------------------------------
NAME: { min: 1, max: 120 }, // Length
DESCRIPTION: { min: 3, max: 1000 }, // Length
SUPPORT: { min: 3, max: 1000 }, // Length
+ EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length
URL: { min: 3, max: 2000 } // Length
},
+ VIDEO_CHANNEL_SYNCS: {
+ EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length
+ },
VIDEO_CAPTIONS: {
CAPTION_FILE: {
EXTNAME: [ '.vtt', '.srt' ],
[VideoImportState.PROCESSING]: 'Processing'
}
+const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = {
+ [VideoChannelSyncState.FAILED]: 'Failed',
+ [VideoChannelSyncState.SYNCED]: 'Synchronized',
+ [VideoChannelSyncState.PROCESSING]: 'Processing',
+ [VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run'
+}
+
const ABUSE_STATES: { [ id in AbuseState ]: string } = {
[AbuseState.PENDING]: 'Pending',
[AbuseState.REJECTED]: 'Rejected',
JOB_COMPLETED_LIFETIME,
HTTP_SIGNATURE,
VIDEO_IMPORT_STATES,
+ VIDEO_CHANNEL_SYNC_STATE,
VIEW_LIFETIME,
CONTACT_FORM_LIFETIME,
VIDEO_PLAYLIST_PRIVACIES,
import { VideoTagModel } from '../models/video/video-tag'
import { VideoViewModel } from '../models/view/video-view'
import { CONFIG } from './config'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
VideoTrackerModel,
PluginModel,
ActorCustomPageModel,
- VideoJobInfoModel
+ VideoJobInfoModel,
+ VideoChannelSyncModel
])
// Check extensions exist in the database
--- /dev/null
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction
+ queryInterface: Sequelize.QueryInterface
+ sequelize: Sequelize.Sequelize
+ db: any
+}): Promise<void> {
+ const query = `
+ CREATE TABLE IF NOT EXISTS "videoChannelSync" (
+ "id" SERIAL,
+ "externalChannelUrl" VARCHAR(2000) NOT NULL DEFAULT NULL,
+ "videoChannelId" INTEGER NOT NULL REFERENCES "videoChannel" ("id")
+ ON DELETE CASCADE
+ ON UPDATE CASCADE,
+ "state" INTEGER NOT NULL DEFAULT 1,
+ "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+ "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+ "lastSyncAt" TIMESTAMP WITH TIME ZONE,
+ PRIMARY KEY ("id")
+ );
+ `
+ await utils.sequelize.query(query, { transaction: utils.transaction })
+}
+
+async function down (utils: {
+ queryInterface: Sequelize.QueryInterface
+ transaction: Sequelize.Transaction
+}) {
+ await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction })
+}
+
+export {
+ up,
+ down
+}
--- /dev/null
+import { Job } from 'bullmq'
+import { logger } from '@server/helpers/logger'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
+
+export async function processAfterVideoChannelImport (job: Job) {
+ const payload = job.data as AfterVideoChannelImportPayload
+ if (!payload.channelSyncId) return
+
+ logger.info('Processing after video channel import in job %s.', job.id)
+
+ const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
+ if (!sync) {
+ logger.error('Unknown sync id %d.', payload.channelSyncId)
+ return
+ }
+
+ const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
+
+ let errors = 0
+ let successes = 0
+
+ for (const value of Object.values(childrenValues)) {
+ if (value.resultType === 'success') successes++
+ else if (value.resultType === 'error') errors++
+ }
+
+ if (errors > 0) {
+ sync.state = VideoChannelSyncState.FAILED
+ logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
+ } else {
+ sync.state = VideoChannelSyncState.SYNCED
+ logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
+ }
+
+ await sync.save()
+}
--- /dev/null
+import { Job } from 'bullmq'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { synchronizeChannel } from '@server/lib/sync-channel'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelImportPayload } from '@shared/models'
+
+export async function processVideoChannelImport (job: Job) {
+ const payload = job.data as VideoChannelImportPayload
+
+ logger.info('Processing video channel import in job %s.', job.id)
+
+ // Channel import requires only http upload to be allowed
+ if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
+ logger.error('Cannot import channel as the HTTP upload is disabled')
+ return
+ }
+
+ if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+ logger.error('Cannot import channel as the synchronization is disabled')
+ return
+ }
+
+ const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
+
+ try {
+ logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
+
+ await synchronizeChannel({
+ channel: videoChannel,
+ externalChannelUrl: payload.externalChannelUrl
+ })
+ } catch (err) {
+ logger.error(`Failed to import channel ${videoChannel.name}`, { err })
+ }
+}
import { Hooks } from '@server/lib/plugins/hooks'
import { ServerConfigManager } from '@server/lib/server-config-manager'
import { isAbleToUploadVideo } from '@server/lib/user'
-import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video'
+import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { buildNextVideoState } from '@server/lib/video-state'
import { ThumbnailModel } from '@server/models/video/thumbnail'
import {
ThumbnailType,
VideoImportPayload,
+ VideoImportPreventExceptionResult,
VideoImportState,
VideoImportTorrentPayload,
VideoImportTorrentPayloadType,
import { generateVideoMiniature } from '../../thumbnail'
import { JobQueue } from '../job-queue'
-async function processVideoImport (job: Job) {
+async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
const payload = job.data as VideoImportPayload
const videoImport = await getVideoImportOrDie(payload)
if (videoImport.state === VideoImportState.CANCELLED) {
logger.info('Do not process import since it has been cancelled', { payload })
- return
+ return { resultType: 'success' }
}
videoImport.state = VideoImportState.PROCESSING
await videoImport.save()
- if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload)
- if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload)
+ try {
+ if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
+ if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
+
+ return { resultType: 'success' }
+ } catch (err) {
+ if (!payload.preventException) throw err
+
+ logger.warn('Catch error in video import to send value to parent job.', { payload, err })
+ return { resultType: 'error' }
+ }
}
// ---------------------------------------------------------------------------
ActivitypubHttpFetcherPayload,
ActivitypubHttpUnicastPayload,
ActorKeysPayload,
+ AfterVideoChannelImportPayload,
DeleteResumableUploadMetaFilePayload,
EmailPayload,
FederateVideoPayload,
MoveObjectStoragePayload,
NotifyPayload,
RefreshPayload,
+ VideoChannelImportPayload,
VideoFileImportPayload,
VideoImportPayload,
VideoLiveEndingPayload,
import { processManageVideoTorrent } from './handlers/manage-video-torrent'
import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
import { processNotify } from './handlers/notify'
+import { processVideoChannelImport } from './handlers/video-channel-import'
import { processVideoFileImport } from './handlers/video-file-import'
import { processVideoImport } from './handlers/video-import'
import { processVideoLiveEnding } from './handlers/video-live-ending'
import { processVideoStudioEdition } from './handlers/video-studio-edition'
import { processVideoTranscoding } from './handlers/video-transcoding'
import { processVideosViewsStats } from './handlers/video-views-stats'
+import { processAfterVideoChannelImport } from './handlers/after-video-channel-import'
export type CreateJobArgument =
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
{ type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
{ type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
{ type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
+ { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
+ { type: 'video-channel-import', payload: VideoChannelImportPayload } |
+ { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
{ type: 'notify', payload: NotifyPayload } |
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
{ type: 'federate-video', payload: FederateVideoPayload }
'video-redundancy': processVideoRedundancy,
'move-to-object-storage': processMoveToObjectStorage,
'manage-video-torrent': processManageVideoTorrent,
- 'notify': processNotify,
'video-studio-edition': processVideoStudioEdition,
+ 'video-channel-import': processVideoChannelImport,
+ 'after-video-channel-import': processAfterVideoChannelImport,
+ 'notify': processNotify,
'federate-video': processFederateVideo
}
'move-to-object-storage',
'manage-video-torrent',
'video-studio-edition',
+ 'video-channel-import',
+ 'after-video-channel-import',
'notify',
'federate-video'
]
.catch(err => logger.error('Cannot create job.', { err, options }))
}
- async createJob (options: CreateJobArgument & CreateJobOptions) {
+ createJob (options: CreateJobArgument & CreateJobOptions) {
const queue: Queue = this.queues[options.type]
if (queue === undefined) {
logger.error('Unknown queue %s: cannot create job.', options.type)
return queue.add('job', options.payload, jobOptions)
}
- async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
+ createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
let lastJob: FlowJob
for (const job of jobs) {
return this.flowProducer.add(lastJob)
}
- async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
+ createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
return this.flowProducer.add({
...this.buildJobFlowOption(parent),
--- /dev/null
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { VideoChannelSyncState } from '@shared/models'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { synchronizeChannel } from '../sync-channel'
+import { AbstractScheduler } from './abstract-scheduler'
+
+export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
+ private static instance: AbstractScheduler
+ protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
+
+ private constructor () {
+ super()
+ }
+
+ protected async internalExecute () {
+ logger.debug('Running %s.%s', this.constructor.name, this.internalExecute.name)
+
+ if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+ logger.info('Discard channels synchronization as the feature is disabled')
+ return
+ }
+
+ const channelSyncs = await VideoChannelSyncModel.listSyncs()
+
+ for (const sync of channelSyncs) {
+ const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
+
+ try {
+ logger.info(
+ 'Creating video import jobs for "%s" sync with external channel "%s"',
+ channel.Actor.preferredUsername, sync.externalChannelUrl
+ )
+
+ const onlyAfter = sync.lastSyncAt || sync.createdAt
+
+ sync.state = VideoChannelSyncState.PROCESSING
+ sync.lastSyncAt = new Date()
+ await sync.save()
+
+ await synchronizeChannel({
+ channel,
+ externalChannelUrl: sync.externalChannelUrl,
+ videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
+ channelSync: sync,
+ onlyAfter
+ })
+ } catch (err) {
+ logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
+ sync.state = VideoChannelSyncState.FAILED
+ await sync.save()
+ }
+ }
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+}
torrent: {
enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
}
+ },
+ videoChannelSynchronization: {
+ enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED
}
},
autoBlacklist: {
--- /dev/null
+import { logger } from '@server/helpers/logger'
+import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
+import { buildYoutubeDLImport } from '@server/lib/video-import'
+import { UserModel } from '@server/models/user/user'
+import { VideoImportModel } from '@server/models/video/video-import'
+import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
+import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
+import { CreateJobArgument, JobQueue } from './job-queue'
+import { ServerConfigManager } from './server-config-manager'
+
+export async function synchronizeChannel (options: {
+ channel: MChannelAccountDefault
+ externalChannelUrl: string
+ channelSync?: MChannelSync
+ videosCountLimit?: number
+ onlyAfter?: Date
+}) {
+ const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
+
+ const user = await UserModel.loadByChannelActorId(channel.actorId)
+ const youtubeDL = new YoutubeDLWrapper(
+ externalChannelUrl,
+ ServerConfigManager.Instance.getEnabledResolutions('vod'),
+ CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+ )
+
+ const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
+
+ const targetUrls = infoList
+ .filter(videoInfo => {
+ if (!onlyAfter) return true
+
+ return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
+ })
+ .map(videoInfo => videoInfo.webpageUrl)
+
+ logger.info(
+ 'Fetched %d candidate URLs for sync channel %s.',
+ targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
+ )
+
+ if (targetUrls.length === 0) {
+ if (channelSync) {
+ channelSync.state = VideoChannelSyncState.SYNCED
+ await channelSync.save()
+ }
+
+ return
+ }
+
+ const children: CreateJobArgument[] = []
+
+ for (const targetUrl of targetUrls) {
+ if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
+ logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
+ continue
+ }
+
+ const { job } = await buildYoutubeDLImport({
+ user,
+ channel,
+ targetUrl,
+ channelSync,
+ importDataOverride: {
+ privacy: VideoPrivacy.PUBLIC
+ }
+ })
+
+ children.push(job)
+ }
+
+ const parent: CreateJobArgument = {
+ type: 'after-video-channel-import',
+ payload: {
+ channelSyncId: channelSync?.id
+ }
+ }
+
+ await JobQueue.Instance.createJobWithChildren(parent, children)
+}
--- /dev/null
+import { remove } from 'fs-extra'
+import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
+import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
+import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
+import { isResolvingToUnicastOnly } from '@server/helpers/dns'
+import { logger } from '@server/helpers/logger'
+import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { ServerConfigManager } from '@server/lib/server-config-manager'
+import { setVideoTags } from '@server/lib/video'
+import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
+import { VideoModel } from '@server/models/video/video'
+import { VideoCaptionModel } from '@server/models/video/video-caption'
+import { VideoImportModel } from '@server/models/video/video-import'
+import { FilteredModelAttributes } from '@server/types'
+import {
+ MChannelAccountDefault,
+ MChannelSync,
+ MThumbnail,
+ MUser,
+ MVideoAccountDefault,
+ MVideoCaption,
+ MVideoImportFormattable,
+ MVideoTag,
+ MVideoThumbnail,
+ MVideoWithBlacklistLight
+} from '@server/types/models'
+import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
+import { getLocalVideoActivityPubUrl } from './activitypub/url'
+import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
+
+class YoutubeDlImportError extends Error {
+ code: YoutubeDlImportError.CODE
+ cause?: Error // Property to remove once ES2022 is used
+ constructor ({ message, code }) {
+ super(message)
+ this.code = code
+ }
+
+ static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
+ const ytDlErr = new this({ message: message ?? err.message, code })
+ ytDlErr.cause = err
+ ytDlErr.stack = err.stack // Useless once ES2022 is used
+ return ytDlErr
+ }
+}
+
+namespace YoutubeDlImportError {
+ export enum CODE {
+ FETCH_ERROR,
+ NOT_ONLY_UNICAST_URL
+ }
+}
+
+// ---------------------------------------------------------------------------
+
+async function insertFromImportIntoDB (parameters: {
+ video: MVideoThumbnail
+ thumbnailModel: MThumbnail
+ previewModel: MThumbnail
+ videoChannel: MChannelAccountDefault
+ tags: string[]
+ videoImportAttributes: FilteredModelAttributes<VideoImportModel>
+ user: MUser
+}): Promise<MVideoImportFormattable> {
+ const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
+
+ const videoImport = await sequelizeTypescript.transaction(async t => {
+ const sequelizeOptions = { transaction: t }
+
+ // Save video object in database
+ const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
+ videoCreated.VideoChannel = videoChannel
+
+ if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
+ if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
+
+ await autoBlacklistVideoIfNeeded({
+ video: videoCreated,
+ user,
+ notify: false,
+ isRemote: false,
+ isNew: true,
+ transaction: t
+ })
+
+ await setVideoTags({ video: videoCreated, tags, transaction: t })
+
+ // Create video import object in database
+ const videoImport = await VideoImportModel.create(
+ Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
+ sequelizeOptions
+ ) as MVideoImportFormattable
+ videoImport.Video = videoCreated
+
+ return videoImport
+ })
+
+ return videoImport
+}
+
+async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
+ channelId: number
+ importData: YoutubeDLInfo
+ importDataOverride?: Partial<VideoImportCreate>
+ importType: 'url' | 'torrent'
+}): Promise<MVideoThumbnail> {
+ let videoData = {
+ name: importDataOverride?.name || importData.name || 'Unknown name',
+ remote: false,
+ category: importDataOverride?.category || importData.category,
+ licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
+ language: importDataOverride?.language || importData.language,
+ commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
+ downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
+ waitTranscoding: importDataOverride?.waitTranscoding || false,
+ state: VideoState.TO_IMPORT,
+ nsfw: importDataOverride?.nsfw || importData.nsfw || false,
+ description: importDataOverride?.description || importData.description,
+ support: importDataOverride?.support || null,
+ privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
+ duration: 0, // duration will be set by the import job
+ channelId,
+ originallyPublishedAt: importDataOverride?.originallyPublishedAt
+ ? new Date(importDataOverride?.originallyPublishedAt)
+ : importData.originallyPublishedAt
+ }
+
+ videoData = await Hooks.wrapObject(
+ videoData,
+ importType === 'url'
+ ? 'filter:api.video.import-url.video-attribute.result'
+ : 'filter:api.video.import-torrent.video-attribute.result'
+ )
+
+ const video = new VideoModel(videoData)
+ video.url = getLocalVideoActivityPubUrl(video)
+
+ return video
+}
+
+async function buildYoutubeDLImport (options: {
+ targetUrl: string
+ channel: MChannelAccountDefault
+ user: MUser
+ channelSync?: MChannelSync
+ importDataOverride?: Partial<VideoImportCreate>
+ thumbnailFilePath?: string
+ previewFilePath?: string
+}) {
+ const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
+
+ const youtubeDL = new YoutubeDLWrapper(
+ targetUrl,
+ ServerConfigManager.Instance.getEnabledResolutions('vod'),
+ CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+ )
+
+ // Get video infos
+ let youtubeDLInfo: YoutubeDLInfo
+ try {
+ youtubeDLInfo = await youtubeDL.getInfoForDownload()
+ } catch (err) {
+ throw YoutubeDlImportError.fromError(
+ err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
+ )
+ }
+
+ if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
+ throw new YoutubeDlImportError({
+ message: 'Cannot use non unicast IP as targetUrl.',
+ code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
+ })
+ }
+
+ const video = await buildVideoFromImport({
+ channelId: channel.id,
+ importData: youtubeDLInfo,
+ importDataOverride,
+ importType: 'url'
+ })
+
+ const thumbnailModel = await forgeThumbnail({
+ inputPath: thumbnailFilePath,
+ downloadUrl: youtubeDLInfo.thumbnailUrl,
+ video,
+ type: ThumbnailType.MINIATURE
+ })
+
+ const previewModel = await forgeThumbnail({
+ inputPath: previewFilePath,
+ downloadUrl: youtubeDLInfo.thumbnailUrl,
+ video,
+ type: ThumbnailType.PREVIEW
+ })
+
+ const videoImport = await insertFromImportIntoDB({
+ video,
+ thumbnailModel,
+ previewModel,
+ videoChannel: channel,
+ tags: importDataOverride?.tags || youtubeDLInfo.tags,
+ user,
+ videoImportAttributes: {
+ targetUrl,
+ state: VideoImportState.PENDING,
+ userId: user.id
+ }
+ })
+
+ // Get video subtitles
+ await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
+
+ let fileExt = `.${youtubeDLInfo.ext}`
+ if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
+
+ const payload: VideoImportPayload = {
+ type: 'youtube-dl' as 'youtube-dl',
+ videoImportId: videoImport.id,
+ fileExt,
+ // If part of a sync process, there is a parent job that will aggregate children results
+ preventException: !!channelSync
+ }
+
+ return {
+ videoImport,
+ job: { type: 'video-import' as 'video-import', payload }
+ }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+ buildYoutubeDLImport,
+ YoutubeDlImportError,
+ insertFromImportIntoDB,
+ buildVideoFromImport
+}
+
+// ---------------------------------------------------------------------------
+
+async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
+ inputPath?: string
+ downloadUrl?: string
+ video: MVideoThumbnail
+ type: ThumbnailType
+}): Promise<MThumbnail> {
+ if (inputPath) {
+ return updateVideoMiniatureFromExisting({
+ inputPath,
+ video,
+ type,
+ automaticallyGenerated: false
+ })
+ } else if (downloadUrl) {
+ try {
+ return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
+ } catch (err) {
+ logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
+ }
+ }
+ return null
+}
+
+async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
+ try {
+ const subtitles = await youtubeDL.getSubtitles()
+
+ logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
+
+ for (const subtitle of subtitles) {
+ if (!await isVTTFileValid(subtitle.path)) {
+ await remove(subtitle.path)
+ continue
+ }
+
+ const videoCaption = new VideoCaptionModel({
+ videoId,
+ language: subtitle.language,
+ filename: VideoCaptionModel.generateCaptionName(subtitle.language)
+ }) as MVideoCaption
+
+ // Move physical file
+ await moveAndProcessCaptionFile(subtitle, videoCaption)
+
+ await sequelizeTypescript.transaction(async t => {
+ await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
+ })
+ }
+ } catch (err) {
+ logger.warn('Cannot get video subtitles.', { err })
+ }
+}
+
+async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
+ const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
+ const uniqHosts = new Set(hosts)
+
+ for (const h of uniqHosts) {
+ if (await isResolvingToUnicastOnly(h) !== true) {
+ return false
+ }
+ }
+
+ return true
+}
body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
+ body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'),
+
body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'),
body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'),
if (areValidationErrors(req, res)) return
if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
if (!checkInvalidTranscodingConfig(req.body, res)) return
+ if (!checkInvalidSynchronizationConfig(req.body, res)) return
if (!checkInvalidLiveConfig(req.body, res)) return
if (!checkInvalidVideoStudioConfig(req.body, res)) return
return true
}
+function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) {
+ if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) {
+ res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' })
+ return false
+ }
+ return true
+}
+
function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
if (customConfig.live.enabled === false) return true
const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS)
const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
+const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS)
const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)
videoPlaylistsSearchSortValidator,
accountsFollowersSortValidator,
videoChannelsFollowersSortValidator,
+ videoChannelSyncsSortValidator,
pluginsSortValidator
}
export * from './video-studio'
export * from './video-transcoding'
export * from './videos'
+export * from './video-channel-sync'
--- /dev/null
+import * as express from 'express'
+import { body, param } from 'express-validator'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
+import { areValidationErrors, doesVideoChannelIdExist } from '../shared'
+
+export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+ return res.fail({
+ status: HttpStatusCode.FORBIDDEN_403,
+ message: 'Synchronization is impossible as video channel synchronization is not enabled on the server'
+ })
+ }
+
+ return next()
+}
+
+export const videoChannelSyncValidator = [
+ body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
+ body('videoChannelId').isInt().withMessage('Should have a valid video channel id'),
+
+ async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ logger.debug('Checking videoChannelSync parameters', { parameters: req.body })
+
+ if (areValidationErrors(req, res)) return
+
+ const body: VideoChannelSyncCreate = req.body
+ if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return
+
+ const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId)
+ if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) {
+ return res.fail({
+ message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations`
+ })
+ }
+
+ return next()
+ }
+]
+
+export const ensureSyncExists = [
+ param('id').exists().isInt().withMessage('Should have an sync id'),
+
+ async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ if (areValidationErrors(req, res)) return
+
+ const syncId = parseInt(req.params.id, 10)
+ const sync = await VideoChannelSyncModel.loadWithChannel(syncId)
+
+ if (!sync) {
+ return res.fail({
+ status: HttpStatusCode.NOT_FOUND_404,
+ message: 'Synchronization not found'
+ })
+ }
+
+ res.locals.videoChannelSync = sync
+ res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
+
+ return next()
+ }
+]
import express from 'express'
import { body, param, query } from 'express-validator'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
import { CONFIG } from '@server/initializers/config'
import { MChannelAccountDefault } from '@server/types/models'
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
import { logger } from '../../../helpers/logger'
import { ActorModel } from '../../../models/actor/actor'
import { VideoChannelModel } from '../../../models/video/video-channel'
-import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared'
+import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared'
-const videoChannelsAddValidator = [
+export const videoChannelsAddValidator = [
body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'),
body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'),
body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'),
}
]
-const videoChannelsUpdateValidator = [
+export const videoChannelsUpdateValidator = [
param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
body('displayName')
.optional()
}
]
-const videoChannelsRemoveValidator = [
+export const videoChannelsRemoveValidator = [
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params })
}
]
-const videoChannelsNameWithHostValidator = [
+export const videoChannelsNameWithHostValidator = [
param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
}
]
-const ensureIsLocalChannel = [
+export const ensureIsLocalChannel = [
(req: express.Request, res: express.Response, next: express.NextFunction) => {
if (res.locals.videoChannel.Actor.isOwned() === false) {
return res.fail({
}
]
-const videoChannelStatsValidator = [
+export const ensureChannelOwnerCanUpload = [
+ async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ const channel = res.locals.videoChannel
+ const user = { id: channel.Account.userId }
+
+ if (!await checkUserQuota(user, 1, res)) return
+
+ next()
+ }
+]
+
+export const videoChannelStatsValidator = [
query('withStats')
.optional()
.customSanitizer(toBooleanOrNull)
}
]
-const videoChannelsListValidator = [
+export const videoChannelsListValidator = [
query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
}
]
-// ---------------------------------------------------------------------------
+export const videoChannelImportVideosValidator = [
+ body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
-export {
- videoChannelsAddValidator,
- videoChannelsUpdateValidator,
- videoChannelsRemoveValidator,
- videoChannelsNameWithHostValidator,
- ensureIsLocalChannel,
- videoChannelsListValidator,
- videoChannelStatsValidator
-}
+ (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ logger.debug('Checking videoChannelImport parameters', { parameters: req.body })
+
+ if (areValidationErrors(req, res)) return
+
+ if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
+ return res.fail({
+ status: HttpStatusCode.FORBIDDEN_403,
+ message: 'Channel import is impossible as video upload via HTTP is not enabled on the server'
+ })
+ }
+
+ return next()
+ }
+]
// ---------------------------------------------------------------------------
return getSort(value, lastSort)
}
+function getChannelSyncSort (value: string): OrderItem[] {
+ const { direction, field } = buildDirectionAndField(value)
+ if (field.toLowerCase() === 'videochannel') {
+ return [
+ [ literal('"VideoChannel.name"'), direction ]
+ ]
+ }
+ return [ [ field, direction ] ]
+}
+
function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
if (!model.createdAt || !model.updatedAt) {
throw new Error('Miss createdAt & updatedAt attributes to model')
getAdminUsersSort,
getVideoSort,
getBlacklistSort,
+ getChannelSyncSort,
createSimilarityAttribute,
throwIfNotValid,
buildServerIdsFollowedBy,
--- /dev/null
+import { Op } from 'sequelize'
+import {
+ AllowNull,
+ BelongsTo,
+ Column,
+ CreatedAt,
+ DataType,
+ Default,
+ DefaultScope,
+ ForeignKey,
+ Is,
+ Model,
+ Table,
+ UpdatedAt
+} from 'sequelize-typescript'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs'
+import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
+import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models'
+import { VideoChannelSync, VideoChannelSyncState } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { AccountModel } from '../account/account'
+import { UserModel } from '../user/user'
+import { getChannelSyncSort, throwIfNotValid } from '../utils'
+import { VideoChannelModel } from './video-channel'
+
+@DefaultScope(() => ({
+ include: [
+ {
+ model: VideoChannelModel, // Default scope includes avatar and server
+ required: true
+ }
+ ]
+}))
+@Table({
+ tableName: 'videoChannelSync',
+ indexes: [
+ {
+ fields: [ 'videoChannelId' ]
+ }
+ ]
+})
+export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> {
+
+ @AllowNull(false)
+ @Default(null)
+ @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true))
+ @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max))
+ externalChannelUrl: string
+
+ @CreatedAt
+ createdAt: Date
+
+ @UpdatedAt
+ updatedAt: Date
+
+ @ForeignKey(() => VideoChannelModel)
+ @Column
+ videoChannelId: number
+
+ @BelongsTo(() => VideoChannelModel, {
+ foreignKey: {
+ allowNull: false
+ },
+ onDelete: 'cascade'
+ })
+ VideoChannel: VideoChannelModel
+
+ @AllowNull(false)
+ @Default(VideoChannelSyncState.WAITING_FIRST_RUN)
+ @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state'))
+ @Column
+ state: VideoChannelSyncState
+
+ @AllowNull(true)
+ @Column(DataType.DATE)
+ lastSyncAt: Date
+
+ static listByAccountForAPI (options: {
+ accountId: number
+ start: number
+ count: number
+ sort: string
+ }) {
+ const getQuery = (forCount: boolean) => {
+ const videoChannelModel = forCount
+ ? VideoChannelModel.unscoped()
+ : VideoChannelModel
+
+ return {
+ offset: options.start,
+ limit: options.count,
+ order: getChannelSyncSort(options.sort),
+ include: [
+ {
+ model: videoChannelModel,
+ required: true,
+ where: {
+ accountId: options.accountId
+ }
+ }
+ ]
+ }
+ }
+
+ return Promise.all([
+ VideoChannelSyncModel.unscoped().count(getQuery(true)),
+ VideoChannelSyncModel.unscoped().findAll(getQuery(false))
+ ]).then(([ total, data ]) => ({ total, data }))
+ }
+
+ static countByAccount (accountId: number) {
+ const query = {
+ include: [
+ {
+ model: VideoChannelModel.unscoped(),
+ required: true,
+ where: {
+ accountId
+ }
+ }
+ ]
+ }
+
+ return VideoChannelSyncModel.unscoped().count(query)
+ }
+
+ static loadWithChannel (id: number): Promise<MChannelSyncChannel> {
+ return VideoChannelSyncModel.findByPk(id)
+ }
+
+ static async listSyncs (): Promise<MChannelSync[]> {
+ const query = {
+ include: [
+ {
+ model: VideoChannelModel.unscoped(),
+ required: true,
+ include: [
+ {
+ model: AccountModel.unscoped(),
+ required: true,
+ include: [ {
+ attributes: [],
+ model: UserModel.unscoped(),
+ required: true,
+ where: {
+ videoQuota: {
+ [Op.ne]: 0
+ },
+ videoQuotaDaily: {
+ [Op.ne]: 0
+ }
+ }
+ } ]
+ }
+ ]
+ }
+ ]
+ }
+ return VideoChannelSyncModel.unscoped().findAll(query)
+ }
+
+ toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync {
+ return {
+ id: this.id,
+ state: {
+ id: this.state,
+ label: VIDEO_CHANNEL_SYNC_STATE[this.state]
+ },
+ externalChannelUrl: this.externalChannelUrl,
+ createdAt: this.createdAt.toISOString(),
+ channel: this.VideoChannel.toFormattedSummaryJSON(),
+ lastSyncAt: this.lastSyncAt?.toISOString()
+ }
+ }
+}
-import { WhereOptions } from 'sequelize'
+import { Op, WhereOptions } from 'sequelize'
import {
AfterUpdate,
AllowNull,
]).then(([ total, data ]) => ({ total, data }))
}
+ static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
+ const element = await VideoImportModel.unscoped().findOne({
+ where: {
+ targetUrl,
+ state: {
+ [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
+ }
+ },
+ include: [
+ {
+ model: VideoModel,
+ required: true,
+ where: {
+ channelId
+ }
+ }
+ ]
+ })
+
+ return !!element
+ }
+
getTargetIdentifier () {
return this.targetUrl || this.magnetUri || this.torrentName
}
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
-import { omit } from 'lodash'
+import { merge, omit } from 'lodash'
+import { CustomConfig, HttpStatusCode } from '@shared/models'
import {
cleanupTests,
createSingleServer,
PeerTubeServer,
setAccessTokensToServers
} from '@shared/server-commands'
-import { CustomConfig, HttpStatusCode } from '@shared/models'
describe('Test config API validators', function () {
const path = '/api/v1/config/custom'
torrent: {
enabled: false
}
+ },
+ videoChannelSynchronization: {
+ enabled: false,
+ maxPerUser: 10
}
},
trending: {
})
})
- it('Should success with the correct parameters', async function () {
+ it('Should fail with a disabled http upload & enabled sync', async function () {
+ const newUpdateParams: CustomConfig = merge({}, updateParams, {
+ import: {
+ videos: {
+ http: { enabled: false }
+ },
+ videoChannelSynchronization: { enabled: true }
+ }
+ })
+
+ await makePutBodyRequest({
+ url: server.url,
+ path,
+ fields: newUpdateParams,
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
+ })
+
+ it('Should succeed with the correct parameters', async function () {
await makePutBodyRequest({
url: server.url,
path,
import './video-comments'
import './video-files'
import './video-imports'
+import './video-channel-syncs'
import './video-playlists'
import './video-source'
import './video-studio'
})
it('Should fail to import with HTTP/Torrent/magnet', async function () {
- this.timeout(120000)
+ this.timeout(120_000)
const baseAttributes = {
channelId: server.store.channel.id,
--- /dev/null
+import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
+import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
+import {
+ ChannelSyncsCommand,
+ createSingleServer,
+ makePostBodyRequest,
+ PeerTubeServer,
+ setAccessTokensToServers,
+ setDefaultVideoChannel
+} from '@shared/server-commands'
+
+describe('Test video channel sync API validator', () => {
+ const path = '/api/v1/video-channel-syncs'
+ let server: PeerTubeServer
+ let command: ChannelSyncsCommand
+ let rootChannelId: number
+ let rootChannelSyncId: number
+ const userInfo = {
+ accessToken: '',
+ username: 'user1',
+ id: -1,
+ channelId: -1,
+ syncId: -1
+ }
+
+ async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> {
+ try {
+ await server.config.disableChannelSync()
+ await callback()
+ } finally {
+ await server.config.enableChannelSync()
+ }
+ }
+
+ async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> {
+ const origConfig = await server.config.getCustomConfig()
+
+ await server.config.updateExistingSubConfig({
+ newConfig: {
+ import: {
+ videoChannelSynchronization: {
+ maxPerUser: maxSync
+ }
+ }
+ }
+ })
+
+ try {
+ await callback()
+ } finally {
+ await server.config.updateCustomConfig({ newCustomConfig: origConfig })
+ }
+ }
+
+ before(async function () {
+ this.timeout(30_000)
+
+ server = await createSingleServer(1)
+
+ await setAccessTokensToServers([ server ])
+ await setDefaultVideoChannel([ server ])
+
+ command = server.channelSyncs
+
+ rootChannelId = server.store.channel.id
+
+ {
+ userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
+
+ const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken })
+ userInfo.id = userId
+ userInfo.channelId = videoChannels[0].id
+ }
+
+ await server.config.enableChannelSync()
+ })
+
+ describe('When creating a sync', function () {
+ let baseCorrectParams: VideoChannelSyncCreate
+
+ before(function () {
+ baseCorrectParams = {
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ videoChannelId: rootChannelId
+ }
+ })
+
+ it('Should fail when sync is disabled', async function () {
+ await withChannelSyncDisabled(async () => {
+ await command.create({
+ token: server.accessToken,
+ attributes: baseCorrectParams,
+ expectedStatus: HttpStatusCode.FORBIDDEN_403
+ })
+ })
+ })
+
+ it('Should fail with nothing', async function () {
+ const fields = {}
+ await makePostBodyRequest({
+ url: server.url,
+ path,
+ token: server.accessToken,
+ fields,
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
+ })
+
+ it('Should fail with no authentication', async function () {
+ await command.create({
+ token: null,
+ attributes: baseCorrectParams,
+ expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+ })
+ })
+
+ it('Should fail without a target url', async function () {
+ const attributes: VideoChannelSyncCreate = {
+ ...baseCorrectParams,
+ externalChannelUrl: null
+ }
+ await command.create({
+ token: server.accessToken,
+ attributes,
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
+ })
+
+ it('Should fail without a channelId', async function () {
+ const attributes: VideoChannelSyncCreate = {
+ ...baseCorrectParams,
+ videoChannelId: null
+ }
+ await command.create({
+ token: server.accessToken,
+ attributes,
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
+ })
+
+ it('Should fail with a channelId refering nothing', async function () {
+ const attributes: VideoChannelSyncCreate = {
+ ...baseCorrectParams,
+ videoChannelId: 42
+ }
+ await command.create({
+ token: server.accessToken,
+ attributes,
+ expectedStatus: HttpStatusCode.NOT_FOUND_404
+ })
+ })
+
+ it('Should fail to create a sync when the user does not own the channel', async function () {
+ await command.create({
+ token: userInfo.accessToken,
+ attributes: baseCorrectParams,
+ expectedStatus: HttpStatusCode.FORBIDDEN_403
+ })
+ })
+
+ it('Should succeed to create a sync with root and for another user\'s channel', async function () {
+ const { videoChannelSync } = await command.create({
+ token: server.accessToken,
+ attributes: {
+ ...baseCorrectParams,
+ videoChannelId: userInfo.channelId
+ },
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ userInfo.syncId = videoChannelSync.id
+ })
+
+ it('Should succeed with the correct parameters', async function () {
+ const { videoChannelSync } = await command.create({
+ token: server.accessToken,
+ attributes: baseCorrectParams,
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ rootChannelSyncId = videoChannelSync.id
+ })
+
+ it('Should fail when the user exceeds allowed number of synchronizations', async function () {
+ await withMaxSyncsPerUser(1, async () => {
+ await command.create({
+ token: server.accessToken,
+ attributes: {
+ ...baseCorrectParams,
+ videoChannelId: userInfo.channelId
+ },
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
+ })
+ })
+ })
+
+ describe('When listing my channel syncs', function () {
+ const myPath = '/api/v1/accounts/root/video-channel-syncs'
+
+ it('Should fail with a bad start pagination', async function () {
+ await checkBadStartPagination(server.url, myPath, server.accessToken)
+ })
+
+ it('Should fail with a bad count pagination', async function () {
+ await checkBadCountPagination(server.url, myPath, server.accessToken)
+ })
+
+ it('Should fail with an incorrect sort', async function () {
+ await checkBadSortPagination(server.url, myPath, server.accessToken)
+ })
+
+ it('Should succeed with the correct parameters', async function () {
+ await command.listByAccount({
+ accountName: 'root',
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ })
+
+ it('Should fail with no authentication', async function () {
+ await command.listByAccount({
+ accountName: 'root',
+ token: null,
+ expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+ })
+ })
+
+ it('Should fail when a simple user lists another user\'s synchronizations', async function () {
+ await command.listByAccount({
+ accountName: 'root',
+ token: userInfo.accessToken,
+ expectedStatus: HttpStatusCode.FORBIDDEN_403
+ })
+ })
+
+ it('Should succeed when root lists another user\'s synchronizations', async function () {
+ await command.listByAccount({
+ accountName: userInfo.username,
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ })
+
+ it('Should succeed even with synchronization disabled', async function () {
+ await withChannelSyncDisabled(async function () {
+ await command.listByAccount({
+ accountName: 'root',
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ })
+ })
+ })
+
+ describe('When triggering deletion', function () {
+ it('should fail with no authentication', async function () {
+ await command.delete({
+ channelSyncId: userInfo.syncId,
+ token: null,
+ expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+ })
+ })
+
+ it('Should fail when channelSyncId does not refer to any sync', async function () {
+ await command.delete({
+ channelSyncId: 42,
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.NOT_FOUND_404
+ })
+ })
+
+ it('Should fail when sync is not owned by the user', async function () {
+ await command.delete({
+ channelSyncId: rootChannelSyncId,
+ token: userInfo.accessToken,
+ expectedStatus: HttpStatusCode.FORBIDDEN_403
+ })
+ })
+
+ it('Should succeed when root delete a sync they do not own', async function () {
+ await command.delete({
+ channelSyncId: userInfo.syncId,
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.NO_CONTENT_204
+ })
+ })
+
+ it('should succeed when user delete a sync they own', async function () {
+ const { videoChannelSync } = await command.create({
+ attributes: {
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ videoChannelId: userInfo.channelId
+ },
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.OK_200
+ })
+
+ await command.delete({
+ channelSyncId: videoChannelSync.id,
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.NO_CONTENT_204
+ })
+ })
+
+ it('Should succeed even when synchronization is disabled', async function () {
+ await withChannelSyncDisabled(async function () {
+ await command.delete({
+ channelSyncId: rootChannelSyncId,
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.NO_CONTENT_204
+ })
+ })
+ })
+ })
+
+ after(async function () {
+ await server?.kill()
+ })
+})
import 'mocha'
import * as chai from 'chai'
import { omit } from 'lodash'
-import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared'
-import { buildAbsoluteFixturePath } from '@shared/core-utils'
+import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils'
import { HttpStatusCode, VideoChannelUpdate } from '@shared/models'
import {
ChannelsCommand,
describe('Test video channels API validator', function () {
const videoChannelPath = '/api/v1/video-channels'
let server: PeerTubeServer
- let accessTokenUser: string
+ const userInfo = {
+ accessToken: '',
+ channelName: 'fake_channel',
+ id: -1,
+ videoQuota: -1,
+ videoQuotaDaily: -1
+ }
let command: ChannelsCommand
// ---------------------------------------------------------------
await setAccessTokensToServers([ server ])
- const user = {
+ const userCreds = {
username: 'fake',
password: 'fake_password'
}
{
- await server.users.create({ username: user.username, password: user.password })
- accessTokenUser = await server.login.getAccessToken(user)
+ const user = await server.users.create({ username: userCreds.username, password: userCreds.password })
+ userInfo.id = user.id
+ userInfo.accessToken = await server.login.getAccessToken(userCreds)
}
command = server.channels
await makePutBodyRequest({
url: server.url,
path,
- token: accessTokenUser,
+ token: userInfo.accessToken,
fields: baseCorrectParams,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should fail with a another user', async function () {
- await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
+ await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should succeed with the correct params', async function () {
})
})
+ describe('When triggering full synchronization', function () {
+
+ it('Should fail when HTTP upload is disabled', async function () {
+ await server.config.disableImports()
+
+ await command.importVideos({
+ channelName: 'super_channel',
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.FORBIDDEN_403
+ })
+
+ await server.config.enableImports()
+ })
+
+ it('Should fail when externalChannelUrl is not provided', async function () {
+ await command.importVideos({
+ channelName: 'super_channel',
+ externalChannelUrl: null,
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
+ })
+
+ it('Should fail when externalChannelUrl is malformed', async function () {
+ await command.importVideos({
+ channelName: 'super_channel',
+ externalChannelUrl: 'not-a-url',
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
+ })
+
+ it('Should fail with no authentication', async function () {
+ await command.importVideos({
+ channelName: 'super_channel',
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ token: null,
+ expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+ })
+ })
+
+ it('Should fail when sync is not owned by the user', async function () {
+ await command.importVideos({
+ channelName: 'super_channel',
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ token: userInfo.accessToken,
+ expectedStatus: HttpStatusCode.FORBIDDEN_403
+ })
+ })
+
+ it('Should fail when the user has no quota', async function () {
+ await server.users.update({
+ userId: userInfo.id,
+ videoQuota: 0
+ })
+
+ await command.importVideos({
+ channelName: 'fake_channel',
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ token: userInfo.accessToken,
+ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
+ })
+
+ await server.users.update({
+ userId: userInfo.id,
+ videoQuota: userInfo.videoQuota
+ })
+ })
+
+ it('Should fail when the user has no daily quota', async function () {
+ await server.users.update({
+ userId: userInfo.id,
+ videoQuotaDaily: 0
+ })
+
+ await command.importVideos({
+ channelName: 'fake_channel',
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ token: userInfo.accessToken,
+ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
+ })
+
+ await server.users.update({
+ userId: userInfo.id,
+ videoQuotaDaily: userInfo.videoQuotaDaily
+ })
+ })
+
+ it('Should succeed when sync is run by its owner', async function () {
+ if (!areHttpImportTestsDisabled()) return
+
+ await command.importVideos({
+ channelName: 'fake_channel',
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ token: userInfo.accessToken
+ })
+ })
+
+ it('Should succeed when sync is run with root and for another user\'s channel', async function () {
+ if (!areHttpImportTestsDisabled()) return
+
+ await command.importVideos({
+ channelName: 'fake_channel',
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel
+ })
+ })
+ })
+
describe('When deleting a video channel', function () {
it('Should fail with a non authenticated user', async function () {
await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
})
it('Should fail with another authenticated user', async function () {
- await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
+ await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should fail with an unknown video channel id', async function () {
it('Should fail with nothing', async function () {
const fields = {}
- await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+ await makePostBodyRequest({
+ url: server.url,
+ path,
+ token: server.accessToken,
+ fields,
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
})
it('Should fail without a target url', async function () {
torrent: {
enabled: false
}
+ },
+ videoChannelSynchronization: {
+ enabled: false,
+ maxPerUser: 10
}
},
trending: {
--- /dev/null
+import { expect } from 'chai'
+import { FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled } from '@shared/core-utils'
+import {
+ createSingleServer,
+ getServerImportConfig,
+ PeerTubeServer,
+ setAccessTokensToServers,
+ setDefaultVideoChannel,
+ waitJobs
+} from '@shared/server-commands'
+
+describe('Test videos import in a channel', function () {
+ if (areHttpImportTestsDisabled()) return
+
+ function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
+
+ describe('Import using ' + mode, function () {
+ let server: PeerTubeServer
+
+ before(async function () {
+ this.timeout(120_000)
+
+ server = await createSingleServer(1, getServerImportConfig(mode))
+
+ await setAccessTokensToServers([ server ])
+ await setDefaultVideoChannel([ server ])
+
+ await server.config.enableChannelSync()
+ })
+
+ it('Should import a whole channel', async function () {
+ this.timeout(240_000)
+
+ await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel })
+ await waitJobs(server)
+
+ const videos = await server.videos.listByChannel({ handle: server.store.channel.name })
+ expect(videos.total).to.equal(2)
+ })
+
+ after(async function () {
+ await server?.kill()
+ })
+ })
+ }
+
+ runSuite('yt-dlp')
+ runSuite('youtube-dl')
+})
import './video-captions'
import './video-change-ownership'
import './video-channels'
+import './channel-import-videos'
+import './video-channel-syncs'
import './video-comments'
import './video-description'
import './video-files'
--- /dev/null
+import 'mocha'
+import { expect } from 'chai'
+import { FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled } from '@shared/core-utils'
+import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
+import {
+ ChannelSyncsCommand,
+ createSingleServer,
+ getServerImportConfig,
+ PeerTubeServer,
+ setAccessTokensToServers,
+ setDefaultAccountAvatar,
+ setDefaultChannelAvatar,
+ setDefaultVideoChannel,
+ waitJobs
+} from '@shared/server-commands'
+
+describe('Test channel synchronizations', function () {
+ if (areHttpImportTestsDisabled()) return
+
+ function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
+
+ describe('Sync using ' + mode, function () {
+ let server: PeerTubeServer
+ let command: ChannelSyncsCommand
+ let startTestDate: Date
+ const userInfo = {
+ accessToken: '',
+ username: 'user1',
+ channelName: 'user1_channel',
+ channelId: -1,
+ syncId: -1
+ }
+
+ async function changeDateForSync (channelSyncId: number, newDate: string) {
+ await server.sql.updateQuery(
+ `UPDATE "videoChannelSync" ` +
+ `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
+ `WHERE id=${channelSyncId}`
+ )
+ }
+
+ before(async function () {
+ this.timeout(120_000)
+
+ startTestDate = new Date()
+
+ server = await createSingleServer(1, getServerImportConfig(mode))
+
+ await setAccessTokensToServers([ server ])
+ await setDefaultVideoChannel([ server ])
+ await setDefaultChannelAvatar([ server ])
+ await setDefaultAccountAvatar([ server ])
+
+ await server.config.enableChannelSync()
+
+ command = server.channelSyncs
+
+ {
+ userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
+
+ const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
+ userInfo.channelId = videoChannels[0].id
+ }
+ })
+
+ it('Should fetch the latest channel videos of a remote channel', async function () {
+ this.timeout(120_000)
+
+ {
+ const { video } = await server.imports.importVideo({
+ attributes: {
+ channelId: server.store.channel.id,
+ privacy: VideoPrivacy.PUBLIC,
+ targetUrl: FIXTURE_URLS.youtube
+ }
+ })
+
+ expect(video.name).to.equal('small video - youtube')
+
+ const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+ expect(total).to.equal(1)
+ }
+
+ const { videoChannelSync } = await command.create({
+ attributes: {
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ videoChannelId: server.store.channel.id
+ },
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.OK_200
+ })
+
+ // Ensure any missing video not already fetched will be considered as new
+ await changeDateForSync(videoChannelSync.id, '1970-01-01')
+
+ await server.debug.sendCommand({
+ body: {
+ command: 'process-video-channel-sync-latest'
+ }
+ })
+
+ {
+ await waitJobs(server)
+
+ const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+ expect(total).to.equal(2)
+ expect(data[0].name).to.equal('test')
+ }
+ })
+
+ it('Should add another synchronization', async function () {
+ const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
+
+ const { videoChannelSync } = await command.create({
+ attributes: {
+ externalChannelUrl,
+ videoChannelId: server.store.channel.id
+ },
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.OK_200
+ })
+
+ expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
+ expect(videoChannelSync.channel).to.include({
+ id: server.store.channel.id,
+ name: 'root_channel'
+ })
+ expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
+ expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
+ })
+
+ it('Should add a synchronization for another user', async function () {
+ const { videoChannelSync } = await command.create({
+ attributes: {
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
+ videoChannelId: userInfo.channelId
+ },
+ token: userInfo.accessToken
+ })
+ userInfo.syncId = videoChannelSync.id
+ })
+
+ it('Should not import a channel if not asked', async function () {
+ await waitJobs(server)
+
+ const { data } = await command.listByAccount({ accountName: userInfo.username })
+
+ expect(data[0].state).to.contain({
+ id: VideoChannelSyncState.WAITING_FIRST_RUN,
+ label: 'Waiting first run'
+ })
+ })
+
+ it('Should only fetch the videos newer than the creation date', async function () {
+ this.timeout(120_000)
+
+ await changeDateForSync(userInfo.syncId, '2019-03-01')
+
+ await server.debug.sendCommand({
+ body: {
+ command: 'process-video-channel-sync-latest'
+ }
+ })
+
+ await waitJobs(server)
+
+ const { data, total } = await server.videos.listByChannel({
+ handle: userInfo.channelName,
+ include: VideoInclude.NOT_PUBLISHED_STATE
+ })
+
+ expect(total).to.equal(1)
+ expect(data[0].name).to.equal('test')
+ })
+
+ it('Should list channel synchronizations', async function () {
+ // Root
+ {
+ const { total, data } = await command.listByAccount({ accountName: 'root' })
+ expect(total).to.equal(2)
+
+ expect(data[0]).to.deep.contain({
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+ state: {
+ id: VideoChannelSyncState.SYNCED,
+ label: 'Synchronized'
+ }
+ })
+
+ expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
+
+ expect(data[0].channel).to.contain({ id: server.store.channel.id })
+ expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
+ }
+
+ // User
+ {
+ const { total, data } = await command.listByAccount({ accountName: userInfo.username })
+ expect(total).to.equal(1)
+ expect(data[0]).to.deep.contain({
+ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
+ state: {
+ id: VideoChannelSyncState.SYNCED,
+ label: 'Synchronized'
+ }
+ })
+ }
+ })
+
+ it('Should remove user\'s channel synchronizations', async function () {
+ await command.delete({ channelSyncId: userInfo.syncId })
+
+ const { total } = await command.listByAccount({ accountName: userInfo.username })
+ expect(total).to.equal(0)
+ })
+
+ after(async function () {
+ await server?.kill()
+ })
+ })
+ }
+
+ runSuite('youtube-dl')
+ runSuite('yt-dlp')
+})
createMultipleServers,
createSingleServer,
doubleFollow,
+ getServerImportConfig,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel,
let servers: PeerTubeServer[] = []
before(async function () {
- this.timeout(30_000)
-
- // Run servers
- servers = await createMultipleServers(2, {
- import: {
- videos: {
- http: {
- youtube_dl_release: {
- url: mode === 'youtube-dl'
- ? 'https://yt-dl.org/downloads/latest/youtube-dl'
- : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
-
- name: mode
- }
- }
- }
- }
- })
+ this.timeout(60_000)
+
+ servers = await createMultipleServers(2, getServerImportConfig(mode))
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
*/
youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4',
+ youtubeChannel: 'https://youtube.com/channel/UCtnlZdXv3-xQzxiqfn6cjIA',
+
// eslint-disable-next-line max-len
magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4',
MActorFollowActorsDefault,
MActorUrl,
MChannelBannerAccountDefault,
+ MChannelSyncChannel,
MStreamingPlaylist,
MVideoChangeOwnershipFull,
MVideoFile,
videoStreamingPlaylist?: MStreamingPlaylist
videoChannel?: MChannelBannerAccountDefault
+ videoChannelSync?: MChannelSyncChannel
videoPlaylistFull?: MVideoPlaylistFull
videoPlaylistSummary?: MVideoPlaylistFullSummary
plugin?: MPlugin
localViewerFull?: MLocalVideoViewerWithWatchSections
+
}
}
}
export * from './video-blacklist'
export * from './video-caption'
export * from './video-change-ownership'
+export * from './video-channel-sync'
export * from './video-channels'
export * from './video-comment'
export * from './video-file'
--- /dev/null
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { FunctionProperties, PickWith } from '@shared/typescript-utils'
+import { MChannelAccountDefault, MChannelFormattable } from './video-channels'
+
+type Use<K extends keyof VideoChannelSyncModel, M> = PickWith<VideoChannelSyncModel, K, M>
+
+export type MChannelSync = Omit<VideoChannelSyncModel, 'VideoChannel'>
+
+export type MChannelSyncChannel =
+ MChannelSync &
+ Use<'VideoChannel', MChannelAccountDefault> &
+ FunctionProperties<VideoChannelSyncModel>
+
+export type MChannelSyncFormattable =
+ FunctionProperties<MChannelSyncChannel> &
+ Use<'VideoChannel', MChannelFormattable> &
+ MChannelSync
enabled: boolean
}
}
+ videoChannelSynchronization: {
+ enabled: boolean
+ maxPerUser: number
+ }
}
trending: {
}
export interface SendDebugCommand {
- command: 'remove-dandling-resumable-uploads' | 'process-video-views-buffer' | 'process-video-viewers'
+ command: 'remove-dandling-resumable-uploads'
+ | 'process-video-views-buffer'
+ | 'process-video-viewers'
+ | 'process-video-channel-sync-latest'
}
| 'manage-video-torrent'
| 'move-to-object-storage'
| 'video-studio-edition'
+ | 'video-channel-import'
+ | 'after-video-channel-import'
| 'notify'
| 'federate-video'
filePath: string
}
+// ---------------------------------------------------------------------------
+
export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file'
export type VideoImportYoutubeDLPayloadType = 'youtube-dl'
-export type VideoImportYoutubeDLPayload = {
+export interface VideoImportYoutubeDLPayload {
type: VideoImportYoutubeDLPayloadType
videoImportId: number
fileExt?: string
}
-export type VideoImportTorrentPayload = {
+
+export interface VideoImportTorrentPayload {
type: VideoImportTorrentPayloadType
videoImportId: number
}
-export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
+
+export type VideoImportPayload = (VideoImportYoutubeDLPayload | VideoImportTorrentPayload) & {
+ preventException: boolean
+}
+
+export interface VideoImportPreventExceptionResult {
+ resultType: 'success' | 'error'
+}
+
+// ---------------------------------------------------------------------------
export type VideoRedundancyPayload = {
videoId: number
// ---------------------------------------------------------------------------
+export interface VideoChannelImportPayload {
+ externalChannelUrl: string
+ videoChannelId: number
+}
+
+export interface AfterVideoChannelImportPayload {
+ channelSyncId: number
+}
+
+// ---------------------------------------------------------------------------
+
export type NotifyPayload =
{
action: 'new-video'
enabled: boolean
}
}
+ videoChannelSynchronization: {
+ enabled: boolean
+ }
}
autoBlacklist: {
--- /dev/null
+export * from './video-channel-sync-state.enum'
+export * from './video-channel-sync.model'
+export * from './video-channel-sync-create.model'
--- /dev/null
+export interface VideoChannelSyncCreate {
+ externalChannelUrl: string
+ videoChannelId: number
+}
--- /dev/null
+export const enum VideoChannelSyncState {
+ WAITING_FIRST_RUN = 1,
+ PROCESSING = 2,
+ SYNCED = 3,
+ FAILED = 4
+}
--- /dev/null
+import { VideoChannelSummary } from '../channel/video-channel.model'
+import { VideoConstant } from '../video-constant.model'
+import { VideoChannelSyncState } from './video-channel-sync-state.enum'
+
+export interface VideoChannelSync {
+ id: number
+
+ externalChannelUrl: string
+
+ createdAt: string
+ channel: VideoChannelSummary
+ state: VideoConstant<VideoChannelSyncState>
+ lastSyncAt: string
+}
export * from './rate'
export * from './stats'
export * from './transcoding'
+export * from './channel-sync'
export * from './nsfw-policy.type'
}
}
+ disableImports () {
+ return this.setImportsEnabled(false)
+ }
+
enableImports () {
+ return this.setImportsEnabled(true)
+ }
+
+ private setImportsEnabled (enabled: boolean) {
return this.updateExistingSubConfig({
newConfig: {
import: {
videos: {
http: {
- enabled: true
+ enabled
},
torrent: {
- enabled: true
+ enabled
}
}
}
})
}
+ private setChannelSyncEnabled (enabled: boolean) {
+ return this.updateExistingSubConfig({
+ newConfig: {
+ import: {
+ videoChannelSynchronization: {
+ enabled
+ }
+ }
+ }
+ })
+ }
+
+ enableChannelSync () {
+ return this.setChannelSyncEnabled(true)
+ }
+
+ disableChannelSync () {
+ return this.setChannelSyncEnabled(false)
+ }
+
enableLive (options: {
allowReplay?: boolean
transcoding?: boolean
torrent: {
enabled: false
}
+ },
+ videoChannelSynchronization: {
+ enabled: false,
+ maxPerUser: 10
}
},
trending: {
CaptionsCommand,
ChangeOwnershipCommand,
ChannelsCommand,
+ ChannelSyncsCommand,
HistoryCommand,
ImportsCommand,
LiveCommand,
playlists?: PlaylistsCommand
history?: HistoryCommand
imports?: ImportsCommand
+ channelSyncs?: ChannelSyncsCommand
streamingPlaylists?: StreamingPlaylistsCommand
channels?: ChannelsCommand
comments?: CommentsCommand
this.playlists = new PlaylistsCommand(this)
this.history = new HistoryCommand(this)
this.imports = new ImportsCommand(this)
+ this.channelSyncs = new ChannelSyncsCommand(this)
this.streamingPlaylists = new StreamingPlaylistsCommand(this)
this.channels = new ChannelsCommand(this)
this.comments = new CommentsCommand(this)
return Promise.all(p)
}
+function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
+ return {
+ import: {
+ videos: {
+ http: {
+ youtube_dl_release: {
+ url: mode === 'youtube-dl'
+ ? 'https://yt-dl.org/downloads/latest/youtube-dl'
+ : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
+
+ name: mode
+ }
+ }
+ }
+ }
+ }
+}
+
// ---------------------------------------------------------------------------
export {
createSingleServer,
createMultipleServers,
cleanupTests,
- killallServers
+ killallServers,
+ getServerImportConfig
}
--- /dev/null
+import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@shared/models'
+import { pick } from '@shared/core-utils'
+import { unwrapBody } from '../requests'
+import { AbstractCommand, OverrideCommandOptions } from '../shared'
+
+export class ChannelSyncsCommand extends AbstractCommand {
+ private static readonly API_PATH = '/api/v1/video-channel-syncs'
+
+ listByAccount (options: OverrideCommandOptions & {
+ accountName: string
+ start?: number
+ count?: number
+ sort?: string
+ }) {
+ const { accountName, sort = 'createdAt' } = options
+
+ const path = `/api/v1/accounts/${accountName}/video-channel-syncs`
+
+ return this.getRequestBody<ResultList<VideoChannelSync>>({
+ ...options,
+
+ path,
+ query: { sort, ...pick(options, [ 'start', 'count' ]) },
+ implicitToken: true,
+ defaultExpectedStatus: HttpStatusCode.OK_200
+ })
+ }
+
+ async create (options: OverrideCommandOptions & {
+ attributes: VideoChannelSyncCreate
+ }) {
+ return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({
+ ...options,
+
+ path: ChannelSyncsCommand.API_PATH,
+ fields: options.attributes,
+ implicitToken: true,
+ defaultExpectedStatus: HttpStatusCode.OK_200
+ }))
+ }
+
+ delete (options: OverrideCommandOptions & {
+ channelSyncId: number
+ }) {
+ const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}`
+
+ return this.deleteRequest({
+ ...options,
+
+ path,
+ implicitToken: true,
+ defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+ })
+ }
+}
defaultExpectedStatus: HttpStatusCode.OK_200
})
}
+
+ importVideos (options: OverrideCommandOptions & {
+ channelName: string
+ externalChannelUrl: string
+ }) {
+ const { channelName, externalChannelUrl } = options
+
+ const path = `/api/v1/video-channels/${channelName}/import-videos`
+
+ return this.postBodyRequest({
+ ...options,
+
+ path,
+ fields: { externalChannelUrl },
+ implicitToken: true,
+ defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+ })
+ }
}
export * from './change-ownership-command'
export * from './channels'
export * from './channels-command'
+export * from './channel-syncs-command'
export * from './comments-command'
export * from './history-command'
export * from './imports-command'
download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have.
- name: Video Imports
description: Operations dealing with listing, adding and removing video imports.
+ - name: Channels Sync
+ description: Operations dealing with synchronizing PeerTube user's channel with channels of other platforms
- name: Video Captions
description: Operations dealing with listing, adding and removing closed captions of a video.
- name: Video Channels
- Video Transcoding
- Live Videos
- Feeds
+ - Channels Sync
- name: Search
tags:
- Search
tags:
- Video Channels
responses:
- '204':
+ '200':
description: successful operation
content:
application/json:
'204':
description: successful operation
+ '/video-channel-syncs':
+ post:
+ summary: Create a synchronization for a video channel
+ operationId: addVideoChannelSync
+ security:
+ - OAuth2: []
+ tags:
+ - Channels Sync
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/VideoChannelSyncCreate'
+ responses:
+ '200':
+ description: successful operation
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ videoChannelSync:
+ $ref: "#/components/schemas/VideoChannelSync"
+
+ '/video-channel-syncs/{channelSyncId}':
+ delete:
+ summary: Delete a video channel synchronization
+ operationId: delVideoChannelSync
+ security:
+ - OAuth2: []
+ tags:
+ - Channels Sync
+ parameters:
+ - $ref: '#/components/parameters/channelSyncId'
+ responses:
+ '204':
+ description: successful operation
+
+ '/video-channel-syncs/{channelSyncId}/sync':
+ post:
+ summary: Triggers the channel synchronization job, fetching all the videos from the remote channel
+ operationId: triggerVideoChannelSync
+ security:
+ - OAuth2: []
+ tags:
+ - Channels Sync
+ parameters:
+ - $ref: '#/components/parameters/channelSyncId'
+ responses:
+ '204':
+ description: successful operation
+
+
/video-playlists/privacies:
get:
summary: List available playlist privacy policies
schema:
$ref: '#/components/schemas/VideoChannelList'
+ '/accounts/{name}/video-channel-syncs':
+ get:
+ summary: List the synchronizations of video channels of an account
+ tags:
+ - Video Channels
+ - Channels Sync
+ - Accounts
+ parameters:
+ - $ref: '#/components/parameters/name'
+ - $ref: '#/components/parameters/start'
+ - $ref: '#/components/parameters/count'
+ - $ref: '#/components/parameters/sort'
+ responses:
+ '200':
+ description: successful operation
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/VideoChannelSyncList'
+
'/accounts/{name}/ratings':
get:
summary: List ratings of an account
schema:
type: string
example: my_username | my_username@example.com
+ channelSyncId:
+ name: channelSyncId
+ in: path
+ required: true
+ description: Channel Sync id
+ schema:
+ $ref: '#/components/schemas/Abuse/properties/id'
subscriptionHandle:
name: subscriptionHandle
in: path
- activitypub-refresher
- video-redundancy
- video-live-ending
+ - video-channel-import
followState:
name: state
in: query
properties:
enabled:
type: boolean
+ videoChannelSynchronization:
+ type: object
+ properties:
+ enabled:
+ type: boolean
autoBlacklist:
type: object
properties:
properties:
enabled:
type: boolean
+ video_channel_synchronization:
+ type: object
+ properties:
+ enabled:
+ type: boolean
autoBlacklist:
type: object
properties:
- videos-views-stats
- activitypub-refresher
- video-redundancy
+ - video-channel-import
data:
type: object
additionalProperties: true
type: integer
uuid:
$ref: '#/components/schemas/UUIDv4'
+
VideoChannelCreate:
allOf:
- $ref: '#/components/schemas/VideoChannel'
- $ref: '#/components/schemas/VideoChannel'
- $ref: '#/components/schemas/Actor'
+ VideoChannelSync:
+ type: object
+ properties:
+ id:
+ $ref: '#/components/schemas/id'
+ state:
+ type: object
+ properties:
+ id:
+ type: integer
+ example: 2
+ label:
+ type: string
+ example: PROCESSING
+ externalChannelUrl:
+ type: string
+ example: 'https://youtube.com/c/UC_myfancychannel'
+ createdAt:
+ type: string
+ format: date-time
+ lastSyncAt:
+ type: string
+ format: date-time
+ nullable: true
+ channel:
+ $ref: '#/components/schemas/VideoChannel'
+ VideoChannelSyncList:
+ type: object
+ properties:
+ total:
+ type: integer
+ example: 1
+ data:
+ type: array
+ items:
+ allOf:
+ - $ref: '#/components/schemas/VideoChannelSync'
+ VideoChannelSyncCreate:
+ type: object
+ properties:
+ externalChannelUrl:
+ type: string
+ example: https://youtube.com/c/UC_myfancychannel
+ videoChannelId:
+ $ref: '#/components/schemas/id'
MRSSPeerLink:
type: object
xml:
projects / github / Chocobozzz / PeerTube.git / commitdiff
