From: Chocobozzz Date: Thu, 2 Aug 2018 13:34:09 +0000 (+0200) Subject: Add ability to import video with youtube-dl X-Git-Tag: delete~64 X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=fbad87b0472f574409f7aa3ae7f8b54927d0cdd6;p=github%2FChocobozzz%2FPeerTube.git Add ability to import video with youtube-dl --- diff --git a/client/package.json b/client/package.json index aae7643c7..34305ee44 100644 --- a/client/package.json +++ b/client/package.json @@ -23,6 +23,7 @@ "ngx-extractor": "ngx-extractor" }, "license": "GPLv3", + "typings": "*.d.ts", "resolutions": { "video.js": "^7", "webtorrent/create-torrent/junk": "^1", diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts index 99df61cdb..62ce97102 100644 --- a/client/src/app/shared/shared.module.ts +++ b/client/src/app/shared/shared.module.ts @@ -51,6 +51,7 @@ import { ScreenService } from '@app/shared/misc/screen.service' import { VideoCaptionsValidatorsService } from '@app/shared/forms/form-validators/video-captions-validators.service' import { VideoCaptionService } from '@app/shared/video-caption' import { PeertubeCheckboxComponent } from '@app/shared/forms/peertube-checkbox.component' +import { VideoImportService } from '@app/shared/video-import/video-import.service' @NgModule({ imports: [ @@ -143,6 +144,7 @@ import { PeertubeCheckboxComponent } from '@app/shared/forms/peertube-checkbox.c VideoCommentValidatorsService, VideoValidatorsService, VideoCaptionsValidatorsService, + VideoImportService, I18nPrimengCalendarService, ScreenService, diff --git a/client/src/app/shared/video-import/index.ts b/client/src/app/shared/video-import/index.ts new file mode 100644 index 000000000..9bb73ec2c --- /dev/null +++ b/client/src/app/shared/video-import/index.ts @@ -0,0 +1 @@ +export * from './video-import.service' diff --git a/client/src/app/shared/video-import/video-import.service.ts b/client/src/app/shared/video-import/video-import.service.ts new file mode 100644 index 000000000..b4709866a --- /dev/null +++ b/client/src/app/shared/video-import/video-import.service.ts @@ -0,0 +1,56 @@ +import { catchError } from 'rxjs/operators' +import { HttpClient } from '@angular/common/http' +import { Injectable } from '@angular/core' +import { Observable } from 'rxjs' +import { VideoImport } from '../../../../../shared' +import { environment } from '../../../environments/environment' +import { RestExtractor, RestService } from '../rest' +import { VideoImportCreate } from '../../../../../shared/models/videos/video-import-create.model' +import { objectToFormData } from '@app/shared/misc/utils' +import { VideoUpdate } from '../../../../../shared/models/videos' + +@Injectable() +export class VideoImportService { + private static BASE_VIDEO_IMPORT_URL = environment.apiUrl + '/api/v1/videos/imports/' + + constructor ( + private authHttp: HttpClient, + private restService: RestService, + private restExtractor: RestExtractor + ) {} + + importVideo (targetUrl: string, video: VideoUpdate): Observable { + const url = VideoImportService.BASE_VIDEO_IMPORT_URL + const language = video.language || null + const licence = video.licence || null + const category = video.category || null + const description = video.description || null + const support = video.support || null + const scheduleUpdate = video.scheduleUpdate || null + + const body: VideoImportCreate = { + targetUrl, + + name: video.name, + category, + licence, + language, + support, + description, + channelId: video.channelId, + privacy: video.privacy, + tags: video.tags, + nsfw: video.nsfw, + waitTranscoding: video.waitTranscoding, + commentsEnabled: video.commentsEnabled, + thumbnailfile: video.thumbnailfile, + previewfile: video.previewfile, + scheduleUpdate + } + + const data = objectToFormData(body) + return this.authHttp.post(url, data) + .pipe(catchError(res => this.restExtractor.handleError(res))) + } + +} diff --git a/client/src/app/shared/video/video-edit.model.ts b/client/src/app/shared/video/video-edit.model.ts index 8562f8d25..0046be964 100644 --- a/client/src/app/shared/video/video-edit.model.ts +++ b/client/src/app/shared/video/video-edit.model.ts @@ -1,7 +1,7 @@ -import { VideoDetails } from './video-details.model' import { VideoPrivacy } from '../../../../../shared/models/videos/video-privacy.enum' import { VideoUpdate } from '../../../../../shared/models/videos' import { VideoScheduleUpdate } from '../../../../../shared/models/videos/video-schedule-update.model' +import { Video } from '../../../../../shared/models/videos/video.model' export class VideoEdit implements VideoUpdate { static readonly SPECIAL_SCHEDULED_PRIVACY = -1 @@ -26,26 +26,26 @@ export class VideoEdit implements VideoUpdate { id?: number scheduleUpdate?: VideoScheduleUpdate - constructor (videoDetails?: VideoDetails) { - if (videoDetails) { - this.id = videoDetails.id - this.uuid = videoDetails.uuid - this.category = videoDetails.category.id - this.licence = videoDetails.licence.id - this.language = videoDetails.language.id - this.description = videoDetails.description - this.name = videoDetails.name - this.tags = videoDetails.tags - this.nsfw = videoDetails.nsfw - this.commentsEnabled = videoDetails.commentsEnabled - this.waitTranscoding = videoDetails.waitTranscoding - this.channelId = videoDetails.channel.id - this.privacy = videoDetails.privacy.id - this.support = videoDetails.support - this.thumbnailUrl = videoDetails.thumbnailUrl - this.previewUrl = videoDetails.previewUrl + constructor (video?: Video & { tags: string[], commentsEnabled: boolean, support: string, thumbnailUrl: string, previewUrl: string }) { + if (video) { + this.id = video.id + this.uuid = video.uuid + this.category = video.category.id + this.licence = video.licence.id + this.language = video.language.id + this.description = video.description + this.name = video.name + this.tags = video.tags + this.nsfw = video.nsfw + this.commentsEnabled = video.commentsEnabled + this.waitTranscoding = video.waitTranscoding + this.channelId = video.channel.id + this.privacy = video.privacy.id + this.support = video.support + this.thumbnailUrl = video.thumbnailUrl + this.previewUrl = video.previewUrl - this.scheduleUpdate = videoDetails.scheduledUpdate + this.scheduleUpdate = video.scheduledUpdate } } diff --git a/client/src/app/videos/+video-edit/video-add.component.html b/client/src/app/videos/+video-edit/video-add.component.html index 9c2c01c65..ed8d91c11 100644 --- a/client/src/app/videos/+video-edit/video-add.component.html +++ b/client/src/app/videos/+video-edit/video-add.component.html @@ -1,65 +1,17 @@
- Upload your video - Upload {{ videoFileName }} + Import {{ videoName }} + Upload {{ videoName }}
-
-
-
+ -
- Select the file to upload - -
- (.mp4, .webm, .ogv) + + + -
- -
- -
-
- -
- -
- -
-
-
-
- -
- - -
- - -
- - -
-
Publish will be available when upload is finished
- -
- - -
-
-
+ + + +
diff --git a/client/src/app/videos/+video-edit/video-add.component.scss b/client/src/app/videos/+video-edit/video-add.component.scss index c0b5f3d07..a811b9cf0 100644 --- a/client/src/app/videos/+video-edit/video-add.component.scss +++ b/client/src/app/videos/+video-edit/video-add.component.scss @@ -1,101 +1,54 @@ @import '_variables'; @import '_mixins'; -.upload-video-container { - border-radius: 3px; - background-color: #F7F7F7; - border: 3px solid #EAEAEA; - width: 100%; - height: 440px; - margin-top: 40px; - display: flex; - justify-content: center; - align-items: center; +$border-width: 3px; +$border-type: solid; +$border-color: #EAEAEA; - .peertube-select-container { - @include peertube-select-container(190px); - } - - .upload-video { - display: flex; - flex-direction: column; - align-items: center; - - .form-group-channel { - margin-bottom: 20px; - } - - .icon.icon-upload { - @include icon(90px); - margin-bottom: 25px; - cursor: default; - - background-image: url('../../../assets/images/video/upload.svg'); - } - - .button-file { - @include peertube-button-file(auto); - - min-width: 190px; - } +$background-color: #F7F7F7; - .button-file-extension { - display: block; - font-size: 12px; - margin-top: 5px; - } - } - - .form-group-channel { - margin-top: 35px; +/deep/ tabset.root-tabset.video-add-tabset { + &.hide-nav .nav { + display: none !important; } -} -.upload-progress-cancel { - display: flex; - margin-top: 25px; - margin-bottom: 40px; + & > .nav { - p-progressBar { - flex-grow: 1; - - /deep/ .ui-progressbar { - font-size: 15px !important; - color: #fff !important; - height: 30px !important; - line-height: 30px !important; - border-radius: 3px !important; - background-color: rgba(11, 204, 41, 0.16) !important; - - .ui-progressbar-value { - background-color: #0BCC29 !important; - } + border-bottom: $border-width $border-type $border-color; + margin: 0 !important; - .ui-progressbar-label { - text-align: left; - padding-left: 18px; - margin-top: 0 !important; - } + & > li { + margin-bottom: -$border-width; } - &.processing { - /deep/ .ui-progressbar-label { - // Same color as background to hide "100%" - color: rgba(11, 204, 41, 0.16) !important; + .nav-link { + height: 40px !important; + padding: 0 30px !important; + font-size: 15px; + + &.active { + border: $border-width $border-type $border-color; + border-bottom: none; + background-color: $background-color !important; - &::before { - content: 'Processing...'; - color: #fff; + span { + border-bottom: 2px solid #F1680D; + font-weight: $font-bold; } } } } - input { - @include peertube-button; - @include grey-button; + .upload-video-container { + border: $border-width $border-type $border-color; + border-top: none; - margin-left: 10px; + background-color: $background-color; + border-radius: 3px; + width: 100%; + height: 440px; + display: flex; + justify-content: center; + align-items: center; } -} - +} \ No newline at end of file diff --git a/client/src/app/videos/+video-edit/video-add.component.ts b/client/src/app/videos/+video-edit/video-add.component.ts index 651ee8dd2..64071b40c 100644 --- a/client/src/app/videos/+video-edit/video-add.component.ts +++ b/client/src/app/videos/+video-edit/video-add.component.ts @@ -1,251 +1,29 @@ -import { HttpEventType, HttpResponse } from '@angular/common/http' -import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core' -import { Router } from '@angular/router' -import { UserService } from '@app/shared' +import { Component, ViewChild } from '@angular/core' import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service' -import { LoadingBarService } from '@ngx-loading-bar/core' -import { NotificationsService } from 'angular2-notifications' -import { BytesPipe } from 'ngx-pipes' -import { Subscription } from 'rxjs' -import { VideoConstant, VideoPrivacy } from '../../../../../shared/models/videos' -import { AuthService, ServerService } from '../../core' -import { FormReactive } from '../../shared' -import { populateAsyncUserVideoChannels } from '../../shared/misc/utils' -import { VideoEdit } from '../../shared/video/video-edit.model' -import { VideoService } from '../../shared/video/video.service' -import { I18n } from '@ngx-translate/i18n-polyfill' -import { FormValidatorService } from '@app/shared/forms/form-validators/form-validator.service' -import { switchMap } from 'rxjs/operators' -import { VideoCaptionService } from '@app/shared/video-caption' -import { VideoCaptionEdit } from '@app/shared/video-caption/video-caption-edit.model' +import { VideoImportComponent } from '@app/videos/+video-edit/video-import.component' +import { VideoUploadComponent } from '@app/videos/+video-edit/video-upload.component' @Component({ selector: 'my-videos-add', templateUrl: './video-add.component.html', - styleUrls: [ - './shared/video-edit.component.scss', - './video-add.component.scss' - ] + styleUrls: [ './video-add.component.scss' ] }) -export class VideoAddComponent extends FormReactive implements OnInit, OnDestroy, CanComponentDeactivate { - @ViewChild('videofileInput') videofileInput +export class VideoAddComponent implements CanComponentDeactivate { + @ViewChild('videoUpload') videoUpload: VideoUploadComponent + @ViewChild('videoImport') videoImport: VideoImportComponent - // So that it can be accessed in the template - readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY + secondStepType: 'upload' | 'import' + videoName: string - isUploadingVideo = false - isUpdatingVideo = false - videoUploaded = false - videoUploadObservable: Subscription = null - videoUploadPercents = 0 - videoUploadedIds = { - id: 0, - uuid: '' - } - videoFileName: string - - userVideoChannels: { id: number, label: string, support: string }[] = [] - userVideoQuotaUsed = 0 - videoPrivacies: VideoConstant[] = [] - firstStepPrivacyId = 0 - firstStepChannelId = 0 - videoCaptions: VideoCaptionEdit[] = [] - - constructor ( - protected formValidatorService: FormValidatorService, - private router: Router, - private notificationsService: NotificationsService, - private authService: AuthService, - private userService: UserService, - private serverService: ServerService, - private videoService: VideoService, - private loadingBar: LoadingBarService, - private i18n: I18n, - private videoCaptionService: VideoCaptionService - ) { - super() - } - - get videoExtensions () { - return this.serverService.getConfig().video.file.extensions.join(',') - } - - ngOnInit () { - this.buildForm({}) - - populateAsyncUserVideoChannels(this.authService, this.userVideoChannels) - .then(() => this.firstStepChannelId = this.userVideoChannels[0].id) - - this.userService.getMyVideoQuotaUsed() - .subscribe(data => this.userVideoQuotaUsed = data.videoQuotaUsed) - - this.serverService.videoPrivaciesLoaded - .subscribe( - () => { - this.videoPrivacies = this.serverService.getVideoPrivacies() - - // Public by default - this.firstStepPrivacyId = VideoPrivacy.PUBLIC - }) - } - - ngOnDestroy () { - if (this.videoUploadObservable) { - this.videoUploadObservable.unsubscribe() - } + onFirstStepDone (type: 'upload' | 'import', videoName: string) { + this.secondStepType = type + this.videoName = videoName } canDeactivate () { - let text = '' - - if (this.videoUploaded === true) { - // FIXME: cannot concatenate strings inside i18n service :/ - text = this.i18n('Your video was uploaded in your account and is private.') + - this.i18n('But associated data (tags, description...) will be lost, are you sure you want to leave this page?') - } else { - text = this.i18n('Your video is not uploaded yet, are you sure you want to leave this page?') - } - - return { - canDeactivate: !this.isUploadingVideo, - text - } - } - - fileChange () { - this.uploadFirstStep() - } - - checkForm () { - this.forceCheck() - - return this.form.valid - } - - cancelUpload () { - if (this.videoUploadObservable !== null) { - this.videoUploadObservable.unsubscribe() - this.isUploadingVideo = false - this.videoUploadPercents = 0 - this.videoUploadObservable = null - this.notificationsService.info(this.i18n('Info'), this.i18n('Upload cancelled')) - } - } - - uploadFirstStep () { - const videofile = this.videofileInput.nativeElement.files[0] as File - if (!videofile) return - - // Cannot upload videos > 8GB for now - if (videofile.size > 8 * 1024 * 1024 * 1024) { - this.notificationsService.error(this.i18n('Error'), this.i18n('We are sorry but PeerTube cannot handle videos > 8GB')) - return - } - - const videoQuota = this.authService.getUser().videoQuota - if (videoQuota !== -1 && (this.userVideoQuotaUsed + videofile.size) > videoQuota) { - const bytePipes = new BytesPipe() - - const msg = this.i18n( - 'Your video quota is exceeded with this video (video size: {{ videoSize }}, used: {{ videoQuotaUsed }}, quota: {{ videoQuota }})', - { - videoSize: bytePipes.transform(videofile.size, 0), - videoQuotaUsed: bytePipes.transform(this.userVideoQuotaUsed, 0), - videoQuota: bytePipes.transform(videoQuota, 0) - } - ) - this.notificationsService.error(this.i18n('Error'), msg) - return - } - - this.videoFileName = videofile.name - - const nameWithoutExtension = videofile.name.replace(/\.[^/.]+$/, '') - let name: string - - // If the name of the file is very small, keep the extension - if (nameWithoutExtension.length < 3) name = videofile.name - else name = nameWithoutExtension - - const privacy = this.firstStepPrivacyId.toString() - const nsfw = false - const waitTranscoding = true - const commentsEnabled = true - const channelId = this.firstStepChannelId.toString() - - const formData = new FormData() - formData.append('name', name) - // Put the video "private" -> we are waiting the user validation of the second step - formData.append('privacy', VideoPrivacy.PRIVATE.toString()) - formData.append('nsfw', '' + nsfw) - formData.append('commentsEnabled', '' + commentsEnabled) - formData.append('waitTranscoding', '' + waitTranscoding) - formData.append('channelId', '' + channelId) - formData.append('videofile', videofile) - - this.isUploadingVideo = true - this.form.patchValue({ - name, - privacy, - nsfw, - channelId - }) - - this.videoUploadObservable = this.videoService.uploadVideo(formData).subscribe( - event => { - if (event.type === HttpEventType.UploadProgress) { - this.videoUploadPercents = Math.round(100 * event.loaded / event.total) - } else if (event instanceof HttpResponse) { - this.videoUploaded = true - - this.videoUploadedIds = event.body.video - - this.videoUploadObservable = null - } - }, - - err => { - // Reset progress - this.isUploadingVideo = false - this.videoUploadPercents = 0 - this.videoUploadObservable = null - this.notificationsService.error(this.i18n('Error'), err.message) - } - ) - } - - updateSecondStep () { - if (this.checkForm() === false) { - return - } - - const video = new VideoEdit() - video.patch(this.form.value) - video.id = this.videoUploadedIds.id - video.uuid = this.videoUploadedIds.uuid - - this.isUpdatingVideo = true - this.loadingBar.start() - this.videoService.updateVideo(video) - .pipe( - // Then update captions - switchMap(() => this.videoCaptionService.updateCaptions(video.id, this.videoCaptions)) - ) - .subscribe( - () => { - this.isUpdatingVideo = false - this.isUploadingVideo = false - this.loadingBar.complete() - - this.notificationsService.success(this.i18n('Success'), this.i18n('Video published.')) - this.router.navigate([ '/videos/watch', video.uuid ]) - }, + if (this.secondStepType === 'upload') return this.videoUpload.canDeactivate() + if (this.secondStepType === 'import') return this.videoImport.canDeactivate() - err => { - this.isUpdatingVideo = false - this.notificationsService.error(this.i18n('Error'), err.message) - console.error(err) - } - ) + return { canDeactivate: true } } } diff --git a/client/src/app/videos/+video-edit/video-add.module.ts b/client/src/app/videos/+video-edit/video-add.module.ts index 1bfedf251..91f544971 100644 --- a/client/src/app/videos/+video-edit/video-add.module.ts +++ b/client/src/app/videos/+video-edit/video-add.module.ts @@ -5,6 +5,8 @@ import { VideoEditModule } from './shared/video-edit.module' import { VideoAddRoutingModule } from './video-add-routing.module' import { VideoAddComponent } from './video-add.component' import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.service' +import { VideoUploadComponent } from '@app/videos/+video-edit/video-upload.component' +import { VideoImportComponent } from '@app/videos/+video-edit/video-import.component' @NgModule({ imports: [ @@ -14,7 +16,9 @@ import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.ser ProgressBarModule ], declarations: [ - VideoAddComponent + VideoAddComponent, + VideoUploadComponent, + VideoImportComponent ], exports: [ VideoAddComponent diff --git a/client/src/app/videos/+video-edit/video-import.component.html b/client/src/app/videos/+video-edit/video-import.component.html new file mode 100644 index 000000000..9d71a0717 --- /dev/null +++ b/client/src/app/videos/+video-edit/video-import.component.html @@ -0,0 +1,55 @@ +
+
+
+ +
+ + +
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+
+ +
+ Congratulations, the video behind {{ targetUrl }} will be imported! You can already add information about this video. +
+ + +
+ + +
+
+ + +
+
+
diff --git a/client/src/app/videos/+video-edit/video-import.component.scss b/client/src/app/videos/+video-edit/video-import.component.scss new file mode 100644 index 000000000..9ada9db19 --- /dev/null +++ b/client/src/app/videos/+video-edit/video-import.component.scss @@ -0,0 +1,37 @@ +@import '_variables'; +@import '_mixins'; + +$width-size: 190px; + +.peertube-select-container { + @include peertube-select-container($width-size); +} + +.import-video { + display: flex; + flex-direction: column; + align-items: center; + + .icon.icon-upload { + @include icon(90px); + margin-bottom: 25px; + cursor: default; + + background-image: url('../../../assets/images/video/upload.svg'); + } + + input[type=text] { + @include peertube-input-text($width-size); + display: block; + } + + input[type=button] { + @include peertube-button; + @include orange-button; + + width: $width-size; + margin-top: 30px; + } +} + + diff --git a/client/src/app/videos/+video-edit/video-import.component.ts b/client/src/app/videos/+video-edit/video-import.component.ts new file mode 100644 index 000000000..bd4482e17 --- /dev/null +++ b/client/src/app/videos/+video-edit/video-import.component.ts @@ -0,0 +1,161 @@ +import { Component, EventEmitter, OnInit, Output } from '@angular/core' +import { Router } from '@angular/router' +import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service' +import { NotificationsService } from 'angular2-notifications' +import { VideoConstant, VideoPrivacy, VideoUpdate } from '../../../../../shared/models/videos' +import { AuthService, ServerService } from '../../core' +import { FormReactive } from '../../shared' +import { populateAsyncUserVideoChannels } from '../../shared/misc/utils' +import { VideoService } from '../../shared/video/video.service' +import { I18n } from '@ngx-translate/i18n-polyfill' +import { FormValidatorService } from '@app/shared/forms/form-validators/form-validator.service' +import { VideoCaptionEdit } from '@app/shared/video-caption/video-caption-edit.model' +import { VideoImportService } from '@app/shared/video-import' +import { VideoEdit } from '@app/shared/video/video-edit.model' +import { switchMap } from 'rxjs/operators' +import { LoadingBarService } from '@ngx-loading-bar/core' +import { VideoCaptionService } from '@app/shared/video-caption' + +@Component({ + selector: 'my-video-import', + templateUrl: './video-import.component.html', + styleUrls: [ + './shared/video-edit.component.scss', + './video-import.component.scss' + ] +}) +export class VideoImportComponent extends FormReactive implements OnInit, CanComponentDeactivate { + @Output() firstStepDone = new EventEmitter() + + targetUrl = '' + videoFileName: string + + isImportingVideo = false + hasImportedVideo = false + isUpdatingVideo = false + + userVideoChannels: { id: number, label: string, support: string }[] = [] + videoPrivacies: VideoConstant[] = [] + videoCaptions: VideoCaptionEdit[] = [] + + firstStepPrivacyId = 0 + firstStepChannelId = 0 + video: VideoEdit + + constructor ( + protected formValidatorService: FormValidatorService, + private router: Router, + private loadingBar: LoadingBarService, + private notificationsService: NotificationsService, + private authService: AuthService, + private serverService: ServerService, + private videoService: VideoService, + private videoImportService: VideoImportService, + private videoCaptionService: VideoCaptionService, + private i18n: I18n + ) { + super() + } + + ngOnInit () { + this.buildForm({}) + + populateAsyncUserVideoChannels(this.authService, this.userVideoChannels) + .then(() => this.firstStepChannelId = this.userVideoChannels[ 0 ].id) + + this.serverService.videoPrivaciesLoaded + .subscribe( + () => { + this.videoPrivacies = this.serverService.getVideoPrivacies() + + // Private by default + this.firstStepPrivacyId = VideoPrivacy.PRIVATE + }) + } + + canDeactivate () { + return { canDeactivate: true } + } + + checkForm () { + this.forceCheck() + + return this.form.valid + } + + isTargetUrlValid () { + return this.targetUrl && this.targetUrl.match(/https?:\/\//) + } + + importVideo () { + this.isImportingVideo = true + + const videoUpdate: VideoUpdate = { + privacy: this.firstStepPrivacyId, + waitTranscoding: false, + commentsEnabled: true, + channelId: this.firstStepChannelId + } + + this.videoImportService.importVideo(this.targetUrl, videoUpdate).subscribe( + res => { + this.firstStepDone.emit(res.video.name) + this.isImportingVideo = false + this.hasImportedVideo = true + + this.video = new VideoEdit(Object.assign(res.video, { + commentsEnabled: videoUpdate.commentsEnabled, + support: null, + thumbnailUrl: null, + previewUrl: null + })) + this.hydrateFormFromVideo() + }, + + err => { + this.isImportingVideo = false + this.notificationsService.error(this.i18n('Error'), err.message) + } + ) + } + + updateSecondStep () { + if (this.checkForm() === false) { + return + } + + this.video.patch(this.form.value) + + this.loadingBar.start() + this.isUpdatingVideo = true + + // Update the video + this.videoService.updateVideo(this.video) + .pipe( + // Then update captions + switchMap(() => this.videoCaptionService.updateCaptions(this.video.id, this.videoCaptions)) + ) + .subscribe( + () => { + this.isUpdatingVideo = false + this.loadingBar.complete() + this.notificationsService.success(this.i18n('Success'), this.i18n('Video to import updated.')) + + // TODO: route to imports list + // this.router.navigate([ '/videos/watch', this.video.uuid ]) + }, + + err => { + this.loadingBar.complete() + this.isUpdatingVideo = false + this.notificationsService.error(this.i18n('Error'), err.message) + console.error(err) + } + ) + + } + + private hydrateFormFromVideo () { + this.form.patchValue(this.video.toFormPatch()) + } +} diff --git a/client/src/app/videos/+video-edit/video-update.component.ts b/client/src/app/videos/+video-edit/video-update.component.ts index 798c48f3c..0c60e3439 100644 --- a/client/src/app/videos/+video-edit/video-update.component.ts +++ b/client/src/app/videos/+video-edit/video-update.component.ts @@ -126,7 +126,6 @@ export class VideoUpdateComponent extends FormReactive implements OnInit { console.error(err) } ) - } private hydrateFormFromVideo () { diff --git a/client/src/app/videos/+video-edit/video-upload.component.html b/client/src/app/videos/+video-edit/video-upload.component.html new file mode 100644 index 000000000..8c0723155 --- /dev/null +++ b/client/src/app/videos/+video-edit/video-upload.component.html @@ -0,0 +1,58 @@ +
+
+
+ +
+ Select the file to upload + +
+ (.mp4, .webm, .ogv) + +
+ +
+ +
+
+ +
+ +
+ +
+
+
+
+ +
+ + +
+ + +
+ + +
+
Publish will be available when upload is finished
+ +
+ + +
+
+
\ No newline at end of file diff --git a/client/src/app/videos/+video-edit/video-upload.component.scss b/client/src/app/videos/+video-edit/video-upload.component.scss new file mode 100644 index 000000000..015835672 --- /dev/null +++ b/client/src/app/videos/+video-edit/video-upload.component.scss @@ -0,0 +1,85 @@ +@import '_variables'; +@import '_mixins'; + +.peertube-select-container { + @include peertube-select-container(190px); +} + +.upload-video { + display: flex; + flex-direction: column; + align-items: center; + + .form-group-channel { + margin-bottom: 20px; + margin-top: 35px; + } + + .icon.icon-upload { + @include icon(90px); + margin-bottom: 25px; + cursor: default; + + background-image: url('../../../assets/images/video/upload.svg'); + } + + .button-file { + @include peertube-button-file(auto); + + min-width: 190px; + } + + .button-file-extension { + display: block; + font-size: 12px; + margin-top: 5px; + } +} + +.upload-progress-cancel { + display: flex; + margin-top: 25px; + margin-bottom: 40px; + + p-progressBar { + flex-grow: 1; + + /deep/ .ui-progressbar { + font-size: 15px !important; + color: #fff !important; + height: 30px !important; + line-height: 30px !important; + border-radius: 3px !important; + background-color: rgba(11, 204, 41, 0.16) !important; + + .ui-progressbar-value { + background-color: #0BCC29 !important; + } + + .ui-progressbar-label { + text-align: left; + padding-left: 18px; + margin-top: 0 !important; + } + } + + &.processing { + /deep/ .ui-progressbar-label { + // Same color as background to hide "100%" + color: rgba(11, 204, 41, 0.16) !important; + + &::before { + content: 'Processing...'; + color: #fff; + } + } + } + } + + input { + @include peertube-button; + @include grey-button; + + margin-left: 10px; + } +} \ No newline at end of file diff --git a/client/src/app/videos/+video-edit/video-upload.component.ts b/client/src/app/videos/+video-edit/video-upload.component.ts new file mode 100644 index 000000000..e6c391d2f --- /dev/null +++ b/client/src/app/videos/+video-edit/video-upload.component.ts @@ -0,0 +1,251 @@ +import { HttpEventType, HttpResponse } from '@angular/common/http' +import { Component, EventEmitter, OnDestroy, OnInit, Output, ViewChild } from '@angular/core' +import { Router } from '@angular/router' +import { UserService } from '@app/shared' +import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service' +import { LoadingBarService } from '@ngx-loading-bar/core' +import { NotificationsService } from 'angular2-notifications' +import { BytesPipe } from 'ngx-pipes' +import { Subscription } from 'rxjs' +import { VideoConstant, VideoPrivacy } from '../../../../../shared/models/videos' +import { AuthService, ServerService } from '../../core' +import { FormReactive } from '../../shared' +import { populateAsyncUserVideoChannels } from '../../shared/misc/utils' +import { VideoEdit } from '../../shared/video/video-edit.model' +import { VideoService } from '../../shared/video/video.service' +import { I18n } from '@ngx-translate/i18n-polyfill' +import { FormValidatorService } from '@app/shared/forms/form-validators/form-validator.service' +import { switchMap } from 'rxjs/operators' +import { VideoCaptionService } from '@app/shared/video-caption' +import { VideoCaptionEdit } from '@app/shared/video-caption/video-caption-edit.model' + +@Component({ + selector: 'my-video-upload', + templateUrl: './video-upload.component.html', + styleUrls: [ + './shared/video-edit.component.scss', + './video-upload.component.scss' + ] +}) +export class VideoUploadComponent extends FormReactive implements OnInit, OnDestroy, CanComponentDeactivate { + @Output() firstStepDone = new EventEmitter() + @ViewChild('videofileInput') videofileInput + + // So that it can be accessed in the template + readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY + + isUploadingVideo = false + isUpdatingVideo = false + videoUploaded = false + videoUploadObservable: Subscription = null + videoUploadPercents = 0 + videoUploadedIds = { + id: 0, + uuid: '' + } + + userVideoChannels: { id: number, label: string, support: string }[] = [] + userVideoQuotaUsed = 0 + videoPrivacies: VideoConstant[] = [] + firstStepPrivacyId = 0 + firstStepChannelId = 0 + videoCaptions: VideoCaptionEdit[] = [] + + constructor ( + protected formValidatorService: FormValidatorService, + private router: Router, + private notificationsService: NotificationsService, + private authService: AuthService, + private userService: UserService, + private serverService: ServerService, + private videoService: VideoService, + private loadingBar: LoadingBarService, + private i18n: I18n, + private videoCaptionService: VideoCaptionService + ) { + super() + } + + get videoExtensions () { + return this.serverService.getConfig().video.file.extensions.join(',') + } + + ngOnInit () { + this.buildForm({}) + + populateAsyncUserVideoChannels(this.authService, this.userVideoChannels) + .then(() => this.firstStepChannelId = this.userVideoChannels[0].id) + + this.userService.getMyVideoQuotaUsed() + .subscribe(data => this.userVideoQuotaUsed = data.videoQuotaUsed) + + this.serverService.videoPrivaciesLoaded + .subscribe( + () => { + this.videoPrivacies = this.serverService.getVideoPrivacies() + + // Public by default + this.firstStepPrivacyId = VideoPrivacy.PUBLIC + }) + } + + ngOnDestroy () { + if (this.videoUploadObservable) { + this.videoUploadObservable.unsubscribe() + } + } + + canDeactivate () { + let text = '' + + if (this.videoUploaded === true) { + // FIXME: cannot concatenate strings inside i18n service :/ + text = this.i18n('Your video was uploaded in your account and is private.') + + this.i18n('But associated data (tags, description...) will be lost, are you sure you want to leave this page?') + } else { + text = this.i18n('Your video is not uploaded yet, are you sure you want to leave this page?') + } + + return { + canDeactivate: !this.isUploadingVideo, + text + } + } + + fileChange () { + this.uploadFirstStep() + } + + checkForm () { + this.forceCheck() + + return this.form.valid + } + + cancelUpload () { + if (this.videoUploadObservable !== null) { + this.videoUploadObservable.unsubscribe() + this.isUploadingVideo = false + this.videoUploadPercents = 0 + this.videoUploadObservable = null + this.notificationsService.info(this.i18n('Info'), this.i18n('Upload cancelled')) + } + } + + uploadFirstStep () { + const videofile = this.videofileInput.nativeElement.files[0] as File + if (!videofile) return + + // Cannot upload videos > 8GB for now + if (videofile.size > 8 * 1024 * 1024 * 1024) { + this.notificationsService.error(this.i18n('Error'), this.i18n('We are sorry but PeerTube cannot handle videos > 8GB')) + return + } + + const videoQuota = this.authService.getUser().videoQuota + if (videoQuota !== -1 && (this.userVideoQuotaUsed + videofile.size) > videoQuota) { + const bytePipes = new BytesPipe() + + const msg = this.i18n( + 'Your video quota is exceeded with this video (video size: {{ videoSize }}, used: {{ videoQuotaUsed }}, quota: {{ videoQuota }})', + { + videoSize: bytePipes.transform(videofile.size, 0), + videoQuotaUsed: bytePipes.transform(this.userVideoQuotaUsed, 0), + videoQuota: bytePipes.transform(videoQuota, 0) + } + ) + this.notificationsService.error(this.i18n('Error'), msg) + return + } + + const nameWithoutExtension = videofile.name.replace(/\.[^/.]+$/, '') + let name: string + + // If the name of the file is very small, keep the extension + if (nameWithoutExtension.length < 3) name = videofile.name + else name = nameWithoutExtension + + const privacy = this.firstStepPrivacyId.toString() + const nsfw = false + const waitTranscoding = true + const commentsEnabled = true + const channelId = this.firstStepChannelId.toString() + + const formData = new FormData() + formData.append('name', name) + // Put the video "private" -> we are waiting the user validation of the second step + formData.append('privacy', VideoPrivacy.PRIVATE.toString()) + formData.append('nsfw', '' + nsfw) + formData.append('commentsEnabled', '' + commentsEnabled) + formData.append('waitTranscoding', '' + waitTranscoding) + formData.append('channelId', '' + channelId) + formData.append('videofile', videofile) + + this.isUploadingVideo = true + this.firstStepDone.emit(name) + + this.form.patchValue({ + name, + privacy, + nsfw, + channelId + }) + + this.videoUploadObservable = this.videoService.uploadVideo(formData).subscribe( + event => { + if (event.type === HttpEventType.UploadProgress) { + this.videoUploadPercents = Math.round(100 * event.loaded / event.total) + } else if (event instanceof HttpResponse) { + this.videoUploaded = true + + this.videoUploadedIds = event.body.video + + this.videoUploadObservable = null + } + }, + + err => { + // Reset progress + this.isUploadingVideo = false + this.videoUploadPercents = 0 + this.videoUploadObservable = null + this.notificationsService.error(this.i18n('Error'), err.message) + } + ) + } + + updateSecondStep () { + if (this.checkForm() === false) { + return + } + + const video = new VideoEdit() + video.patch(this.form.value) + video.id = this.videoUploadedIds.id + video.uuid = this.videoUploadedIds.uuid + + this.isUpdatingVideo = true + this.loadingBar.start() + this.videoService.updateVideo(video) + .pipe( + // Then update captions + switchMap(() => this.videoCaptionService.updateCaptions(video.id, this.videoCaptions)) + ) + .subscribe( + () => { + this.isUpdatingVideo = false + this.isUploadingVideo = false + this.loadingBar.complete() + + this.notificationsService.success(this.i18n('Success'), this.i18n('Video published.')) + this.router.navigate([ '/videos/watch', video.uuid ]) + }, + + err => { + this.isUpdatingVideo = false + this.notificationsService.error(this.i18n('Error'), err.message) + console.error(err) + } + ) + } +} diff --git a/config/default.yaml b/config/default.yaml index 3b877476d..722b33db3 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -92,6 +92,12 @@ transcoding: 720p: false 1080p: false +import: + # Add ability for your users to import remote videos (from YouTube, torrent...) + videos: + http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html + enabled: true + instance: name: 'PeerTube' short_description: 'PeerTube, a federated (ActivityPub) video streaming platform using P2P (BitTorrent) directly in the web browser with WebTorrent and Angular.' diff --git a/package.json b/package.json index cd0a3c5b8..8627fe9ba 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "type": "git", "url": "git://github.com/Chocobozzz/PeerTube.git" }, + "typings": "*.d.ts", "scripts": { "e2e": "scripty", "build": "SCRIPTY_PARALLEL=true scripty", @@ -132,7 +133,8 @@ "validator": "^10.2.0", "webfinger.js": "^2.6.6", "winston": "3.0.0", - "ws": "^5.0.0" + "ws": "^5.0.0", + "youtube-dl": "^1.12.2" }, "devDependencies": { "@types/async": "^2.0.40", @@ -184,8 +186,7 @@ "tslint-config-standard": "^7.0.0", "typescript": "^2.5.2", "webtorrent": "^0.100.0", - "xliff": "^3.0.1", - "youtube-dl": "^1.12.2" + "xliff": "^3.0.1" }, "scripty": { "silent": true diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts new file mode 100644 index 000000000..9761cdbcf --- /dev/null +++ b/server/controllers/api/videos/import.ts @@ -0,0 +1,151 @@ +import * as express from 'express' +import { auditLoggerFactory } from '../../../helpers/audit-logger' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + videoImportAddValidator, + videoImportDeleteValidator +} from '../../../middlewares' +import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers' +import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' +import { createReqFiles } from '../../../helpers/express-utils' +import { logger } from '../../../helpers/logger' +import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' +import { VideoModel } from '../../../models/video/video' +import { getVideoActivityPubUrl } from '../../../lib/activitypub' +import { TagModel } from '../../../models/video/tag' +import { VideoImportModel } from '../../../models/video/video-import' +import { JobQueue } from '../../../lib/job-queue/job-queue' +import { processImage } from '../../../helpers/image-utils' +import { join } from 'path' + +const auditLogger = auditLoggerFactory('video-imports') +const videoImportsRouter = express.Router() + +const reqVideoFileImport = createReqFiles( + [ 'thumbnailfile', 'previewfile' ], + IMAGE_MIMETYPE_EXT, + { + thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, + previewfile: CONFIG.STORAGE.PREVIEWS_DIR + } +) + +videoImportsRouter.post('/imports', + authenticate, + reqVideoFileImport, + asyncMiddleware(videoImportAddValidator), + asyncRetryTransactionMiddleware(addVideoImport) +) + +videoImportsRouter.delete('/imports/:id', + authenticate, + videoImportDeleteValidator, + asyncRetryTransactionMiddleware(deleteVideoImport) +) + +// --------------------------------------------------------------------------- + +export { + videoImportsRouter +} + +// --------------------------------------------------------------------------- + +async function addVideoImport (req: express.Request, res: express.Response) { + const body: VideoImportCreate = req.body + const targetUrl = body.targetUrl + + let youtubeDLInfo: YoutubeDLInfo + try { + youtubeDLInfo = await getYoutubeDLInfo(targetUrl) + } catch (err) { + logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) + + return res.status(400).json({ + error: 'Cannot fetch remote information of this URL.' + }).end() + } + + // Create video DB object + const videoData = { + name: body.name || youtubeDLInfo.name, + remote: false, + category: body.category || youtubeDLInfo.category, + licence: body.licence || youtubeDLInfo.licence, + language: undefined, + commentsEnabled: body.commentsEnabled || true, + waitTranscoding: body.waitTranscoding || false, + state: VideoState.TO_IMPORT, + nsfw: body.nsfw || youtubeDLInfo.nsfw || false, + description: body.description || youtubeDLInfo.description, + support: body.support || null, + privacy: body.privacy || VideoPrivacy.PRIVATE, + duration: 0, // duration will be set by the import job + channelId: res.locals.videoChannel.id + } + const video = new VideoModel(videoData) + video.url = getVideoActivityPubUrl(video) + + // Process thumbnail file? + const thumbnailField = req.files['thumbnailfile'] + let downloadThumbnail = true + if (thumbnailField) { + const thumbnailPhysicalFile = thumbnailField[ 0 ] + await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE) + downloadThumbnail = false + } + + // Process preview file? + const previewField = req.files['previewfile'] + let downloadPreview = true + if (previewField) { + const previewPhysicalFile = previewField[0] + await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE) + downloadPreview = false + } + + const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + // Save video object in database + const videoCreated = await video.save(sequelizeOptions) + videoCreated.VideoChannel = res.locals.videoChannel + + // Set tags to the video + if (youtubeDLInfo.tags !== undefined) { + const tagInstances = await TagModel.findOrCreateTags(youtubeDLInfo.tags, t) + + await videoCreated.$set('Tags', tagInstances, sequelizeOptions) + videoCreated.Tags = tagInstances + } + + // Create video import object in database + const videoImport = await VideoImportModel.create({ + targetUrl, + state: VideoImportState.PENDING, + videoId: videoCreated.id + }, sequelizeOptions) + + videoImport.Video = videoCreated + + return videoImport + }) + + // Create job to import the video + const payload = { + type: 'youtube-dl' as 'youtube-dl', + videoImportId: videoImport.id, + thumbnailUrl: youtubeDLInfo.thumbnailUrl, + downloadThumbnail, + downloadPreview + } + await JobQueue.Instance.createJob({ type: 'video-import', payload }) + + return res.json(videoImport.toFormattedJSON()) +} + +async function deleteVideoImport (req: express.Request, res: express.Response) { + // TODO: delete video import +} diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index e396ee6be..c9365da08 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -54,6 +54,7 @@ import { VideoFilter } from '../../../../shared/models/videos/video-query.type' import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils' import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' import { videoCaptionsRouter } from './captions' +import { videoImportsRouter } from './import' const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() @@ -81,6 +82,7 @@ videosRouter.use('/', blacklistRouter) videosRouter.use('/', rateVideoRouter) videosRouter.use('/', videoCommentRouter) videosRouter.use('/', videoCaptionsRouter) +videosRouter.use('/', videoImportsRouter) videosRouter.get('/categories', listVideoCategories) videosRouter.get('/licences', listVideoLicences) @@ -160,7 +162,6 @@ async function addVideo (req: express.Request, res: express.Response) { const videoData = { name: videoInfo.name, remote: false, - extname: extname(videoPhysicalFile.filename), category: videoInfo.category, licence: videoInfo.licence, language: videoInfo.language, diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index d97bbd2a9..c6a350236 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) { } function sanitizeAndCheckVideoTorrentObject (video: any) { - if (video.type !== 'Video') return false + if (!video || video.type !== 'Video') return false if (!setValidRemoteTags(video)) return false if (!setValidRemoteVideoUrls(video)) return false diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts new file mode 100644 index 000000000..36c0559fd --- /dev/null +++ b/server/helpers/custom-validators/video-imports.ts @@ -0,0 +1,30 @@ +import 'express-validator' +import 'multer' +import * as validator from 'validator' +import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers' +import { exists } from './misc' + +function isVideoImportTargetUrlValid (url: string) { + const isURLOptions = { + require_host: true, + require_tld: true, + require_protocol: true, + require_valid_protocol: true, + protocols: [ 'http', 'https' ] + } + + return exists(url) && + validator.isURL('' + url, isURLOptions) && + validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL) +} + +function isVideoImportStateValid (value: any) { + return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined +} + +// --------------------------------------------------------------------------- + +export { + isVideoImportStateValid, + isVideoImportTargetUrlValid +} diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 04a19a9c6..480c5b49e 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) { } const consoleLoggerFormat = winston.format.printf(info => { - let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2) + let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2) if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' else additionalInfos = ' ' + additionalInfos diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts new file mode 100644 index 000000000..74d3e213b --- /dev/null +++ b/server/helpers/youtube-dl.ts @@ -0,0 +1,142 @@ +import * as youtubeDL from 'youtube-dl' +import { truncate } from 'lodash' +import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' +import { join } from 'path' +import * as crypto from 'crypto' +import { logger } from './logger' + +export type YoutubeDLInfo = { + name: string + description: string + category: number + licence: number + nsfw: boolean + tags: string[] + thumbnailUrl: string +} + +function getYoutubeDLInfo (url: string): Promise { + return new Promise((res, rej) => { + const options = [ '-j', '--flat-playlist' ] + + youtubeDL.getInfo(url, options, (err, info) => { + if (err) return rej(err) + + const obj = normalizeObject(info) + + return res(buildVideoInfo(obj)) + }) + }) +} + +function downloadYoutubeDLVideo (url: string) { + const hash = crypto.createHash('sha256').update(url).digest('base64') + const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') + + logger.info('Importing video %s', url) + + const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] + + return new Promise((res, rej) => { + youtubeDL.exec(url, options, async (err, output) => { + if (err) return rej(err) + + return res(path) + }) + }) +} + +// --------------------------------------------------------------------------- + +export { + downloadYoutubeDLVideo, + getYoutubeDLInfo +} + +// --------------------------------------------------------------------------- + +function normalizeObject (obj: any) { + const newObj: any = {} + + for (const key of Object.keys(obj)) { + // Deprecated key + if (key === 'resolution') continue + + const value = obj[key] + + if (typeof value === 'string') { + newObj[key] = value.normalize() + } else { + newObj[key] = value + } + } + + return newObj +} + +function buildVideoInfo (obj: any) { + return { + name: titleTruncation(obj.title), + description: descriptionTruncation(obj.description), + category: getCategory(obj.categories), + licence: getLicence(obj.license), + nsfw: isNSFW(obj), + tags: getTags(obj.tags), + thumbnailUrl: obj.thumbnail || undefined + } +} + +function titleTruncation (title: string) { + return truncate(title, { + 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max, + 'separator': /,? +/, + 'omission': ' […]' + }) +} + +function descriptionTruncation (description: string) { + if (!description) return undefined + + return truncate(description, { + 'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, + 'separator': /,? +/, + 'omission': ' […]' + }) +} + +function isNSFW (info: any) { + return info.age_limit && info.age_limit >= 16 +} + +function getTags (tags: any) { + if (Array.isArray(tags) === false) return [] + + return tags + .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) + .map(t => t.normalize()) + .slice(0, 5) +} + +function getLicence (licence: string) { + if (!licence) return undefined + + if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 + + return undefined +} + +function getCategory (categories: string[]) { + if (!categories) return undefined + + const categoryString = categories[0] + if (!categoryString || typeof categoryString !== 'string') return undefined + + if (categoryString === 'News & Politics') return 11 + + for (const key of Object.keys(VIDEO_CATEGORIES)) { + const category = VIDEO_CATEGORIES[key] + if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) + } + + return undefined +} diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index bec343bb7..fdd772d84 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -8,6 +8,7 @@ import { VideoPrivacy } from '../../shared/models/videos' import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import { invert } from 'lodash' +import { VideoImportState } from '../../shared/models/videos/video-import-state.enum' // Use a variable to reload the configuration if we need let config: IConfig = require('config') @@ -85,6 +86,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = { 'activitypub-follow': 5, 'video-file-import': 1, 'video-file': 1, + 'video-import': 1, 'email': 5 } const JOB_CONCURRENCY: { [ id in JobType ]: number } = { @@ -94,6 +96,7 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = { 'activitypub-follow': 3, 'video-file-import': 1, 'video-file': 1, + 'video-import': 1, 'email': 5 } const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job @@ -248,6 +251,9 @@ const CONSTRAINTS_FIELDS = { } } }, + VIDEO_IMPORTS: { + URL: { min: 3, max: 2000 } // Length + }, VIDEOS: { NAME: { min: 3, max: 120 }, // Length LANGUAGE: { min: 1, max: 10 }, // Length @@ -262,7 +268,7 @@ const CONSTRAINTS_FIELDS = { }, EXTNAME: [ '.mp4', '.ogv', '.webm' ], INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 - DURATION: { min: 1 }, // Number + DURATION: { min: 0 }, // Number TAGS: { min: 0, max: 5 }, // Number of total tags TAG: { min: 2, max: 30 }, // Length THUMBNAIL: { min: 2, max: 30 }, @@ -363,7 +369,14 @@ const VIDEO_PRIVACIES = { const VIDEO_STATES = { [VideoState.PUBLISHED]: 'Published', - [VideoState.TO_TRANSCODE]: 'To transcode' + [VideoState.TO_TRANSCODE]: 'To transcode', + [VideoState.TO_IMPORT]: 'To import' +} + +const VIDEO_IMPORT_STATES = { + [VideoImportState.FAILED]: 'Failed', + [VideoImportState.PENDING]: 'Pending', + [VideoImportState.SUCCESS]: 'Success' } const VIDEO_MIMETYPE_EXT = { @@ -585,6 +598,7 @@ export { RATES_LIMIT, VIDEO_EXT_MIMETYPE, JOB_COMPLETED_LIFETIME, + VIDEO_IMPORT_STATES, VIDEO_VIEW_LIFETIME, buildLanguages } diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 21c083084..0be752363 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -24,6 +24,7 @@ import { VideoTagModel } from '../models/video/video-tag' import { CONFIG } from './constants' import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' import { VideoCaptionModel } from '../models/video/video-caption' +import { VideoImportModel } from '../models/video/video-import' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string @@ -81,7 +82,8 @@ async function initDatabaseModels (silent: boolean) { VideoTagModel, VideoModel, VideoCommentModel, - ScheduleVideoUpdateModel + ScheduleVideoUpdateModel, + VideoImportModel ]) // Check extensions exist in the database diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts new file mode 100644 index 000000000..2f219e986 --- /dev/null +++ b/server/lib/job-queue/handlers/video-import.ts @@ -0,0 +1,129 @@ +import * as Bull from 'bull' +import { logger } from '../../../helpers/logger' +import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' +import { VideoImportModel } from '../../../models/video/video-import' +import { VideoImportState } from '../../../../shared/models/videos' +import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' +import { extname, join } from 'path' +import { VideoFileModel } from '../../../models/video/video-file' +import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils' +import { CONFIG, sequelizeTypescript } from '../../../initializers' +import { doRequestAndSaveToFile } from '../../../helpers/requests' +import { VideoState } from '../../../../shared' +import { JobQueue } from '../index' +import { federateVideoIfNeeded } from '../../activitypub' + +export type VideoImportPayload = { + type: 'youtube-dl' + videoImportId: number + thumbnailUrl: string + downloadThumbnail: boolean + downloadPreview: boolean +} + +async function processVideoImport (job: Bull.Job) { + const payload = job.data as VideoImportPayload + logger.info('Processing video import in job %d.', job.id) + + const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) + if (!videoImport) throw new Error('Cannot import video %s: the video import entry does not exist anymore.') + + let tempVideoPath: string + try { + // Download video from youtubeDL + tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl) + + // Get information about this video + const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) + const fps = await getVideoFileFPS(tempVideoPath) + const stats = await statPromise(tempVideoPath) + const duration = await getDurationFromVideoFile(tempVideoPath) + + // Create video file object in database + const videoFileData = { + extname: extname(tempVideoPath), + resolution: videoFileResolution, + size: stats.size, + fps, + videoId: videoImport.videoId + } + const videoFile = new VideoFileModel(videoFileData) + + // Move file + const destination = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile)) + await renamePromise(tempVideoPath, destination) + + // Process thumbnail + if (payload.downloadThumbnail) { + if (payload.thumbnailUrl) { + const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) + await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath) + } else { + await videoImport.Video.createThumbnail(videoFile) + } + } + + // Process preview + if (payload.downloadPreview) { + if (payload.thumbnailUrl) { + const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) + await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath) + } else { + await videoImport.Video.createPreview(videoFile) + } + } + + // Create torrent + await videoImport.Video.createTorrentAndSetInfoHash(videoFile) + + const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => { + await videoFile.save({ transaction: t }) + + // Update video DB object + videoImport.Video.duration = duration + videoImport.Video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED + const videoUpdated = await videoImport.Video.save({ transaction: t }) + + // Now we can federate the video + await federateVideoIfNeeded(videoImport.Video, true, t) + + // Update video import object + videoImport.state = VideoImportState.SUCCESS + const videoImportUpdated = await videoImport.save({ transaction: t }) + + logger.info('Video %s imported.', videoImport.targetUrl) + + videoImportUpdated.Video = videoUpdated + return videoImportUpdated + }) + + // Create transcoding jobs? + if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { + // Put uuid because we don't have id auto incremented for now + const dataInput = { + videoUUID: videoImportUpdated.Video.uuid, + isNewVideo: true + } + + await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) + } + + } catch (err) { + try { + if (tempVideoPath) await unlinkPromise(tempVideoPath) + } catch (errUnlink) { + logger.error('Cannot cleanup files after a video import error.', { err: errUnlink }) + } + + videoImport.state = VideoImportState.FAILED + await videoImport.save() + + throw err + } +} + +// --------------------------------------------------------------------------- + +export { + processVideoImport +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 8ff0c169e..2e14867f2 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -9,6 +9,7 @@ import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './ import { EmailPayload, processEmail } from './handlers/email' import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' +import { processVideoImport, VideoImportPayload } from './handlers/video-import' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -17,7 +18,8 @@ type CreateJobArgument = { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | { type: 'video-file-import', payload: VideoFileImportPayload } | { type: 'video-file', payload: VideoFilePayload } | - { type: 'email', payload: EmailPayload } + { type: 'email', payload: EmailPayload } | + { type: 'video-import', payload: VideoImportPayload } const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'activitypub-http-broadcast': processActivityPubHttpBroadcast, @@ -26,7 +28,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'activitypub-follow': processActivityPubFollow, 'video-file-import': processVideoFileImport, 'video-file': processVideoFile, - 'email': processEmail + 'email': processEmail, + 'video-import': processVideoImport } const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { @@ -43,7 +46,8 @@ const jobTypes: JobType[] = [ 'activitypub-http-unicast', 'email', 'video-file', - 'video-file-import' + 'video-file-import', + 'video-import' ] class JobQueue { diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index e3f0f5963..c5400c8f5 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -11,3 +11,4 @@ export * from './video-blacklist' export * from './video-channels' export * from './webfinger' export * from './search' +export * from './video-imports' diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts new file mode 100644 index 000000000..0ba759ff0 --- /dev/null +++ b/server/middlewares/validators/video-imports.ts @@ -0,0 +1,51 @@ +import * as express from 'express' +import { body, param } from 'express-validator/check' +import { isIdValid } from '../../helpers/custom-validators/misc' +import { logger } from '../../helpers/logger' +import { areValidationErrors } from './utils' +import { getCommonVideoAttributes } from './videos' +import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' +import { cleanUpReqFiles } from '../../helpers/utils' +import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos' + +const videoImportAddValidator = getCommonVideoAttributes().concat([ + body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'), + body('channelId') + .toInt() + .custom(isIdValid).withMessage('Should have correct video channel id'), + body('name') + .optional() + .custom(isVideoNameValid).withMessage('Should have a valid name'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body }) + + const user = res.locals.oauth.token.User + + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) + + return next() + } +]) + +const videoImportDeleteValidator = [ + param('id').custom(isIdValid).not().isEmpty().withMessage('Should have a valid id'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoImportDeleteValidator parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoImportAddValidator, + videoImportDeleteValidator +} + +// --------------------------------------------------------------------------- diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts index 9357c1e39..c812d4677 100644 --- a/server/middlewares/validators/videos.ts +++ b/server/middlewares/validators/videos.ts @@ -223,36 +223,6 @@ const videosShareValidator = [ } ] -// --------------------------------------------------------------------------- - -export { - videosAddValidator, - videosUpdateValidator, - videosGetValidator, - videosRemoveValidator, - videosShareValidator, - - videoAbuseReportValidator, - - videoRateValidator -} - -// --------------------------------------------------------------------------- - -function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) { - if (req.body.scheduleUpdate) { - if (!req.body.scheduleUpdate.updateAt) { - res.status(400) - .json({ error: 'Schedule update at is mandatory.' }) - .end() - - return true - } - } - - return false -} - function getCommonVideoAttributes () { return [ body('thumbnailfile') @@ -319,3 +289,35 @@ function getCommonVideoAttributes () { .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy') ] as (ValidationChain | express.Handler)[] } + +// --------------------------------------------------------------------------- + +export { + videosAddValidator, + videosUpdateValidator, + videosGetValidator, + videosRemoveValidator, + videosShareValidator, + + videoAbuseReportValidator, + + videoRateValidator, + + getCommonVideoAttributes +} + +// --------------------------------------------------------------------------- + +function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) { + if (req.body.scheduleUpdate) { + if (!req.body.scheduleUpdate.updateAt) { + res.status(400) + .json({ error: 'Schedule update at is mandatory.' }) + .end() + + return true + } + } + + return false +} diff --git a/server/models/account/account.ts b/server/models/account/account.ts index d674d8d22..66f5dcf2e 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -16,7 +16,6 @@ import { } from 'sequelize-typescript' import { Account } from '../../../shared/models/actors' import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts' -import { logger } from '../../helpers/logger' import { sendDeleteActor } from '../../lib/activitypub/send' import { ActorModel } from '../activitypub/actor' import { ApplicationModel } from '../application/application' diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts new file mode 100644 index 000000000..89eeafd6a --- /dev/null +++ b/server/models/video/video-import.ts @@ -0,0 +1,105 @@ +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + ForeignKey, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { CONSTRAINTS_FIELDS } from '../../initializers' +import { throwIfNotValid } from '../utils' +import { VideoModel } from './video' +import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' +import { VideoImport, VideoImportState } from '../../../shared' +import { VideoChannelModel } from './video-channel' +import { AccountModel } from '../account/account' + +@DefaultScope({ + include: [ + { + model: () => VideoModel, + required: true, + include: [ + { + model: () => VideoChannelModel, + required: true, + include: [ + { + model: () => AccountModel, + required: true + } + ] + } + ] + } + ] +}) + +@Table({ + tableName: 'videoImport', + indexes: [ + { + fields: [ 'videoId' ], + unique: true + } + ] +}) +export class VideoImportModel extends Model { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) + targetUrl: string + + @AllowNull(false) + @Default(null) + @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) + @Column + state: VideoImportState + + @AllowNull(true) + @Default(null) + @Column(DataType.TEXT) + error: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: VideoModel + + static loadAndPopulateVideo (id: number) { + return VideoImportModel.findById(id) + } + + toFormattedJSON (): VideoImport { + const videoFormatOptions = { + additionalAttributes: { state: true, waitTranscoding: true, scheduledUpdate: true } + } + const video = Object.assign(this.Video.toFormattedJSON(videoFormatOptions), { + tags: this.Video.Tags.map(t => t.name) + }) + + return { + targetUrl: this.targetUrl, + video + } + } +} diff --git a/server/models/video/video.ts b/server/models/video/video.ts index a6c4620b2..459fcb31e 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -377,7 +377,7 @@ type AvailableForListOptions = { include: [ { model: () => VideoFileModel.unscoped(), - required: true + required: false } ] }, diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index a38a8aa3b..2565479f6 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -6,7 +6,8 @@ export type JobType = 'activitypub-http-unicast' | 'activitypub-follow' | 'video-file-import' | 'video-file' | - 'email' + 'email' | + 'video-import' export interface Job { id: number diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index cb9669772..1b135e26a 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts @@ -15,4 +15,8 @@ export * from './video-update.model' export * from './video.model' export * from './video-state.enum' export * from './video-caption-update.model' +export * from './video-import-create.model' +export * from './video-import-update.model' +export * from './video-import-state.enum' +export * from './video-import.model' export { VideoConstant } from './video-constant.model' diff --git a/shared/models/videos/video-import-create.model.ts b/shared/models/videos/video-import-create.model.ts new file mode 100644 index 000000000..65d142c2b --- /dev/null +++ b/shared/models/videos/video-import-create.model.ts @@ -0,0 +1,6 @@ +import { VideoUpdate } from './video-update.model' + +export interface VideoImportCreate extends VideoUpdate { + targetUrl: string + channelId: number // Required +} diff --git a/shared/models/videos/video-import-state.enum.ts b/shared/models/videos/video-import-state.enum.ts new file mode 100644 index 000000000..b178fbf3a --- /dev/null +++ b/shared/models/videos/video-import-state.enum.ts @@ -0,0 +1,5 @@ +export enum VideoImportState { + PENDING = 1, + SUCCESS = 2, + FAILED = 3 +} diff --git a/shared/models/videos/video-import-update.model.ts b/shared/models/videos/video-import-update.model.ts new file mode 100644 index 000000000..5ae244683 --- /dev/null +++ b/shared/models/videos/video-import-update.model.ts @@ -0,0 +1,5 @@ +import { VideoUpdate } from './video-update.model' + +export interface VideoImportUpdate extends VideoUpdate { + targetUrl: string +} diff --git a/shared/models/videos/video-import.model.ts b/shared/models/videos/video-import.model.ts new file mode 100644 index 000000000..858108599 --- /dev/null +++ b/shared/models/videos/video-import.model.ts @@ -0,0 +1,7 @@ +import { Video } from './video.model' + +export interface VideoImport { + targetUrl: string + + video: Video & { tags: string[] } +} diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts index 625aefae1..a50e14e4b 100644 --- a/shared/models/videos/video-state.enum.ts +++ b/shared/models/videos/video-state.enum.ts @@ -1,4 +1,5 @@ export enum VideoState { PUBLISHED = 1, - TO_TRANSCODE = 2 + TO_TRANSCODE = 2, + TO_IMPORT = 3 } diff --git a/tsconfig.json b/tsconfig.json index 4254010e7..7633465b2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,6 +20,7 @@ ] }, "exclude": [ + "client/node_modules", "node_modules", "dist", "storage",