From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- .../my-account-settings.component.ts | 4 +- .../my-video-channel-update.component.ts | 6 +- .../video-add-components/uploaderx-form-data.ts | 48 ++++ .../video-upload.component.html | 20 +- .../video-upload.component.scss | 4 - .../video-add-components/video-upload.component.ts | 295 ++++++++++++--------- .../app/+videos/+video-edit/video-add.module.ts | 5 +- client/src/app/helpers/utils.ts | 9 +- 8 files changed, 243 insertions(+), 148 deletions(-) create mode 100644 client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts (limited to 'client/src') diff --git a/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts b/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts index c16368952..a0f2f28f8 100644 --- a/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts +++ b/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts @@ -2,7 +2,7 @@ import { ViewportScroller } from '@angular/common' import { HttpErrorResponse } from '@angular/common/http' import { AfterViewChecked, Component, OnInit } from '@angular/core' import { AuthService, Notifier, User, UserService } from '@app/core' -import { uploadErrorHandler } from '@app/helpers' +import { genericUploadErrorHandler } from '@app/helpers' @Component({ selector: 'my-account-settings', @@ -46,7 +46,7 @@ export class MyAccountSettingsComponent implements OnInit, AfterViewChecked { this.user.updateAccountAvatar(data.avatar) }, - (err: HttpErrorResponse) => uploadErrorHandler({ + (err: HttpErrorResponse) => genericUploadErrorHandler({ err, name: $localize`avatar`, notifier: this.notifier diff --git a/client/src/app/+my-library/+my-video-channels/my-video-channel-update.component.ts b/client/src/app/+my-library/+my-video-channels/my-video-channel-update.component.ts index a29af176c..c9173039a 100644 --- a/client/src/app/+my-library/+my-video-channels/my-video-channel-update.component.ts +++ b/client/src/app/+my-library/+my-video-channels/my-video-channel-update.component.ts @@ -3,7 +3,7 @@ import { HttpErrorResponse } from '@angular/common/http' import { Component, OnDestroy, OnInit } from '@angular/core' import { ActivatedRoute, Router } from '@angular/router' import { AuthService, Notifier, ServerService } from '@app/core' -import { uploadErrorHandler } from '@app/helpers' +import { genericUploadErrorHandler } from '@app/helpers' import { VIDEO_CHANNEL_DESCRIPTION_VALIDATOR, VIDEO_CHANNEL_DISPLAY_NAME_VALIDATOR, @@ -109,7 +109,7 @@ export class MyVideoChannelUpdateComponent extends MyVideoChannelEdit implements this.videoChannel.updateAvatar(data.avatar) }, - (err: HttpErrorResponse) => uploadErrorHandler({ + (err: HttpErrorResponse) => genericUploadErrorHandler({ err, name: $localize`avatar`, notifier: this.notifier @@ -139,7 +139,7 @@ export class MyVideoChannelUpdateComponent extends MyVideoChannelEdit implements this.videoChannel.updateBanner(data.banner) }, - (err: HttpErrorResponse) => uploadErrorHandler({ + (err: HttpErrorResponse) => genericUploadErrorHandler({ err, name: $localize`banner`, notifier: this.notifier diff --git a/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts b/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts new file mode 100644 index 000000000..3392a0d8a --- /dev/null +++ b/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts @@ -0,0 +1,48 @@ +import { objectToFormData } from '@app/helpers' +import { resolveUrl, UploaderX } from 'ngx-uploadx' + +/** + * multipart/form-data uploader extending the UploaderX implementation of Google Resumable + * for use with multer + * + * @see https://github.com/kukhariev/ngx-uploadx/blob/637e258fe366b8095203f387a6101a230ee4f8e6/src/uploadx/lib/uploaderx.ts + * @example + * + * options: UploadxOptions = { + * uploaderClass: UploaderXFormData + * }; + */ +export class UploaderXFormData extends UploaderX { + + async getFileUrl (): Promise { + const headers = { + 'X-Upload-Content-Length': this.size.toString(), + 'X-Upload-Content-Type': this.file.type || 'application/octet-stream' + } + + const previewfile = this.metadata.previewfile as any as File + delete this.metadata.previewfile + + const data = objectToFormData(this.metadata) + if (previewfile !== undefined) { + data.append('previewfile', previewfile, previewfile.name) + data.append('thumbnailfile', previewfile, previewfile.name) + } + + await this.request({ + method: 'POST', + body: data, + url: this.endpoint, + headers + }) + + const location = this.getValueFromResponse('location') + if (!location) { + throw new Error('Invalid or missing Location header') + } + + this.offset = this.responseStatus === 201 ? 0 : undefined + + return resolveUrl(location, this.endpoint) + } +} diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html index 4c0b09894..86a779f8a 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html +++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html @@ -1,12 +1,17 @@ -
+
Select the file to upload
@@ -41,7 +46,13 @@
- + +
@@ -64,6 +75,7 @@ {{ error }}
+
diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss index 9549257f6..d9f348a70 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss +++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss @@ -47,8 +47,4 @@ margin-left: 10px; } - - .btn-group > input:not(:first-child) { - margin-left: 0; - } } diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts index effb37077..2d3fc3578 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts +++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts @@ -1,15 +1,16 @@ -import { Subscription } from 'rxjs' -import { HttpErrorResponse, HttpEventType, HttpResponse } from '@angular/common/http' import { AfterViewInit, Component, ElementRef, EventEmitter, OnDestroy, OnInit, Output, ViewChild } from '@angular/core' import { Router } from '@angular/router' +import { UploadxOptions, UploadState, UploadxService } from 'ngx-uploadx' +import { UploaderXFormData } from './uploaderx-form-data' import { AuthService, CanComponentDeactivate, HooksService, Notifier, ServerService, UserService } from '@app/core' -import { scrollToTop, uploadErrorHandler } from '@app/helpers' +import { scrollToTop, genericUploadErrorHandler } from '@app/helpers' import { FormValidatorService } from '@app/shared/shared-forms' import { BytesPipe, VideoCaptionService, VideoEdit, VideoService } from '@app/shared/shared-main' import { LoadingBarService } from '@ngx-loading-bar/core' import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes' import { VideoPrivacy } from '@shared/models' import { VideoSend } from './video-send' +import { HttpErrorResponse, HttpEventType, HttpHeaders } from '@angular/common/http' @Component({ selector: 'my-video-upload', @@ -20,23 +21,18 @@ import { VideoSend } from './video-send' './video-send.scss' ] }) -export class VideoUploadComponent extends VideoSend implements OnInit, AfterViewInit, OnDestroy, CanComponentDeactivate { +export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy, AfterViewInit, CanComponentDeactivate { @Output() firstStepDone = new EventEmitter() @Output() firstStepError = new EventEmitter() @ViewChild('videofileInput') videofileInput: ElementRef - // So that it can be accessed in the template - readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY - userVideoQuotaUsed = 0 userVideoQuotaUsedDaily = 0 isUploadingAudioFile = false isUploadingVideo = false - isUpdatingVideo = false videoUploaded = false - videoUploadObservable: Subscription = null videoUploadPercents = 0 videoUploadedIds = { id: 0, @@ -49,7 +45,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView error: string enableRetryAfterError: boolean + // So that it can be accessed in the template protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC + protected readonly BASE_VIDEO_UPLOAD_URL = VideoService.BASE_VIDEO_URL + 'upload-resumable' + + private uploadxOptions: UploadxOptions + private isUpdatingVideo = false + private fileToUpload: File constructor ( protected formValidatorService: FormValidatorService, @@ -61,15 +63,77 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView protected videoCaptionService: VideoCaptionService, private userService: UserService, private router: Router, - private hooks: HooksService - ) { + private hooks: HooksService, + private resumableUploadService: UploadxService + ) { super() + + this.uploadxOptions = { + endpoint: this.BASE_VIDEO_UPLOAD_URL, + multiple: false, + token: this.authService.getAccessToken(), + uploaderClass: UploaderXFormData, + retryConfig: { + maxAttempts: 6, + shouldRetry: (code: number) => { + return code < 400 || code >= 501 + } + } + } } get videoExtensions () { return this.serverConfig.video.file.extensions.join(', ') } + onUploadVideoOngoing (state: UploadState) { + switch (state.status) { + case 'error': + const error = state.response?.error || 'Unknow error' + + this.handleUploadError({ + error: new Error(error), + name: 'HttpErrorResponse', + message: error, + ok: false, + headers: new HttpHeaders(state.responseHeaders), + status: +state.responseStatus, + statusText: error, + type: HttpEventType.Response, + url: state.url + }) + break + + case 'cancelled': + this.isUploadingVideo = false + this.videoUploadPercents = 0 + + this.firstStepError.emit() + this.enableRetryAfterError = false + this.error = '' + break + + case 'queue': + this.closeFirstStep(state.name) + break + + case 'uploading': + this.videoUploadPercents = state.progress + break + + case 'paused': + this.notifier.info($localize`Upload cancelled`) + break + + case 'complete': + this.videoUploaded = true + this.videoUploadPercents = 100 + + this.videoUploadedIds = state?.response.video + break + } + } + ngOnInit () { super.ngOnInit() @@ -78,6 +142,9 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView this.userVideoQuotaUsed = data.videoQuotaUsed this.userVideoQuotaUsedDaily = data.videoQuotaUsedDaily }) + + this.resumableUploadService.events + .subscribe(state => this.onUploadVideoOngoing(state)) } ngAfterViewInit () { @@ -85,7 +152,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView } ngOnDestroy () { - if (this.videoUploadObservable) this.videoUploadObservable.unsubscribe() + this.cancelUpload() } canDeactivate () { @@ -105,137 +172,43 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView } } - getVideoFile () { - return this.videofileInput.nativeElement.files[0] - } - - setVideoFile (files: FileList) { + onFileDropped (files: FileList) { this.videofileInput.nativeElement.files = files - this.fileChange() - } - - getAudioUploadLabel () { - const videofile = this.getVideoFile() - if (!videofile) return $localize`Upload` - return $localize`Upload ${videofile.name}` + this.onFileChange({ target: this.videofileInput.nativeElement }) } - fileChange () { - this.uploadFirstStep() - } - - retryUpload () { - this.enableRetryAfterError = false - this.error = '' - this.uploadVideo() - } - - cancelUpload () { - if (this.videoUploadObservable !== null) { - this.videoUploadObservable.unsubscribe() - } - - this.isUploadingVideo = false - this.videoUploadPercents = 0 - this.videoUploadObservable = null + onFileChange (event: Event | { target: HTMLInputElement }) { + const file = (event.target as HTMLInputElement).files[0] - this.firstStepError.emit() - this.enableRetryAfterError = false - this.error = '' + if (!file) return - this.notifier.info($localize`Upload cancelled`) - } + if (!this.checkGlobalUserQuota(file)) return + if (!this.checkDailyUserQuota(file)) return - uploadFirstStep (clickedOnButton = false) { - const videofile = this.getVideoFile() - if (!videofile) return - - if (!this.checkGlobalUserQuota(videofile)) return - if (!this.checkDailyUserQuota(videofile)) return - - if (clickedOnButton === false && this.isAudioFile(videofile.name)) { + if (this.isAudioFile(file.name)) { this.isUploadingAudioFile = true return } - // Build name field - const nameWithoutExtension = videofile.name.replace(/\.[^/.]+$/, '') - let name: string - - // If the name of the file is very small, keep the extension - if (nameWithoutExtension.length < 3) name = videofile.name - else name = nameWithoutExtension - - const nsfw = this.serverConfig.instance.isNSFW - const waitTranscoding = true - const commentsEnabled = true - const downloadEnabled = true - const channelId = this.firstStepChannelId.toString() - - this.formData = new FormData() - this.formData.append('name', name) - // Put the video "private" -> we are waiting the user validation of the second step - this.formData.append('privacy', VideoPrivacy.PRIVATE.toString()) - this.formData.append('nsfw', '' + nsfw) - this.formData.append('commentsEnabled', '' + commentsEnabled) - this.formData.append('downloadEnabled', '' + downloadEnabled) - this.formData.append('waitTranscoding', '' + waitTranscoding) - this.formData.append('channelId', '' + channelId) - this.formData.append('videofile', videofile) - - if (this.previewfileUpload) { - this.formData.append('previewfile', this.previewfileUpload) - this.formData.append('thumbnailfile', this.previewfileUpload) - } - this.isUploadingVideo = true - this.firstStepDone.emit(name) - - this.form.patchValue({ - name, - privacy: this.firstStepPrivacyId, - nsfw, - channelId: this.firstStepChannelId, - previewfile: this.previewfileUpload - }) + this.fileToUpload = file - this.uploadVideo() + this.uploadFile(file) } - uploadVideo () { - this.videoUploadObservable = this.videoService.uploadVideo(this.formData).subscribe( - event => { - if (event.type === HttpEventType.UploadProgress) { - this.videoUploadPercents = Math.round(100 * event.loaded / event.total) - } else if (event instanceof HttpResponse) { - this.videoUploaded = true - - this.videoUploadedIds = event.body.video - - this.videoUploadObservable = null - } - }, + uploadAudio () { + this.uploadFile(this.getInputVideoFile(), this.previewfileUpload) + } - (err: HttpErrorResponse) => { - // Reset progress (but keep isUploadingVideo true) - this.videoUploadPercents = 0 - this.videoUploadObservable = null - this.enableRetryAfterError = true - - this.error = uploadErrorHandler({ - err, - name: $localize`video`, - notifier: this.notifier, - sticky: false - }) + retryUpload () { + this.enableRetryAfterError = false + this.error = '' + this.uploadFile(this.fileToUpload) + } - if (err.status === HttpStatusCode.PAYLOAD_TOO_LARGE_413 || - err.status === HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) { - this.cancelUpload() - } - } - ) + cancelUpload () { + this.resumableUploadService.control({ action: 'cancel' }) } isPublishingButtonDisabled () { @@ -245,6 +218,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView !this.videoUploadedIds.id } + getAudioUploadLabel () { + const videofile = this.getInputVideoFile() + if (!videofile) return $localize`Upload` + + return $localize`Upload ${videofile.name}` + } + updateSecondStep () { if (this.isPublishingButtonDisabled() || !this.checkForm()) { return @@ -275,6 +255,62 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView ) } + private getInputVideoFile () { + return this.videofileInput.nativeElement.files[0] + } + + private uploadFile (file: File, previewfile?: File) { + const metadata = { + waitTranscoding: true, + commentsEnabled: true, + downloadEnabled: true, + channelId: this.firstStepChannelId, + nsfw: this.serverConfig.instance.isNSFW, + privacy: VideoPrivacy.PRIVATE.toString(), + filename: file.name, + previewfile: previewfile as any + } + + this.resumableUploadService.handleFiles(file, { + ...this.uploadxOptions, + metadata + }) + + this.isUploadingVideo = true + } + + private handleUploadError (err: HttpErrorResponse) { + // Reset progress (but keep isUploadingVideo true) + this.videoUploadPercents = 0 + this.enableRetryAfterError = true + + this.error = genericUploadErrorHandler({ + err, + name: $localize`video`, + notifier: this.notifier, + sticky: false + }) + + if (err.status === HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) { + this.cancelUpload() + } + } + + private closeFirstStep (filename: string) { + const nameWithoutExtension = filename.replace(/\.[^/.]+$/, '') + const name = nameWithoutExtension.length < 3 ? filename : nameWithoutExtension + + this.form.patchValue({ + name, + privacy: this.firstStepPrivacyId, + nsfw: this.serverConfig.instance.isNSFW, + channelId: this.firstStepChannelId, + previewfile: this.previewfileUpload + }) + + this.firstStepDone.emit(name) + } + private checkGlobalUserQuota (videofile: File) { const bytePipes = new BytesPipe() @@ -285,8 +321,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView const videoQuotaUsedBytes = bytePipes.transform(this.userVideoQuotaUsed, 0) const videoQuotaBytes = bytePipes.transform(videoQuota, 0) - const msg = $localize`Your video quota is exceeded with this video ( -video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})` + const msg = $localize`Your video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})` this.notifier.error(msg) return false @@ -304,9 +339,7 @@ video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuota const videoSizeBytes = bytePipes.transform(videofile.size, 0) const quotaUsedDailyBytes = bytePipes.transform(this.userVideoQuotaUsedDaily, 0) const quotaDailyBytes = bytePipes.transform(videoQuotaDaily, 0) - - const msg = $localize`Your daily video quota is exceeded with this video ( -video size: ${videoSizeBytes}, used: ${quotaUsedDailyBytes}, quota: ${quotaDailyBytes})` + const msg = $localize`Your daily video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${quotaUsedDailyBytes}, quota: ${quotaDailyBytes})` this.notifier.error(msg) return false diff --git a/client/src/app/+videos/+video-edit/video-add.module.ts b/client/src/app/+videos/+video-edit/video-add.module.ts index da651119b..e836cf81e 100644 --- a/client/src/app/+videos/+video-edit/video-add.module.ts +++ b/client/src/app/+videos/+video-edit/video-add.module.ts @@ -1,5 +1,6 @@ import { NgModule } from '@angular/core' import { CanDeactivateGuard } from '@app/core' +import { UploadxModule } from 'ngx-uploadx' import { VideoEditModule } from './shared/video-edit.module' import { DragDropDirective } from './video-add-components/drag-drop.directive' import { VideoImportTorrentComponent } from './video-add-components/video-import-torrent.component' @@ -13,7 +14,9 @@ import { VideoAddComponent } from './video-add.component' imports: [ VideoAddRoutingModule, - VideoEditModule + VideoEditModule, + + UploadxModule ], declarations: [ diff --git a/client/src/app/helpers/utils.ts b/client/src/app/helpers/utils.ts index 17eb5effc..d6ac5b9b4 100644 --- a/client/src/app/helpers/utils.ts +++ b/client/src/app/helpers/utils.ts @@ -173,8 +173,8 @@ function isXPercentInViewport (el: HTMLElement, percentVisible: number) { ) } -function uploadErrorHandler (parameters: { - err: HttpErrorResponse +function genericUploadErrorHandler (parameters: { + err: Pick name: string notifier: Notifier sticky?: boolean @@ -186,6 +186,9 @@ function uploadErrorHandler (parameters: { if (err instanceof ErrorEvent) { // network error message = $localize`The connection was interrupted` notifier.error(message, title, null, sticky) + } else if (err.status === HttpStatusCode.INTERNAL_SERVER_ERROR_500) { + message = $localize`The server encountered an error` + notifier.error(message, title, null, sticky) } else if (err.status === HttpStatusCode.REQUEST_TIMEOUT_408) { message = $localize`Your ${name} file couldn't be transferred before the set timeout (usually 10min)` notifier.error(message, title, null, sticky) @@ -216,5 +219,5 @@ export { isInViewport, isXPercentInViewport, listUserChannels, - uploadErrorHandler + genericUploadErrorHandler } -- cgit v1.2.3