Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

5 files changed, 232 insertions, 140 deletions

diff --git a/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts b/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts
new file mode 100644
index 000000000..3392a0d8a
--- /dev/null
+++ b/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts
@@ -0,0 +1,48 @@
+import { objectToFormData } from '@app/helpers'
+import { resolveUrl, UploaderX } from 'ngx-uploadx'
+
+/**
+ * multipart/form-data uploader extending the UploaderX implementation of Google Resumable
+ * for use with multer
+ *
+ * @see https://github.com/kukhariev/ngx-uploadx/blob/637e258fe366b8095203f387a6101a230ee4f8e6/src/uploadx/lib/uploaderx.ts
+ * @example
+ *
+ *   options: UploadxOptions = {
+ *     uploaderClass: UploaderXFormData
+ *   };
+ */
+export class UploaderXFormData extends UploaderX {
+
+  async getFileUrl (): Promise<string> {
+    const headers = {
+      'X-Upload-Content-Length': this.size.toString(),
+      'X-Upload-Content-Type': this.file.type || 'application/octet-stream'
+    }
+
+    const previewfile = this.metadata.previewfile as any as File
+    delete this.metadata.previewfile
+
+    const data = objectToFormData(this.metadata)
+    if (previewfile !== undefined) {
+      data.append('previewfile', previewfile, previewfile.name)
+      data.append('thumbnailfile', previewfile, previewfile.name)
+    }
+
+    await this.request({
+      method: 'POST',
+      body: data,
+      url: this.endpoint,
+      headers
+    })
+
+    const location = this.getValueFromResponse('location')
+    if (!location) {
+      throw new Error('Invalid or missing Location header')
+    }
+
+    this.offset = this.responseStatus === 201 ? 0 : undefined
+
+    return resolveUrl(location, this.endpoint)
+  }
+}
diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html
index 4c0b09894..86a779f8a 100644
--- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html
+++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html
@@ -1,12 +1,17 @@
-<div *ngIf="!isUploadingVideo" class="upload-video-container" dragDrop (fileDropped)="setVideoFile($event)">
+<div *ngIf="!isUploadingVideo" class="upload-video-container" dragDrop (fileDropped)="onFileDropped($event)">
   <div class="first-step-block">
     <my-global-icon class="upload-icon" iconName="upload" aria-hidden="true"></my-global-icon>
 
     <div class="button-file form-control" [ngbTooltip]="'(extensions: ' + videoExtensions + ')'">
       <span i18n>Select the file to upload</span>
       <input
-        aria-label="Select the file to upload" i18n-aria-label
-        #videofileInput type="file" name="videofile" id="videofile" [accept]="videoExtensions" (change)="fileChange()" autofocus
+        aria-label="Select the file to upload"
+        i18n-aria-label
+        #videofileInput
+        [accept]="videoExtensions"
+        (change)="onFileChange($event)"
+        id="videofile"
+        type="file"
       />
     </div>
 
@@ -41,7 +46,13 @@
       </div>
 
       <div class="form-group upload-audio-button">
-        <my-button className="orange-button" i18n-label [label]="getAudioUploadLabel()" icon="upload" (click)="uploadFirstStep(true)"></my-button>
+        <my-button
+          className="orange-button"
+          [label]="getAudioUploadLabel()"
+          icon="upload"
+          (click)="uploadAudio()"
+        >
+        </my-button>
       </div>
     </ng-container>
   </div>
@@ -64,6 +75,7 @@
       <span>{{ error }}</span>
     </div>
   </div>
+
   <div class="btn-group" role="group">
     <input type="button" class="btn" i18n-value="Retry failed upload of a video" value="Retry" (click)="retryUpload()" />
     <input type="button" class="btn" i18n-value="Cancel ongoing upload of a video" value="Cancel" (click)="cancelUpload()" />
diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss
index 9549257f6..d9f348a70 100644
--- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss
+++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss
@@ -47,8 +47,4 @@
 
     margin-left: 10px;
   }
-
-  .btn-group > input:not(:first-child) {
-    margin-left: 0;
-  }
 }
diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts
index effb37077..2d3fc3578 100644
--- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts
+++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts
@@ -1,15 +1,16 @@
-import { Subscription } from 'rxjs'
-import { HttpErrorResponse, HttpEventType, HttpResponse } from '@angular/common/http'
 import { AfterViewInit, Component, ElementRef, EventEmitter, OnDestroy, OnInit, Output, ViewChild } from '@angular/core'
 import { Router } from '@angular/router'
+import { UploadxOptions, UploadState, UploadxService } from 'ngx-uploadx'
+import { UploaderXFormData } from './uploaderx-form-data'
 import { AuthService, CanComponentDeactivate, HooksService, Notifier, ServerService, UserService } from '@app/core'
-import { scrollToTop, uploadErrorHandler } from '@app/helpers'
+import { scrollToTop, genericUploadErrorHandler } from '@app/helpers'
 import { FormValidatorService } from '@app/shared/shared-forms'
 import { BytesPipe, VideoCaptionService, VideoEdit, VideoService } from '@app/shared/shared-main'
 import { LoadingBarService } from '@ngx-loading-bar/core'
 import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
 import { VideoPrivacy } from '@shared/models'
 import { VideoSend } from './video-send'
+import { HttpErrorResponse, HttpEventType, HttpHeaders } from '@angular/common/http'
 
 @Component({
   selector: 'my-video-upload',
@@ -20,23 +21,18 @@ import { VideoSend } from './video-send'
     './video-send.scss'
   ]
 })
-export class VideoUploadComponent extends VideoSend implements OnInit, AfterViewInit, OnDestroy, CanComponentDeactivate {
+export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy, AfterViewInit, CanComponentDeactivate {
   @Output() firstStepDone = new EventEmitter<string>()
   @Output() firstStepError = new EventEmitter<void>()
   @ViewChild('videofileInput') videofileInput: ElementRef<HTMLInputElement>
 
-  // So that it can be accessed in the template
-  readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY
-
   userVideoQuotaUsed = 0
   userVideoQuotaUsedDaily = 0
 
   isUploadingAudioFile = false
   isUploadingVideo = false
-  isUpdatingVideo = false
 
   videoUploaded = false
-  videoUploadObservable: Subscription = null
   videoUploadPercents = 0
   videoUploadedIds = {
     id: 0,
@@ -49,7 +45,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
   error: string
   enableRetryAfterError: boolean
 
+  // So that it can be accessed in the template
   protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC
+  protected readonly BASE_VIDEO_UPLOAD_URL = VideoService.BASE_VIDEO_URL + 'upload-resumable'
+
+  private uploadxOptions: UploadxOptions
+  private isUpdatingVideo = false
+  private fileToUpload: File
 
   constructor (
     protected formValidatorService: FormValidatorService,
@@ -61,15 +63,77 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
     protected videoCaptionService: VideoCaptionService,
     private userService: UserService,
     private router: Router,
-    private hooks: HooksService
-    ) {
+    private hooks: HooksService,
+    private resumableUploadService: UploadxService
+  ) {
     super()
+
+    this.uploadxOptions = {
+      endpoint: this.BASE_VIDEO_UPLOAD_URL,
+      multiple: false,
+      token: this.authService.getAccessToken(),
+      uploaderClass: UploaderXFormData,
+      retryConfig: {
+        maxAttempts: 6,
+        shouldRetry: (code: number) => {
+          return code < 400 || code >= 501
+        }
+      }
+    }
   }
 
   get videoExtensions () {
     return this.serverConfig.video.file.extensions.join(', ')
   }
 
+  onUploadVideoOngoing (state: UploadState) {
+    switch (state.status) {
+      case 'error':
+        const error = state.response?.error || 'Unknow error'
+
+        this.handleUploadError({
+          error: new Error(error),
+          name: 'HttpErrorResponse',
+          message: error,
+          ok: false,
+          headers: new HttpHeaders(state.responseHeaders),
+          status: +state.responseStatus,
+          statusText: error,
+          type: HttpEventType.Response,
+          url: state.url
+        })
+        break
+
+      case 'cancelled':
+        this.isUploadingVideo = false
+        this.videoUploadPercents = 0
+
+        this.firstStepError.emit()
+        this.enableRetryAfterError = false
+        this.error = ''
+        break
+
+      case 'queue':
+        this.closeFirstStep(state.name)
+        break
+
+      case 'uploading':
+        this.videoUploadPercents = state.progress
+        break
+
+      case 'paused':
+        this.notifier.info($localize`Upload cancelled`)
+        break
+
+      case 'complete':
+        this.videoUploaded = true
+        this.videoUploadPercents = 100
+
+        this.videoUploadedIds = state?.response.video
+        break
+    }
+  }
+
   ngOnInit () {
     super.ngOnInit()
 
@@ -78,6 +142,9 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
           this.userVideoQuotaUsed = data.videoQuotaUsed
           this.userVideoQuotaUsedDaily = data.videoQuotaUsedDaily
         })
+
+    this.resumableUploadService.events
+      .subscribe(state => this.onUploadVideoOngoing(state))
   }
 
   ngAfterViewInit () {
@@ -85,7 +152,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
   }
 
   ngOnDestroy () {
-    if (this.videoUploadObservable) this.videoUploadObservable.unsubscribe()
+    this.cancelUpload()
   }
 
   canDeactivate () {
@@ -105,137 +172,43 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
     }
   }
 
-  getVideoFile () {
-    return this.videofileInput.nativeElement.files[0]
-  }
-
-  setVideoFile (files: FileList) {
+  onFileDropped (files: FileList) {
     this.videofileInput.nativeElement.files = files
-    this.fileChange()
-  }
-
-  getAudioUploadLabel () {
-    const videofile = this.getVideoFile()
-    if (!videofile) return $localize`Upload`
 
-    return $localize`Upload ${videofile.name}`
+    this.onFileChange({ target: this.videofileInput.nativeElement })
   }
 
-  fileChange () {
-    this.uploadFirstStep()
-  }
-
-  retryUpload () {
-    this.enableRetryAfterError = false
-    this.error = ''
-    this.uploadVideo()
-  }
-
-  cancelUpload () {
-    if (this.videoUploadObservable !== null) {
-      this.videoUploadObservable.unsubscribe()
-    }
-
-    this.isUploadingVideo = false
-    this.videoUploadPercents = 0
-    this.videoUploadObservable = null
+  onFileChange (event: Event | { target: HTMLInputElement }) {
+    const file = (event.target as HTMLInputElement).files[0]
 
-    this.firstStepError.emit()
-    this.enableRetryAfterError = false
-    this.error = ''
+    if (!file) return
 
-    this.notifier.info($localize`Upload cancelled`)
-  }
+    if (!this.checkGlobalUserQuota(file)) return
+    if (!this.checkDailyUserQuota(file)) return
 
-  uploadFirstStep (clickedOnButton = false) {
-    const videofile = this.getVideoFile()
-    if (!videofile) return
-
-    if (!this.checkGlobalUserQuota(videofile)) return
-    if (!this.checkDailyUserQuota(videofile)) return
-
-    if (clickedOnButton === false && this.isAudioFile(videofile.name)) {
+    if (this.isAudioFile(file.name)) {
       this.isUploadingAudioFile = true
       return
     }
 
-    // Build name field
-    const nameWithoutExtension = videofile.name.replace(/\.[^/.]+$/, '')
-    let name: string
-
-    // If the name of the file is very small, keep the extension
-    if (nameWithoutExtension.length < 3) name = videofile.name
-    else name = nameWithoutExtension
-
-    const nsfw = this.serverConfig.instance.isNSFW
-    const waitTranscoding = true
-    const commentsEnabled = true
-    const downloadEnabled = true
-    const channelId = this.firstStepChannelId.toString()
-
-    this.formData = new FormData()
-    this.formData.append('name', name)
-    // Put the video "private" -> we are waiting the user validation of the second step
-    this.formData.append('privacy', VideoPrivacy.PRIVATE.toString())
-    this.formData.append('nsfw', '' + nsfw)
-    this.formData.append('commentsEnabled', '' + commentsEnabled)
-    this.formData.append('downloadEnabled', '' + downloadEnabled)
-    this.formData.append('waitTranscoding', '' + waitTranscoding)
-    this.formData.append('channelId', '' + channelId)
-    this.formData.append('videofile', videofile)
-
-    if (this.previewfileUpload) {
-      this.formData.append('previewfile', this.previewfileUpload)
-      this.formData.append('thumbnailfile', this.previewfileUpload)
-    }
-
     this.isUploadingVideo = true
-    this.firstStepDone.emit(name)
-
-    this.form.patchValue({
-      name,
-      privacy: this.firstStepPrivacyId,
-      nsfw,
-      channelId: this.firstStepChannelId,
-      previewfile: this.previewfileUpload
-    })
+    this.fileToUpload = file
 
-    this.uploadVideo()
+    this.uploadFile(file)
   }
 
-  uploadVideo () {
-    this.videoUploadObservable = this.videoService.uploadVideo(this.formData).subscribe(
-      event => {
-        if (event.type === HttpEventType.UploadProgress) {
-          this.videoUploadPercents = Math.round(100 * event.loaded / event.total)
-        } else if (event instanceof HttpResponse) {
-          this.videoUploaded = true
-
-          this.videoUploadedIds = event.body.video
-
-          this.videoUploadObservable = null
-        }
-      },
+  uploadAudio () {
+    this.uploadFile(this.getInputVideoFile(), this.previewfileUpload)
+  }
 
-      (err: HttpErrorResponse) => {
-        // Reset progress (but keep isUploadingVideo true)
-        this.videoUploadPercents = 0
-        this.videoUploadObservable = null
-        this.enableRetryAfterError = true
-
-        this.error = uploadErrorHandler({
-          err,
-          name: $localize`video`,
-          notifier: this.notifier,
-          sticky: false
-        })
+  retryUpload () {
+    this.enableRetryAfterError = false
+    this.error = ''
+    this.uploadFile(this.fileToUpload)
+  }
 
-        if (err.status === HttpStatusCode.PAYLOAD_TOO_LARGE_413 ||
-            err.status === HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) {
-          this.cancelUpload()
-        }
-      }
-    )
+  cancelUpload () {
+    this.resumableUploadService.control({ action: 'cancel' })
   }
 
   isPublishingButtonDisabled () {
@@ -245,6 +218,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
       !this.videoUploadedIds.id
   }
 
+  getAudioUploadLabel () {
+    const videofile = this.getInputVideoFile()
+    if (!videofile) return $localize`Upload`
+
+    return $localize`Upload ${videofile.name}`
+  }
+
   updateSecondStep () {
     if (this.isPublishingButtonDisabled() || !this.checkForm()) {
       return
@@ -275,6 +255,62 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
         )
   }
 
+  private getInputVideoFile () {
+    return this.videofileInput.nativeElement.files[0]
+  }
+
+  private uploadFile (file: File, previewfile?: File) {
+    const metadata = {
+      waitTranscoding: true,
+      commentsEnabled: true,
+      downloadEnabled: true,
+      channelId: this.firstStepChannelId,
+      nsfw: this.serverConfig.instance.isNSFW,
+      privacy: VideoPrivacy.PRIVATE.toString(),
+      filename: file.name,
+      previewfile: previewfile as any
+    }
+
+    this.resumableUploadService.handleFiles(file, {
+      ...this.uploadxOptions,
+      metadata
+    })
+
+    this.isUploadingVideo = true
+  }
+
+  private handleUploadError (err: HttpErrorResponse) {
+    // Reset progress (but keep isUploadingVideo true)
+    this.videoUploadPercents = 0
+    this.enableRetryAfterError = true
+
+    this.error = genericUploadErrorHandler({
+      err,
+      name: $localize`video`,
+      notifier: this.notifier,
+      sticky: false
+    })
+
+    if (err.status === HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) {
+      this.cancelUpload()
+    }
+  }
+
+  private closeFirstStep (filename: string) {
+    const nameWithoutExtension = filename.replace(/\.[^/.]+$/, '')
+    const name = nameWithoutExtension.length < 3 ? filename : nameWithoutExtension
+
+    this.form.patchValue({
+      name,
+      privacy: this.firstStepPrivacyId,
+      nsfw: this.serverConfig.instance.isNSFW,
+      channelId: this.firstStepChannelId,
+      previewfile: this.previewfileUpload
+    })
+
+    this.firstStepDone.emit(name)
+  }
+
   private checkGlobalUserQuota (videofile: File) {
     const bytePipes = new BytesPipe()
 
@@ -285,8 +321,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView
       const videoQuotaUsedBytes = bytePipes.transform(this.userVideoQuotaUsed, 0)
       const videoQuotaBytes = bytePipes.transform(videoQuota, 0)
 
-      const msg = $localize`Your video quota is exceeded with this video (
-video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})`
+      const msg = $localize`Your video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})`
       this.notifier.error(msg)
 
       return false
@@ -304,9 +339,7 @@ video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuota
       const videoSizeBytes = bytePipes.transform(videofile.size, 0)
       const quotaUsedDailyBytes = bytePipes.transform(this.userVideoQuotaUsedDaily, 0)
       const quotaDailyBytes = bytePipes.transform(videoQuotaDaily, 0)
-
-      const msg = $localize`Your daily video quota is exceeded with this video (
-video size: ${videoSizeBytes}, used: ${quotaUsedDailyBytes}, quota: ${quotaDailyBytes})`
+      const msg = $localize`Your daily video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${quotaUsedDailyBytes}, quota: ${quotaDailyBytes})`
       this.notifier.error(msg)
 
       return false
diff --git a/client/src/app/+videos/+video-edit/video-add.module.ts b/client/src/app/+videos/+video-edit/video-add.module.ts
index da651119b..e836cf81e 100644
--- a/client/src/app/+videos/+video-edit/video-add.module.ts
+++ b/client/src/app/+videos/+video-edit/video-add.module.ts
@@ -1,5 +1,6 @@
 import { NgModule } from '@angular/core'
 import { CanDeactivateGuard } from '@app/core'
+import { UploadxModule } from 'ngx-uploadx'
 import { VideoEditModule } from './shared/video-edit.module'
 import { DragDropDirective } from './video-add-components/drag-drop.directive'
 import { VideoImportTorrentComponent } from './video-add-components/video-import-torrent.component'
@@ -13,7 +14,9 @@ import { VideoAddComponent } from './video-add.component'
   imports: [
     VideoAddRoutingModule,
 
-    VideoEditModule
+    VideoEditModule,
+
+    UploadxModule
   ],
 
   declarations: [