aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers
diff options
context:
space:
mode:
authorkontrollanten <6680299+kontrollanten@users.noreply.github.com>2021-05-10 11:13:41 +0200
committerGitHub <noreply@github.com>2021-05-10 11:13:41 +0200
commitf6d6e7f861189a4446f406efb775a29688764b48 (patch)
treec3dda9958c3f189d4c39e8743c738d8c1fef4c2d /server/helpers
parentd29ced1a8582d99b776f664475a157adcf555d98 (diff)
downloadPeerTube-f6d6e7f861189a4446f406efb775a29688764b48.tar.gz
PeerTube-f6d6e7f861189a4446f406efb775a29688764b48.tar.zst
PeerTube-f6d6e7f861189a4446f406efb775a29688764b48.zip
Resumable video uploads (#3933)
* WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/helpers')
-rw-r--r--server/helpers/custom-validators/misc.ts5
-rw-r--r--server/helpers/custom-validators/videos.ts9
-rw-r--r--server/helpers/express-utils.ts8
-rw-r--r--server/helpers/upload.ts21
-rw-r--r--server/helpers/utils.ts4
5 files changed, 35 insertions, 12 deletions
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index effdd98cb..fd3b45804 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -1,6 +1,7 @@
1import 'multer' 1import 'multer'
2import validator from 'validator' 2import { UploadFilesForCheck } from 'express'
3import { sep } from 'path' 3import { sep } from 'path'
4import validator from 'validator'
4 5
5function exists (value: any) { 6function exists (value: any) {
6 return value !== undefined && value !== null 7 return value !== undefined && value !== null
@@ -108,7 +109,7 @@ function isFileFieldValid (
108} 109}
109 110
110function isFileMimeTypeValid ( 111function isFileMimeTypeValid (
111 files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[], 112 files: UploadFilesForCheck,
112 mimeTypeRegex: string, 113 mimeTypeRegex: string,
113 field: string, 114 field: string,
114 optional = false 115 optional = false
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 87966798f..b33e088eb 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -1,4 +1,6 @@
1import { UploadFilesForCheck } from 'express'
1import { values } from 'lodash' 2import { values } from 'lodash'
3import * as magnetUtil from 'magnet-uri'
2import validator from 'validator' 4import validator from 'validator'
3import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared' 5import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
4import { 6import {
@@ -6,13 +8,12 @@ import {
6 MIMETYPES, 8 MIMETYPES,
7 VIDEO_CATEGORIES, 9 VIDEO_CATEGORIES,
8 VIDEO_LICENCES, 10 VIDEO_LICENCES,
11 VIDEO_LIVE,
9 VIDEO_PRIVACIES, 12 VIDEO_PRIVACIES,
10 VIDEO_RATE_TYPES, 13 VIDEO_RATE_TYPES,
11 VIDEO_STATES, 14 VIDEO_STATES
12 VIDEO_LIVE
13} from '../../initializers/constants' 15} from '../../initializers/constants'
14import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc' 16import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc'
15import * as magnetUtil from 'magnet-uri'
16 17
17const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS 18const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
18 19
@@ -81,7 +82,7 @@ function isVideoFileExtnameValid (value: string) {
81 return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined) 82 return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined)
82} 83}
83 84
84function isVideoFileMimeTypeValid (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { 85function isVideoFileMimeTypeValid (files: UploadFilesForCheck) {
85 return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile') 86 return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile')
86} 87}
87 88
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index c0d3f8f32..ede22a3cc 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -2,7 +2,7 @@ import * as express from 'express'
2import * as multer from 'multer' 2import * as multer from 'multer'
3import { REMOTE_SCHEME } from '../initializers/constants' 3import { REMOTE_SCHEME } from '../initializers/constants'
4import { logger } from './logger' 4import { logger } from './logger'
5import { deleteFileAsync, generateRandomString } from './utils' 5import { deleteFileAndCatch, generateRandomString } from './utils'
6import { extname } from 'path' 6import { extname } from 'path'
7import { isArray } from './custom-validators/misc' 7import { isArray } from './custom-validators/misc'
8import { CONFIG } from '../initializers/config' 8import { CONFIG } from '../initializers/config'
@@ -36,15 +36,15 @@ function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.Fi
36 if (!files) return 36 if (!files) return
37 37
38 if (isArray(files)) { 38 if (isArray(files)) {
39 (files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path)) 39 (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path))
40 return 40 return
41 } 41 }
42 42
43 for (const key of Object.keys(files)) { 43 for (const key of Object.keys(files)) {
44 const file = files[key] 44 const file = files[key]
45 45
46 if (isArray(file)) file.forEach(f => deleteFileAsync(f.path)) 46 if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path))
47 else deleteFileAsync(file.path) 47 else deleteFileAndCatch(file.path)
48 } 48 }
49} 49}
50 50
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts
new file mode 100644
index 000000000..030a6b7d5
--- /dev/null
+++ b/server/helpers/upload.ts
@@ -0,0 +1,21 @@
1import { METAFILE_EXTNAME } from '@uploadx/core'
2import { remove } from 'fs-extra'
3import { join } from 'path'
4import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
5
6function getResumableUploadPath (filename?: string) {
7 if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename)
8
9 return RESUMABLE_UPLOAD_DIRECTORY
10}
11
12function deleteResumableUploadMetaFile (filepath: string) {
13 return remove(filepath + METAFILE_EXTNAME)
14}
15
16// ---------------------------------------------------------------------------
17
18export {
19 getResumableUploadPath,
20 deleteResumableUploadMetaFile
21}
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 0545e8996..6c95a43b6 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -6,7 +6,7 @@ import { CONFIG } from '../initializers/config'
6import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' 6import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
7import { logger } from './logger' 7import { logger } from './logger'
8 8
9function deleteFileAsync (path: string) { 9function deleteFileAndCatch (path: string) {
10 remove(path) 10 remove(path)
11 .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) 11 .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
12} 12}
@@ -83,7 +83,7 @@ function getUUIDFromFilename (filename: string) {
83// --------------------------------------------------------------------------- 83// ---------------------------------------------------------------------------
84 84
85export { 85export {
86 deleteFileAsync, 86 deleteFileAndCatch,
87 generateRandomString, 87 generateRandomString,
88 getFormattedObjects, 88 getFormattedObjects,
89 getSecureTorrentName, 89 getSecureTorrentName,