Merge branch 'develop' into shorter-URLs-channels-accounts

8 files changed, 180 insertions, 62 deletions

diff --git a/server/middlewares/validators/follows.ts b/server/middlewares/validators/follows.ts
index bb849dc72..1d18de8cd 100644
--- a/server/middlewares/validators/follows.ts
+++ b/server/middlewares/validators/follows.ts
@@ -1,18 +1,18 @@
 import * as express from 'express'
 import { body, param, query } from 'express-validator'
+import { isFollowStateValid } from '@server/helpers/custom-validators/follows'
+import { getServerActor } from '@server/models/application/application'
+import { MActorFollowActorsDefault } from '@server/types/models'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 import { isTestInstance } from '../../helpers/core-utils'
+import { isActorTypeValid, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor'
 import { isEachUniqueHostValid, isHostValid } from '../../helpers/custom-validators/servers'
 import { logger } from '../../helpers/logger'
+import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
 import { SERVER_ACTOR_NAME, WEBSERVER } from '../../initializers/constants'
-import { ActorFollowModel } from '../../models/activitypub/actor-follow'
+import { ActorModel } from '../../models/actor/actor'
+import { ActorFollowModel } from '../../models/actor/actor-follow'
 import { areValidationErrors } from './utils'
-import { ActorModel } from '../../models/activitypub/actor'
-import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
-import { isActorTypeValid, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor'
-import { MActorFollowActorsDefault } from '@server/types/models'
-import { isFollowStateValid } from '@server/helpers/custom-validators/follows'
-import { getServerActor } from '@server/models/application/application'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 
 const listFollowsValidator = [
   query('state')
diff --git a/server/middlewares/validators/plugins.ts b/server/middlewares/validators/plugins.ts
index ab87fe720..2c47ec5bb 100644
--- a/server/middlewares/validators/plugins.ts
+++ b/server/middlewares/validators/plugins.ts
@@ -1,15 +1,15 @@
 import * as express from 'express'
 import { body, param, query, ValidationChain } from 'express-validator'
-import { logger } from '../../helpers/logger'
-import { areValidationErrors } from './utils'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { PluginType } from '../../../shared/models/plugins/plugin.type'
+import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/server/api/install-plugin.model'
+import { exists, isBooleanValid, isSafePath, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc'
 import { isNpmPluginNameValid, isPluginNameValid, isPluginTypeValid, isPluginVersionValid } from '../../helpers/custom-validators/plugins'
+import { logger } from '../../helpers/logger'
+import { CONFIG } from '../../initializers/config'
 import { PluginManager } from '../../lib/plugins/plugin-manager'
-import { isBooleanValid, isSafePath, toBooleanOrNull, exists, toIntOrNull } from '../../helpers/custom-validators/misc'
 import { PluginModel } from '../../models/server/plugin'
-import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/install-plugin.model'
-import { PluginType } from '../../../shared/models/plugins/plugin.type'
-import { CONFIG } from '../../initializers/config'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { areValidationErrors } from './utils'
 
 const getPluginValidator = (pluginType: PluginType, withVersion = true) => {
   const validators: (ValidationChain | express.Handler)[] = [
diff --git a/server/middlewares/validators/user-subscriptions.ts b/server/middlewares/validators/user-subscriptions.ts
index 0d0c8ccbf..1823892b6 100644
--- a/server/middlewares/validators/user-subscriptions.ts
+++ b/server/middlewares/validators/user-subscriptions.ts
@@ -1,12 +1,12 @@
 import * as express from 'express'
 import { body, param, query } from 'express-validator'
-import { logger } from '../../helpers/logger'
-import { areValidationErrors } from './utils'
-import { ActorFollowModel } from '../../models/activitypub/actor-follow'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 import { areValidActorHandles, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor'
 import { toArray } from '../../helpers/custom-validators/misc'
+import { logger } from '../../helpers/logger'
 import { WEBSERVER } from '../../initializers/constants'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { ActorFollowModel } from '../../models/actor/actor-follow'
+import { areValidationErrors } from './utils'
 
 const userSubscriptionListValidator = [
   query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts
index 9cff51d45..548d5df4d 100644
--- a/server/middlewares/validators/users.ts
+++ b/server/middlewares/validators/users.ts
@@ -34,8 +34,8 @@ import { doesVideoExist } from '../../helpers/middlewares'
 import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/signup'
 import { isThemeRegistered } from '../../lib/plugins/theme-utils'
 import { Redis } from '../../lib/redis'
-import { UserModel } from '../../models/account/user'
-import { ActorModel } from '../../models/activitypub/actor'
+import { UserModel } from '../../models/user/user'
+import { ActorModel } from '../../models/actor/actor'
 import { areValidationErrors } from './utils'
 
 const usersListValidator = [
@@ -196,6 +196,7 @@ const deleteMeValidator = [
 
 const usersUpdateValidator = [
   param('id').isInt().not().isEmpty().withMessage('Should have a valid id'),
+
   body('password').optional().custom(isUserPasswordValid).withMessage('Should have a valid password'),
   body('email').optional().isEmail().withMessage('Should have a valid email attribute'),
   body('emailVerified').optional().isBoolean().withMessage('Should have a valid email verified attribute'),
diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts
index 2463d281c..e881f0d3e 100644
--- a/server/middlewares/validators/videos/video-channels.ts
+++ b/server/middlewares/validators/videos/video-channels.ts
@@ -3,6 +3,7 @@ import { body, param, query } from 'express-validator'
 import { VIDEO_CHANNELS } from '@server/initializers/constants'
 import { MChannelAccountDefault, MUser } from '@server/types/models'
 import { UserRight } from '../../../../shared'
+import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
 import { isActorPreferredUsernameValid } from '../../../helpers/custom-validators/activitypub/actor'
 import { isBooleanValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc'
 import {
@@ -12,10 +13,9 @@ import {
 } from '../../../helpers/custom-validators/video-channels'
 import { logger } from '../../../helpers/logger'
 import { doesLocalVideoChannelNameExist, doesVideoChannelNameWithHostExist } from '../../../helpers/middlewares'
-import { ActorModel } from '../../../models/activitypub/actor'
+import { ActorModel } from '../../../models/actor/actor'
 import { VideoChannelModel } from '../../../models/video/video-channel'
 import { areValidationErrors } from '../utils'
-import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
 
 const videoChannelsAddValidator = [
   body('name').custom(isActorPreferredUsernameValid).withMessage('Should have a valid channel name'),
diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts
index c53af3861..d0643ff26 100644
--- a/server/middlewares/validators/videos/video-imports.ts
+++ b/server/middlewares/validators/videos/video-imports.ts
@@ -47,14 +47,12 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([
       cleanUpReqFiles(req)
       return res.status(HttpStatusCode.CONFLICT_409)
         .json({ error: 'HTTP import is not enabled on this instance.' })
-        .end()
     }
 
     if (CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED !== true && (req.body.magnetUri || torrentFile)) {
       cleanUpReqFiles(req)
       return res.status(HttpStatusCode.CONFLICT_409)
                 .json({ error: 'Torrent/magnet URI import is not enabled on this instance.' })
-                .end()
     }
 
     if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
@@ -65,7 +63,6 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([
 
       return res.status(HttpStatusCode.BAD_REQUEST_400)
         .json({ error: 'Should have a magnetUri or a targetUrl or a torrent file.' })
-        .end()
     }
 
     if (!await isImportAccepted(req, res)) return cleanUpReqFiles(req)
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index bb617d77c..3219e10d4 100644
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -1,12 +1,13 @@
 import * as express from 'express'
-import { body, param, query, ValidationChain } from 'express-validator'
+import { body, header, param, query, ValidationChain } from 'express-validator'
+import { getResumableUploadPath } from '@server/helpers/upload'
 import { isAbleToUploadVideo } from '@server/lib/user'
 import { getServerActor } from '@server/models/application/application'
 import { ExpressPromiseHandler } from '@server/types/express'
-import { MVideoWithRights } from '@server/types/models'
+import { MUserAccountId, MVideoWithRights } from '@server/types/models'
 import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared'
 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
-import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model'
+import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/change-ownership/video-change-ownership-accept.model'
 import {
   exists,
   isBooleanValid,
@@ -47,6 +48,7 @@ import {
   doesVideoExist,
   doesVideoFileOfVideoExist
 } from '../../../helpers/middlewares'
+import { deleteFileAndCatch } from '../../../helpers/utils'
 import { getVideoWithAttributes } from '../../../helpers/video'
 import { CONFIG } from '../../../initializers/config'
 import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants'
@@ -57,7 +59,7 @@ import { VideoModel } from '../../../models/video/video'
 import { authenticatePromiseIfNeeded } from '../../auth'
 import { areValidationErrors } from '../utils'
 
-const videosAddValidator = getCommonVideoEditAttributes().concat([
+const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([
   body('videofile')
     .custom((value, { req }) => isFileFieldValid(req.files, 'videofile'))
     .withMessage('Should have a file'),
@@ -73,54 +75,117 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([
     logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files })
 
     if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
-    if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
 
-    const videoFile: Express.Multer.File & { duration?: number } = req.files['videofile'][0]
+    const videoFile: express.VideoUploadFile = req.files['videofile'][0]
     const user = res.locals.oauth.token.User
 
-    if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
-
-    if (!isVideoFileMimeTypeValid(req.files)) {
-      res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415)
-         .json({
-           error: 'This file is not supported. Please, make sure it is of the following type: ' +
-                  CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')
-         })
-
+    if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files })) {
       return cleanUpReqFiles(req)
     }
 
-    if (!isVideoFileSizeValid(videoFile.size.toString())) {
-      res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
-         .json({
-           error: 'This file is too large.'
-         })
+    try {
+      if (!videoFile.duration) await addDurationToVideo(videoFile)
+    } catch (err) {
+      logger.error('Invalid input file in videosAddLegacyValidator.', { err })
+      res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422)
+         .json({ error: 'Video file unreadable.' })
 
       return cleanUpReqFiles(req)
     }
 
-    if (await isAbleToUploadVideo(user.id, videoFile.size) === false) {
-      res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
-         .json({ error: 'The user video quota is exceeded with this video.' })
+    if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req)
 
-      return cleanUpReqFiles(req)
-    }
+    return next()
+  }
+])
+
+/**
+ * Gets called after the last PUT request
+ */
+const videosAddResumableValidator = [
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    const user = res.locals.oauth.token.User
+
+    const body: express.CustomUploadXFile<express.UploadXFileMetadata> = req.body
+    const file = { ...body, duration: undefined, path: getResumableUploadPath(body.id), filename: body.metadata.filename }
+
+    const cleanup = () => deleteFileAndCatch(file.path)
 
-    let duration: number
+    if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup()
 
     try {
-      duration = await getDurationFromVideoFile(videoFile.path)
+      if (!file.duration) await addDurationToVideo(file)
     } catch (err) {
-      logger.error('Invalid input file in videosAddValidator.', { err })
+      logger.error('Invalid input file in videosAddResumableValidator.', { err })
       res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422)
          .json({ error: 'Video file unreadable.' })
 
-      return cleanUpReqFiles(req)
+      return cleanup()
     }
 
-    videoFile.duration = duration
+    if (!await isVideoAccepted(req, res, file)) return cleanup()
 
-    if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req)
+    res.locals.videoFileResumable = file
+
+    return next()
+  }
+]
+
+/**
+ * File is created in POST initialisation, and its body is saved as a 'metadata' field is saved by uploadx for later use.
+ * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/uploadx.ts
+ *
+ * Uploadx doesn't use next() until the upload completes, so this middleware has to be placed before uploadx
+ * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/base-handler.ts
+ *
+ */
+const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([
+  body('filename')
+    .isString()
+    .exists()
+    .withMessage('Should have a valid filename'),
+  body('name')
+    .trim()
+    .custom(isVideoNameValid)
+    .withMessage('Should have a valid name'),
+  body('channelId')
+    .customSanitizer(toIntOrNull)
+    .custom(isIdValid).withMessage('Should have correct video channel id'),
+
+  header('x-upload-content-length')
+    .isNumeric()
+    .exists()
+    .withMessage('Should specify the file length'),
+  header('x-upload-content-type')
+    .isString()
+    .exists()
+    .withMessage('Should specify the file mimetype'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    const videoFileMetadata = {
+      mimetype: req.headers['x-upload-content-type'] as string,
+      size: +req.headers['x-upload-content-length'],
+      originalname: req.body.name
+    }
+
+    const user = res.locals.oauth.token.User
+    const cleanup = () => cleanUpReqFiles(req)
+
+    logger.debug('Checking videosAddResumableInitValidator parameters and headers', {
+      parameters: req.body,
+      headers: req.headers,
+      files: req.files
+    })
+
+    if (areValidationErrors(req, res)) return cleanup()
+
+    const files = { videofile: [ videoFileMetadata ] }
+    if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup()
+
+    // multer required unsetting the Content-Type, now we can set it for node-uploadx
+    req.headers['content-type'] = 'application/json; charset=utf-8'
+    // place previewfile in metadata so that uploadx saves it in .META
+    if (req.files['previewfile']) req.body.previewfile = req.files['previewfile']
 
     return next()
   }
@@ -478,7 +543,10 @@ const commonVideosFiltersValidator = [
 // ---------------------------------------------------------------------------
 
 export {
-  videosAddValidator,
+  videosAddLegacyValidator,
+  videosAddResumableValidator,
+  videosAddResumableInitValidator,
+
   videosUpdateValidator,
   videosGetValidator,
   videoFileMetadataGetValidator,
@@ -515,7 +583,51 @@ function areErrorsInScheduleUpdate (req: express.Request, res: express.Response)
   return false
 }
 
-async function isVideoAccepted (req: express.Request, res: express.Response, videoFile: Express.Multer.File & { duration?: number }) {
+async function commonVideoChecksPass (parameters: {
+  req: express.Request
+  res: express.Response
+  user: MUserAccountId
+  videoFileSize: number
+  files: express.UploadFilesForCheck
+}): Promise<boolean> {
+  const { req, res, user, videoFileSize, files } = parameters
+
+  if (areErrorsInScheduleUpdate(req, res)) return false
+
+  if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false
+
+  if (!isVideoFileMimeTypeValid(files)) {
+    res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415)
+        .json({
+          error: 'This file is not supported. Please, make sure it is of the following type: ' +
+                CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')
+        })
+
+    return false
+  }
+
+  if (!isVideoFileSizeValid(videoFileSize.toString())) {
+    res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
+        .json({ error: 'This file is too large. It exceeds the maximum file size authorized.' })
+
+    return false
+  }
+
+  if (await isAbleToUploadVideo(user.id, videoFileSize) === false) {
+    res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
+        .json({ error: 'The user video quota is exceeded with this video.' })
+
+    return false
+  }
+
+  return true
+}
+
+export async function isVideoAccepted (
+  req: express.Request,
+  res: express.Response,
+  videoFile: express.VideoUploadFile
+) {
   // Check we accept this video
   const acceptParameters = {
     videoBody: req.body,
@@ -538,3 +650,11 @@ async function isVideoAccepted (req: express.Request, res: express.Response, vid
 
   return true
 }
+
+async function addDurationToVideo (videoFile: { path: string, duration?: number }) {
+  const duration: number = await getDurationFromVideoFile(videoFile.path)
+
+  if (isNaN(duration)) throw new Error(`Couldn't get video duration`)
+
+  videoFile.duration = duration
+}
diff --git a/server/middlewares/validators/webfinger.ts b/server/middlewares/validators/webfinger.ts
index a71422ed8..c2dfccc96 100644
--- a/server/middlewares/validators/webfinger.ts
+++ b/server/middlewares/validators/webfinger.ts
@@ -1,11 +1,11 @@
 import * as express from 'express'
 import { query } from 'express-validator'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 import { isWebfingerLocalResourceValid } from '../../helpers/custom-validators/webfinger'
+import { getHostWithPort } from '../../helpers/express-utils'
 import { logger } from '../../helpers/logger'
-import { ActorModel } from '../../models/activitypub/actor'
+import { ActorModel } from '../../models/actor/actor'
 import { areValidationErrors } from './utils'
-import { getHostWithPort } from '../../helpers/express-utils'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 
 const webfingerValidator = [
   query('resource').custom(isWebfingerLocalResourceValid).withMessage('Should have a valid webfinger resource'),