Merge branch 'develop' into shorter-URLs-channels-accounts

10 files changed, 318 insertions, 69 deletions

diff --git a/shared/extra-utils/custom-pages/custom-pages.ts b/shared/extra-utils/custom-pages/custom-pages.ts
new file mode 100644
index 000000000..bf2d16c70
--- /dev/null
+++ b/shared/extra-utils/custom-pages/custom-pages.ts
@@ -0,0 +1,31 @@
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { makeGetRequest, makePutBodyRequest } from '../requests/requests'
+
+function getInstanceHomepage (url: string, statusCodeExpected = HttpStatusCode.OK_200) {
+  const path = '/api/v1/custom-pages/homepage/instance'
+
+  return makeGetRequest({
+    url,
+    path,
+    statusCodeExpected
+  })
+}
+
+function updateInstanceHomepage (url: string, token: string, content: string) {
+  const path = '/api/v1/custom-pages/homepage/instance'
+
+  return makePutBodyRequest({
+    url,
+    path,
+    token,
+    fields: { content },
+    statusCodeExpected: HttpStatusCode.NO_CONTENT_204
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getInstanceHomepage,
+  updateInstanceHomepage
+}
diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts
index 5c520d0c2..9f5b5bb28 100644
--- a/shared/extra-utils/index.ts
+++ b/shared/extra-utils/index.ts
@@ -1,16 +1,27 @@
 export * from './actors/actors'
 export * from './bulk/bulk'
+
 export * from './cli/cli'
+
+export * from './custom-pages/custom-pages'
+
 export * from './feeds/feeds'
+
 export * from './mock-servers/mock-instances-index'
-export * from './miscs/miscs'
+
+export * from './miscs/email'
 export * from './miscs/sql'
+export * from './miscs/miscs'
 export * from './miscs/stubs'
+
 export * from './moderation/abuses'
 export * from './plugins/mock-blocklist'
+
 export * from './requests/check-api-params'
 export * from './requests/requests'
+
 export * from './search/videos'
+
 export * from './server/activitypub'
 export * from './server/clients'
 export * from './server/config'
@@ -19,9 +30,14 @@ export * from './server/follows'
 export * from './server/jobs'
 export * from './server/plugins'
 export * from './server/servers'
+
 export * from './users/accounts'
+export * from './users/blocklist'
 export * from './users/login'
+export * from './users/user-notifications'
+export * from './users/user-subscriptions'
 export * from './users/users'
+
 export * from './videos/live'
 export * from './videos/services'
 export * from './videos/video-blacklist'
diff --git a/shared/extra-utils/server/config.ts b/shared/extra-utils/server/config.ts
index 026a5e61c..b70110852 100644
--- a/shared/extra-utils/server/config.ts
+++ b/shared/extra-utils/server/config.ts
@@ -223,6 +223,18 @@ function updateCustomSubConfig (url: string, token: string, newConfig: DeepParti
   return updateCustomConfig(url, token, updateParams)
 }
 
+function getCustomConfigResolutions (enabled: boolean) {
+  return {
+    '240p': enabled,
+    '360p': enabled,
+    '480p': enabled,
+    '720p': enabled,
+    '1080p': enabled,
+    '1440p': enabled,
+    '2160p': enabled
+  }
+}
+
 function deleteCustomConfig (url: string, token: string, statusCodeExpected = HttpStatusCode.OK_200) {
   const path = '/api/v1/config/custom'
 
@@ -242,5 +254,6 @@ export {
   updateCustomConfig,
   getAbout,
   deleteCustomConfig,
-  updateCustomSubConfig
+  updateCustomSubConfig,
+  getCustomConfigResolutions
 }
diff --git a/shared/extra-utils/server/debug.ts b/shared/extra-utils/server/debug.ts
index 5cf80a5fb..f196812b7 100644
--- a/shared/extra-utils/server/debug.ts
+++ b/shared/extra-utils/server/debug.ts
@@ -1,5 +1,6 @@
-import { makeGetRequest } from '../requests/requests'
+import { makeGetRequest, makePostBodyRequest } from '../requests/requests'
 import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes'
+import { SendDebugCommand } from '@shared/models'
 
 function getDebug (url: string, token: string) {
   const path = '/api/v1/server/debug'
@@ -12,8 +13,21 @@ function getDebug (url: string, token: string) {
   })
 }
 
+function sendDebugCommand (url: string, token: string, body: SendDebugCommand) {
+  const path = '/api/v1/server/debug/run-command'
+
+  return makePostBodyRequest({
+    url,
+    path,
+    token,
+    fields: body,
+    statusCodeExpected: HttpStatusCode.NO_CONTENT_204
+  })
+}
+
 // ---------------------------------------------------------------------------
 
 export {
-  getDebug
+  getDebug,
+  sendDebugCommand
 }
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts
index 704929bd4..763374e03 100644
--- a/shared/extra-utils/server/jobs.ts
+++ b/shared/extra-utils/server/jobs.ts
@@ -55,7 +55,7 @@ function getJobsListPaginationAndSort (options: {
 async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
   const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT
     ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10)
-    : 500
+    : 250
 
   let servers: ServerInfo[]
 
@@ -115,7 +115,7 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
     }
 
     if (pendingRequests) {
-      await wait(1000)
+      await wait(pendingJobWait)
     }
   } while (pendingRequests)
 }
diff --git a/shared/extra-utils/server/plugins.ts b/shared/extra-utils/server/plugins.ts
index 864954ee7..d53e5b382 100644
--- a/shared/extra-utils/server/plugins.ts
+++ b/shared/extra-utils/server/plugins.ts
@@ -4,12 +4,12 @@ import { expect } from 'chai'
 import { readJSON, writeJSON } from 'fs-extra'
 import { join } from 'path'
 import { RegisteredServerSettings } from '@shared/models'
-import { PeertubePluginIndexList } from '../../models/plugins/peertube-plugin-index-list.model'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { PeertubePluginIndexList } from '../../models/plugins/plugin-index/peertube-plugin-index-list.model'
 import { PluginType } from '../../models/plugins/plugin.type'
 import { buildServerDirectory, root } from '../miscs/miscs'
 import { makeGetRequest, makePostBodyRequest, makePutBodyRequest } from '../requests/requests'
 import { ServerInfo } from './servers'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 
 function listPlugins (parameters: {
   url: string
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts
index 779a3cc36..d04757470 100644
--- a/shared/extra-utils/server/servers.ts
+++ b/shared/extra-utils/server/servers.ts
@@ -45,9 +45,12 @@ interface ServerInfo {
     uuid: string
     name?: string
     url?: string
+
     account?: {
       name: string
     }
+
+    embedPath?: string
   }
 
   remoteVideo?: {
@@ -274,7 +277,7 @@ async function reRunServer (server: ServerInfo, configOverride?: any) {
 }
 
 async function checkTmpIsEmpty (server: ServerInfo) {
-  await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls' ])
+  await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ])
 
   if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) {
     await checkDirectoryIsEmpty(server, 'tmp/hls')
diff --git a/shared/extra-utils/users/users.ts b/shared/extra-utils/users/users.ts
index 6040dd9c0..0f15962ad 100644
--- a/shared/extra-utils/users/users.ts
+++ b/shared/extra-utils/users/users.ts
@@ -1,5 +1,6 @@
 import { omit } from 'lodash'
 import * as request from 'supertest'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 import { UserUpdateMe } from '../../models/users'
 import { UserAdminFlag } from '../../models/users/user-flag.model'
 import { UserRegister } from '../../models/users/user-register.model'
@@ -7,9 +8,8 @@ import { UserRole } from '../../models/users/user-role'
 import { makeGetRequest, makePostBodyRequest, makePutBodyRequest, updateImageRequest } from '../requests/requests'
 import { ServerInfo } from '../server/servers'
 import { userLogin } from './login'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 
-type CreateUserArgs = {
+function createUser (parameters: {
   url: string
   accessToken: string
   username: string
@@ -19,8 +19,7 @@ type CreateUserArgs = {
   role?: UserRole
   adminFlags?: UserAdminFlag
   specialStatus?: number
-}
-function createUser (parameters: CreateUserArgs) {
+}) {
   const {
     url,
     accessToken,
@@ -52,6 +51,21 @@ function createUser (parameters: CreateUserArgs) {
           .expect(specialStatus)
 }
 
+async function generateUser (server: ServerInfo, username: string) {
+  const password = 'my super password'
+  const resCreate = await createUser({ url: server.url, accessToken: server.accessToken, username: username, password: password })
+
+  const token = await userLogin(server, { username, password })
+
+  const resMe = await getMyUserInformation(server.url, token)
+
+  return {
+    token,
+    userId: resCreate.body.user.id,
+    userChannelId: resMe.body.videoChannels[0].id
+  }
+}
+
 async function generateUserAccessToken (server: ServerInfo, username: string) {
   const password = 'my super password'
   await createUser({ url: server.url, accessToken: server.accessToken, username: username, password: password })
@@ -393,6 +407,7 @@ export {
   resetPassword,
   renewUserScopedTokens,
   updateMyAvatar,
+  generateUser,
   askSendVerifyEmail,
   generateUserAccessToken,
   verifyEmail,
diff --git a/shared/extra-utils/videos/video-channels.ts b/shared/extra-utils/videos/video-channels.ts
index d0dfb5856..0aab93e52 100644
--- a/shared/extra-utils/videos/video-channels.ts
+++ b/shared/extra-utils/videos/video-channels.ts
@@ -5,7 +5,7 @@ import { VideoChannelUpdate } from '../../models/videos/channel/video-channel-up
 import { VideoChannelCreate } from '../../models/videos/channel/video-channel-create.model'
 import { makeDeleteRequest, makeGetRequest, updateImageRequest } from '../requests/requests'
 import { ServerInfo } from '../server/servers'
-import { User } from '../../models/users/user.model'
+import { MyUser, User } from '../../models/users/user.model'
 import { getMyUserInformation } from '../users/users'
 import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 
@@ -170,6 +170,12 @@ function setDefaultVideoChannel (servers: ServerInfo[]) {
   return Promise.all(tasks)
 }
 
+async function getDefaultVideoChannel (url: string, token: string) {
+  const res = await getMyUserInformation(url, token)
+
+  return (res.body as MyUser).videoChannels[0].id
+}
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -181,5 +187,6 @@ export {
   deleteVideoChannel,
   getVideoChannel,
   setDefaultVideoChannel,
-  deleteVideoChannelImage
+  deleteVideoChannelImage,
+  getDefaultVideoChannel
 }
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts
index a0143b0ef..e88256ac0 100644
--- a/shared/extra-utils/videos/videos.ts
+++ b/shared/extra-utils/videos/videos.ts
@@ -1,7 +1,8 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
 
 import { expect } from 'chai'
-import { pathExists, readdir, readFile } from 'fs-extra'
+import { createReadStream, pathExists, readdir, readFile, stat } from 'fs-extra'
+import got, { Response as GotResponse } from 'got/dist/source'
 import * as parseTorrent from 'parse-torrent'
 import { extname, join } from 'path'
 import * as request from 'supertest'
@@ -42,6 +43,7 @@ type VideoAttributes = {
   channelId?: number
   privacy?: VideoPrivacy
   fixture?: string
+  support?: string
   thumbnailfile?: string
   previewfile?: string
   scheduleUpdate?: {
@@ -364,8 +366,13 @@ async function checkVideoFilesWereRemoved (
   }
 }
 
-async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
-  const path = '/api/v1/videos/upload'
+async function uploadVideo (
+  url: string,
+  accessToken: string,
+  videoAttributesArg: VideoAttributes,
+  specialStatus = HttpStatusCode.OK_200,
+  mode: 'legacy' | 'resumable' = 'legacy'
+) {
   let defaultChannelId = '1'
 
   try {
@@ -391,74 +398,170 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
     fixture: 'video_short.webm'
   }, videoAttributesArg)
 
+  const res = mode === 'legacy'
+    ? await buildLegacyUpload(url, accessToken, attributes, specialStatus)
+    : await buildResumeUpload(url, accessToken, attributes, specialStatus)
+
+  // Wait torrent generation
+  if (specialStatus === HttpStatusCode.OK_200) {
+    let video: VideoDetails
+    do {
+      const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid)
+      video = resVideo.body
+
+      await wait(50)
+    } while (!video.files[0].torrentUrl)
+  }
+
+  return res
+}
+
+function checkUploadVideoParam (
+  url: string,
+  token: string,
+  attributes: Partial<VideoAttributes>,
+  specialStatus = HttpStatusCode.OK_200,
+  mode: 'legacy' | 'resumable' = 'legacy'
+) {
+  return mode === 'legacy'
+    ? buildLegacyUpload(url, token, attributes, specialStatus)
+    : buildResumeUpload(url, token, attributes, specialStatus)
+}
+
+async function buildLegacyUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
+  const path = '/api/v1/videos/upload'
   const req = request(url)
               .post(path)
               .set('Accept', 'application/json')
-              .set('Authorization', 'Bearer ' + accessToken)
-              .field('name', attributes.name)
-              .field('nsfw', JSON.stringify(attributes.nsfw))
-              .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled))
-              .field('downloadEnabled', JSON.stringify(attributes.downloadEnabled))
-              .field('waitTranscoding', JSON.stringify(attributes.waitTranscoding))
-              .field('privacy', attributes.privacy.toString())
-              .field('channelId', attributes.channelId)
-
-  if (attributes.support !== undefined) {
-    req.field('support', attributes.support)
-  }
+              .set('Authorization', 'Bearer ' + token)
 
-  if (attributes.description !== undefined) {
-    req.field('description', attributes.description)
-  }
-  if (attributes.language !== undefined) {
-    req.field('language', attributes.language.toString())
-  }
-  if (attributes.category !== undefined) {
-    req.field('category', attributes.category.toString())
-  }
-  if (attributes.licence !== undefined) {
-    req.field('licence', attributes.licence.toString())
-  }
+  buildUploadReq(req, attributes)
 
-  const tags = attributes.tags || []
-  for (let i = 0; i < tags.length; i++) {
-    req.field('tags[' + i + ']', attributes.tags[i])
+  if (attributes.fixture !== undefined) {
+    req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture))
   }
 
-  if (attributes.thumbnailfile !== undefined) {
-    req.attach('thumbnailfile', buildAbsoluteFixturePath(attributes.thumbnailfile))
-  }
-  if (attributes.previewfile !== undefined) {
-    req.attach('previewfile', buildAbsoluteFixturePath(attributes.previewfile))
-  }
+  return req.expect(specialStatus)
+}
 
-  if (attributes.scheduleUpdate) {
-    req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt)
+async function buildResumeUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
+  let size = 0
+  let videoFilePath: string
+  let mimetype = 'video/mp4'
 
-    if (attributes.scheduleUpdate.privacy) {
-      req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy)
+  if (attributes.fixture) {
+    videoFilePath = buildAbsoluteFixturePath(attributes.fixture)
+    size = (await stat(videoFilePath)).size
+
+    if (videoFilePath.endsWith('.mkv')) {
+      mimetype = 'video/x-matroska'
+    } else if (videoFilePath.endsWith('.webm')) {
+      mimetype = 'video/webm'
     }
   }
 
-  if (attributes.originallyPublishedAt !== undefined) {
-    req.field('originallyPublishedAt', attributes.originallyPublishedAt)
+  const initializeSessionRes = await prepareResumableUpload({ url, token, attributes, size, mimetype })
+  const initStatus = initializeSessionRes.status
+
+  if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) {
+    const locationHeader = initializeSessionRes.header['location']
+    expect(locationHeader).to.not.be.undefined
+
+    const pathUploadId = locationHeader.split('?')[1]
+
+    return sendResumableChunks({ url, token, pathUploadId, videoFilePath, size, specialStatus })
   }
 
-  const res = await req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture))
-            .expect(specialStatus)
+  const expectedInitStatus = specialStatus === HttpStatusCode.OK_200
+    ? HttpStatusCode.CREATED_201
+    : specialStatus
 
-  // Wait torrent generation
-  if (specialStatus === HttpStatusCode.OK_200) {
-    let video: VideoDetails
-    do {
-      const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid)
-      video = resVideo.body
+  expect(initStatus).to.equal(expectedInitStatus)
 
-      await wait(50)
-    } while (!video.files[0].torrentUrl)
+  return initializeSessionRes
+}
+
+async function prepareResumableUpload (options: {
+  url: string
+  token: string
+  attributes: VideoAttributes
+  size: number
+  mimetype: string
+}) {
+  const { url, token, attributes, size, mimetype } = options
+
+  const path = '/api/v1/videos/upload-resumable'
+
+  const req = request(url)
+              .post(path)
+              .set('Authorization', 'Bearer ' + token)
+              .set('X-Upload-Content-Type', mimetype)
+              .set('X-Upload-Content-Length', size.toString())
+
+  buildUploadReq(req, attributes)
+
+  if (attributes.fixture) {
+    req.field('filename', attributes.fixture)
   }
 
-  return res
+  return req
+}
+
+function sendResumableChunks (options: {
+  url: string
+  token: string
+  pathUploadId: string
+  videoFilePath: string
+  size: number
+  specialStatus?: HttpStatusCode
+  contentLength?: number
+  contentRangeBuilder?: (start: number, chunk: any) => string
+}) {
+  const { url, token, pathUploadId, videoFilePath, size, specialStatus, contentLength, contentRangeBuilder } = options
+
+  const expectedStatus = specialStatus || HttpStatusCode.OK_200
+
+  const path = '/api/v1/videos/upload-resumable'
+  let start = 0
+
+  const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 })
+  return new Promise<GotResponse>((resolve, reject) => {
+    readable.on('data', async function onData (chunk) {
+      readable.pause()
+
+      const headers = {
+        'Authorization': 'Bearer ' + token,
+        'Content-Type': 'application/octet-stream',
+        'Content-Range': contentRangeBuilder
+          ? contentRangeBuilder(start, chunk)
+          : `bytes ${start}-${start + chunk.length - 1}/${size}`,
+        'Content-Length': contentLength ? contentLength + '' : chunk.length + ''
+      }
+
+      const res = await got({
+        url,
+        method: 'put',
+        headers,
+        path: path + '?' + pathUploadId,
+        body: chunk,
+        responseType: 'json',
+        throwHttpErrors: false
+      })
+
+      start += chunk.length
+
+      if (res.statusCode === expectedStatus) {
+        return resolve(res)
+      }
+
+      if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) {
+        readable.off('data', onData)
+        return reject(new Error('Incorrect transient behaviour sending intermediary chunks'))
+      }
+
+      readable.resume()
+    })
+  })
 }
 
 function updateVideo (
@@ -749,11 +852,13 @@ export {
   getVideoWithToken,
   getVideosList,
   removeAllVideos,
+  checkUploadVideoParam,
   getVideosListPagination,
   getVideosListSort,
   removeVideo,
   getVideosListWithToken,
   uploadVideo,
+  sendResumableChunks,
   getVideosWithFilters,
   uploadRandomVideoOnServers,
   updateVideo,
@@ -767,5 +872,50 @@ export {
   getMyVideosWithFilter,
   uploadVideoAndGetId,
   getLocalIdByUUID,
-  getVideoIdFromUUID
+  getVideoIdFromUUID,
+  prepareResumableUpload
+}
+
+// ---------------------------------------------------------------------------
+
+function buildUploadReq (req: request.Test, attributes: VideoAttributes) {
+
+  for (const key of [ 'name', 'support', 'channelId', 'description', 'originallyPublishedAt' ]) {
+    if (attributes[key] !== undefined) {
+      req.field(key, attributes[key])
+    }
+  }
+
+  for (const key of [ 'nsfw', 'commentsEnabled', 'downloadEnabled', 'waitTranscoding' ]) {
+    if (attributes[key] !== undefined) {
+      req.field(key, JSON.stringify(attributes[key]))
+    }
+  }
+
+  for (const key of [ 'language', 'privacy', 'category', 'licence' ]) {
+    if (attributes[key] !== undefined) {
+      req.field(key, attributes[key].toString())
+    }
+  }
+
+  const tags = attributes.tags || []
+  for (let i = 0; i < tags.length; i++) {
+    req.field('tags[' + i + ']', attributes.tags[i])
+  }
+
+  for (const key of [ 'thumbnailfile', 'previewfile' ]) {
+    if (attributes[key] !== undefined) {
+      req.attach(key, buildAbsoluteFixturePath(attributes[key]))
+    }
+  }
+
+  if (attributes.scheduleUpdate) {
+    if (attributes.scheduleUpdate.updateAt) {
+      req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt)
+    }
+
+    if (attributes.scheduleUpdate.privacy) {
+      req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy)
+    }
+  }
 }