11 files changed, 166 insertions, 33 deletions

diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index c762f6a29..73bd994c1 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -6,7 +6,7 @@
 */
 
 import { exec, ExecOptions } from 'child_process'
-import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions } from 'crypto'
+import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
 import { truncate } from 'lodash'
 import { pipeline } from 'stream'
 import { URL } from 'url'
@@ -311,7 +311,17 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A)
   }
 }
 
+// eslint-disable-next-line max-len
+function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> {
+  return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> {
+    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
+      func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ])
+    })
+  }
+}
+
 const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
+const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
 const execPromise2 = promisify2<string, any, string>(exec)
 const execPromise = promisify1<string, string>(exec)
 const pipelinePromise = promisify(pipeline)
@@ -339,6 +349,8 @@ export {
   promisify1,
   promisify2,
 
+  scryptPromise,
+
   randomBytesPromise,
 
   generateRSAKeyPairPromise,
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 2a2f008b9..97b3577af 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -7,11 +7,11 @@ import { peertubeTruncate } from '../../core-utils'
 import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
 import { isLiveLatencyModeValid } from '../video-lives'
 import {
+  isVideoDescriptionValid,
   isVideoDurationValid,
   isVideoNameValid,
   isVideoStateValid,
   isVideoTagValid,
-  isVideoTruncatedDescriptionValid,
   isVideoViewsValid
 } from '../videos'
 import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc'
@@ -32,7 +32,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
     logger.debug('Video has invalid urls', { video })
     return false
   }
-  if (!setRemoteVideoTruncatedContent(video)) {
+  if (!setRemoteVideoContent(video)) {
     logger.debug('Video has invalid content', { video })
     return false
   }
@@ -168,7 +168,7 @@ function isRemoteStringIdentifierValid (data: any) {
 }
 
 function isRemoteVideoContentValid (mediaType: string, content: string) {
-  return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content)
+  return mediaType === 'text/markdown' && isVideoDescriptionValid(content)
 }
 
 function setValidRemoteIcon (video: any) {
@@ -194,9 +194,9 @@ function setValidRemoteVideoUrls (video: any) {
   return true
 }
 
-function setRemoteVideoTruncatedContent (video: any) {
+function setRemoteVideoContent (video: any) {
   if (video.content) {
-    video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.TRUNCATED_DESCRIPTION.max })
+    video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max })
   }
 
   return true
diff --git a/server/helpers/custom-validators/servers.ts b/server/helpers/custom-validators/servers.ts
index b9f45c282..94fda05aa 100644
--- a/server/helpers/custom-validators/servers.ts
+++ b/server/helpers/custom-validators/servers.ts
@@ -1,6 +1,6 @@
 import validator from 'validator'
+import { CONFIG } from '@server/initializers/config'
 import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
-import { isTestOrDevInstance } from '../core-utils'
 import { exists, isArray } from './misc'
 
 function isHostValid (host: string) {
@@ -10,7 +10,7 @@ function isHostValid (host: string) {
   }
 
   // We validate 'localhost', so we don't have the top level domain
-  if (isTestOrDevInstance()) {
+  if (CONFIG.WEBSERVER.HOSTNAME === 'localhost') {
     isURLOptions.require_tld = false
   }
 
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 3ebfe2937..9e8177f77 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -45,10 +45,6 @@ function isVideoDurationValid (value: string) {
   return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION)
 }
 
-function isVideoTruncatedDescriptionValid (value: string) {
-  return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.TRUNCATED_DESCRIPTION)
-}
-
 function isVideoDescriptionValid (value: string) {
   return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION))
 }
@@ -151,7 +147,6 @@ export {
   isVideoCategoryValid,
   isVideoLicenceValid,
   isVideoLanguageValid,
-  isVideoTruncatedDescriptionValid,
   isVideoDescriptionValid,
   isVideoFileInfoHashValid,
   isVideoNameValid,
diff --git a/server/helpers/ffmpeg/ffmpeg-vod.ts b/server/helpers/ffmpeg/ffmpeg-vod.ts
index 7a81a1313..d84703eb9 100644
--- a/server/helpers/ffmpeg/ffmpeg-vod.ts
+++ b/server/helpers/ffmpeg/ffmpeg-vod.ts
@@ -1,14 +1,15 @@
+import { MutexInterface } from 'async-mutex'
 import { Job } from 'bullmq'
 import { FfmpegCommand } from 'fluent-ffmpeg'
 import { readFile, writeFile } from 'fs-extra'
 import { dirname } from 'path'
+import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
 import { pick } from '@shared/core-utils'
 import { AvailableEncoders, VideoResolution } from '@shared/models'
 import { logger, loggerTagsFactory } from '../logger'
 import { getFFmpeg, runCommand } from './ffmpeg-commons'
 import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets'
 import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils'
-import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
 
 const lTags = loggerTagsFactory('ffmpeg')
 
@@ -22,6 +23,10 @@ interface BaseTranscodeVODOptions {
   inputPath: string
   outputPath: string
 
+  // Will be released after the ffmpeg started
+  // To prevent a bug where the input file does not exist anymore when running ffmpeg
+  inputFileMutexReleaser: MutexInterface.Releaser
+
   availableEncoders: AvailableEncoders
   profile: string
 
@@ -94,6 +99,12 @@ async function transcodeVOD (options: TranscodeVODOptions) {
 
   command = await builders[options.type](command, options)
 
+  command.on('start', () => {
+    setTimeout(() => {
+      options.inputFileMutexReleaser()
+    }, 1000)
+  })
+
   await runCommand({ command, job: options.job })
 
   await fixHLSPlaylistIfNeeded(options)
diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts
index 2c6253d44..fb270b3cb 100644
--- a/server/helpers/ffmpeg/ffprobe-utils.ts
+++ b/server/helpers/ffmpeg/ffprobe-utils.ts
@@ -15,6 +15,7 @@ import {
 import { VideoResolution, VideoTranscodingFPS } from '@shared/models'
 import { CONFIG } from '../../initializers/config'
 import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants'
+import { toEven } from '../core-utils'
 import { logger } from '../logger'
 
 /**
@@ -96,8 +97,9 @@ function computeResolutionsToTranscode (options: {
   type: 'vod' | 'live'
   includeInput: boolean
   strictLower: boolean
+  hasAudio: boolean
 }) {
-  const { input, type, includeInput, strictLower } = options
+  const { input, type, includeInput, strictLower, hasAudio } = options
 
   const configResolutions = type === 'vod'
     ? CONFIG.TRANSCODING.RESOLUTIONS
@@ -125,12 +127,15 @@ function computeResolutionsToTranscode (options: {
     if (input < resolution) continue
     // We only want lower resolutions than input file
     if (strictLower && input === resolution) continue
+    // Audio resolutio but no audio in the video
+    if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue
 
     resolutionsEnabled.add(resolution)
   }
 
   if (includeInput) {
-    resolutionsEnabled.add(input)
+    // Always use an even resolution to avoid issues with ffmpeg
+    resolutionsEnabled.add(toEven(input))
   }
 
   return Array.from(resolutionsEnabled)
diff --git a/server/helpers/otp.ts b/server/helpers/otp.ts
new file mode 100644
index 000000000..a32cc9621
--- /dev/null
+++ b/server/helpers/otp.ts
@@ -0,0 +1,58 @@
+import { Secret, TOTP } from 'otpauth'
+import { CONFIG } from '@server/initializers/config'
+import { WEBSERVER } from '@server/initializers/constants'
+import { decrypt } from './peertube-crypto'
+
+async function isOTPValid (options: {
+  encryptedSecret: string
+  token: string
+}) {
+  const { token, encryptedSecret } = options
+
+  const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE)
+
+  const totp = new TOTP({
+    ...baseOTPOptions(),
+
+    secret
+  })
+
+  const delta = totp.validate({
+    token,
+    window: 1
+  })
+
+  if (delta === null) return false
+
+  return true
+}
+
+function generateOTPSecret (email: string) {
+  const totp = new TOTP({
+    ...baseOTPOptions(),
+
+    label: email,
+    secret: new Secret()
+  })
+
+  return {
+    secret: totp.secret.base32,
+    uri: totp.toString()
+  }
+}
+
+export {
+  isOTPValid,
+  generateOTPSecret
+}
+
+// ---------------------------------------------------------------------------
+
+function baseOTPOptions () {
+  return {
+    issuer: WEBSERVER.HOST,
+    algorithm: 'SHA1',
+    digits: 6,
+    period: 30
+  }
+}
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts
index 8aca50900..ae7d11800 100644
--- a/server/helpers/peertube-crypto.ts
+++ b/server/helpers/peertube-crypto.ts
@@ -1,11 +1,11 @@
 import { compare, genSalt, hash } from 'bcrypt'
-import { createSign, createVerify } from 'crypto'
+import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto'
 import { Request } from 'express'
 import { cloneDeep } from 'lodash'
 import { sha256 } from '@shared/extra-utils'
-import { BCRYPT_SALT_SIZE, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
+import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
 import { MActor } from '../types/models'
-import { generateRSAKeyPairPromise, promisify1, promisify2 } from './core-utils'
+import { generateRSAKeyPairPromise, promisify1, promisify2, randomBytesPromise, scryptPromise } from './core-utils'
 import { jsonld } from './custom-jsonld-signature'
 import { logger } from './logger'
 
@@ -21,9 +21,13 @@ function createPrivateAndPublicKeys () {
   return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE)
 }
 
+// ---------------------------------------------------------------------------
 // User password checks
+// ---------------------------------------------------------------------------
 
 function comparePassword (plainPassword: string, hashPassword: string) {
+  if (!plainPassword) return Promise.resolve(false)
+
   return bcryptComparePromise(plainPassword, hashPassword)
 }
 
@@ -33,7 +37,9 @@ async function cryptPassword (password: string) {
   return bcryptHashPromise(password, salt)
 }
 
+// ---------------------------------------------------------------------------
 // HTTP Signature
+// ---------------------------------------------------------------------------
 
 function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean {
   if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) {
@@ -62,7 +68,9 @@ function parseHTTPSignature (req: Request, clockSkew?: number) {
   return parsed
 }
 
+// ---------------------------------------------------------------------------
 // JSONLD
+// ---------------------------------------------------------------------------
 
 function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> {
   if (signedDocument.signature.type === 'RsaSignature2017') {
@@ -112,6 +120,8 @@ async function signJsonLDObject <T> (byActor: MActor, data: T) {
   return Object.assign(data, { signature })
 }
 
+// ---------------------------------------------------------------------------
+
 function buildDigest (body: any) {
   const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
 
@@ -119,6 +129,34 @@ function buildDigest (body: any) {
 }
 
 // ---------------------------------------------------------------------------
+// Encryption
+// ---------------------------------------------------------------------------
+
+async function encrypt (str: string, secret: string) {
+  const iv = await randomBytesPromise(ENCRYPTION.IV)
+
+  const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
+  const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv)
+
+  let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':'
+  encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING)
+  encrypted += cipher.final(ENCRYPTION.ENCODING)
+
+  return encrypted
+}
+
+async function decrypt (encryptedArg: string, secret: string) {
+  const [ ivStr, encryptedStr ] = encryptedArg.split(':')
+
+  const iv = Buffer.from(ivStr, 'hex')
+  const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
+
+  const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv)
+
+  return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8')
+}
+
+// ---------------------------------------------------------------------------
 
 export {
   isHTTPSignatureDigestValid,
@@ -129,7 +167,10 @@ export {
   comparePassword,
   createPrivateAndPublicKeys,
   cryptPassword,
-  signJsonLDObject
+  signJsonLDObject,
+
+  encrypt,
+  decrypt
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts
index 3cb17edd0..f5f476913 100644
--- a/server/helpers/upload.ts
+++ b/server/helpers/upload.ts
@@ -1,10 +1,10 @@
 import { join } from 'path'
-import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
+import { DIRECTORIES } from '@server/initializers/constants'
 
 function getResumableUploadPath (filename?: string) {
-  if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename)
+  if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename)
 
-  return RESUMABLE_UPLOAD_DIRECTORY
+  return DIRECTORIES.RESUMABLE_UPLOAD
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 88bdb16b6..a3c93e6fe 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -1,6 +1,6 @@
 import { decode, encode } from 'bencode'
 import createTorrent from 'create-torrent'
-import { createWriteStream, ensureDir, readFile, remove, writeFile } from 'fs-extra'
+import { createWriteStream, ensureDir, pathExists, readFile, remove, writeFile } from 'fs-extra'
 import magnetUtil from 'magnet-uri'
 import parseTorrent from 'parse-torrent'
 import { dirname, join } from 'path'
@@ -134,6 +134,11 @@ async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlayli
 
   const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
 
+  if (!await pathExists(oldTorrentPath)) {
+    logger.info('Do not update torrent metadata %s of video %s because the file does not exist anymore.', video.uuid, oldTorrentPath)
+    return
+  }
+
   const torrentContent = await readFile(oldTorrentPath)
   const decoded = decode(torrentContent)
 
@@ -151,7 +156,7 @@ async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlayli
   logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath)
 
   await writeFile(newTorrentPath, encode(decoded))
-  await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
+  await remove(oldTorrentPath)
 
   videoFile.torrentFilename = newTorrentFilename
   videoFile.infoHash = sha1(encode(decoded.info))
@@ -164,7 +169,10 @@ function generateMagnetUri (
 ) {
   const xs = videoFile.getTorrentUrl()
   const announce = trackerUrls
-  let urlList = [ videoFile.getFileUrl(video) ]
+
+  let urlList = video.hasPrivateStaticPath()
+    ? []
+    : [ videoFile.getFileUrl(video) ]
 
   const redundancies = videoFile.RedundancyVideos
   if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
@@ -240,6 +248,8 @@ function buildAnnounceList () {
 }
 
 function buildUrlList (video: MVideo, videoFile: MVideoFile) {
+  if (video.hasPrivateStaticPath()) return []
+
   return [ videoFile.getFileUrl(video) ]
 }
 
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index fc4c40787..a2f630953 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -128,14 +128,14 @@ export class YoutubeDLCLI {
     const data = await this.run({ url, args: completeArgs, processOptions })
     if (!data) return undefined
 
-    const info = data.map(this.parseInfo)
+    const info = data.map(d => JSON.parse(d))
 
     return info.length === 1
       ? info[0]
       : info
   }
 
-  getListInfo (options: {
+  async getListInfo (options: {
     url: string
     latestVideosCount?: number
     processOptions: execa.NodeOptions
@@ -151,12 +151,17 @@ export class YoutubeDLCLI {
       additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
     }
 
-    return this.getInfo({
+    const result = await this.getInfo({
       url: options.url,
       format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
       processOptions: options.processOptions,
       additionalYoutubeDLArgs
     })
+
+    if (!result) return result
+    if (!Array.isArray(result)) return [ result ]
+
+    return result
   }
 
   async getSubs (options: {
@@ -241,8 +246,4 @@ export class YoutubeDLCLI {
 
     return args
   }
-
-  private parseInfo (data: string) {
-    return JSON.parse(data)
-  }
 }