Use got instead of request

6 files changed, 165 insertions, 100 deletions

diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index 08aef2908..e0754b501 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -3,7 +3,6 @@ import { URL } from 'url'
 import validator from 'validator'
 import { ContextType } from '@shared/models/activitypub/context'
 import { ResultList } from '../../shared/models'
-import { Activity } from '../../shared/models/activitypub'
 import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants'
 import { MActor, MVideoWithHost } from '../types/models'
 import { pageToStartAndCount } from './core-utils'
@@ -182,10 +181,10 @@ async function activityPubCollectionPagination (
 
 }
 
-function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) {
+function buildSignedActivity <T> (byActor: MActor, data: T, contextType?: ContextType) {
   const activity = activityPubContextify(data, contextType)
 
-  return signJsonLDObject(byActor, activity) as Promise<Activity>
+  return signJsonLDObject(byActor, activity)
 }
 
 function getAPId (activity: string | { id: string }) {
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 935fd22d9..7ba7d865a 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -10,7 +10,9 @@ import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto'
 import { truncate } from 'lodash'
 import { basename, isAbsolute, join, resolve } from 'path'
 import * as pem from 'pem'
+import { pipeline } from 'stream'
 import { URL } from 'url'
+import { promisify } from 'util'
 
 const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
   if (!oldObject || typeof oldObject !== 'object') {
@@ -254,6 +256,7 @@ const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKe
 const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey)
 const execPromise2 = promisify2<string, any, string>(exec)
 const execPromise = promisify1<string, string>(exec)
+const pipelinePromise = promisify(pipeline)
 
 // ---------------------------------------------------------------------------
 
@@ -284,5 +287,6 @@ export {
   createPrivateKey,
   getPublicKey,
   execPromise2,
-  execPromise
+  execPromise,
+  pipelinePromise
 }
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts
index 46126da57..69558e358 100644
--- a/server/helpers/custom-validators/activitypub/activity.ts
+++ b/server/helpers/custom-validators/activitypub/activity.ts
@@ -41,7 +41,7 @@ const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean
 }
 
 function isActivityValid (activity: any) {
-  const checker = activityCheckers[activity.tswype]
+  const checker = activityCheckers[activity.type]
   // Unknown activity type
   if (!checker) return false
 
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts
index 994f725d8..bc6f1d074 100644
--- a/server/helpers/peertube-crypto.ts
+++ b/server/helpers/peertube-crypto.ts
@@ -84,7 +84,7 @@ async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any)
   return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64')
 }
 
-async function signJsonLDObject (byActor: MActor, data: any) {
+async function signJsonLDObject <T> (byActor: MActor, data: T) {
   const signature = {
     type: 'RsaSignature2017',
     creator: byActor.url,
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index b556c392e..2c9da213c 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -1,58 +1,124 @@
-import * as Bluebird from 'bluebird'
 import { createWriteStream, remove } from 'fs-extra'
-import * as request from 'request'
+import got, { CancelableRequest, Options as GotOptions } from 'got'
+import { join } from 'path'
+import { CONFIG } from '../initializers/config'
 import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
+import { pipelinePromise } from './core-utils'
 import { processImage } from './image-utils'
-import { join } from 'path'
 import { logger } from './logger'
-import { CONFIG } from '../initializers/config'
 
-function doRequest <T> (
-  requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean },
-  bodyKBLimit = 1000 // 1MB
-): Bluebird<{ response: request.RequestResponse, body: T }> {
-  if (!(requestOptions.headers)) requestOptions.headers = {}
-  requestOptions.headers['User-Agent'] = getUserAgent()
+const httpSignature = require('http-signature')
+
+type PeerTubeRequestOptions = {
+  activityPub?: boolean
+  bodyKBLimit?: number // 1MB
+  httpSignature?: {
+    algorithm: string
+    authorizationHeaderName: string
+    keyId: string
+    key: string
+    headers: string[]
+  }
+  jsonResponse?: boolean
+} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
+
+const peertubeGot = got.extend({
+  headers: {
+    'user-agent': getUserAgent()
+  },
+
+  handlers: [
+    (options, next) => {
+      const promiseOrStream = next(options) as CancelableRequest<any>
+      const bodyKBLimit = options.context?.bodyKBLimit
+      if (!bodyKBLimit) throw new Error('No KB limit for this request')
+
+      /* eslint-disable @typescript-eslint/no-floating-promises */
+      promiseOrStream.on('downloadProgress', progress => {
+        if (progress.transferred * 1000 > bodyKBLimit && progress.percent !== 1) {
+          promiseOrStream.cancel(`Exceeded the download limit of ${bodyKBLimit} bytes`)
+        }
+      })
 
-  if (requestOptions.activityPub === true) {
-    requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER
+      return promiseOrStream
+    }
+  ],
+
+  hooks: {
+    beforeRequest: [
+      options => {
+        const headers = options.headers || {}
+        headers['host'] = options.url.host
+      },
+
+      options => {
+        const httpSignatureOptions = options.context?.httpSignature
+
+        if (httpSignatureOptions) {
+          const method = options.method ?? 'GET'
+          const path = options.path ?? options.url.pathname
+
+          if (!method || !path) {
+            throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
+          }
+
+          httpSignature.signRequest({
+            getHeader: function (header) {
+              return options.headers[header]
+            },
+
+            setHeader: function (header, value) {
+              options.headers[header] = value
+            },
+
+            method,
+            path
+          }, httpSignatureOptions)
+        }
+      }
+    ]
   }
+})
 
-  return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => {
-    request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body }))
-      .on('data', onRequestDataLengthCheck(bodyKBLimit))
-  })
+function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
+  const gotOptions = buildGotOptions(options)
+
+  return peertubeGot(url, gotOptions)
+    .catch(err => { throw buildRequestError(err) })
+}
+
+function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
+  const gotOptions = buildGotOptions(options)
+
+  return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
+    .catch(err => { throw buildRequestError(err) })
 }
 
-function doRequestAndSaveToFile (
-  requestOptions: request.CoreOptions & request.UriOptions,
+async function doRequestAndSaveToFile (
+  url: string,
   destPath: string,
-  bodyKBLimit = 10000 // 10MB
+  options: PeerTubeRequestOptions = {}
 ) {
-  if (!requestOptions.headers) requestOptions.headers = {}
-  requestOptions.headers['User-Agent'] = getUserAgent()
-
-  return new Bluebird<void>((res, rej) => {
-    const file = createWriteStream(destPath)
-    file.on('finish', () => res())
+  const gotOptions = buildGotOptions(options)
 
-    request(requestOptions)
-      .on('data', onRequestDataLengthCheck(bodyKBLimit))
-      .on('error', err => {
-        file.close()
+  const outFile = createWriteStream(destPath)
 
-        remove(destPath)
-          .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
+  try {
+    await pipelinePromise(
+      peertubeGot.stream(url, gotOptions),
+      outFile
+    )
+  } catch (err) {
+    remove(destPath)
+      .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
 
-        return rej(err)
-      })
-      .pipe(file)
-  })
+    throw buildRequestError(err)
+  }
 }
 
 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
   const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
-  await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath)
+  await doRequestAndSaveToFile(url, tmpPath)
 
   const destPath = join(destDir, destName)
 
@@ -73,24 +139,43 @@ function getUserAgent () {
 
 export {
   doRequest,
+  doJSONRequest,
   doRequestAndSaveToFile,
   downloadImage
 }
 
 // ---------------------------------------------------------------------------
 
-// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3
-function onRequestDataLengthCheck (bodyKBLimit: number) {
-  let bufferLength = 0
-  const bytesLimit = bodyKBLimit * 1000
+function buildGotOptions (options: PeerTubeRequestOptions) {
+  const { activityPub, bodyKBLimit = 1000 } = options
 
-  return function (chunk) {
-    bufferLength += chunk.length
-    if (bufferLength > bytesLimit) {
-      this.abort()
+  const context = { bodyKBLimit, httpSignature: options.httpSignature }
 
-      const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`)
-      this.emit('error', error)
-    }
+  let headers = options.headers || {}
+
+  headers = { ...headers, date: new Date().toUTCString() }
+
+  if (activityPub) {
+    headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
   }
+
+  return {
+    method: options.method,
+    json: options.json,
+    searchParams: options.searchParams,
+    headers,
+    context
+  }
+}
+
+function buildRequestError (error: any) {
+  const newError = new Error(error.message)
+  newError.name = error.name
+  newError.stack = error.stack
+
+  if (error.response?.body) {
+    error.responseBody = error.response.body
+  }
+
+  return newError
 }
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 8537a5772..9d2e54fb5 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,13 +1,13 @@
 import { createWriteStream } from 'fs'
 import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra'
+import got from 'got'
 import { join } from 'path'
-import * as request from 'request'
 import { CONFIG } from '@server/initializers/config'
 import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
 import { VideoResolution } from '../../shared/models/videos'
 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
 import { getEnabledResolutions } from '../lib/video-transcoding'
-import { peertubeTruncate, root } from './core-utils'
+import { peertubeTruncate, pipelinePromise, root } from './core-utils'
 import { isVideoFileExtnameValid } from './custom-validators/videos'
 import { logger } from './logger'
 import { generateVideoImportTmpPath } from './utils'
@@ -195,55 +195,32 @@ async function updateYoutubeDLBinary () {
 
   await ensureDir(binDirectory)
 
-  return new Promise<void>(res => {
-    request.get(url, { followRedirect: false }, (err, result) => {
-      if (err) {
-        logger.error('Cannot update youtube-dl.', { err })
-        return res()
-      }
-
-      if (result.statusCode !== HttpStatusCode.FOUND_302) {
-        logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
-        return res()
-      }
-
-      const url = result.headers.location
-      const downloadFile = request.get(url)
-      const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
-
-      downloadFile.on('response', result => {
-        if (result.statusCode !== HttpStatusCode.OK_200) {
-          logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
-          return res()
-        }
-
-        const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => {
-          logger.error('youtube-dl update error in write stream', { err })
-          return res()
-        })
+  try {
+    const result = await got(url, { followRedirect: false })
 
-        downloadFile.pipe(writeStream)
-      })
+    if (result.statusCode !== HttpStatusCode.FOUND_302) {
+      logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
+      return
+    }
 
-      downloadFile.on('error', err => {
-        logger.error('youtube-dl update error.', { err })
-        return res()
-      })
+    const newUrl = result.headers.location
+    const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1]
 
-      downloadFile.on('end', () => {
-        const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
-        writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
-          if (err) {
-            logger.error('youtube-dl update error: cannot write details.', { err })
-            return res()
-          }
+    const downloadFileStream = got.stream(newUrl)
+    const writeStream = createWriteStream(bin, { mode: 493 })
 
-          logger.info('youtube-dl updated to version %s.', newVersion)
-          return res()
-        })
-      })
-    })
-  })
+    await pipelinePromise(
+      downloadFileStream,
+      writeStream
+    )
+
+    const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
+    await writeFile(detailsPath, details, { encoding: 'utf8' })
+
+    logger.info('youtube-dl updated to version %s.', newVersion)
+  } catch (err) {
+    logger.error('Cannot update youtube-dl.', { err })
+  }
 }
 
 async function safeGetYoutubeDL () {