Use got instead of request

7 files changed, 71 insertions, 125 deletions

diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index a726f9e20..52b6c1f56 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -1,26 +1,28 @@
 import * as Bluebird from 'bluebird'
+import { extname } from 'path'
 import { Op, Transaction } from 'sequelize'
 import { URL } from 'url'
 import { v4 as uuidv4 } from 'uuid'
+import { getServerActor } from '@server/models/application/application'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
+import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
 import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
-import { doRequest } from '../../helpers/requests'
+import { doJSONRequest } from '../../helpers/requests'
 import { getUrlFromWebfinger } from '../../helpers/webfinger'
 import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
+import { sequelizeTypescript } from '../../initializers/database'
 import { AccountModel } from '../../models/account/account'
 import { ActorModel } from '../../models/activitypub/actor'
 import { AvatarModel } from '../../models/avatar/avatar'
 import { ServerModel } from '../../models/server/server'
 import { VideoChannelModel } from '../../models/video/video-channel'
-import { JobQueue } from '../job-queue'
-import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
-import { sequelizeTypescript } from '../../initializers/database'
 import {
   MAccount,
   MAccountDefault,
@@ -34,9 +36,7 @@ import {
   MActorId,
   MChannel
 } from '../../types/models'
-import { extname } from 'path'
-import { getServerActor } from '@server/models/application/application'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { JobQueue } from '../job-queue'
 
 // Set account keys, this could be long so process after the account creation and do not block the client
 async function generateAndSaveActorKeys <T extends MActor> (actor: T) {
@@ -209,16 +209,10 @@ async function deleteActorAvatarInstance (actor: MActorDefault, t: Transaction)
 }
 
 async function fetchActorTotalItems (url: string) {
-  const options = {
-    uri: url,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
   try {
-    const { body } = await doRequest<ActivityPubOrderedCollection<unknown>>(options)
-    return body.totalItems ? body.totalItems : 0
+    const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true })
+
+    return body.totalItems || 0
   } catch (err) {
     logger.warn('Cannot fetch remote actor count %s.', url, { err })
     return 0
@@ -449,26 +443,19 @@ type FetchRemoteActorResult = {
   attributedTo: ActivityPubAttributedTo[]
 }
 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
-  const options = {
-    uri: actorUrl,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
   logger.info('Fetching remote actor %s.', actorUrl)
 
-  const requestResult = await doRequest<ActivityPubActor>(options)
+  const requestResult = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true })
   const actorJSON = requestResult.body
 
   if (sanitizeAndCheckActorObject(actorJSON) === false) {
     logger.debug('Remote actor JSON is not valid.', { actorJSON })
-    return { result: undefined, statusCode: requestResult.response.statusCode }
+    return { result: undefined, statusCode: requestResult.statusCode }
   }
 
   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
     logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
-    return { result: undefined, statusCode: requestResult.response.statusCode }
+    return { result: undefined, statusCode: requestResult.statusCode }
   }
 
   const followersCount = await fetchActorTotalItems(actorJSON.followers)
@@ -496,7 +483,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
 
   const name = actorJSON.name || actorJSON.preferredUsername
   return {
-    statusCode: requestResult.response.statusCode,
+    statusCode: requestResult.statusCode,
     result: {
       actor,
       name,
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index 1ed105bbe..278abf7de 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -1,27 +1,26 @@
-import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants'
-import { doRequest } from '../../helpers/requests'
-import { logger } from '../../helpers/logger'
 import * as Bluebird from 'bluebird'
-import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
 import { URL } from 'url'
+import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
+import { logger } from '../../helpers/logger'
+import { doJSONRequest } from '../../helpers/requests'
+import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants'
 
 type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
 type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
 
-async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
-  logger.info('Crawling ActivityPub data on %s.', uri)
+async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
+  let url = argUrl
+
+  logger.info('Crawling ActivityPub data on %s.', url)
 
   const options = {
-    method: 'GET',
-    uri,
-    json: true,
     activityPub: true,
     timeout: REQUEST_TIMEOUT
   }
 
   const startDate = new Date()
 
-  const response = await doRequest<ActivityPubOrderedCollection<T>>(options)
+  const response = await doJSONRequest<ActivityPubOrderedCollection<T>>(url, options)
   const firstBody = response.body
 
   const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
@@ -35,9 +34,9 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>
       const remoteHost = new URL(nextLink).host
       if (remoteHost === WEBSERVER.HOST) continue
 
-      options.uri = nextLink
+      url = nextLink
 
-      const res = await doRequest<ActivityPubOrderedCollection<T>>(options)
+      const res = await doJSONRequest<ActivityPubOrderedCollection<T>>(url, options)
       body = res.body
     } else {
       // nextLink is already the object we want
@@ -49,7 +48,7 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>
 
     if (Array.isArray(body.orderedItems)) {
       const items = body.orderedItems
-      logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri)
+      logger.info('Processing %i ActivityPub items for %s.', items.length, url)
 
       await handler(items)
     }
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
index d5a3ef7c8..795be60d7 100644
--- a/server/lib/activitypub/playlist.ts
+++ b/server/lib/activitypub/playlist.ts
@@ -1,24 +1,24 @@
+import * as Bluebird from 'bluebird'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
-import { crawlCollectionPage } from './crawl'
-import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
+import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
+import { checkUrlsSameHost } from '../../helpers/activitypub'
+import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
 import { isArray } from '../../helpers/custom-validators/misc'
-import { getOrCreateActorAndServerAndModel } from './actor'
 import { logger } from '../../helpers/logger'
+import { doJSONRequest } from '../../helpers/requests'
+import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
+import { sequelizeTypescript } from '../../initializers/database'
 import { VideoPlaylistModel } from '../../models/video/video-playlist'
-import { doRequest } from '../../helpers/requests'
-import { checkUrlsSameHost } from '../../helpers/activitypub'
-import * as Bluebird from 'bluebird'
-import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
-import { getOrCreateVideoAndAccountAndChannel } from './videos'
-import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
-import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
-import { sequelizeTypescript } from '../../initializers/database'
-import { createPlaylistMiniatureFromUrl } from '../thumbnail'
-import { FilteredModelAttributes } from '../../types/sequelize'
 import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
 import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { FilteredModelAttributes } from '../../types/sequelize'
+import { createPlaylistMiniatureFromUrl } from '../thumbnail'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { crawlCollectionPage } from './crawl'
+import { getOrCreateVideoAndAccountAndChannel } from './videos'
 
 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
@@ -56,11 +56,7 @@ async function createAccountPlaylists (playlistUrls: string[], account: MAccount
       if (exists === true) return
 
       // Fetch url
-      const { body } = await doRequest<PlaylistObject>({
-        uri: playlistUrl,
-        json: true,
-        activityPub: true
-      })
+      const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
 
       if (!isPlaylistObjectValid(body)) {
         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
@@ -164,12 +160,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid
 
   await Bluebird.map(elementUrls, async elementUrl => {
     try {
-      // Fetch url
-      const { body } = await doRequest<PlaylistElementObject>({
-        uri: elementUrl,
-        json: true,
-        activityPub: true
-      })
+      const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
 
       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 
@@ -199,21 +190,14 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid
 }
 
 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
-  const options = {
-    uri: playlistUrl,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
   logger.info('Fetching remote playlist %s.', playlistUrl)
 
-  const { response, body } = await doRequest<any>(options)
+  const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
 
   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
     logger.debug('Remote video playlist JSON is not valid.', { body })
-    return { statusCode: response.statusCode, playlistObject: undefined }
+    return { statusCode, playlistObject: undefined }
   }
 
-  return { statusCode: response.statusCode, playlistObject: body }
+  return { statusCode, playlistObject: body }
 }
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index dde0c628e..c22fa0893 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -3,7 +3,7 @@ import { Transaction } from 'sequelize'
 import { getServerActor } from '@server/models/application/application'
 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
 import { logger, loggerTagsFactory } from '../../helpers/logger'
-import { doRequest } from '../../helpers/requests'
+import { doJSONRequest } from '../../helpers/requests'
 import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
 import { VideoShareModel } from '../../models/video/video-share'
 import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video'
@@ -40,12 +40,7 @@ async function changeVideoChannelShare (
 async function addVideoShares (shareUrls: string[], video: MVideoId) {
   await Bluebird.map(shareUrls, async shareUrl => {
     try {
-      // Fetch url
-      const { body } = await doRequest<any>({
-        uri: shareUrl,
-        json: true,
-        activityPub: true
-      })
+      const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
       if (!body || !body.actor) throw new Error('Body or body actor is invalid')
 
       const actorUrl = getAPId(body.actor)
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index d025ed7f1..f1edfb0ac 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -1,13 +1,13 @@
+import * as Bluebird from 'bluebird'
+import { checkUrlsSameHost } from '../../helpers/activitypub'
 import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
 import { logger } from '../../helpers/logger'
-import { doRequest } from '../../helpers/requests'
+import { doJSONRequest } from '../../helpers/requests'
 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
 import { VideoCommentModel } from '../../models/video/video-comment'
+import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
 import { getOrCreateActorAndServerAndModel } from './actor'
 import { getOrCreateVideoAndAccountAndChannel } from './videos'
-import * as Bluebird from 'bluebird'
-import { checkUrlsSameHost } from '../../helpers/activitypub'
-import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
 
 type ResolveThreadParams = {
   url: string
@@ -126,11 +126,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
     throw new Error('Recursion limit reached when resolving a thread')
   }
 
-  const { body } = await doRequest<any>({
-    uri: url,
-    json: true,
-    activityPub: true
-  })
+  const { body } = await doJSONRequest<any>(url, { activityPub: true })
 
   if (sanitizeAndCheckVideoCommentObject(body) === false) {
     throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body))
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index e246b1313..f40c07fea 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -1,26 +1,22 @@
+import * as Bluebird from 'bluebird'
 import { Transaction } from 'sequelize'
-import { sendLike, sendUndoDislike, sendUndoLike } from './send'
+import { doJSONRequest } from '@server/helpers/requests'
 import { VideoRateType } from '../../../shared/models/videos'
-import * as Bluebird from 'bluebird'
-import { getOrCreateActorAndServerAndModel } from './actor'
-import { AccountVideoRateModel } from '../../models/account/account-video-rate'
+import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
 import { logger } from '../../helpers/logger'
 import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
-import { doRequest } from '../../helpers/requests'
-import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
-import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url'
-import { sendDislike } from './send/send-dislike'
+import { AccountVideoRateModel } from '../../models/account/account-video-rate'
 import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { sendLike, sendUndoDislike, sendUndoLike } from './send'
+import { sendDislike } from './send/send-dislike'
+import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url'
 
 async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) {
   await Bluebird.map(ratesUrl, async rateUrl => {
     try {
       // Fetch url
-      const { body } = await doRequest<any>({
-        uri: rateUrl,
-        json: true,
-        activityPub: true
-      })
+      const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
       if (!body || !body.actor) throw new Error('Body or body actor is invalid')
 
       const actorUrl = getAPId(body.actor)
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index c02578aad..a5f58dd01 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -2,7 +2,6 @@ import * as Bluebird from 'bluebird'
 import { maxBy, minBy } from 'lodash'
 import * as magnetUtil from 'magnet-uri'
 import { basename, join } from 'path'
-import * as request from 'request'
 import { Transaction } from 'sequelize/types'
 import { TrackerModel } from '@server/models/server/tracker'
 import { VideoLiveModel } from '@server/models/video/video-live'
@@ -31,7 +30,7 @@ import { isArray } from '../../helpers/custom-validators/misc'
 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
 import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
-import { doRequest } from '../../helpers/requests'
+import { doJSONRequest } from '../../helpers/requests'
 import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
 import {
   ACTIVITY_PUB,
@@ -115,36 +114,26 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid
   }
 }
 
-async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoObject }> {
-  const options = {
-    uri: videoUrl,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
+async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
   logger.info('Fetching remote video %s.', videoUrl)
 
-  const { response, body } = await doRequest<any>(options)
+  const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
 
   if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
     logger.debug('Remote video JSON is not valid.', { body })
-    return { response, videoObject: undefined }
+    return { statusCode, videoObject: undefined }
   }
 
-  return { response, videoObject: body }
+  return { statusCode, videoObject: body }
 }
 
 async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
   const host = video.VideoChannel.Account.Actor.Server.host
   const path = video.getDescriptionAPIPath()
-  const options = {
-    uri: REMOTE_SCHEME.HTTP + '://' + host + path,
-    json: true
-  }
+  const url = REMOTE_SCHEME.HTTP + '://' + host + path
 
-  const { body } = await doRequest<any>(options)
-  return body.description ? body.description : ''
+  const { body } = await doJSONRequest<any>(url)
+  return body.description || ''
 }
 
 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) {
@@ -534,8 +523,8 @@ async function refreshVideoIfNeeded (options: {
     : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
 
   try {
-    const { response, videoObject } = await fetchRemoteVideo(video.url)
-    if (response.statusCode === HttpStatusCode.NOT_FOUND_404) {
+    const { statusCode, videoObject } = await fetchRemoteVideo(video.url)
+    if (statusCode === HttpStatusCode.NOT_FOUND_404) {
       logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
 
       // Video does not exist anymore