aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers
diff options
context:
space:
mode:
Diffstat (limited to 'server/helpers')
-rw-r--r--server/helpers/activitypub.ts5
-rw-r--r--server/helpers/core-utils.ts19
-rw-r--r--server/helpers/custom-validators/activitypub/activity.ts110
-rw-r--r--server/helpers/custom-validators/activitypub/flag.ts14
-rw-r--r--server/helpers/custom-validators/activitypub/rate.ts18
-rw-r--r--server/helpers/custom-validators/activitypub/share.ts11
-rw-r--r--server/helpers/custom-validators/activitypub/view.ts13
-rw-r--r--server/helpers/custom-validators/user-notifications.ts5
-rw-r--r--server/helpers/ffmpeg-utils.ts21
-rw-r--r--server/helpers/logger.ts10
-rw-r--r--server/helpers/peertube-crypto.ts2
-rw-r--r--server/helpers/requests.ts199
-rw-r--r--server/helpers/youtube-dl.ts71
13 files changed, 289 insertions, 209 deletions
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index 08aef2908..e0754b501 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -3,7 +3,6 @@ import { URL } from 'url'
3import validator from 'validator' 3import validator from 'validator'
4import { ContextType } from '@shared/models/activitypub/context' 4import { ContextType } from '@shared/models/activitypub/context'
5import { ResultList } from '../../shared/models' 5import { ResultList } from '../../shared/models'
6import { Activity } from '../../shared/models/activitypub'
7import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants' 6import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants'
8import { MActor, MVideoWithHost } from '../types/models' 7import { MActor, MVideoWithHost } from '../types/models'
9import { pageToStartAndCount } from './core-utils' 8import { pageToStartAndCount } from './core-utils'
@@ -182,10 +181,10 @@ async function activityPubCollectionPagination (
182 181
183} 182}
184 183
185function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) { 184function buildSignedActivity <T> (byActor: MActor, data: T, contextType?: ContextType) {
186 const activity = activityPubContextify(data, contextType) 185 const activity = activityPubContextify(data, contextType)
187 186
188 return signJsonLDObject(byActor, activity) as Promise<Activity> 187 return signJsonLDObject(byActor, activity)
189} 188}
190 189
191function getAPId (activity: string | { id: string }) { 190function getAPId (activity: string | { id: string }) {
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 935fd22d9..0bd84ffaa 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -10,7 +10,9 @@ import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto'
10import { truncate } from 'lodash' 10import { truncate } from 'lodash'
11import { basename, isAbsolute, join, resolve } from 'path' 11import { basename, isAbsolute, join, resolve } from 'path'
12import * as pem from 'pem' 12import * as pem from 'pem'
13import { pipeline } from 'stream'
13import { URL } from 'url' 14import { URL } from 'url'
15import { promisify } from 'util'
14 16
15const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { 17const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
16 if (!oldObject || typeof oldObject !== 'object') { 18 if (!oldObject || typeof oldObject !== 'object') {
@@ -249,11 +251,23 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A)
249 } 251 }
250} 252}
251 253
254type SemVersion = { major: number, minor: number, patch: number }
255function parseSemVersion (s: string) {
256 const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i)
257
258 return {
259 major: parseInt(parsed[1]),
260 minor: parseInt(parsed[2]),
261 patch: parseInt(parsed[3])
262 } as SemVersion
263}
264
252const randomBytesPromise = promisify1<number, Buffer>(randomBytes) 265const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
253const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey) 266const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey)
254const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey) 267const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey)
255const execPromise2 = promisify2<string, any, string>(exec) 268const execPromise2 = promisify2<string, any, string>(exec)
256const execPromise = promisify1<string, string>(exec) 269const execPromise = promisify1<string, string>(exec)
270const pipelinePromise = promisify(pipeline)
257 271
258// --------------------------------------------------------------------------- 272// ---------------------------------------------------------------------------
259 273
@@ -284,5 +298,8 @@ export {
284 createPrivateKey, 298 createPrivateKey,
285 getPublicKey, 299 getPublicKey,
286 execPromise2, 300 execPromise2,
287 execPromise 301 execPromise,
302 pipelinePromise,
303
304 parseSemVersion
288} 305}
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts
index da79b2782..b5c96f6e7 100644
--- a/server/helpers/custom-validators/activitypub/activity.ts
+++ b/server/helpers/custom-validators/activitypub/activity.ts
@@ -1,16 +1,13 @@
1import validator from 'validator' 1import validator from 'validator'
2import { Activity, ActivityType } from '../../../../shared/models/activitypub' 2import { Activity, ActivityType } from '../../../../shared/models/activitypub'
3import { isAbuseReasonValid } from '../abuses'
3import { exists } from '../misc' 4import { exists } from '../misc'
4import { sanitizeAndCheckActorObject } from './actor' 5import { sanitizeAndCheckActorObject } from './actor'
5import { isCacheFileObjectValid } from './cache-file' 6import { isCacheFileObjectValid } from './cache-file'
6import { isFlagActivityValid } from './flag'
7import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' 7import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc'
8import { isPlaylistObjectValid } from './playlist' 8import { isPlaylistObjectValid } from './playlist'
9import { isDislikeActivityValid, isLikeActivityValid } from './rate'
10import { isShareActivityValid } from './share'
11import { sanitizeAndCheckVideoCommentObject } from './video-comments' 9import { sanitizeAndCheckVideoCommentObject } from './video-comments'
12import { sanitizeAndCheckVideoTorrentObject } from './videos' 10import { sanitizeAndCheckVideoTorrentObject } from './videos'
13import { isViewActivityValid } from './view'
14 11
15function isRootActivityValid (activity: any) { 12function isRootActivityValid (activity: any) {
16 return isCollection(activity) || isActivity(activity) 13 return isCollection(activity) || isActivity(activity)
@@ -29,18 +26,18 @@ function isActivity (activity: any) {
29} 26}
30 27
31const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = { 28const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = {
32 Create: checkCreateActivity, 29 Create: isCreateActivityValid,
33 Update: checkUpdateActivity, 30 Update: isUpdateActivityValid,
34 Delete: checkDeleteActivity, 31 Delete: isDeleteActivityValid,
35 Follow: checkFollowActivity, 32 Follow: isFollowActivityValid,
36 Accept: checkAcceptActivity, 33 Accept: isAcceptActivityValid,
37 Reject: checkRejectActivity, 34 Reject: isRejectActivityValid,
38 Announce: checkAnnounceActivity, 35 Announce: isAnnounceActivityValid,
39 Undo: checkUndoActivity, 36 Undo: isUndoActivityValid,
40 Like: checkLikeActivity, 37 Like: isLikeActivityValid,
41 View: checkViewActivity, 38 View: isViewActivityValid,
42 Flag: checkFlagActivity, 39 Flag: isFlagActivityValid,
43 Dislike: checkDislikeActivity 40 Dislike: isDislikeActivityValid
44} 41}
45 42
46function isActivityValid (activity: any) { 43function isActivityValid (activity: any) {
@@ -51,34 +48,34 @@ function isActivityValid (activity: any) {
51 return checker(activity) 48 return checker(activity)
52} 49}
53 50
54// --------------------------------------------------------------------------- 51function isFlagActivityValid (activity: any) {
55 52 return isBaseActivityValid(activity, 'Flag') &&
56export { 53 isAbuseReasonValid(activity.content) &&
57 isRootActivityValid, 54 isActivityPubUrlValid(activity.object)
58 isActivityValid
59} 55}
60 56
61// --------------------------------------------------------------------------- 57function isLikeActivityValid (activity: any) {
62 58 return isBaseActivityValid(activity, 'Like') &&
63function checkViewActivity (activity: any) { 59 isObjectValid(activity.object)
64 return isBaseActivityValid(activity, 'View') &&
65 isViewActivityValid(activity)
66} 60}
67 61
68function checkFlagActivity (activity: any) { 62function isDislikeActivityValid (activity: any) {
69 return isBaseActivityValid(activity, 'Flag') && 63 return isBaseActivityValid(activity, 'Dislike') &&
70 isFlagActivityValid(activity) 64 isObjectValid(activity.object)
71} 65}
72 66
73function checkDislikeActivity (activity: any) { 67function isAnnounceActivityValid (activity: any) {
74 return isDislikeActivityValid(activity) 68 return isBaseActivityValid(activity, 'Announce') &&
69 isObjectValid(activity.object)
75} 70}
76 71
77function checkLikeActivity (activity: any) { 72function isViewActivityValid (activity: any) {
78 return isLikeActivityValid(activity) 73 return isBaseActivityValid(activity, 'View') &&
74 isActivityPubUrlValid(activity.actor) &&
75 isActivityPubUrlValid(activity.object)
79} 76}
80 77
81function checkCreateActivity (activity: any) { 78function isCreateActivityValid (activity: any) {
82 return isBaseActivityValid(activity, 'Create') && 79 return isBaseActivityValid(activity, 'Create') &&
83 ( 80 (
84 isViewActivityValid(activity.object) || 81 isViewActivityValid(activity.object) ||
@@ -92,7 +89,7 @@ function checkCreateActivity (activity: any) {
92 ) 89 )
93} 90}
94 91
95function checkUpdateActivity (activity: any) { 92function isUpdateActivityValid (activity: any) {
96 return isBaseActivityValid(activity, 'Update') && 93 return isBaseActivityValid(activity, 'Update') &&
97 ( 94 (
98 isCacheFileObjectValid(activity.object) || 95 isCacheFileObjectValid(activity.object) ||
@@ -102,36 +99,51 @@ function checkUpdateActivity (activity: any) {
102 ) 99 )
103} 100}
104 101
105function checkDeleteActivity (activity: any) { 102function isDeleteActivityValid (activity: any) {
106 // We don't really check objects 103 // We don't really check objects
107 return isBaseActivityValid(activity, 'Delete') && 104 return isBaseActivityValid(activity, 'Delete') &&
108 isObjectValid(activity.object) 105 isObjectValid(activity.object)
109} 106}
110 107
111function checkFollowActivity (activity: any) { 108function isFollowActivityValid (activity: any) {
112 return isBaseActivityValid(activity, 'Follow') && 109 return isBaseActivityValid(activity, 'Follow') &&
113 isObjectValid(activity.object) 110 isObjectValid(activity.object)
114} 111}
115 112
116function checkAcceptActivity (activity: any) { 113function isAcceptActivityValid (activity: any) {
117 return isBaseActivityValid(activity, 'Accept') 114 return isBaseActivityValid(activity, 'Accept')
118} 115}
119 116
120function checkRejectActivity (activity: any) { 117function isRejectActivityValid (activity: any) {
121 return isBaseActivityValid(activity, 'Reject') 118 return isBaseActivityValid(activity, 'Reject')
122} 119}
123 120
124function checkAnnounceActivity (activity: any) { 121function isUndoActivityValid (activity: any) {
125 return isShareActivityValid(activity)
126}
127
128function checkUndoActivity (activity: any) {
129 return isBaseActivityValid(activity, 'Undo') && 122 return isBaseActivityValid(activity, 'Undo') &&
130 ( 123 (
131 checkFollowActivity(activity.object) || 124 isFollowActivityValid(activity.object) ||
132 checkLikeActivity(activity.object) || 125 isLikeActivityValid(activity.object) ||
133 checkDislikeActivity(activity.object) || 126 isDislikeActivityValid(activity.object) ||
134 checkAnnounceActivity(activity.object) || 127 isAnnounceActivityValid(activity.object) ||
135 checkCreateActivity(activity.object) 128 isCreateActivityValid(activity.object)
136 ) 129 )
137} 130}
131
132// ---------------------------------------------------------------------------
133
134export {
135 isRootActivityValid,
136 isActivityValid,
137 isFlagActivityValid,
138 isLikeActivityValid,
139 isDislikeActivityValid,
140 isAnnounceActivityValid,
141 isViewActivityValid,
142 isCreateActivityValid,
143 isUpdateActivityValid,
144 isDeleteActivityValid,
145 isFollowActivityValid,
146 isAcceptActivityValid,
147 isRejectActivityValid,
148 isUndoActivityValid
149}
diff --git a/server/helpers/custom-validators/activitypub/flag.ts b/server/helpers/custom-validators/activitypub/flag.ts
deleted file mode 100644
index dc90b3667..000000000
--- a/server/helpers/custom-validators/activitypub/flag.ts
+++ /dev/null
@@ -1,14 +0,0 @@
1import { isActivityPubUrlValid } from './misc'
2import { isAbuseReasonValid } from '../abuses'
3
4function isFlagActivityValid (activity: any) {
5 return activity.type === 'Flag' &&
6 isAbuseReasonValid(activity.content) &&
7 isActivityPubUrlValid(activity.object)
8}
9
10// ---------------------------------------------------------------------------
11
12export {
13 isFlagActivityValid
14}
diff --git a/server/helpers/custom-validators/activitypub/rate.ts b/server/helpers/custom-validators/activitypub/rate.ts
deleted file mode 100644
index aafdda443..000000000
--- a/server/helpers/custom-validators/activitypub/rate.ts
+++ /dev/null
@@ -1,18 +0,0 @@
1import { isBaseActivityValid, isObjectValid } from './misc'
2
3function isLikeActivityValid (activity: any) {
4 return isBaseActivityValid(activity, 'Like') &&
5 isObjectValid(activity.object)
6}
7
8function isDislikeActivityValid (activity: any) {
9 return isBaseActivityValid(activity, 'Dislike') &&
10 isObjectValid(activity.object)
11}
12
13// ---------------------------------------------------------------------------
14
15export {
16 isDislikeActivityValid,
17 isLikeActivityValid
18}
diff --git a/server/helpers/custom-validators/activitypub/share.ts b/server/helpers/custom-validators/activitypub/share.ts
deleted file mode 100644
index fb5e4c05e..000000000
--- a/server/helpers/custom-validators/activitypub/share.ts
+++ /dev/null
@@ -1,11 +0,0 @@
1import { isBaseActivityValid, isObjectValid } from './misc'
2
3function isShareActivityValid (activity: any) {
4 return isBaseActivityValid(activity, 'Announce') &&
5 isObjectValid(activity.object)
6}
7// ---------------------------------------------------------------------------
8
9export {
10 isShareActivityValid
11}
diff --git a/server/helpers/custom-validators/activitypub/view.ts b/server/helpers/custom-validators/activitypub/view.ts
deleted file mode 100644
index 41d16469f..000000000
--- a/server/helpers/custom-validators/activitypub/view.ts
+++ /dev/null
@@ -1,13 +0,0 @@
1import { isActivityPubUrlValid } from './misc'
2
3function isViewActivityValid (activity: any) {
4 return activity.type === 'View' &&
5 isActivityPubUrlValid(activity.actor) &&
6 isActivityPubUrlValid(activity.object)
7}
8
9// ---------------------------------------------------------------------------
10
11export {
12 isViewActivityValid
13}
diff --git a/server/helpers/custom-validators/user-notifications.ts b/server/helpers/custom-validators/user-notifications.ts
index 8a33b895b..252c107db 100644
--- a/server/helpers/custom-validators/user-notifications.ts
+++ b/server/helpers/custom-validators/user-notifications.ts
@@ -1,10 +1,9 @@
1import { exists } from './misc'
2import validator from 'validator' 1import validator from 'validator'
3import { UserNotificationType } from '../../../shared/models/users'
4import { UserNotificationSettingValue } from '../../../shared/models/users/user-notification-setting.model' 2import { UserNotificationSettingValue } from '../../../shared/models/users/user-notification-setting.model'
3import { exists } from './misc'
5 4
6function isUserNotificationTypeValid (value: any) { 5function isUserNotificationTypeValid (value: any) {
7 return exists(value) && validator.isInt('' + value) && UserNotificationType[value] !== undefined 6 return exists(value) && validator.isInt('' + value)
8} 7}
9 8
10function isUserNotificationSettingValid (value: any) { 9function isUserNotificationSettingValid (value: any) {
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 620025966..69cd397b9 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -5,7 +5,7 @@ import { dirname, join } from 'path'
5import { FFMPEG_NICE, VIDEO_LIVE } from '@server/initializers/constants' 5import { FFMPEG_NICE, VIDEO_LIVE } from '@server/initializers/constants'
6import { AvailableEncoders, EncoderOptionsBuilder, EncoderProfile, VideoResolution } from '../../shared/models/videos' 6import { AvailableEncoders, EncoderOptionsBuilder, EncoderProfile, VideoResolution } from '../../shared/models/videos'
7import { CONFIG } from '../initializers/config' 7import { CONFIG } from '../initializers/config'
8import { promisify0 } from './core-utils' 8import { execPromise, promisify0 } from './core-utils'
9import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils' 9import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils'
10import { processImage } from './image-utils' 10import { processImage } from './image-utils'
11import { logger } from './logger' 11import { logger } from './logger'
@@ -649,6 +649,24 @@ function getFFmpeg (input: string, type: 'live' | 'vod') {
649 return command 649 return command
650} 650}
651 651
652function getFFmpegVersion () {
653 return new Promise<string>((res, rej) => {
654 (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => {
655 if (err) return rej(err)
656 if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path'))
657
658 return execPromise(`${ffmpegPath} -version`)
659 .then(stdout => {
660 const parsed = stdout.match(/ffmpeg version .(\d+\.\d+\.\d+)/)
661 if (!parsed || !parsed[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`))
662
663 return res(parsed[1])
664 })
665 .catch(err => rej(err))
666 })
667 })
668}
669
652async function runCommand (options: { 670async function runCommand (options: {
653 command: ffmpeg.FfmpegCommand 671 command: ffmpeg.FfmpegCommand
654 silent?: boolean // false 672 silent?: boolean // false
@@ -695,6 +713,7 @@ export {
695 TranscodeOptionsType, 713 TranscodeOptionsType,
696 transcode, 714 transcode,
697 runCommand, 715 runCommand,
716 getFFmpegVersion,
698 717
699 resetSupportedEncoders, 718 resetSupportedEncoders,
700 719
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 6917a64d9..a112fd300 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -48,7 +48,7 @@ function getLoggerReplacer () {
48} 48}
49 49
50const consoleLoggerFormat = winston.format.printf(info => { 50const consoleLoggerFormat = winston.format.printf(info => {
51 const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql' ] 51 const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ]
52 52
53 const obj = omit(info, ...toOmit) 53 const obj = omit(info, ...toOmit)
54 54
@@ -150,6 +150,13 @@ const bunyanLogger = {
150 error: bunyanLogFactory('error'), 150 error: bunyanLogFactory('error'),
151 fatal: bunyanLogFactory('error') 151 fatal: bunyanLogFactory('error')
152} 152}
153
154function loggerTagsFactory (...defaultTags: string[]) {
155 return (...tags: string[]) => {
156 return { tags: defaultTags.concat(tags) }
157 }
158}
159
153// --------------------------------------------------------------------------- 160// ---------------------------------------------------------------------------
154 161
155export { 162export {
@@ -159,5 +166,6 @@ export {
159 consoleLoggerFormat, 166 consoleLoggerFormat,
160 jsonLoggerFormat, 167 jsonLoggerFormat,
161 logger, 168 logger,
169 loggerTagsFactory,
162 bunyanLogger 170 bunyanLogger
163} 171}
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts
index 994f725d8..bc6f1d074 100644
--- a/server/helpers/peertube-crypto.ts
+++ b/server/helpers/peertube-crypto.ts
@@ -84,7 +84,7 @@ async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any)
84 return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') 84 return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64')
85} 85}
86 86
87async function signJsonLDObject (byActor: MActor, data: any) { 87async function signJsonLDObject <T> (byActor: MActor, data: T) {
88 const signature = { 88 const signature = {
89 type: 'RsaSignature2017', 89 type: 'RsaSignature2017',
90 creator: byActor.url, 90 creator: byActor.url,
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index b556c392e..fd2a56f30 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -1,58 +1,141 @@
1import * as Bluebird from 'bluebird'
2import { createWriteStream, remove } from 'fs-extra' 1import { createWriteStream, remove } from 'fs-extra'
3import * as request from 'request' 2import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
3import { join } from 'path'
4import { CONFIG } from '../initializers/config'
4import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' 5import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
6import { pipelinePromise } from './core-utils'
5import { processImage } from './image-utils' 7import { processImage } from './image-utils'
6import { join } from 'path'
7import { logger } from './logger' 8import { logger } from './logger'
8import { CONFIG } from '../initializers/config'
9 9
10function doRequest <T> ( 10export interface PeerTubeRequestError extends Error {
11 requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }, 11 statusCode?: number
12 bodyKBLimit = 1000 // 1MB 12 responseBody?: any
13): Bluebird<{ response: request.RequestResponse, body: T }> { 13}
14 if (!(requestOptions.headers)) requestOptions.headers = {}
15 requestOptions.headers['User-Agent'] = getUserAgent()
16 14
17 if (requestOptions.activityPub === true) { 15const httpSignature = require('http-signature')
18 requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER 16
17type PeerTubeRequestOptions = {
18 activityPub?: boolean
19 bodyKBLimit?: number // 1MB
20 httpSignature?: {
21 algorithm: string
22 authorizationHeaderName: string
23 keyId: string
24 key: string
25 headers: string[]
19 } 26 }
27 jsonResponse?: boolean
28} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
29
30const peertubeGot = got.extend({
31 headers: {
32 'user-agent': getUserAgent()
33 },
34
35 handlers: [
36 (options, next) => {
37 const promiseOrStream = next(options) as CancelableRequest<any>
38 const bodyKBLimit = options.context?.bodyKBLimit as number
39 if (!bodyKBLimit) throw new Error('No KB limit for this request')
40
41 const bodyLimit = bodyKBLimit * 1000
42
43 /* eslint-disable @typescript-eslint/no-floating-promises */
44 promiseOrStream.on('downloadProgress', progress => {
45 if (progress.transferred > bodyLimit && progress.percent !== 1) {
46 const message = `Exceeded the download limit of ${bodyLimit} B`
47 logger.warn(message)
48
49 // CancelableRequest
50 if (promiseOrStream.cancel) {
51 promiseOrStream.cancel()
52 return
53 }
54
55 // Stream
56 (promiseOrStream as any).destroy()
57 }
58 })
20 59
21 return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => { 60 return promiseOrStream
22 request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) 61 }
23 .on('data', onRequestDataLengthCheck(bodyKBLimit)) 62 ],
24 }) 63
64 hooks: {
65 beforeRequest: [
66 options => {
67 const headers = options.headers || {}
68 headers['host'] = options.url.host
69 },
70
71 options => {
72 const httpSignatureOptions = options.context?.httpSignature
73
74 if (httpSignatureOptions) {
75 const method = options.method ?? 'GET'
76 const path = options.path ?? options.url.pathname
77
78 if (!method || !path) {
79 throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
80 }
81
82 httpSignature.signRequest({
83 getHeader: function (header) {
84 return options.headers[header]
85 },
86
87 setHeader: function (header, value) {
88 options.headers[header] = value
89 },
90
91 method,
92 path
93 }, httpSignatureOptions)
94 }
95 }
96 ]
97 }
98})
99
100function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
101 const gotOptions = buildGotOptions(options)
102
103 return peertubeGot(url, gotOptions)
104 .catch(err => { throw buildRequestError(err) })
25} 105}
26 106
27function doRequestAndSaveToFile ( 107function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
28 requestOptions: request.CoreOptions & request.UriOptions, 108 const gotOptions = buildGotOptions(options)
109
110 return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
111 .catch(err => { throw buildRequestError(err) })
112}
113
114async function doRequestAndSaveToFile (
115 url: string,
29 destPath: string, 116 destPath: string,
30 bodyKBLimit = 10000 // 10MB 117 options: PeerTubeRequestOptions = {}
31) { 118) {
32 if (!requestOptions.headers) requestOptions.headers = {} 119 const gotOptions = buildGotOptions(options)
33 requestOptions.headers['User-Agent'] = getUserAgent()
34
35 return new Bluebird<void>((res, rej) => {
36 const file = createWriteStream(destPath)
37 file.on('finish', () => res())
38 120
39 request(requestOptions) 121 const outFile = createWriteStream(destPath)
40 .on('data', onRequestDataLengthCheck(bodyKBLimit))
41 .on('error', err => {
42 file.close()
43 122
44 remove(destPath) 123 try {
45 .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) 124 await pipelinePromise(
125 peertubeGot.stream(url, gotOptions),
126 outFile
127 )
128 } catch (err) {
129 remove(destPath)
130 .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
46 131
47 return rej(err) 132 throw buildRequestError(err)
48 }) 133 }
49 .pipe(file)
50 })
51} 134}
52 135
53async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { 136async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
54 const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) 137 const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
55 await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath) 138 await doRequestAndSaveToFile(url, tmpPath)
56 139
57 const destPath = join(destDir, destName) 140 const destPath = join(destDir, destName)
58 141
@@ -73,24 +156,46 @@ function getUserAgent () {
73 156
74export { 157export {
75 doRequest, 158 doRequest,
159 doJSONRequest,
76 doRequestAndSaveToFile, 160 doRequestAndSaveToFile,
77 downloadImage 161 downloadImage
78} 162}
79 163
80// --------------------------------------------------------------------------- 164// ---------------------------------------------------------------------------
81 165
82// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3 166function buildGotOptions (options: PeerTubeRequestOptions) {
83function onRequestDataLengthCheck (bodyKBLimit: number) { 167 const { activityPub, bodyKBLimit = 1000 } = options
84 let bufferLength = 0
85 const bytesLimit = bodyKBLimit * 1000
86 168
87 return function (chunk) { 169 const context = { bodyKBLimit, httpSignature: options.httpSignature }
88 bufferLength += chunk.length
89 if (bufferLength > bytesLimit) {
90 this.abort()
91 170
92 const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`) 171 let headers = options.headers || {}
93 this.emit('error', error) 172
94 } 173 if (!headers.date) {
174 headers = { ...headers, date: new Date().toUTCString() }
175 }
176
177 if (activityPub && !headers.accept) {
178 headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
95 } 179 }
180
181 return {
182 method: options.method,
183 json: options.json,
184 searchParams: options.searchParams,
185 headers,
186 context
187 }
188}
189
190function buildRequestError (error: RequestError) {
191 const newError: PeerTubeRequestError = new Error(error.message)
192 newError.name = error.name
193 newError.stack = error.stack
194
195 if (error.response) {
196 newError.responseBody = error.response.body
197 newError.statusCode = error.response.statusCode
198 }
199
200 return newError
96} 201}
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 8537a5772..9d2e54fb5 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,13 +1,13 @@
1import { createWriteStream } from 'fs' 1import { createWriteStream } from 'fs'
2import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra' 2import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra'
3import got from 'got'
3import { join } from 'path' 4import { join } from 'path'
4import * as request from 'request'
5import { CONFIG } from '@server/initializers/config' 5import { CONFIG } from '@server/initializers/config'
6import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' 6import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
7import { VideoResolution } from '../../shared/models/videos' 7import { VideoResolution } from '../../shared/models/videos'
8import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' 8import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
9import { getEnabledResolutions } from '../lib/video-transcoding' 9import { getEnabledResolutions } from '../lib/video-transcoding'
10import { peertubeTruncate, root } from './core-utils' 10import { peertubeTruncate, pipelinePromise, root } from './core-utils'
11import { isVideoFileExtnameValid } from './custom-validators/videos' 11import { isVideoFileExtnameValid } from './custom-validators/videos'
12import { logger } from './logger' 12import { logger } from './logger'
13import { generateVideoImportTmpPath } from './utils' 13import { generateVideoImportTmpPath } from './utils'
@@ -195,55 +195,32 @@ async function updateYoutubeDLBinary () {
195 195
196 await ensureDir(binDirectory) 196 await ensureDir(binDirectory)
197 197
198 return new Promise<void>(res => { 198 try {
199 request.get(url, { followRedirect: false }, (err, result) => { 199 const result = await got(url, { followRedirect: false })
200 if (err) {
201 logger.error('Cannot update youtube-dl.', { err })
202 return res()
203 }
204
205 if (result.statusCode !== HttpStatusCode.FOUND_302) {
206 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
207 return res()
208 }
209
210 const url = result.headers.location
211 const downloadFile = request.get(url)
212 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
213
214 downloadFile.on('response', result => {
215 if (result.statusCode !== HttpStatusCode.OK_200) {
216 logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
217 return res()
218 }
219
220 const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => {
221 logger.error('youtube-dl update error in write stream', { err })
222 return res()
223 })
224 200
225 downloadFile.pipe(writeStream) 201 if (result.statusCode !== HttpStatusCode.FOUND_302) {
226 }) 202 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
203 return
204 }
227 205
228 downloadFile.on('error', err => { 206 const newUrl = result.headers.location
229 logger.error('youtube-dl update error.', { err }) 207 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1]
230 return res()
231 })
232 208
233 downloadFile.on('end', () => { 209 const downloadFileStream = got.stream(newUrl)
234 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) 210 const writeStream = createWriteStream(bin, { mode: 493 })
235 writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
236 if (err) {
237 logger.error('youtube-dl update error: cannot write details.', { err })
238 return res()
239 }
240 211
241 logger.info('youtube-dl updated to version %s.', newVersion) 212 await pipelinePromise(
242 return res() 213 downloadFileStream,
243 }) 214 writeStream
244 }) 215 )
245 }) 216
246 }) 217 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
218 await writeFile(detailsPath, details, { encoding: 'utf8' })
219
220 logger.info('youtube-dl updated to version %s.', newVersion)
221 } catch (err) {
222 logger.error('Cannot update youtube-dl.', { err })
223 }
247} 224}
248 225
249async function safeGetYoutubeDL () { 226async function safeGetYoutubeDL () {