diff options
Diffstat (limited to 'server/helpers')
-rw-r--r-- | server/helpers/activitypub.ts | 5 | ||||
-rw-r--r-- | server/helpers/core-utils.ts | 19 | ||||
-rw-r--r-- | server/helpers/custom-validators/activitypub/activity.ts | 110 | ||||
-rw-r--r-- | server/helpers/custom-validators/activitypub/flag.ts | 14 | ||||
-rw-r--r-- | server/helpers/custom-validators/activitypub/rate.ts | 18 | ||||
-rw-r--r-- | server/helpers/custom-validators/activitypub/share.ts | 11 | ||||
-rw-r--r-- | server/helpers/custom-validators/activitypub/view.ts | 13 | ||||
-rw-r--r-- | server/helpers/custom-validators/user-notifications.ts | 5 | ||||
-rw-r--r-- | server/helpers/ffmpeg-utils.ts | 21 | ||||
-rw-r--r-- | server/helpers/logger.ts | 10 | ||||
-rw-r--r-- | server/helpers/peertube-crypto.ts | 2 | ||||
-rw-r--r-- | server/helpers/requests.ts | 199 | ||||
-rw-r--r-- | server/helpers/youtube-dl.ts | 71 |
13 files changed, 289 insertions, 209 deletions
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 08aef2908..e0754b501 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts | |||
@@ -3,7 +3,6 @@ import { URL } from 'url' | |||
3 | import validator from 'validator' | 3 | import validator from 'validator' |
4 | import { ContextType } from '@shared/models/activitypub/context' | 4 | import { ContextType } from '@shared/models/activitypub/context' |
5 | import { ResultList } from '../../shared/models' | 5 | import { ResultList } from '../../shared/models' |
6 | import { Activity } from '../../shared/models/activitypub' | ||
7 | import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants' | 6 | import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants' |
8 | import { MActor, MVideoWithHost } from '../types/models' | 7 | import { MActor, MVideoWithHost } from '../types/models' |
9 | import { pageToStartAndCount } from './core-utils' | 8 | import { pageToStartAndCount } from './core-utils' |
@@ -182,10 +181,10 @@ async function activityPubCollectionPagination ( | |||
182 | 181 | ||
183 | } | 182 | } |
184 | 183 | ||
185 | function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) { | 184 | function buildSignedActivity <T> (byActor: MActor, data: T, contextType?: ContextType) { |
186 | const activity = activityPubContextify(data, contextType) | 185 | const activity = activityPubContextify(data, contextType) |
187 | 186 | ||
188 | return signJsonLDObject(byActor, activity) as Promise<Activity> | 187 | return signJsonLDObject(byActor, activity) |
189 | } | 188 | } |
190 | 189 | ||
191 | function getAPId (activity: string | { id: string }) { | 190 | function getAPId (activity: string | { id: string }) { |
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 935fd22d9..0bd84ffaa 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -10,7 +10,9 @@ import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto' | |||
10 | import { truncate } from 'lodash' | 10 | import { truncate } from 'lodash' |
11 | import { basename, isAbsolute, join, resolve } from 'path' | 11 | import { basename, isAbsolute, join, resolve } from 'path' |
12 | import * as pem from 'pem' | 12 | import * as pem from 'pem' |
13 | import { pipeline } from 'stream' | ||
13 | import { URL } from 'url' | 14 | import { URL } from 'url' |
15 | import { promisify } from 'util' | ||
14 | 16 | ||
15 | const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { | 17 | const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { |
16 | if (!oldObject || typeof oldObject !== 'object') { | 18 | if (!oldObject || typeof oldObject !== 'object') { |
@@ -249,11 +251,23 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) | |||
249 | } | 251 | } |
250 | } | 252 | } |
251 | 253 | ||
254 | type SemVersion = { major: number, minor: number, patch: number } | ||
255 | function parseSemVersion (s: string) { | ||
256 | const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i) | ||
257 | |||
258 | return { | ||
259 | major: parseInt(parsed[1]), | ||
260 | minor: parseInt(parsed[2]), | ||
261 | patch: parseInt(parsed[3]) | ||
262 | } as SemVersion | ||
263 | } | ||
264 | |||
252 | const randomBytesPromise = promisify1<number, Buffer>(randomBytes) | 265 | const randomBytesPromise = promisify1<number, Buffer>(randomBytes) |
253 | const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey) | 266 | const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey) |
254 | const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey) | 267 | const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey) |
255 | const execPromise2 = promisify2<string, any, string>(exec) | 268 | const execPromise2 = promisify2<string, any, string>(exec) |
256 | const execPromise = promisify1<string, string>(exec) | 269 | const execPromise = promisify1<string, string>(exec) |
270 | const pipelinePromise = promisify(pipeline) | ||
257 | 271 | ||
258 | // --------------------------------------------------------------------------- | 272 | // --------------------------------------------------------------------------- |
259 | 273 | ||
@@ -284,5 +298,8 @@ export { | |||
284 | createPrivateKey, | 298 | createPrivateKey, |
285 | getPublicKey, | 299 | getPublicKey, |
286 | execPromise2, | 300 | execPromise2, |
287 | execPromise | 301 | execPromise, |
302 | pipelinePromise, | ||
303 | |||
304 | parseSemVersion | ||
288 | } | 305 | } |
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index da79b2782..b5c96f6e7 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts | |||
@@ -1,16 +1,13 @@ | |||
1 | import validator from 'validator' | 1 | import validator from 'validator' |
2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' | 2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' |
3 | import { isAbuseReasonValid } from '../abuses' | ||
3 | import { exists } from '../misc' | 4 | import { exists } from '../misc' |
4 | import { sanitizeAndCheckActorObject } from './actor' | 5 | import { sanitizeAndCheckActorObject } from './actor' |
5 | import { isCacheFileObjectValid } from './cache-file' | 6 | import { isCacheFileObjectValid } from './cache-file' |
6 | import { isFlagActivityValid } from './flag' | ||
7 | import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' | 7 | import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' |
8 | import { isPlaylistObjectValid } from './playlist' | 8 | import { isPlaylistObjectValid } from './playlist' |
9 | import { isDislikeActivityValid, isLikeActivityValid } from './rate' | ||
10 | import { isShareActivityValid } from './share' | ||
11 | import { sanitizeAndCheckVideoCommentObject } from './video-comments' | 9 | import { sanitizeAndCheckVideoCommentObject } from './video-comments' |
12 | import { sanitizeAndCheckVideoTorrentObject } from './videos' | 10 | import { sanitizeAndCheckVideoTorrentObject } from './videos' |
13 | import { isViewActivityValid } from './view' | ||
14 | 11 | ||
15 | function isRootActivityValid (activity: any) { | 12 | function isRootActivityValid (activity: any) { |
16 | return isCollection(activity) || isActivity(activity) | 13 | return isCollection(activity) || isActivity(activity) |
@@ -29,18 +26,18 @@ function isActivity (activity: any) { | |||
29 | } | 26 | } |
30 | 27 | ||
31 | const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = { | 28 | const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = { |
32 | Create: checkCreateActivity, | 29 | Create: isCreateActivityValid, |
33 | Update: checkUpdateActivity, | 30 | Update: isUpdateActivityValid, |
34 | Delete: checkDeleteActivity, | 31 | Delete: isDeleteActivityValid, |
35 | Follow: checkFollowActivity, | 32 | Follow: isFollowActivityValid, |
36 | Accept: checkAcceptActivity, | 33 | Accept: isAcceptActivityValid, |
37 | Reject: checkRejectActivity, | 34 | Reject: isRejectActivityValid, |
38 | Announce: checkAnnounceActivity, | 35 | Announce: isAnnounceActivityValid, |
39 | Undo: checkUndoActivity, | 36 | Undo: isUndoActivityValid, |
40 | Like: checkLikeActivity, | 37 | Like: isLikeActivityValid, |
41 | View: checkViewActivity, | 38 | View: isViewActivityValid, |
42 | Flag: checkFlagActivity, | 39 | Flag: isFlagActivityValid, |
43 | Dislike: checkDislikeActivity | 40 | Dislike: isDislikeActivityValid |
44 | } | 41 | } |
45 | 42 | ||
46 | function isActivityValid (activity: any) { | 43 | function isActivityValid (activity: any) { |
@@ -51,34 +48,34 @@ function isActivityValid (activity: any) { | |||
51 | return checker(activity) | 48 | return checker(activity) |
52 | } | 49 | } |
53 | 50 | ||
54 | // --------------------------------------------------------------------------- | 51 | function isFlagActivityValid (activity: any) { |
55 | 52 | return isBaseActivityValid(activity, 'Flag') && | |
56 | export { | 53 | isAbuseReasonValid(activity.content) && |
57 | isRootActivityValid, | 54 | isActivityPubUrlValid(activity.object) |
58 | isActivityValid | ||
59 | } | 55 | } |
60 | 56 | ||
61 | // --------------------------------------------------------------------------- | 57 | function isLikeActivityValid (activity: any) { |
62 | 58 | return isBaseActivityValid(activity, 'Like') && | |
63 | function checkViewActivity (activity: any) { | 59 | isObjectValid(activity.object) |
64 | return isBaseActivityValid(activity, 'View') && | ||
65 | isViewActivityValid(activity) | ||
66 | } | 60 | } |
67 | 61 | ||
68 | function checkFlagActivity (activity: any) { | 62 | function isDislikeActivityValid (activity: any) { |
69 | return isBaseActivityValid(activity, 'Flag') && | 63 | return isBaseActivityValid(activity, 'Dislike') && |
70 | isFlagActivityValid(activity) | 64 | isObjectValid(activity.object) |
71 | } | 65 | } |
72 | 66 | ||
73 | function checkDislikeActivity (activity: any) { | 67 | function isAnnounceActivityValid (activity: any) { |
74 | return isDislikeActivityValid(activity) | 68 | return isBaseActivityValid(activity, 'Announce') && |
69 | isObjectValid(activity.object) | ||
75 | } | 70 | } |
76 | 71 | ||
77 | function checkLikeActivity (activity: any) { | 72 | function isViewActivityValid (activity: any) { |
78 | return isLikeActivityValid(activity) | 73 | return isBaseActivityValid(activity, 'View') && |
74 | isActivityPubUrlValid(activity.actor) && | ||
75 | isActivityPubUrlValid(activity.object) | ||
79 | } | 76 | } |
80 | 77 | ||
81 | function checkCreateActivity (activity: any) { | 78 | function isCreateActivityValid (activity: any) { |
82 | return isBaseActivityValid(activity, 'Create') && | 79 | return isBaseActivityValid(activity, 'Create') && |
83 | ( | 80 | ( |
84 | isViewActivityValid(activity.object) || | 81 | isViewActivityValid(activity.object) || |
@@ -92,7 +89,7 @@ function checkCreateActivity (activity: any) { | |||
92 | ) | 89 | ) |
93 | } | 90 | } |
94 | 91 | ||
95 | function checkUpdateActivity (activity: any) { | 92 | function isUpdateActivityValid (activity: any) { |
96 | return isBaseActivityValid(activity, 'Update') && | 93 | return isBaseActivityValid(activity, 'Update') && |
97 | ( | 94 | ( |
98 | isCacheFileObjectValid(activity.object) || | 95 | isCacheFileObjectValid(activity.object) || |
@@ -102,36 +99,51 @@ function checkUpdateActivity (activity: any) { | |||
102 | ) | 99 | ) |
103 | } | 100 | } |
104 | 101 | ||
105 | function checkDeleteActivity (activity: any) { | 102 | function isDeleteActivityValid (activity: any) { |
106 | // We don't really check objects | 103 | // We don't really check objects |
107 | return isBaseActivityValid(activity, 'Delete') && | 104 | return isBaseActivityValid(activity, 'Delete') && |
108 | isObjectValid(activity.object) | 105 | isObjectValid(activity.object) |
109 | } | 106 | } |
110 | 107 | ||
111 | function checkFollowActivity (activity: any) { | 108 | function isFollowActivityValid (activity: any) { |
112 | return isBaseActivityValid(activity, 'Follow') && | 109 | return isBaseActivityValid(activity, 'Follow') && |
113 | isObjectValid(activity.object) | 110 | isObjectValid(activity.object) |
114 | } | 111 | } |
115 | 112 | ||
116 | function checkAcceptActivity (activity: any) { | 113 | function isAcceptActivityValid (activity: any) { |
117 | return isBaseActivityValid(activity, 'Accept') | 114 | return isBaseActivityValid(activity, 'Accept') |
118 | } | 115 | } |
119 | 116 | ||
120 | function checkRejectActivity (activity: any) { | 117 | function isRejectActivityValid (activity: any) { |
121 | return isBaseActivityValid(activity, 'Reject') | 118 | return isBaseActivityValid(activity, 'Reject') |
122 | } | 119 | } |
123 | 120 | ||
124 | function checkAnnounceActivity (activity: any) { | 121 | function isUndoActivityValid (activity: any) { |
125 | return isShareActivityValid(activity) | ||
126 | } | ||
127 | |||
128 | function checkUndoActivity (activity: any) { | ||
129 | return isBaseActivityValid(activity, 'Undo') && | 122 | return isBaseActivityValid(activity, 'Undo') && |
130 | ( | 123 | ( |
131 | checkFollowActivity(activity.object) || | 124 | isFollowActivityValid(activity.object) || |
132 | checkLikeActivity(activity.object) || | 125 | isLikeActivityValid(activity.object) || |
133 | checkDislikeActivity(activity.object) || | 126 | isDislikeActivityValid(activity.object) || |
134 | checkAnnounceActivity(activity.object) || | 127 | isAnnounceActivityValid(activity.object) || |
135 | checkCreateActivity(activity.object) | 128 | isCreateActivityValid(activity.object) |
136 | ) | 129 | ) |
137 | } | 130 | } |
131 | |||
132 | // --------------------------------------------------------------------------- | ||
133 | |||
134 | export { | ||
135 | isRootActivityValid, | ||
136 | isActivityValid, | ||
137 | isFlagActivityValid, | ||
138 | isLikeActivityValid, | ||
139 | isDislikeActivityValid, | ||
140 | isAnnounceActivityValid, | ||
141 | isViewActivityValid, | ||
142 | isCreateActivityValid, | ||
143 | isUpdateActivityValid, | ||
144 | isDeleteActivityValid, | ||
145 | isFollowActivityValid, | ||
146 | isAcceptActivityValid, | ||
147 | isRejectActivityValid, | ||
148 | isUndoActivityValid | ||
149 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/flag.ts b/server/helpers/custom-validators/activitypub/flag.ts deleted file mode 100644 index dc90b3667..000000000 --- a/server/helpers/custom-validators/activitypub/flag.ts +++ /dev/null | |||
@@ -1,14 +0,0 @@ | |||
1 | import { isActivityPubUrlValid } from './misc' | ||
2 | import { isAbuseReasonValid } from '../abuses' | ||
3 | |||
4 | function isFlagActivityValid (activity: any) { | ||
5 | return activity.type === 'Flag' && | ||
6 | isAbuseReasonValid(activity.content) && | ||
7 | isActivityPubUrlValid(activity.object) | ||
8 | } | ||
9 | |||
10 | // --------------------------------------------------------------------------- | ||
11 | |||
12 | export { | ||
13 | isFlagActivityValid | ||
14 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/rate.ts b/server/helpers/custom-validators/activitypub/rate.ts deleted file mode 100644 index aafdda443..000000000 --- a/server/helpers/custom-validators/activitypub/rate.ts +++ /dev/null | |||
@@ -1,18 +0,0 @@ | |||
1 | import { isBaseActivityValid, isObjectValid } from './misc' | ||
2 | |||
3 | function isLikeActivityValid (activity: any) { | ||
4 | return isBaseActivityValid(activity, 'Like') && | ||
5 | isObjectValid(activity.object) | ||
6 | } | ||
7 | |||
8 | function isDislikeActivityValid (activity: any) { | ||
9 | return isBaseActivityValid(activity, 'Dislike') && | ||
10 | isObjectValid(activity.object) | ||
11 | } | ||
12 | |||
13 | // --------------------------------------------------------------------------- | ||
14 | |||
15 | export { | ||
16 | isDislikeActivityValid, | ||
17 | isLikeActivityValid | ||
18 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/share.ts b/server/helpers/custom-validators/activitypub/share.ts deleted file mode 100644 index fb5e4c05e..000000000 --- a/server/helpers/custom-validators/activitypub/share.ts +++ /dev/null | |||
@@ -1,11 +0,0 @@ | |||
1 | import { isBaseActivityValid, isObjectValid } from './misc' | ||
2 | |||
3 | function isShareActivityValid (activity: any) { | ||
4 | return isBaseActivityValid(activity, 'Announce') && | ||
5 | isObjectValid(activity.object) | ||
6 | } | ||
7 | // --------------------------------------------------------------------------- | ||
8 | |||
9 | export { | ||
10 | isShareActivityValid | ||
11 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/view.ts b/server/helpers/custom-validators/activitypub/view.ts deleted file mode 100644 index 41d16469f..000000000 --- a/server/helpers/custom-validators/activitypub/view.ts +++ /dev/null | |||
@@ -1,13 +0,0 @@ | |||
1 | import { isActivityPubUrlValid } from './misc' | ||
2 | |||
3 | function isViewActivityValid (activity: any) { | ||
4 | return activity.type === 'View' && | ||
5 | isActivityPubUrlValid(activity.actor) && | ||
6 | isActivityPubUrlValid(activity.object) | ||
7 | } | ||
8 | |||
9 | // --------------------------------------------------------------------------- | ||
10 | |||
11 | export { | ||
12 | isViewActivityValid | ||
13 | } | ||
diff --git a/server/helpers/custom-validators/user-notifications.ts b/server/helpers/custom-validators/user-notifications.ts index 8a33b895b..252c107db 100644 --- a/server/helpers/custom-validators/user-notifications.ts +++ b/server/helpers/custom-validators/user-notifications.ts | |||
@@ -1,10 +1,9 @@ | |||
1 | import { exists } from './misc' | ||
2 | import validator from 'validator' | 1 | import validator from 'validator' |
3 | import { UserNotificationType } from '../../../shared/models/users' | ||
4 | import { UserNotificationSettingValue } from '../../../shared/models/users/user-notification-setting.model' | 2 | import { UserNotificationSettingValue } from '../../../shared/models/users/user-notification-setting.model' |
3 | import { exists } from './misc' | ||
5 | 4 | ||
6 | function isUserNotificationTypeValid (value: any) { | 5 | function isUserNotificationTypeValid (value: any) { |
7 | return exists(value) && validator.isInt('' + value) && UserNotificationType[value] !== undefined | 6 | return exists(value) && validator.isInt('' + value) |
8 | } | 7 | } |
9 | 8 | ||
10 | function isUserNotificationSettingValid (value: any) { | 9 | function isUserNotificationSettingValid (value: any) { |
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 620025966..69cd397b9 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts | |||
@@ -5,7 +5,7 @@ import { dirname, join } from 'path' | |||
5 | import { FFMPEG_NICE, VIDEO_LIVE } from '@server/initializers/constants' | 5 | import { FFMPEG_NICE, VIDEO_LIVE } from '@server/initializers/constants' |
6 | import { AvailableEncoders, EncoderOptionsBuilder, EncoderProfile, VideoResolution } from '../../shared/models/videos' | 6 | import { AvailableEncoders, EncoderOptionsBuilder, EncoderProfile, VideoResolution } from '../../shared/models/videos' |
7 | import { CONFIG } from '../initializers/config' | 7 | import { CONFIG } from '../initializers/config' |
8 | import { promisify0 } from './core-utils' | 8 | import { execPromise, promisify0 } from './core-utils' |
9 | import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils' | 9 | import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils' |
10 | import { processImage } from './image-utils' | 10 | import { processImage } from './image-utils' |
11 | import { logger } from './logger' | 11 | import { logger } from './logger' |
@@ -649,6 +649,24 @@ function getFFmpeg (input: string, type: 'live' | 'vod') { | |||
649 | return command | 649 | return command |
650 | } | 650 | } |
651 | 651 | ||
652 | function getFFmpegVersion () { | ||
653 | return new Promise<string>((res, rej) => { | ||
654 | (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => { | ||
655 | if (err) return rej(err) | ||
656 | if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path')) | ||
657 | |||
658 | return execPromise(`${ffmpegPath} -version`) | ||
659 | .then(stdout => { | ||
660 | const parsed = stdout.match(/ffmpeg version .(\d+\.\d+\.\d+)/) | ||
661 | if (!parsed || !parsed[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`)) | ||
662 | |||
663 | return res(parsed[1]) | ||
664 | }) | ||
665 | .catch(err => rej(err)) | ||
666 | }) | ||
667 | }) | ||
668 | } | ||
669 | |||
652 | async function runCommand (options: { | 670 | async function runCommand (options: { |
653 | command: ffmpeg.FfmpegCommand | 671 | command: ffmpeg.FfmpegCommand |
654 | silent?: boolean // false | 672 | silent?: boolean // false |
@@ -695,6 +713,7 @@ export { | |||
695 | TranscodeOptionsType, | 713 | TranscodeOptionsType, |
696 | transcode, | 714 | transcode, |
697 | runCommand, | 715 | runCommand, |
716 | getFFmpegVersion, | ||
698 | 717 | ||
699 | resetSupportedEncoders, | 718 | resetSupportedEncoders, |
700 | 719 | ||
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 6917a64d9..a112fd300 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts | |||
@@ -48,7 +48,7 @@ function getLoggerReplacer () { | |||
48 | } | 48 | } |
49 | 49 | ||
50 | const consoleLoggerFormat = winston.format.printf(info => { | 50 | const consoleLoggerFormat = winston.format.printf(info => { |
51 | const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql' ] | 51 | const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ] |
52 | 52 | ||
53 | const obj = omit(info, ...toOmit) | 53 | const obj = omit(info, ...toOmit) |
54 | 54 | ||
@@ -150,6 +150,13 @@ const bunyanLogger = { | |||
150 | error: bunyanLogFactory('error'), | 150 | error: bunyanLogFactory('error'), |
151 | fatal: bunyanLogFactory('error') | 151 | fatal: bunyanLogFactory('error') |
152 | } | 152 | } |
153 | |||
154 | function loggerTagsFactory (...defaultTags: string[]) { | ||
155 | return (...tags: string[]) => { | ||
156 | return { tags: defaultTags.concat(tags) } | ||
157 | } | ||
158 | } | ||
159 | |||
153 | // --------------------------------------------------------------------------- | 160 | // --------------------------------------------------------------------------- |
154 | 161 | ||
155 | export { | 162 | export { |
@@ -159,5 +166,6 @@ export { | |||
159 | consoleLoggerFormat, | 166 | consoleLoggerFormat, |
160 | jsonLoggerFormat, | 167 | jsonLoggerFormat, |
161 | logger, | 168 | logger, |
169 | loggerTagsFactory, | ||
162 | bunyanLogger | 170 | bunyanLogger |
163 | } | 171 | } |
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index 994f725d8..bc6f1d074 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts | |||
@@ -84,7 +84,7 @@ async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any) | |||
84 | return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') | 84 | return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') |
85 | } | 85 | } |
86 | 86 | ||
87 | async function signJsonLDObject (byActor: MActor, data: any) { | 87 | async function signJsonLDObject <T> (byActor: MActor, data: T) { |
88 | const signature = { | 88 | const signature = { |
89 | type: 'RsaSignature2017', | 89 | type: 'RsaSignature2017', |
90 | creator: byActor.url, | 90 | creator: byActor.url, |
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index b556c392e..fd2a56f30 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts | |||
@@ -1,58 +1,141 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { createWriteStream, remove } from 'fs-extra' | 1 | import { createWriteStream, remove } from 'fs-extra' |
3 | import * as request from 'request' | 2 | import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got' |
3 | import { join } from 'path' | ||
4 | import { CONFIG } from '../initializers/config' | ||
4 | import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' | 5 | import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' |
6 | import { pipelinePromise } from './core-utils' | ||
5 | import { processImage } from './image-utils' | 7 | import { processImage } from './image-utils' |
6 | import { join } from 'path' | ||
7 | import { logger } from './logger' | 8 | import { logger } from './logger' |
8 | import { CONFIG } from '../initializers/config' | ||
9 | 9 | ||
10 | function doRequest <T> ( | 10 | export interface PeerTubeRequestError extends Error { |
11 | requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }, | 11 | statusCode?: number |
12 | bodyKBLimit = 1000 // 1MB | 12 | responseBody?: any |
13 | ): Bluebird<{ response: request.RequestResponse, body: T }> { | 13 | } |
14 | if (!(requestOptions.headers)) requestOptions.headers = {} | ||
15 | requestOptions.headers['User-Agent'] = getUserAgent() | ||
16 | 14 | ||
17 | if (requestOptions.activityPub === true) { | 15 | const httpSignature = require('http-signature') |
18 | requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER | 16 | |
17 | type PeerTubeRequestOptions = { | ||
18 | activityPub?: boolean | ||
19 | bodyKBLimit?: number // 1MB | ||
20 | httpSignature?: { | ||
21 | algorithm: string | ||
22 | authorizationHeaderName: string | ||
23 | keyId: string | ||
24 | key: string | ||
25 | headers: string[] | ||
19 | } | 26 | } |
27 | jsonResponse?: boolean | ||
28 | } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'> | ||
29 | |||
30 | const peertubeGot = got.extend({ | ||
31 | headers: { | ||
32 | 'user-agent': getUserAgent() | ||
33 | }, | ||
34 | |||
35 | handlers: [ | ||
36 | (options, next) => { | ||
37 | const promiseOrStream = next(options) as CancelableRequest<any> | ||
38 | const bodyKBLimit = options.context?.bodyKBLimit as number | ||
39 | if (!bodyKBLimit) throw new Error('No KB limit for this request') | ||
40 | |||
41 | const bodyLimit = bodyKBLimit * 1000 | ||
42 | |||
43 | /* eslint-disable @typescript-eslint/no-floating-promises */ | ||
44 | promiseOrStream.on('downloadProgress', progress => { | ||
45 | if (progress.transferred > bodyLimit && progress.percent !== 1) { | ||
46 | const message = `Exceeded the download limit of ${bodyLimit} B` | ||
47 | logger.warn(message) | ||
48 | |||
49 | // CancelableRequest | ||
50 | if (promiseOrStream.cancel) { | ||
51 | promiseOrStream.cancel() | ||
52 | return | ||
53 | } | ||
54 | |||
55 | // Stream | ||
56 | (promiseOrStream as any).destroy() | ||
57 | } | ||
58 | }) | ||
20 | 59 | ||
21 | return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => { | 60 | return promiseOrStream |
22 | request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) | 61 | } |
23 | .on('data', onRequestDataLengthCheck(bodyKBLimit)) | 62 | ], |
24 | }) | 63 | |
64 | hooks: { | ||
65 | beforeRequest: [ | ||
66 | options => { | ||
67 | const headers = options.headers || {} | ||
68 | headers['host'] = options.url.host | ||
69 | }, | ||
70 | |||
71 | options => { | ||
72 | const httpSignatureOptions = options.context?.httpSignature | ||
73 | |||
74 | if (httpSignatureOptions) { | ||
75 | const method = options.method ?? 'GET' | ||
76 | const path = options.path ?? options.url.pathname | ||
77 | |||
78 | if (!method || !path) { | ||
79 | throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) | ||
80 | } | ||
81 | |||
82 | httpSignature.signRequest({ | ||
83 | getHeader: function (header) { | ||
84 | return options.headers[header] | ||
85 | }, | ||
86 | |||
87 | setHeader: function (header, value) { | ||
88 | options.headers[header] = value | ||
89 | }, | ||
90 | |||
91 | method, | ||
92 | path | ||
93 | }, httpSignatureOptions) | ||
94 | } | ||
95 | } | ||
96 | ] | ||
97 | } | ||
98 | }) | ||
99 | |||
100 | function doRequest (url: string, options: PeerTubeRequestOptions = {}) { | ||
101 | const gotOptions = buildGotOptions(options) | ||
102 | |||
103 | return peertubeGot(url, gotOptions) | ||
104 | .catch(err => { throw buildRequestError(err) }) | ||
25 | } | 105 | } |
26 | 106 | ||
27 | function doRequestAndSaveToFile ( | 107 | function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) { |
28 | requestOptions: request.CoreOptions & request.UriOptions, | 108 | const gotOptions = buildGotOptions(options) |
109 | |||
110 | return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' }) | ||
111 | .catch(err => { throw buildRequestError(err) }) | ||
112 | } | ||
113 | |||
114 | async function doRequestAndSaveToFile ( | ||
115 | url: string, | ||
29 | destPath: string, | 116 | destPath: string, |
30 | bodyKBLimit = 10000 // 10MB | 117 | options: PeerTubeRequestOptions = {} |
31 | ) { | 118 | ) { |
32 | if (!requestOptions.headers) requestOptions.headers = {} | 119 | const gotOptions = buildGotOptions(options) |
33 | requestOptions.headers['User-Agent'] = getUserAgent() | ||
34 | |||
35 | return new Bluebird<void>((res, rej) => { | ||
36 | const file = createWriteStream(destPath) | ||
37 | file.on('finish', () => res()) | ||
38 | 120 | ||
39 | request(requestOptions) | 121 | const outFile = createWriteStream(destPath) |
40 | .on('data', onRequestDataLengthCheck(bodyKBLimit)) | ||
41 | .on('error', err => { | ||
42 | file.close() | ||
43 | 122 | ||
44 | remove(destPath) | 123 | try { |
45 | .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) | 124 | await pipelinePromise( |
125 | peertubeGot.stream(url, gotOptions), | ||
126 | outFile | ||
127 | ) | ||
128 | } catch (err) { | ||
129 | remove(destPath) | ||
130 | .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) | ||
46 | 131 | ||
47 | return rej(err) | 132 | throw buildRequestError(err) |
48 | }) | 133 | } |
49 | .pipe(file) | ||
50 | }) | ||
51 | } | 134 | } |
52 | 135 | ||
53 | async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { | 136 | async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { |
54 | const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) | 137 | const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) |
55 | await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath) | 138 | await doRequestAndSaveToFile(url, tmpPath) |
56 | 139 | ||
57 | const destPath = join(destDir, destName) | 140 | const destPath = join(destDir, destName) |
58 | 141 | ||
@@ -73,24 +156,46 @@ function getUserAgent () { | |||
73 | 156 | ||
74 | export { | 157 | export { |
75 | doRequest, | 158 | doRequest, |
159 | doJSONRequest, | ||
76 | doRequestAndSaveToFile, | 160 | doRequestAndSaveToFile, |
77 | downloadImage | 161 | downloadImage |
78 | } | 162 | } |
79 | 163 | ||
80 | // --------------------------------------------------------------------------- | 164 | // --------------------------------------------------------------------------- |
81 | 165 | ||
82 | // Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3 | 166 | function buildGotOptions (options: PeerTubeRequestOptions) { |
83 | function onRequestDataLengthCheck (bodyKBLimit: number) { | 167 | const { activityPub, bodyKBLimit = 1000 } = options |
84 | let bufferLength = 0 | ||
85 | const bytesLimit = bodyKBLimit * 1000 | ||
86 | 168 | ||
87 | return function (chunk) { | 169 | const context = { bodyKBLimit, httpSignature: options.httpSignature } |
88 | bufferLength += chunk.length | ||
89 | if (bufferLength > bytesLimit) { | ||
90 | this.abort() | ||
91 | 170 | ||
92 | const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`) | 171 | let headers = options.headers || {} |
93 | this.emit('error', error) | 172 | |
94 | } | 173 | if (!headers.date) { |
174 | headers = { ...headers, date: new Date().toUTCString() } | ||
175 | } | ||
176 | |||
177 | if (activityPub && !headers.accept) { | ||
178 | headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } | ||
95 | } | 179 | } |
180 | |||
181 | return { | ||
182 | method: options.method, | ||
183 | json: options.json, | ||
184 | searchParams: options.searchParams, | ||
185 | headers, | ||
186 | context | ||
187 | } | ||
188 | } | ||
189 | |||
190 | function buildRequestError (error: RequestError) { | ||
191 | const newError: PeerTubeRequestError = new Error(error.message) | ||
192 | newError.name = error.name | ||
193 | newError.stack = error.stack | ||
194 | |||
195 | if (error.response) { | ||
196 | newError.responseBody = error.response.body | ||
197 | newError.statusCode = error.response.statusCode | ||
198 | } | ||
199 | |||
200 | return newError | ||
96 | } | 201 | } |
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 8537a5772..9d2e54fb5 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts | |||
@@ -1,13 +1,13 @@ | |||
1 | import { createWriteStream } from 'fs' | 1 | import { createWriteStream } from 'fs' |
2 | import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra' | 2 | import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra' |
3 | import got from 'got' | ||
3 | import { join } from 'path' | 4 | import { join } from 'path' |
4 | import * as request from 'request' | ||
5 | import { CONFIG } from '@server/initializers/config' | 5 | import { CONFIG } from '@server/initializers/config' |
6 | import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' | 6 | import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' |
7 | import { VideoResolution } from '../../shared/models/videos' | 7 | import { VideoResolution } from '../../shared/models/videos' |
8 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' | 8 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' |
9 | import { getEnabledResolutions } from '../lib/video-transcoding' | 9 | import { getEnabledResolutions } from '../lib/video-transcoding' |
10 | import { peertubeTruncate, root } from './core-utils' | 10 | import { peertubeTruncate, pipelinePromise, root } from './core-utils' |
11 | import { isVideoFileExtnameValid } from './custom-validators/videos' | 11 | import { isVideoFileExtnameValid } from './custom-validators/videos' |
12 | import { logger } from './logger' | 12 | import { logger } from './logger' |
13 | import { generateVideoImportTmpPath } from './utils' | 13 | import { generateVideoImportTmpPath } from './utils' |
@@ -195,55 +195,32 @@ async function updateYoutubeDLBinary () { | |||
195 | 195 | ||
196 | await ensureDir(binDirectory) | 196 | await ensureDir(binDirectory) |
197 | 197 | ||
198 | return new Promise<void>(res => { | 198 | try { |
199 | request.get(url, { followRedirect: false }, (err, result) => { | 199 | const result = await got(url, { followRedirect: false }) |
200 | if (err) { | ||
201 | logger.error('Cannot update youtube-dl.', { err }) | ||
202 | return res() | ||
203 | } | ||
204 | |||
205 | if (result.statusCode !== HttpStatusCode.FOUND_302) { | ||
206 | logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) | ||
207 | return res() | ||
208 | } | ||
209 | |||
210 | const url = result.headers.location | ||
211 | const downloadFile = request.get(url) | ||
212 | const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1] | ||
213 | |||
214 | downloadFile.on('response', result => { | ||
215 | if (result.statusCode !== HttpStatusCode.OK_200) { | ||
216 | logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) | ||
217 | return res() | ||
218 | } | ||
219 | |||
220 | const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => { | ||
221 | logger.error('youtube-dl update error in write stream', { err }) | ||
222 | return res() | ||
223 | }) | ||
224 | 200 | ||
225 | downloadFile.pipe(writeStream) | 201 | if (result.statusCode !== HttpStatusCode.FOUND_302) { |
226 | }) | 202 | logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) |
203 | return | ||
204 | } | ||
227 | 205 | ||
228 | downloadFile.on('error', err => { | 206 | const newUrl = result.headers.location |
229 | logger.error('youtube-dl update error.', { err }) | 207 | const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1] |
230 | return res() | ||
231 | }) | ||
232 | 208 | ||
233 | downloadFile.on('end', () => { | 209 | const downloadFileStream = got.stream(newUrl) |
234 | const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) | 210 | const writeStream = createWriteStream(bin, { mode: 493 }) |
235 | writeFile(detailsPath, details, { encoding: 'utf8' }, err => { | ||
236 | if (err) { | ||
237 | logger.error('youtube-dl update error: cannot write details.', { err }) | ||
238 | return res() | ||
239 | } | ||
240 | 211 | ||
241 | logger.info('youtube-dl updated to version %s.', newVersion) | 212 | await pipelinePromise( |
242 | return res() | 213 | downloadFileStream, |
243 | }) | 214 | writeStream |
244 | }) | 215 | ) |
245 | }) | 216 | |
246 | }) | 217 | const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) |
218 | await writeFile(detailsPath, details, { encoding: 'utf8' }) | ||
219 | |||
220 | logger.info('youtube-dl updated to version %s.', newVersion) | ||
221 | } catch (err) { | ||
222 | logger.error('Cannot update youtube-dl.', { err }) | ||
223 | } | ||
247 | } | 224 | } |
248 | 225 | ||
249 | async function safeGetYoutubeDL () { | 226 | async function safeGetYoutubeDL () { |