diff options
author | Chocobozzz <me@florianbigard.com> | 2021-03-08 14:24:11 +0100 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2021-03-24 18:18:40 +0100 |
commit | db4b15f21fbf4e33434e930ffc7fb768cdcf9d42 (patch) | |
tree | 418117b84a5c2ee578c210288b698d155422d608 /server | |
parent | 71926aae0762facb25243f27eaf45933b5a37353 (diff) | |
download | PeerTube-db4b15f21fbf4e33434e930ffc7fb768cdcf9d42.tar.gz PeerTube-db4b15f21fbf4e33434e930ffc7fb768cdcf9d42.tar.zst PeerTube-db4b15f21fbf4e33434e930ffc7fb768cdcf9d42.zip |
Use got instead of request
Diffstat (limited to 'server')
28 files changed, 308 insertions, 297 deletions
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 7e1b7b230..317b24fe9 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { sanitizeUrl } from '@server/helpers/core-utils' | 2 | import { sanitizeUrl } from '@server/helpers/core-utils' |
3 | import { doRequest } from '@server/helpers/requests' | 3 | import { doJSONRequest } from '@server/helpers/requests' |
4 | import { CONFIG } from '@server/initializers/config' | 4 | import { CONFIG } from '@server/initializers/config' |
5 | import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' | 5 | import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' |
6 | import { AccountBlocklistModel } from '@server/models/account/account-blocklist' | 6 | import { AccountBlocklistModel } from '@server/models/account/account-blocklist' |
@@ -94,9 +94,9 @@ async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: e | |||
94 | try { | 94 | try { |
95 | logger.debug('Doing video channels search index request on %s.', url, { body }) | 95 | logger.debug('Doing video channels search index request on %s.', url, { body }) |
96 | 96 | ||
97 | const searchIndexResult = await doRequest<ResultList<VideoChannel>>({ uri: url, body, json: true }) | 97 | const searchIndexResult = await doJSONRequest<ResultList<VideoChannel>>(url, { json: body }) |
98 | 98 | ||
99 | return res.json(searchIndexResult.body) | 99 | return res.json(searchIndexResult) |
100 | } catch (err) { | 100 | } catch (err) { |
101 | logger.warn('Cannot use search index to make video channels search.', { err }) | 101 | logger.warn('Cannot use search index to make video channels search.', { err }) |
102 | 102 | ||
@@ -186,9 +186,9 @@ async function searchVideosIndex (query: VideosSearchQuery, res: express.Respons | |||
186 | try { | 186 | try { |
187 | logger.debug('Doing videos search index request on %s.', url, { body }) | 187 | logger.debug('Doing videos search index request on %s.', url, { body }) |
188 | 188 | ||
189 | const searchIndexResult = await doRequest<ResultList<Video>>({ uri: url, body, json: true }) | 189 | const searchIndexResult = await doJSONRequest<ResultList<Video>>(url, { json: body }) |
190 | 190 | ||
191 | return res.json(searchIndexResult.body) | 191 | return res.json(searchIndexResult) |
192 | } catch (err) { | 192 | } catch (err) { |
193 | logger.warn('Cannot use search index to make video search.', { err }) | 193 | logger.warn('Cannot use search index to make video search.', { err }) |
194 | 194 | ||
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 08aef2908..e0754b501 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts | |||
@@ -3,7 +3,6 @@ import { URL } from 'url' | |||
3 | import validator from 'validator' | 3 | import validator from 'validator' |
4 | import { ContextType } from '@shared/models/activitypub/context' | 4 | import { ContextType } from '@shared/models/activitypub/context' |
5 | import { ResultList } from '../../shared/models' | 5 | import { ResultList } from '../../shared/models' |
6 | import { Activity } from '../../shared/models/activitypub' | ||
7 | import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants' | 6 | import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants' |
8 | import { MActor, MVideoWithHost } from '../types/models' | 7 | import { MActor, MVideoWithHost } from '../types/models' |
9 | import { pageToStartAndCount } from './core-utils' | 8 | import { pageToStartAndCount } from './core-utils' |
@@ -182,10 +181,10 @@ async function activityPubCollectionPagination ( | |||
182 | 181 | ||
183 | } | 182 | } |
184 | 183 | ||
185 | function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) { | 184 | function buildSignedActivity <T> (byActor: MActor, data: T, contextType?: ContextType) { |
186 | const activity = activityPubContextify(data, contextType) | 185 | const activity = activityPubContextify(data, contextType) |
187 | 186 | ||
188 | return signJsonLDObject(byActor, activity) as Promise<Activity> | 187 | return signJsonLDObject(byActor, activity) |
189 | } | 188 | } |
190 | 189 | ||
191 | function getAPId (activity: string | { id: string }) { | 190 | function getAPId (activity: string | { id: string }) { |
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 935fd22d9..7ba7d865a 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -10,7 +10,9 @@ import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto' | |||
10 | import { truncate } from 'lodash' | 10 | import { truncate } from 'lodash' |
11 | import { basename, isAbsolute, join, resolve } from 'path' | 11 | import { basename, isAbsolute, join, resolve } from 'path' |
12 | import * as pem from 'pem' | 12 | import * as pem from 'pem' |
13 | import { pipeline } from 'stream' | ||
13 | import { URL } from 'url' | 14 | import { URL } from 'url' |
15 | import { promisify } from 'util' | ||
14 | 16 | ||
15 | const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { | 17 | const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { |
16 | if (!oldObject || typeof oldObject !== 'object') { | 18 | if (!oldObject || typeof oldObject !== 'object') { |
@@ -254,6 +256,7 @@ const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKe | |||
254 | const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey) | 256 | const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey) |
255 | const execPromise2 = promisify2<string, any, string>(exec) | 257 | const execPromise2 = promisify2<string, any, string>(exec) |
256 | const execPromise = promisify1<string, string>(exec) | 258 | const execPromise = promisify1<string, string>(exec) |
259 | const pipelinePromise = promisify(pipeline) | ||
257 | 260 | ||
258 | // --------------------------------------------------------------------------- | 261 | // --------------------------------------------------------------------------- |
259 | 262 | ||
@@ -284,5 +287,6 @@ export { | |||
284 | createPrivateKey, | 287 | createPrivateKey, |
285 | getPublicKey, | 288 | getPublicKey, |
286 | execPromise2, | 289 | execPromise2, |
287 | execPromise | 290 | execPromise, |
291 | pipelinePromise | ||
288 | } | 292 | } |
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index 46126da57..69558e358 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts | |||
@@ -41,7 +41,7 @@ const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean | |||
41 | } | 41 | } |
42 | 42 | ||
43 | function isActivityValid (activity: any) { | 43 | function isActivityValid (activity: any) { |
44 | const checker = activityCheckers[activity.tswype] | 44 | const checker = activityCheckers[activity.type] |
45 | // Unknown activity type | 45 | // Unknown activity type |
46 | if (!checker) return false | 46 | if (!checker) return false |
47 | 47 | ||
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index 994f725d8..bc6f1d074 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts | |||
@@ -84,7 +84,7 @@ async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any) | |||
84 | return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') | 84 | return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') |
85 | } | 85 | } |
86 | 86 | ||
87 | async function signJsonLDObject (byActor: MActor, data: any) { | 87 | async function signJsonLDObject <T> (byActor: MActor, data: T) { |
88 | const signature = { | 88 | const signature = { |
89 | type: 'RsaSignature2017', | 89 | type: 'RsaSignature2017', |
90 | creator: byActor.url, | 90 | creator: byActor.url, |
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index b556c392e..2c9da213c 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts | |||
@@ -1,58 +1,124 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { createWriteStream, remove } from 'fs-extra' | 1 | import { createWriteStream, remove } from 'fs-extra' |
3 | import * as request from 'request' | 2 | import got, { CancelableRequest, Options as GotOptions } from 'got' |
3 | import { join } from 'path' | ||
4 | import { CONFIG } from '../initializers/config' | ||
4 | import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' | 5 | import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' |
6 | import { pipelinePromise } from './core-utils' | ||
5 | import { processImage } from './image-utils' | 7 | import { processImage } from './image-utils' |
6 | import { join } from 'path' | ||
7 | import { logger } from './logger' | 8 | import { logger } from './logger' |
8 | import { CONFIG } from '../initializers/config' | ||
9 | 9 | ||
10 | function doRequest <T> ( | 10 | const httpSignature = require('http-signature') |
11 | requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }, | 11 | |
12 | bodyKBLimit = 1000 // 1MB | 12 | type PeerTubeRequestOptions = { |
13 | ): Bluebird<{ response: request.RequestResponse, body: T }> { | 13 | activityPub?: boolean |
14 | if (!(requestOptions.headers)) requestOptions.headers = {} | 14 | bodyKBLimit?: number // 1MB |
15 | requestOptions.headers['User-Agent'] = getUserAgent() | 15 | httpSignature?: { |
16 | algorithm: string | ||
17 | authorizationHeaderName: string | ||
18 | keyId: string | ||
19 | key: string | ||
20 | headers: string[] | ||
21 | } | ||
22 | jsonResponse?: boolean | ||
23 | } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'> | ||
24 | |||
25 | const peertubeGot = got.extend({ | ||
26 | headers: { | ||
27 | 'user-agent': getUserAgent() | ||
28 | }, | ||
29 | |||
30 | handlers: [ | ||
31 | (options, next) => { | ||
32 | const promiseOrStream = next(options) as CancelableRequest<any> | ||
33 | const bodyKBLimit = options.context?.bodyKBLimit | ||
34 | if (!bodyKBLimit) throw new Error('No KB limit for this request') | ||
35 | |||
36 | /* eslint-disable @typescript-eslint/no-floating-promises */ | ||
37 | promiseOrStream.on('downloadProgress', progress => { | ||
38 | if (progress.transferred * 1000 > bodyKBLimit && progress.percent !== 1) { | ||
39 | promiseOrStream.cancel(`Exceeded the download limit of ${bodyKBLimit} bytes`) | ||
40 | } | ||
41 | }) | ||
16 | 42 | ||
17 | if (requestOptions.activityPub === true) { | 43 | return promiseOrStream |
18 | requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER | 44 | } |
45 | ], | ||
46 | |||
47 | hooks: { | ||
48 | beforeRequest: [ | ||
49 | options => { | ||
50 | const headers = options.headers || {} | ||
51 | headers['host'] = options.url.host | ||
52 | }, | ||
53 | |||
54 | options => { | ||
55 | const httpSignatureOptions = options.context?.httpSignature | ||
56 | |||
57 | if (httpSignatureOptions) { | ||
58 | const method = options.method ?? 'GET' | ||
59 | const path = options.path ?? options.url.pathname | ||
60 | |||
61 | if (!method || !path) { | ||
62 | throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) | ||
63 | } | ||
64 | |||
65 | httpSignature.signRequest({ | ||
66 | getHeader: function (header) { | ||
67 | return options.headers[header] | ||
68 | }, | ||
69 | |||
70 | setHeader: function (header, value) { | ||
71 | options.headers[header] = value | ||
72 | }, | ||
73 | |||
74 | method, | ||
75 | path | ||
76 | }, httpSignatureOptions) | ||
77 | } | ||
78 | } | ||
79 | ] | ||
19 | } | 80 | } |
81 | }) | ||
20 | 82 | ||
21 | return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => { | 83 | function doRequest (url: string, options: PeerTubeRequestOptions = {}) { |
22 | request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) | 84 | const gotOptions = buildGotOptions(options) |
23 | .on('data', onRequestDataLengthCheck(bodyKBLimit)) | 85 | |
24 | }) | 86 | return peertubeGot(url, gotOptions) |
87 | .catch(err => { throw buildRequestError(err) }) | ||
88 | } | ||
89 | |||
90 | function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) { | ||
91 | const gotOptions = buildGotOptions(options) | ||
92 | |||
93 | return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' }) | ||
94 | .catch(err => { throw buildRequestError(err) }) | ||
25 | } | 95 | } |
26 | 96 | ||
27 | function doRequestAndSaveToFile ( | 97 | async function doRequestAndSaveToFile ( |
28 | requestOptions: request.CoreOptions & request.UriOptions, | 98 | url: string, |
29 | destPath: string, | 99 | destPath: string, |
30 | bodyKBLimit = 10000 // 10MB | 100 | options: PeerTubeRequestOptions = {} |
31 | ) { | 101 | ) { |
32 | if (!requestOptions.headers) requestOptions.headers = {} | 102 | const gotOptions = buildGotOptions(options) |
33 | requestOptions.headers['User-Agent'] = getUserAgent() | ||
34 | |||
35 | return new Bluebird<void>((res, rej) => { | ||
36 | const file = createWriteStream(destPath) | ||
37 | file.on('finish', () => res()) | ||
38 | 103 | ||
39 | request(requestOptions) | 104 | const outFile = createWriteStream(destPath) |
40 | .on('data', onRequestDataLengthCheck(bodyKBLimit)) | ||
41 | .on('error', err => { | ||
42 | file.close() | ||
43 | 105 | ||
44 | remove(destPath) | 106 | try { |
45 | .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) | 107 | await pipelinePromise( |
108 | peertubeGot.stream(url, gotOptions), | ||
109 | outFile | ||
110 | ) | ||
111 | } catch (err) { | ||
112 | remove(destPath) | ||
113 | .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) | ||
46 | 114 | ||
47 | return rej(err) | 115 | throw buildRequestError(err) |
48 | }) | 116 | } |
49 | .pipe(file) | ||
50 | }) | ||
51 | } | 117 | } |
52 | 118 | ||
53 | async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { | 119 | async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { |
54 | const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) | 120 | const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) |
55 | await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath) | 121 | await doRequestAndSaveToFile(url, tmpPath) |
56 | 122 | ||
57 | const destPath = join(destDir, destName) | 123 | const destPath = join(destDir, destName) |
58 | 124 | ||
@@ -73,24 +139,43 @@ function getUserAgent () { | |||
73 | 139 | ||
74 | export { | 140 | export { |
75 | doRequest, | 141 | doRequest, |
142 | doJSONRequest, | ||
76 | doRequestAndSaveToFile, | 143 | doRequestAndSaveToFile, |
77 | downloadImage | 144 | downloadImage |
78 | } | 145 | } |
79 | 146 | ||
80 | // --------------------------------------------------------------------------- | 147 | // --------------------------------------------------------------------------- |
81 | 148 | ||
82 | // Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3 | 149 | function buildGotOptions (options: PeerTubeRequestOptions) { |
83 | function onRequestDataLengthCheck (bodyKBLimit: number) { | 150 | const { activityPub, bodyKBLimit = 1000 } = options |
84 | let bufferLength = 0 | ||
85 | const bytesLimit = bodyKBLimit * 1000 | ||
86 | 151 | ||
87 | return function (chunk) { | 152 | const context = { bodyKBLimit, httpSignature: options.httpSignature } |
88 | bufferLength += chunk.length | ||
89 | if (bufferLength > bytesLimit) { | ||
90 | this.abort() | ||
91 | 153 | ||
92 | const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`) | 154 | let headers = options.headers || {} |
93 | this.emit('error', error) | 155 | |
94 | } | 156 | headers = { ...headers, date: new Date().toUTCString() } |
157 | |||
158 | if (activityPub) { | ||
159 | headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } | ||
95 | } | 160 | } |
161 | |||
162 | return { | ||
163 | method: options.method, | ||
164 | json: options.json, | ||
165 | searchParams: options.searchParams, | ||
166 | headers, | ||
167 | context | ||
168 | } | ||
169 | } | ||
170 | |||
171 | function buildRequestError (error: any) { | ||
172 | const newError = new Error(error.message) | ||
173 | newError.name = error.name | ||
174 | newError.stack = error.stack | ||
175 | |||
176 | if (error.response?.body) { | ||
177 | error.responseBody = error.response.body | ||
178 | } | ||
179 | |||
180 | return newError | ||
96 | } | 181 | } |
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 8537a5772..9d2e54fb5 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts | |||
@@ -1,13 +1,13 @@ | |||
1 | import { createWriteStream } from 'fs' | 1 | import { createWriteStream } from 'fs' |
2 | import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra' | 2 | import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra' |
3 | import got from 'got' | ||
3 | import { join } from 'path' | 4 | import { join } from 'path' |
4 | import * as request from 'request' | ||
5 | import { CONFIG } from '@server/initializers/config' | 5 | import { CONFIG } from '@server/initializers/config' |
6 | import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' | 6 | import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' |
7 | import { VideoResolution } from '../../shared/models/videos' | 7 | import { VideoResolution } from '../../shared/models/videos' |
8 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' | 8 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' |
9 | import { getEnabledResolutions } from '../lib/video-transcoding' | 9 | import { getEnabledResolutions } from '../lib/video-transcoding' |
10 | import { peertubeTruncate, root } from './core-utils' | 10 | import { peertubeTruncate, pipelinePromise, root } from './core-utils' |
11 | import { isVideoFileExtnameValid } from './custom-validators/videos' | 11 | import { isVideoFileExtnameValid } from './custom-validators/videos' |
12 | import { logger } from './logger' | 12 | import { logger } from './logger' |
13 | import { generateVideoImportTmpPath } from './utils' | 13 | import { generateVideoImportTmpPath } from './utils' |
@@ -195,55 +195,32 @@ async function updateYoutubeDLBinary () { | |||
195 | 195 | ||
196 | await ensureDir(binDirectory) | 196 | await ensureDir(binDirectory) |
197 | 197 | ||
198 | return new Promise<void>(res => { | 198 | try { |
199 | request.get(url, { followRedirect: false }, (err, result) => { | 199 | const result = await got(url, { followRedirect: false }) |
200 | if (err) { | ||
201 | logger.error('Cannot update youtube-dl.', { err }) | ||
202 | return res() | ||
203 | } | ||
204 | |||
205 | if (result.statusCode !== HttpStatusCode.FOUND_302) { | ||
206 | logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) | ||
207 | return res() | ||
208 | } | ||
209 | |||
210 | const url = result.headers.location | ||
211 | const downloadFile = request.get(url) | ||
212 | const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1] | ||
213 | |||
214 | downloadFile.on('response', result => { | ||
215 | if (result.statusCode !== HttpStatusCode.OK_200) { | ||
216 | logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) | ||
217 | return res() | ||
218 | } | ||
219 | |||
220 | const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => { | ||
221 | logger.error('youtube-dl update error in write stream', { err }) | ||
222 | return res() | ||
223 | }) | ||
224 | 200 | ||
225 | downloadFile.pipe(writeStream) | 201 | if (result.statusCode !== HttpStatusCode.FOUND_302) { |
226 | }) | 202 | logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) |
203 | return | ||
204 | } | ||
227 | 205 | ||
228 | downloadFile.on('error', err => { | 206 | const newUrl = result.headers.location |
229 | logger.error('youtube-dl update error.', { err }) | 207 | const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1] |
230 | return res() | ||
231 | }) | ||
232 | 208 | ||
233 | downloadFile.on('end', () => { | 209 | const downloadFileStream = got.stream(newUrl) |
234 | const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) | 210 | const writeStream = createWriteStream(bin, { mode: 493 }) |
235 | writeFile(detailsPath, details, { encoding: 'utf8' }, err => { | ||
236 | if (err) { | ||
237 | logger.error('youtube-dl update error: cannot write details.', { err }) | ||
238 | return res() | ||
239 | } | ||
240 | 211 | ||
241 | logger.info('youtube-dl updated to version %s.', newVersion) | 212 | await pipelinePromise( |
242 | return res() | 213 | downloadFileStream, |
243 | }) | 214 | writeStream |
244 | }) | 215 | ) |
245 | }) | 216 | |
246 | }) | 217 | const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) |
218 | await writeFile(detailsPath, details, { encoding: 'utf8' }) | ||
219 | |||
220 | logger.info('youtube-dl updated to version %s.', newVersion) | ||
221 | } catch (err) { | ||
222 | logger.error('Cannot update youtube-dl.', { err }) | ||
223 | } | ||
247 | } | 224 | } |
248 | 225 | ||
249 | async function safeGetYoutubeDL () { | 226 | async function safeGetYoutubeDL () { |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 1623e6f42..ea98e8a38 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -29,7 +29,7 @@ const LAST_MIGRATION_VERSION = 610 | |||
29 | // --------------------------------------------------------------------------- | 29 | // --------------------------------------------------------------------------- |
30 | 30 | ||
31 | const API_VERSION = 'v1' | 31 | const API_VERSION = 'v1' |
32 | const PEERTUBE_VERSION = require(join(root(), 'package.json')).version | 32 | const PEERTUBE_VERSION: string = require(join(root(), 'package.json')).version |
33 | 33 | ||
34 | const PAGINATION = { | 34 | const PAGINATION = { |
35 | GLOBAL: { | 35 | GLOBAL: { |
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index a726f9e20..52b6c1f56 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -1,26 +1,28 @@ | |||
1 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
2 | import { extname } from 'path' | ||
2 | import { Op, Transaction } from 'sequelize' | 3 | import { Op, Transaction } from 'sequelize' |
3 | import { URL } from 'url' | 4 | import { URL } from 'url' |
4 | import { v4 as uuidv4 } from 'uuid' | 5 | import { v4 as uuidv4 } from 'uuid' |
6 | import { getServerActor } from '@server/models/application/application' | ||
7 | import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' | ||
5 | import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | 8 | import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub' |
6 | import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' | 9 | import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' |
7 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | 10 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
11 | import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' | ||
8 | import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor' | 12 | import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor' |
9 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 13 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
10 | import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' | 14 | import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' |
11 | import { logger } from '../../helpers/logger' | 15 | import { logger } from '../../helpers/logger' |
12 | import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' | 16 | import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' |
13 | import { doRequest } from '../../helpers/requests' | 17 | import { doJSONRequest } from '../../helpers/requests' |
14 | import { getUrlFromWebfinger } from '../../helpers/webfinger' | 18 | import { getUrlFromWebfinger } from '../../helpers/webfinger' |
15 | import { MIMETYPES, WEBSERVER } from '../../initializers/constants' | 19 | import { MIMETYPES, WEBSERVER } from '../../initializers/constants' |
20 | import { sequelizeTypescript } from '../../initializers/database' | ||
16 | import { AccountModel } from '../../models/account/account' | 21 | import { AccountModel } from '../../models/account/account' |
17 | import { ActorModel } from '../../models/activitypub/actor' | 22 | import { ActorModel } from '../../models/activitypub/actor' |
18 | import { AvatarModel } from '../../models/avatar/avatar' | 23 | import { AvatarModel } from '../../models/avatar/avatar' |
19 | import { ServerModel } from '../../models/server/server' | 24 | import { ServerModel } from '../../models/server/server' |
20 | import { VideoChannelModel } from '../../models/video/video-channel' | 25 | import { VideoChannelModel } from '../../models/video/video-channel' |
21 | import { JobQueue } from '../job-queue' | ||
22 | import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' | ||
23 | import { sequelizeTypescript } from '../../initializers/database' | ||
24 | import { | 26 | import { |
25 | MAccount, | 27 | MAccount, |
26 | MAccountDefault, | 28 | MAccountDefault, |
@@ -34,9 +36,7 @@ import { | |||
34 | MActorId, | 36 | MActorId, |
35 | MChannel | 37 | MChannel |
36 | } from '../../types/models' | 38 | } from '../../types/models' |
37 | import { extname } from 'path' | 39 | import { JobQueue } from '../job-queue' |
38 | import { getServerActor } from '@server/models/application/application' | ||
39 | import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' | ||
40 | 40 | ||
41 | // Set account keys, this could be long so process after the account creation and do not block the client | 41 | // Set account keys, this could be long so process after the account creation and do not block the client |
42 | async function generateAndSaveActorKeys <T extends MActor> (actor: T) { | 42 | async function generateAndSaveActorKeys <T extends MActor> (actor: T) { |
@@ -209,16 +209,10 @@ async function deleteActorAvatarInstance (actor: MActorDefault, t: Transaction) | |||
209 | } | 209 | } |
210 | 210 | ||
211 | async function fetchActorTotalItems (url: string) { | 211 | async function fetchActorTotalItems (url: string) { |
212 | const options = { | ||
213 | uri: url, | ||
214 | method: 'GET', | ||
215 | json: true, | ||
216 | activityPub: true | ||
217 | } | ||
218 | |||
219 | try { | 212 | try { |
220 | const { body } = await doRequest<ActivityPubOrderedCollection<unknown>>(options) | 213 | const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true }) |
221 | return body.totalItems ? body.totalItems : 0 | 214 | |
215 | return body.totalItems || 0 | ||
222 | } catch (err) { | 216 | } catch (err) { |
223 | logger.warn('Cannot fetch remote actor count %s.', url, { err }) | 217 | logger.warn('Cannot fetch remote actor count %s.', url, { err }) |
224 | return 0 | 218 | return 0 |
@@ -449,26 +443,19 @@ type FetchRemoteActorResult = { | |||
449 | attributedTo: ActivityPubAttributedTo[] | 443 | attributedTo: ActivityPubAttributedTo[] |
450 | } | 444 | } |
451 | async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { | 445 | async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { |
452 | const options = { | ||
453 | uri: actorUrl, | ||
454 | method: 'GET', | ||
455 | json: true, | ||
456 | activityPub: true | ||
457 | } | ||
458 | |||
459 | logger.info('Fetching remote actor %s.', actorUrl) | 446 | logger.info('Fetching remote actor %s.', actorUrl) |
460 | 447 | ||
461 | const requestResult = await doRequest<ActivityPubActor>(options) | 448 | const requestResult = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true }) |
462 | const actorJSON = requestResult.body | 449 | const actorJSON = requestResult.body |
463 | 450 | ||
464 | if (sanitizeAndCheckActorObject(actorJSON) === false) { | 451 | if (sanitizeAndCheckActorObject(actorJSON) === false) { |
465 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) | 452 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) |
466 | return { result: undefined, statusCode: requestResult.response.statusCode } | 453 | return { result: undefined, statusCode: requestResult.statusCode } |
467 | } | 454 | } |
468 | 455 | ||
469 | if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { | 456 | if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { |
470 | logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) | 457 | logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) |
471 | return { result: undefined, statusCode: requestResult.response.statusCode } | 458 | return { result: undefined, statusCode: requestResult.statusCode } |
472 | } | 459 | } |
473 | 460 | ||
474 | const followersCount = await fetchActorTotalItems(actorJSON.followers) | 461 | const followersCount = await fetchActorTotalItems(actorJSON.followers) |
@@ -496,7 +483,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe | |||
496 | 483 | ||
497 | const name = actorJSON.name || actorJSON.preferredUsername | 484 | const name = actorJSON.name || actorJSON.preferredUsername |
498 | return { | 485 | return { |
499 | statusCode: requestResult.response.statusCode, | 486 | statusCode: requestResult.statusCode, |
500 | result: { | 487 | result: { |
501 | actor, | 488 | actor, |
502 | name, | 489 | name, |
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 1ed105bbe..278abf7de 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts | |||
@@ -1,27 +1,26 @@ | |||
1 | import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' | ||
2 | import { doRequest } from '../../helpers/requests' | ||
3 | import { logger } from '../../helpers/logger' | ||
4 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
5 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | ||
6 | import { URL } from 'url' | 2 | import { URL } from 'url' |
3 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | ||
4 | import { logger } from '../../helpers/logger' | ||
5 | import { doJSONRequest } from '../../helpers/requests' | ||
6 | import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' | ||
7 | 7 | ||
8 | type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) | 8 | type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) |
9 | type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) | 9 | type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) |
10 | 10 | ||
11 | async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) { | 11 | async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) { |
12 | logger.info('Crawling ActivityPub data on %s.', uri) | 12 | let url = argUrl |
13 | |||
14 | logger.info('Crawling ActivityPub data on %s.', url) | ||
13 | 15 | ||
14 | const options = { | 16 | const options = { |
15 | method: 'GET', | ||
16 | uri, | ||
17 | json: true, | ||
18 | activityPub: true, | 17 | activityPub: true, |
19 | timeout: REQUEST_TIMEOUT | 18 | timeout: REQUEST_TIMEOUT |
20 | } | 19 | } |
21 | 20 | ||
22 | const startDate = new Date() | 21 | const startDate = new Date() |
23 | 22 | ||
24 | const response = await doRequest<ActivityPubOrderedCollection<T>>(options) | 23 | const response = await doJSONRequest<ActivityPubOrderedCollection<T>>(url, options) |
25 | const firstBody = response.body | 24 | const firstBody = response.body |
26 | 25 | ||
27 | const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT | 26 | const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT |
@@ -35,9 +34,9 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T> | |||
35 | const remoteHost = new URL(nextLink).host | 34 | const remoteHost = new URL(nextLink).host |
36 | if (remoteHost === WEBSERVER.HOST) continue | 35 | if (remoteHost === WEBSERVER.HOST) continue |
37 | 36 | ||
38 | options.uri = nextLink | 37 | url = nextLink |
39 | 38 | ||
40 | const res = await doRequest<ActivityPubOrderedCollection<T>>(options) | 39 | const res = await doJSONRequest<ActivityPubOrderedCollection<T>>(url, options) |
41 | body = res.body | 40 | body = res.body |
42 | } else { | 41 | } else { |
43 | // nextLink is already the object we want | 42 | // nextLink is already the object we want |
@@ -49,7 +48,7 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T> | |||
49 | 48 | ||
50 | if (Array.isArray(body.orderedItems)) { | 49 | if (Array.isArray(body.orderedItems)) { |
51 | const items = body.orderedItems | 50 | const items = body.orderedItems |
52 | logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri) | 51 | logger.info('Processing %i ActivityPub items for %s.', items.length, url) |
53 | 52 | ||
54 | await handler(items) | 53 | await handler(items) |
55 | } | 54 | } |
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts index d5a3ef7c8..795be60d7 100644 --- a/server/lib/activitypub/playlist.ts +++ b/server/lib/activitypub/playlist.ts | |||
@@ -1,24 +1,24 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' | ||
3 | import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' | ||
1 | import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' | 4 | import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' |
2 | import { crawlCollectionPage } from './crawl' | 5 | import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' |
3 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | 6 | import { checkUrlsSameHost } from '../../helpers/activitypub' |
7 | import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' | ||
4 | import { isArray } from '../../helpers/custom-validators/misc' | 8 | import { isArray } from '../../helpers/custom-validators/misc' |
5 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
6 | import { logger } from '../../helpers/logger' | 9 | import { logger } from '../../helpers/logger' |
10 | import { doJSONRequest } from '../../helpers/requests' | ||
11 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | ||
12 | import { sequelizeTypescript } from '../../initializers/database' | ||
7 | import { VideoPlaylistModel } from '../../models/video/video-playlist' | 13 | import { VideoPlaylistModel } from '../../models/video/video-playlist' |
8 | import { doRequest } from '../../helpers/requests' | ||
9 | import { checkUrlsSameHost } from '../../helpers/activitypub' | ||
10 | import * as Bluebird from 'bluebird' | ||
11 | import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' | ||
12 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | ||
13 | import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' | ||
14 | import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' | 14 | import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' |
15 | import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' | ||
16 | import { sequelizeTypescript } from '../../initializers/database' | ||
17 | import { createPlaylistMiniatureFromUrl } from '../thumbnail' | ||
18 | import { FilteredModelAttributes } from '../../types/sequelize' | ||
19 | import { MAccountDefault, MAccountId, MVideoId } from '../../types/models' | 15 | import { MAccountDefault, MAccountId, MVideoId } from '../../types/models' |
20 | import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist' | 16 | import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist' |
21 | import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' | 17 | import { FilteredModelAttributes } from '../../types/sequelize' |
18 | import { createPlaylistMiniatureFromUrl } from '../thumbnail' | ||
19 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
20 | import { crawlCollectionPage } from './crawl' | ||
21 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | ||
22 | 22 | ||
23 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { | 23 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { |
24 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | 24 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) |
@@ -56,11 +56,7 @@ async function createAccountPlaylists (playlistUrls: string[], account: MAccount | |||
56 | if (exists === true) return | 56 | if (exists === true) return |
57 | 57 | ||
58 | // Fetch url | 58 | // Fetch url |
59 | const { body } = await doRequest<PlaylistObject>({ | 59 | const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true }) |
60 | uri: playlistUrl, | ||
61 | json: true, | ||
62 | activityPub: true | ||
63 | }) | ||
64 | 60 | ||
65 | if (!isPlaylistObjectValid(body)) { | 61 | if (!isPlaylistObjectValid(body)) { |
66 | throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`) | 62 | throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`) |
@@ -164,12 +160,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid | |||
164 | 160 | ||
165 | await Bluebird.map(elementUrls, async elementUrl => { | 161 | await Bluebird.map(elementUrls, async elementUrl => { |
166 | try { | 162 | try { |
167 | // Fetch url | 163 | const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true }) |
168 | const { body } = await doRequest<PlaylistElementObject>({ | ||
169 | uri: elementUrl, | ||
170 | json: true, | ||
171 | activityPub: true | ||
172 | }) | ||
173 | 164 | ||
174 | if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`) | 165 | if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`) |
175 | 166 | ||
@@ -199,21 +190,14 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid | |||
199 | } | 190 | } |
200 | 191 | ||
201 | async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { | 192 | async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { |
202 | const options = { | ||
203 | uri: playlistUrl, | ||
204 | method: 'GET', | ||
205 | json: true, | ||
206 | activityPub: true | ||
207 | } | ||
208 | |||
209 | logger.info('Fetching remote playlist %s.', playlistUrl) | 193 | logger.info('Fetching remote playlist %s.', playlistUrl) |
210 | 194 | ||
211 | const { response, body } = await doRequest<any>(options) | 195 | const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true }) |
212 | 196 | ||
213 | if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { | 197 | if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { |
214 | logger.debug('Remote video playlist JSON is not valid.', { body }) | 198 | logger.debug('Remote video playlist JSON is not valid.', { body }) |
215 | return { statusCode: response.statusCode, playlistObject: undefined } | 199 | return { statusCode, playlistObject: undefined } |
216 | } | 200 | } |
217 | 201 | ||
218 | return { statusCode: response.statusCode, playlistObject: body } | 202 | return { statusCode, playlistObject: body } |
219 | } | 203 | } |
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index dde0c628e..c22fa0893 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts | |||
@@ -3,7 +3,7 @@ import { Transaction } from 'sequelize' | |||
3 | import { getServerActor } from '@server/models/application/application' | 3 | import { getServerActor } from '@server/models/application/application' |
4 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | 4 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
5 | import { logger, loggerTagsFactory } from '../../helpers/logger' | 5 | import { logger, loggerTagsFactory } from '../../helpers/logger' |
6 | import { doRequest } from '../../helpers/requests' | 6 | import { doJSONRequest } from '../../helpers/requests' |
7 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | 7 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
8 | import { VideoShareModel } from '../../models/video/video-share' | 8 | import { VideoShareModel } from '../../models/video/video-share' |
9 | import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' | 9 | import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' |
@@ -40,12 +40,7 @@ async function changeVideoChannelShare ( | |||
40 | async function addVideoShares (shareUrls: string[], video: MVideoId) { | 40 | async function addVideoShares (shareUrls: string[], video: MVideoId) { |
41 | await Bluebird.map(shareUrls, async shareUrl => { | 41 | await Bluebird.map(shareUrls, async shareUrl => { |
42 | try { | 42 | try { |
43 | // Fetch url | 43 | const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true }) |
44 | const { body } = await doRequest<any>({ | ||
45 | uri: shareUrl, | ||
46 | json: true, | ||
47 | activityPub: true | ||
48 | }) | ||
49 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | 44 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') |
50 | 45 | ||
51 | const actorUrl = getAPId(body.actor) | 46 | const actorUrl = getAPId(body.actor) |
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index d025ed7f1..f1edfb0ac 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -1,13 +1,13 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { checkUrlsSameHost } from '../../helpers/activitypub' | ||
1 | import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' | 3 | import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' |
2 | import { logger } from '../../helpers/logger' | 4 | import { logger } from '../../helpers/logger' |
3 | import { doRequest } from '../../helpers/requests' | 5 | import { doJSONRequest } from '../../helpers/requests' |
4 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | 6 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
5 | import { VideoCommentModel } from '../../models/video/video-comment' | 7 | import { VideoCommentModel } from '../../models/video/video-comment' |
8 | import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' | ||
6 | import { getOrCreateActorAndServerAndModel } from './actor' | 9 | import { getOrCreateActorAndServerAndModel } from './actor' |
7 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | 10 | import { getOrCreateVideoAndAccountAndChannel } from './videos' |
8 | import * as Bluebird from 'bluebird' | ||
9 | import { checkUrlsSameHost } from '../../helpers/activitypub' | ||
10 | import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' | ||
11 | 11 | ||
12 | type ResolveThreadParams = { | 12 | type ResolveThreadParams = { |
13 | url: string | 13 | url: string |
@@ -126,11 +126,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) { | |||
126 | throw new Error('Recursion limit reached when resolving a thread') | 126 | throw new Error('Recursion limit reached when resolving a thread') |
127 | } | 127 | } |
128 | 128 | ||
129 | const { body } = await doRequest<any>({ | 129 | const { body } = await doJSONRequest<any>(url, { activityPub: true }) |
130 | uri: url, | ||
131 | json: true, | ||
132 | activityPub: true | ||
133 | }) | ||
134 | 130 | ||
135 | if (sanitizeAndCheckVideoCommentObject(body) === false) { | 131 | if (sanitizeAndCheckVideoCommentObject(body) === false) { |
136 | throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) | 132 | throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) |
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index e246b1313..f40c07fea 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts | |||
@@ -1,26 +1,22 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
1 | import { Transaction } from 'sequelize' | 2 | import { Transaction } from 'sequelize' |
2 | import { sendLike, sendUndoDislike, sendUndoLike } from './send' | 3 | import { doJSONRequest } from '@server/helpers/requests' |
3 | import { VideoRateType } from '../../../shared/models/videos' | 4 | import { VideoRateType } from '../../../shared/models/videos' |
4 | import * as Bluebird from 'bluebird' | 5 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
5 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
6 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | ||
7 | import { logger } from '../../helpers/logger' | 6 | import { logger } from '../../helpers/logger' |
8 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | 7 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
9 | import { doRequest } from '../../helpers/requests' | 8 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' |
10 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | ||
11 | import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' | ||
12 | import { sendDislike } from './send/send-dislike' | ||
13 | import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' | 9 | import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' |
10 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
11 | import { sendLike, sendUndoDislike, sendUndoLike } from './send' | ||
12 | import { sendDislike } from './send/send-dislike' | ||
13 | import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' | ||
14 | 14 | ||
15 | async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { | 15 | async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { |
16 | await Bluebird.map(ratesUrl, async rateUrl => { | 16 | await Bluebird.map(ratesUrl, async rateUrl => { |
17 | try { | 17 | try { |
18 | // Fetch url | 18 | // Fetch url |
19 | const { body } = await doRequest<any>({ | 19 | const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true }) |
20 | uri: rateUrl, | ||
21 | json: true, | ||
22 | activityPub: true | ||
23 | }) | ||
24 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | 20 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') |
25 | 21 | ||
26 | const actorUrl = getAPId(body.actor) | 22 | const actorUrl = getAPId(body.actor) |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index c02578aad..a5f58dd01 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -2,7 +2,6 @@ import * as Bluebird from 'bluebird' | |||
2 | import { maxBy, minBy } from 'lodash' | 2 | import { maxBy, minBy } from 'lodash' |
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import { basename, join } from 'path' | 4 | import { basename, join } from 'path' |
5 | import * as request from 'request' | ||
6 | import { Transaction } from 'sequelize/types' | 5 | import { Transaction } from 'sequelize/types' |
7 | import { TrackerModel } from '@server/models/server/tracker' | 6 | import { TrackerModel } from '@server/models/server/tracker' |
8 | import { VideoLiveModel } from '@server/models/video/video-live' | 7 | import { VideoLiveModel } from '@server/models/video/video-live' |
@@ -31,7 +30,7 @@ import { isArray } from '../../helpers/custom-validators/misc' | |||
31 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | 30 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' |
32 | import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' | 31 | import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' |
33 | import { logger } from '../../helpers/logger' | 32 | import { logger } from '../../helpers/logger' |
34 | import { doRequest } from '../../helpers/requests' | 33 | import { doJSONRequest } from '../../helpers/requests' |
35 | import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video' | 34 | import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video' |
36 | import { | 35 | import { |
37 | ACTIVITY_PUB, | 36 | ACTIVITY_PUB, |
@@ -115,36 +114,26 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid | |||
115 | } | 114 | } |
116 | } | 115 | } |
117 | 116 | ||
118 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoObject }> { | 117 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { |
119 | const options = { | ||
120 | uri: videoUrl, | ||
121 | method: 'GET', | ||
122 | json: true, | ||
123 | activityPub: true | ||
124 | } | ||
125 | |||
126 | logger.info('Fetching remote video %s.', videoUrl) | 118 | logger.info('Fetching remote video %s.', videoUrl) |
127 | 119 | ||
128 | const { response, body } = await doRequest<any>(options) | 120 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) |
129 | 121 | ||
130 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | 122 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { |
131 | logger.debug('Remote video JSON is not valid.', { body }) | 123 | logger.debug('Remote video JSON is not valid.', { body }) |
132 | return { response, videoObject: undefined } | 124 | return { statusCode, videoObject: undefined } |
133 | } | 125 | } |
134 | 126 | ||
135 | return { response, videoObject: body } | 127 | return { statusCode, videoObject: body } |
136 | } | 128 | } |
137 | 129 | ||
138 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { | 130 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { |
139 | const host = video.VideoChannel.Account.Actor.Server.host | 131 | const host = video.VideoChannel.Account.Actor.Server.host |
140 | const path = video.getDescriptionAPIPath() | 132 | const path = video.getDescriptionAPIPath() |
141 | const options = { | 133 | const url = REMOTE_SCHEME.HTTP + '://' + host + path |
142 | uri: REMOTE_SCHEME.HTTP + '://' + host + path, | ||
143 | json: true | ||
144 | } | ||
145 | 134 | ||
146 | const { body } = await doRequest<any>(options) | 135 | const { body } = await doJSONRequest<any>(url) |
147 | return body.description ? body.description : '' | 136 | return body.description || '' |
148 | } | 137 | } |
149 | 138 | ||
150 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { | 139 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { |
@@ -534,8 +523,8 @@ async function refreshVideoIfNeeded (options: { | |||
534 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | 523 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) |
535 | 524 | ||
536 | try { | 525 | try { |
537 | const { response, videoObject } = await fetchRemoteVideo(video.url) | 526 | const { statusCode, videoObject } = await fetchRemoteVideo(video.url) |
538 | if (response.statusCode === HttpStatusCode.NOT_FOUND_404) { | 527 | if (statusCode === HttpStatusCode.NOT_FOUND_404) { |
539 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | 528 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) |
540 | 529 | ||
541 | // Video does not exist anymore | 530 | // Video does not exist anymore |
diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts index ee0447010..58e2260b6 100644 --- a/server/lib/files-cache/videos-caption-cache.ts +++ b/server/lib/files-cache/videos-caption-cache.ts | |||
@@ -41,7 +41,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> { | |||
41 | const remoteUrl = videoCaption.getFileUrl(video) | 41 | const remoteUrl = videoCaption.getFileUrl(video) |
42 | const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename) | 42 | const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename) |
43 | 43 | ||
44 | await doRequestAndSaveToFile({ uri: remoteUrl }, destPath) | 44 | await doRequestAndSaveToFile(remoteUrl, destPath) |
45 | 45 | ||
46 | return { isOwned: false, path: destPath } | 46 | return { isOwned: false, path: destPath } |
47 | } | 47 | } |
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts index ee72cd3f9..dd3a84aca 100644 --- a/server/lib/files-cache/videos-preview-cache.ts +++ b/server/lib/files-cache/videos-preview-cache.ts | |||
@@ -39,7 +39,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { | |||
39 | const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename) | 39 | const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename) |
40 | 40 | ||
41 | const remoteUrl = preview.getFileUrl(video) | 41 | const remoteUrl = preview.getFileUrl(video) |
42 | await doRequestAndSaveToFile({ uri: remoteUrl }, destPath) | 42 | await doRequestAndSaveToFile(remoteUrl, destPath) |
43 | 43 | ||
44 | logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath) | 44 | logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath) |
45 | 45 | ||
diff --git a/server/lib/files-cache/videos-torrent-cache.ts b/server/lib/files-cache/videos-torrent-cache.ts index ca0e1770d..881fa9ced 100644 --- a/server/lib/files-cache/videos-torrent-cache.ts +++ b/server/lib/files-cache/videos-torrent-cache.ts | |||
@@ -41,7 +41,7 @@ class VideosTorrentCache extends AbstractVideoStaticFileCache <string> { | |||
41 | const remoteUrl = file.getRemoteTorrentUrl(video) | 41 | const remoteUrl = file.getRemoteTorrentUrl(video) |
42 | const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) | 42 | const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) |
43 | 43 | ||
44 | await doRequestAndSaveToFile({ uri: remoteUrl }, destPath) | 44 | await doRequestAndSaveToFile(remoteUrl, destPath) |
45 | 45 | ||
46 | const downloadName = `${video.name}-${file.resolution}p.torrent` | 46 | const downloadName = `${video.name}-${file.resolution}p.torrent` |
47 | 47 | ||
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 04187668c..84539e2c1 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -135,7 +135,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, | |||
135 | const destPath = join(tmpDirectory, basename(fileUrl)) | 135 | const destPath = join(tmpDirectory, basename(fileUrl)) |
136 | 136 | ||
137 | const bodyKBLimit = 10 * 1000 * 1000 // 10GB | 137 | const bodyKBLimit = 10 * 1000 * 1000 // 10GB |
138 | await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit) | 138 | await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit }) |
139 | } | 139 | } |
140 | 140 | ||
141 | clearTimeout(timer) | 141 | clearTimeout(timer) |
@@ -156,7 +156,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, | |||
156 | } | 156 | } |
157 | 157 | ||
158 | async function fetchUniqUrls (playlistUrl: string) { | 158 | async function fetchUniqUrls (playlistUrl: string) { |
159 | const { body } = await doRequest<string>({ uri: playlistUrl }) | 159 | const { body } = await doRequest(playlistUrl) |
160 | 160 | ||
161 | if (!body) return [] | 161 | if (!body) return [] |
162 | 162 | ||
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts index 0e75b0a6e..9dcc778fa 100644 --- a/server/lib/job-queue/handlers/activitypub-cleaner.ts +++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts | |||
@@ -7,7 +7,7 @@ import { | |||
7 | isLikeActivityValid | 7 | isLikeActivityValid |
8 | } from '@server/helpers/custom-validators/activitypub/activity' | 8 | } from '@server/helpers/custom-validators/activitypub/activity' |
9 | import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' | 9 | import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' |
10 | import { doRequest } from '@server/helpers/requests' | 10 | import { doJSONRequest } from '@server/helpers/requests' |
11 | import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants' | 11 | import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants' |
12 | import { VideoModel } from '@server/models/video/video' | 12 | import { VideoModel } from '@server/models/video/video' |
13 | import { VideoCommentModel } from '@server/models/video/video-comment' | 13 | import { VideoCommentModel } from '@server/models/video/video-comment' |
@@ -81,15 +81,10 @@ async function updateObjectIfNeeded <T> ( | |||
81 | updater: (url: string, newUrl: string) => Promise<T>, | 81 | updater: (url: string, newUrl: string) => Promise<T>, |
82 | deleter: (url: string) => Promise<T> | 82 | deleter: (url: string) => Promise<T> |
83 | ): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { | 83 | ): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { |
84 | // Fetch url | 84 | const { statusCode, body } = await doJSONRequest<any>(url, { activityPub: true }) |
85 | const { response, body } = await doRequest<any>({ | ||
86 | uri: url, | ||
87 | json: true, | ||
88 | activityPub: true | ||
89 | }) | ||
90 | 85 | ||
91 | // Does not exist anymore, remove entry | 86 | // Does not exist anymore, remove entry |
92 | if (response.statusCode === HttpStatusCode.NOT_FOUND_404) { | 87 | if (statusCode === HttpStatusCode.NOT_FOUND_404) { |
93 | logger.info('Removing remote AP object %s.', url) | 88 | logger.info('Removing remote AP object %s.', url) |
94 | const data = await deleter(url) | 89 | const data = await deleter(url) |
95 | 90 | ||
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 7174786d6..c69ff9e83 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts | |||
@@ -16,8 +16,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { | |||
16 | const httpSignatureOptions = await buildSignedRequestOptions(payload) | 16 | const httpSignatureOptions = await buildSignedRequestOptions(payload) |
17 | 17 | ||
18 | const options = { | 18 | const options = { |
19 | method: 'POST', | 19 | method: 'POST' as 'POST', |
20 | uri: '', | ||
21 | json: body, | 20 | json: body, |
22 | httpSignature: httpSignatureOptions, | 21 | httpSignature: httpSignatureOptions, |
23 | timeout: REQUEST_TIMEOUT, | 22 | timeout: REQUEST_TIMEOUT, |
@@ -28,7 +27,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { | |||
28 | const goodUrls: string[] = [] | 27 | const goodUrls: string[] = [] |
29 | 28 | ||
30 | await Bluebird.map(payload.uris, uri => { | 29 | await Bluebird.map(payload.uris, uri => { |
31 | return doRequest(Object.assign({}, options, { uri })) | 30 | return doRequest(uri, options) |
32 | .then(() => goodUrls.push(uri)) | 31 | .then(() => goodUrls.push(uri)) |
33 | .catch(() => badUrls.push(uri)) | 32 | .catch(() => badUrls.push(uri)) |
34 | }, { concurrency: BROADCAST_CONCURRENCY }) | 33 | }, { concurrency: BROADCAST_CONCURRENCY }) |
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index 74989d62e..585dad671 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts | |||
@@ -16,8 +16,7 @@ async function processActivityPubHttpUnicast (job: Bull.Job) { | |||
16 | const httpSignatureOptions = await buildSignedRequestOptions(payload) | 16 | const httpSignatureOptions = await buildSignedRequestOptions(payload) |
17 | 17 | ||
18 | const options = { | 18 | const options = { |
19 | method: 'POST', | 19 | method: 'POST' as 'POST', |
20 | uri, | ||
21 | json: body, | 20 | json: body, |
22 | httpSignature: httpSignatureOptions, | 21 | httpSignature: httpSignatureOptions, |
23 | timeout: REQUEST_TIMEOUT, | 22 | timeout: REQUEST_TIMEOUT, |
@@ -25,7 +24,7 @@ async function processActivityPubHttpUnicast (job: Bull.Job) { | |||
25 | } | 24 | } |
26 | 25 | ||
27 | try { | 26 | try { |
28 | await doRequest(options) | 27 | await doRequest(uri, options) |
29 | ActorFollowScoreCache.Instance.updateActorFollowsScore([ uri ], []) | 28 | ActorFollowScoreCache.Instance.updateActorFollowsScore([ uri ], []) |
30 | } catch (err) { | 29 | } catch (err) { |
31 | ActorFollowScoreCache.Instance.updateActorFollowsScore([], [ uri ]) | 30 | ActorFollowScoreCache.Instance.updateActorFollowsScore([], [ uri ]) |
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index c030d31ef..4116a9c0e 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts | |||
@@ -6,21 +6,24 @@ import { getServerActor } from '@server/models/application/application' | |||
6 | import { buildDigest } from '@server/helpers/peertube-crypto' | 6 | import { buildDigest } from '@server/helpers/peertube-crypto' |
7 | import { ContextType } from '@shared/models/activitypub/context' | 7 | import { ContextType } from '@shared/models/activitypub/context' |
8 | 8 | ||
9 | type Payload = { body: any, contextType?: ContextType, signatureActorId?: number } | 9 | type Payload <T> = { body: T, contextType?: ContextType, signatureActorId?: number } |
10 | 10 | ||
11 | async function computeBody (payload: Payload) { | 11 | async function computeBody <T> ( |
12 | payload: Payload<T> | ||
13 | ): Promise<T | T & { type: 'RsaSignature2017', creator: string, created: string }> { | ||
12 | let body = payload.body | 14 | let body = payload.body |
13 | 15 | ||
14 | if (payload.signatureActorId) { | 16 | if (payload.signatureActorId) { |
15 | const actorSignature = await ActorModel.load(payload.signatureActorId) | 17 | const actorSignature = await ActorModel.load(payload.signatureActorId) |
16 | if (!actorSignature) throw new Error('Unknown signature actor id.') | 18 | if (!actorSignature) throw new Error('Unknown signature actor id.') |
19 | |||
17 | body = await buildSignedActivity(actorSignature, payload.body, payload.contextType) | 20 | body = await buildSignedActivity(actorSignature, payload.body, payload.contextType) |
18 | } | 21 | } |
19 | 22 | ||
20 | return body | 23 | return body |
21 | } | 24 | } |
22 | 25 | ||
23 | async function buildSignedRequestOptions (payload: Payload) { | 26 | async function buildSignedRequestOptions (payload: Payload<any>) { |
24 | let actor: MActor | null | 27 | let actor: MActor | null |
25 | 28 | ||
26 | if (payload.signatureActorId) { | 29 | if (payload.signatureActorId) { |
diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts index 7bcb6ed4c..624f5da1d 100644 --- a/server/lib/plugins/plugin-index.ts +++ b/server/lib/plugins/plugin-index.ts | |||
@@ -1,22 +1,22 @@ | |||
1 | import { doRequest } from '../../helpers/requests' | 1 | import { sanitizeUrl } from '@server/helpers/core-utils' |
2 | import { CONFIG } from '../../initializers/config' | 2 | import { ResultList } from '../../../shared/models' |
3 | import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model' | ||
4 | import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model' | ||
3 | import { | 5 | import { |
4 | PeertubePluginLatestVersionRequest, | 6 | PeertubePluginLatestVersionRequest, |
5 | PeertubePluginLatestVersionResponse | 7 | PeertubePluginLatestVersionResponse |
6 | } from '../../../shared/models/plugins/peertube-plugin-latest-version.model' | 8 | } from '../../../shared/models/plugins/peertube-plugin-latest-version.model' |
7 | import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model' | ||
8 | import { ResultList } from '../../../shared/models' | ||
9 | import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model' | ||
10 | import { PluginModel } from '../../models/server/plugin' | ||
11 | import { PluginManager } from './plugin-manager' | ||
12 | import { logger } from '../../helpers/logger' | 9 | import { logger } from '../../helpers/logger' |
10 | import { doJSONRequest } from '../../helpers/requests' | ||
11 | import { CONFIG } from '../../initializers/config' | ||
13 | import { PEERTUBE_VERSION } from '../../initializers/constants' | 12 | import { PEERTUBE_VERSION } from '../../initializers/constants' |
14 | import { sanitizeUrl } from '@server/helpers/core-utils' | 13 | import { PluginModel } from '../../models/server/plugin' |
14 | import { PluginManager } from './plugin-manager' | ||
15 | 15 | ||
16 | async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { | 16 | async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { |
17 | const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options | 17 | const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options |
18 | 18 | ||
19 | const qs: PeertubePluginIndexList = { | 19 | const searchParams: PeertubePluginIndexList & Record<string, string | number> = { |
20 | start, | 20 | start, |
21 | count, | 21 | count, |
22 | sort, | 22 | sort, |
@@ -28,7 +28,7 @@ async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) | |||
28 | const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins' | 28 | const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins' |
29 | 29 | ||
30 | try { | 30 | try { |
31 | const { body } = await doRequest<any>({ uri, qs, json: true }) | 31 | const { body } = await doJSONRequest<any>(uri, { searchParams }) |
32 | 32 | ||
33 | logger.debug('Got result from PeerTube index.', { body }) | 33 | logger.debug('Got result from PeerTube index.', { body }) |
34 | 34 | ||
@@ -58,7 +58,11 @@ async function getLatestPluginsVersion (npmNames: string[]): Promise<PeertubePlu | |||
58 | 58 | ||
59 | const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version' | 59 | const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version' |
60 | 60 | ||
61 | const { body } = await doRequest<any>({ uri, body: bodyRequest, json: true, method: 'POST' }) | 61 | const options = { |
62 | json: bodyRequest, | ||
63 | method: 'POST' as 'POST' | ||
64 | } | ||
65 | const { body } = await doJSONRequest<PeertubePluginLatestVersionResponse>(uri, options) | ||
62 | 66 | ||
63 | return body | 67 | return body |
64 | } | 68 | } |
diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts index f62f52f9c..0b8cd1389 100644 --- a/server/lib/schedulers/auto-follow-index-instances.ts +++ b/server/lib/schedulers/auto-follow-index-instances.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { chunk } from 'lodash' | 1 | import { chunk } from 'lodash' |
2 | import { doRequest } from '@server/helpers/requests' | 2 | import { doJSONRequest } from '@server/helpers/requests' |
3 | import { JobQueue } from '@server/lib/job-queue' | 3 | import { JobQueue } from '@server/lib/job-queue' |
4 | import { ActorFollowModel } from '@server/models/activitypub/actor-follow' | 4 | import { ActorFollowModel } from '@server/models/activitypub/actor-follow' |
5 | import { getServerActor } from '@server/models/application/application' | 5 | import { getServerActor } from '@server/models/application/application' |
@@ -34,12 +34,12 @@ export class AutoFollowIndexInstances extends AbstractScheduler { | |||
34 | try { | 34 | try { |
35 | const serverActor = await getServerActor() | 35 | const serverActor = await getServerActor() |
36 | 36 | ||
37 | const qs = { count: 1000 } | 37 | const searchParams = { count: 1000 } |
38 | if (this.lastCheck) Object.assign(qs, { since: this.lastCheck.toISOString() }) | 38 | if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() }) |
39 | 39 | ||
40 | this.lastCheck = new Date() | 40 | this.lastCheck = new Date() |
41 | 41 | ||
42 | const { body } = await doRequest<any>({ uri: indexUrl, qs, json: true }) | 42 | const { body } = await doJSONRequest<any>(indexUrl, { searchParams }) |
43 | if (!body.data || Array.isArray(body.data) === false) { | 43 | if (!body.data || Array.isArray(body.data) === false) { |
44 | logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body }) | 44 | logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body }) |
45 | return | 45 | return |
diff --git a/server/tests/api/activitypub/security.ts b/server/tests/api/activitypub/security.ts index 8bde54a40..26b4545ac 100644 --- a/server/tests/api/activitypub/security.ts +++ b/server/tests/api/activitypub/security.ts | |||
@@ -79,9 +79,9 @@ describe('Test ActivityPub security', function () { | |||
79 | Digest: buildDigest({ hello: 'coucou' }) | 79 | Digest: buildDigest({ hello: 'coucou' }) |
80 | } | 80 | } |
81 | 81 | ||
82 | const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) | 82 | const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) |
83 | 83 | ||
84 | expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) | 84 | expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) |
85 | }) | 85 | }) |
86 | 86 | ||
87 | it('Should fail with an invalid date', async function () { | 87 | it('Should fail with an invalid date', async function () { |
@@ -89,9 +89,9 @@ describe('Test ActivityPub security', function () { | |||
89 | const headers = buildGlobalHeaders(body) | 89 | const headers = buildGlobalHeaders(body) |
90 | headers['date'] = 'Wed, 21 Oct 2015 07:28:00 GMT' | 90 | headers['date'] = 'Wed, 21 Oct 2015 07:28:00 GMT' |
91 | 91 | ||
92 | const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) | 92 | const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) |
93 | 93 | ||
94 | expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) | 94 | expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) |
95 | }) | 95 | }) |
96 | 96 | ||
97 | it('Should fail with bad keys', async function () { | 97 | it('Should fail with bad keys', async function () { |
@@ -101,9 +101,9 @@ describe('Test ActivityPub security', function () { | |||
101 | const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) | 101 | const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) |
102 | const headers = buildGlobalHeaders(body) | 102 | const headers = buildGlobalHeaders(body) |
103 | 103 | ||
104 | const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) | 104 | const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) |
105 | 105 | ||
106 | expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) | 106 | expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) |
107 | }) | 107 | }) |
108 | 108 | ||
109 | it('Should reject requests without appropriate signed headers', async function () { | 109 | it('Should reject requests without appropriate signed headers', async function () { |
@@ -123,8 +123,8 @@ describe('Test ActivityPub security', function () { | |||
123 | for (const badHeaders of badHeadersMatrix) { | 123 | for (const badHeaders of badHeadersMatrix) { |
124 | signatureOptions.headers = badHeaders | 124 | signatureOptions.headers = badHeaders |
125 | 125 | ||
126 | const { response } = await makePOSTAPRequest(url, body, signatureOptions, headers) | 126 | const { statusCode } = await makePOSTAPRequest(url, body, signatureOptions, headers) |
127 | expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) | 127 | expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) |
128 | } | 128 | } |
129 | }) | 129 | }) |
130 | 130 | ||
@@ -132,9 +132,9 @@ describe('Test ActivityPub security', function () { | |||
132 | const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) | 132 | const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) |
133 | const headers = buildGlobalHeaders(body) | 133 | const headers = buildGlobalHeaders(body) |
134 | 134 | ||
135 | const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) | 135 | const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) |
136 | 136 | ||
137 | expect(response.statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) | 137 | expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) |
138 | }) | 138 | }) |
139 | 139 | ||
140 | it('Should refresh the actor keys', async function () { | 140 | it('Should refresh the actor keys', async function () { |
@@ -150,9 +150,9 @@ describe('Test ActivityPub security', function () { | |||
150 | const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) | 150 | const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) |
151 | const headers = buildGlobalHeaders(body) | 151 | const headers = buildGlobalHeaders(body) |
152 | 152 | ||
153 | const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) | 153 | const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) |
154 | 154 | ||
155 | expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) | 155 | expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) |
156 | }) | 156 | }) |
157 | }) | 157 | }) |
158 | 158 | ||
@@ -183,9 +183,9 @@ describe('Test ActivityPub security', function () { | |||
183 | 183 | ||
184 | const headers = buildGlobalHeaders(signedBody) | 184 | const headers = buildGlobalHeaders(signedBody) |
185 | 185 | ||
186 | const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) | 186 | const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) |
187 | 187 | ||
188 | expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) | 188 | expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) |
189 | }) | 189 | }) |
190 | 190 | ||
191 | it('Should fail with an altered body', async function () { | 191 | it('Should fail with an altered body', async function () { |
@@ -204,9 +204,9 @@ describe('Test ActivityPub security', function () { | |||
204 | 204 | ||
205 | const headers = buildGlobalHeaders(signedBody) | 205 | const headers = buildGlobalHeaders(signedBody) |
206 | 206 | ||
207 | const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) | 207 | const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) |
208 | 208 | ||
209 | expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) | 209 | expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) |
210 | }) | 210 | }) |
211 | 211 | ||
212 | it('Should succeed with a valid signature', async function () { | 212 | it('Should succeed with a valid signature', async function () { |
@@ -220,9 +220,9 @@ describe('Test ActivityPub security', function () { | |||
220 | 220 | ||
221 | const headers = buildGlobalHeaders(signedBody) | 221 | const headers = buildGlobalHeaders(signedBody) |
222 | 222 | ||
223 | const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) | 223 | const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) |
224 | 224 | ||
225 | expect(response.statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) | 225 | expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) |
226 | }) | 226 | }) |
227 | 227 | ||
228 | it('Should refresh the actor keys', async function () { | 228 | it('Should refresh the actor keys', async function () { |
@@ -243,9 +243,9 @@ describe('Test ActivityPub security', function () { | |||
243 | 243 | ||
244 | const headers = buildGlobalHeaders(signedBody) | 244 | const headers = buildGlobalHeaders(signedBody) |
245 | 245 | ||
246 | const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) | 246 | const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) |
247 | 247 | ||
248 | expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) | 248 | expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) |
249 | }) | 249 | }) |
250 | }) | 250 | }) |
251 | 251 | ||
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index 043754e70..f3ba11950 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts | |||
@@ -348,8 +348,8 @@ describe('Test handle downs', function () { | |||
348 | 348 | ||
349 | for (let i = 0; i < 3; i++) { | 349 | for (let i = 0; i < 3; i++) { |
350 | await getVideo(servers[1].url, videoIdsServer1[i]) | 350 | await getVideo(servers[1].url, videoIdsServer1[i]) |
351 | await wait(1000) | ||
352 | await waitJobs([ servers[1] ]) | 351 | await waitJobs([ servers[1] ]) |
352 | await wait(1500) | ||
353 | } | 353 | } |
354 | 354 | ||
355 | for (const id of videoIdsServer1) { | 355 | for (const id of videoIdsServer1) { |
diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts index f8b2d599b..5e77f129e 100644 --- a/server/tests/helpers/request.ts +++ b/server/tests/helpers/request.ts | |||
@@ -1,11 +1,11 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' | ||
5 | import { get4KFileUrl, root, wait } from '../../../shared/extra-utils' | ||
6 | import { join } from 'path' | ||
7 | import { pathExists, remove } from 'fs-extra' | ||
8 | import { expect } from 'chai' | 4 | import { expect } from 'chai' |
5 | import { pathExists, remove } from 'fs-extra' | ||
6 | import { join } from 'path' | ||
7 | import { get4KFileUrl, root, wait } from '../../../shared/extra-utils' | ||
8 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' | ||
9 | 9 | ||
10 | describe('Request helpers', function () { | 10 | describe('Request helpers', function () { |
11 | const destPath1 = join(root(), 'test-output-1.txt') | 11 | const destPath1 = join(root(), 'test-output-1.txt') |
@@ -13,7 +13,7 @@ describe('Request helpers', function () { | |||
13 | 13 | ||
14 | it('Should throw an error when the bytes limit is exceeded for request', async function () { | 14 | it('Should throw an error when the bytes limit is exceeded for request', async function () { |
15 | try { | 15 | try { |
16 | await doRequest({ uri: get4KFileUrl() }, 3) | 16 | await doRequest(get4KFileUrl(), { bodyKBLimit: 3 }) |
17 | } catch { | 17 | } catch { |
18 | return | 18 | return |
19 | } | 19 | } |
@@ -23,7 +23,7 @@ describe('Request helpers', function () { | |||
23 | 23 | ||
24 | it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { | 24 | it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { |
25 | try { | 25 | try { |
26 | await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath1, 3) | 26 | await doRequestAndSaveToFile(get4KFileUrl(), destPath1, { bodyKBLimit: 3 }) |
27 | } catch { | 27 | } catch { |
28 | 28 | ||
29 | await wait(500) | 29 | await wait(500) |
@@ -35,8 +35,8 @@ describe('Request helpers', function () { | |||
35 | }) | 35 | }) |
36 | 36 | ||
37 | it('Should succeed if the file is below the limit', async function () { | 37 | it('Should succeed if the file is below the limit', async function () { |
38 | await doRequest({ uri: get4KFileUrl() }, 5) | 38 | await doRequest(get4KFileUrl(), { bodyKBLimit: 5 }) |
39 | await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath2, 5) | 39 | await doRequestAndSaveToFile(get4KFileUrl(), destPath2, { bodyKBLimit: 5 }) |
40 | 40 | ||
41 | expect(await pathExists(destPath2)).to.be.true | 41 | expect(await pathExists(destPath2)).to.be.true |
42 | }) | 42 | }) |