From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- packages/server-commands/src/requests/requests.ts | 260 ++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 packages/server-commands/src/requests/requests.ts (limited to 'packages/server-commands/src/requests/requests.ts') diff --git a/packages/server-commands/src/requests/requests.ts b/packages/server-commands/src/requests/requests.ts new file mode 100644 index 000000000..ac143ea5d --- /dev/null +++ b/packages/server-commands/src/requests/requests.ts @@ -0,0 +1,260 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ + +import { decode } from 'querystring' +import request from 'supertest' +import { URL } from 'url' +import { pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' + +export type CommonRequestParams = { + url: string + path?: string + contentType?: string + responseType?: string + range?: string + redirects?: number + accept?: string + host?: string + token?: string + headers?: { [ name: string ]: string } + type?: string + xForwardedFor?: string + expectedStatus?: HttpStatusCodeType +} + +function makeRawRequest (options: { + url: string + token?: string + expectedStatus?: HttpStatusCodeType + range?: string + query?: { [ id: string ]: string } + method?: 'GET' | 'POST' + headers?: { [ name: string ]: string } +}) { + const { host, protocol, pathname } = new URL(options.url) + + const reqOptions = { + url: `${protocol}//${host}`, + path: pathname, + contentType: undefined, + + ...pick(options, [ 'expectedStatus', 'range', 'token', 'query', 'headers' ]) + } + + if (options.method === 'POST') { + return makePostBodyRequest(reqOptions) + } + + return makeGetRequest(reqOptions) +} + +function makeGetRequest (options: CommonRequestParams & { + query?: any + rawQuery?: string +}) { + const req = request(options.url).get(options.path) + + if (options.query) req.query(options.query) + if (options.rawQuery) req.query(options.rawQuery) + + return buildRequest(req, { contentType: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) +} + +function makeHTMLRequest (url: string, path: string) { + return makeGetRequest({ + url, + path, + accept: 'text/html', + expectedStatus: HttpStatusCode.OK_200 + }) +} + +function makeActivityPubGetRequest (url: string, path: string, expectedStatus: HttpStatusCodeType = HttpStatusCode.OK_200) { + return makeGetRequest({ + url, + path, + expectedStatus, + accept: 'application/activity+json,text/html;q=0.9,\\*/\\*;q=0.8' + }) +} + +function makeDeleteRequest (options: CommonRequestParams & { + query?: any + rawQuery?: string +}) { + const req = request(options.url).delete(options.path) + + if (options.query) req.query(options.query) + if (options.rawQuery) req.query(options.rawQuery) + + return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) +} + +function makeUploadRequest (options: CommonRequestParams & { + method?: 'POST' | 'PUT' + + fields: { [ fieldName: string ]: any } + attaches?: { [ attachName: string ]: any | any[] } +}) { + let req = options.method === 'PUT' + ? request(options.url).put(options.path) + : request(options.url).post(options.path) + + req = buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) + + buildFields(req, options.fields) + + Object.keys(options.attaches || {}).forEach(attach => { + const value = options.attaches[attach] + if (!value) return + + if (Array.isArray(value)) { + req.attach(attach, buildAbsoluteFixturePath(value[0]), value[1]) + } else { + req.attach(attach, buildAbsoluteFixturePath(value)) + } + }) + + return req +} + +function makePostBodyRequest (options: CommonRequestParams & { + fields?: { [ fieldName: string ]: any } +}) { + const req = request(options.url).post(options.path) + .send(options.fields) + + return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) +} + +function makePutBodyRequest (options: { + url: string + path: string + token?: string + fields: { [ fieldName: string ]: any } + expectedStatus?: HttpStatusCodeType + headers?: { [name: string]: string } +}) { + const req = request(options.url).put(options.path) + .send(options.fields) + + return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) +} + +function decodeQueryString (path: string) { + return decode(path.split('?')[1]) +} + +// --------------------------------------------------------------------------- + +function unwrapBody (test: request.Test): Promise { + return test.then(res => res.body) +} + +function unwrapText (test: request.Test): Promise { + return test.then(res => res.text) +} + +function unwrapBodyOrDecodeToJSON (test: request.Test): Promise { + return test.then(res => { + if (res.body instanceof Buffer) { + try { + return JSON.parse(new TextDecoder().decode(res.body)) + } catch (err) { + console.error('Cannot decode JSON.', { res, body: res.body instanceof Buffer ? res.body.toString() : res.body }) + throw err + } + } + + if (res.text) { + try { + return JSON.parse(res.text) + } catch (err) { + console.error('Cannot decode json', { res, text: res.text }) + throw err + } + } + + return res.body + }) +} + +function unwrapTextOrDecode (test: request.Test): Promise { + return test.then(res => res.text || new TextDecoder().decode(res.body)) +} + +// --------------------------------------------------------------------------- + +export { + makeHTMLRequest, + makeGetRequest, + decodeQueryString, + makeUploadRequest, + makePostBodyRequest, + makePutBodyRequest, + makeDeleteRequest, + makeRawRequest, + makeActivityPubGetRequest, + unwrapBody, + unwrapTextOrDecode, + unwrapBodyOrDecodeToJSON, + unwrapText +} + +// --------------------------------------------------------------------------- + +function buildRequest (req: request.Test, options: CommonRequestParams) { + if (options.contentType) req.set('Accept', options.contentType) + if (options.responseType) req.responseType(options.responseType) + if (options.token) req.set('Authorization', 'Bearer ' + options.token) + if (options.range) req.set('Range', options.range) + if (options.accept) req.set('Accept', options.accept) + if (options.host) req.set('Host', options.host) + if (options.redirects) req.redirects(options.redirects) + if (options.xForwardedFor) req.set('X-Forwarded-For', options.xForwardedFor) + if (options.type) req.type(options.type) + + Object.keys(options.headers || {}).forEach(name => { + req.set(name, options.headers[name]) + }) + + return req.expect(res => { + if (options.expectedStatus && res.status !== options.expectedStatus) { + const err = new Error(`Expected status ${options.expectedStatus}, got ${res.status}. ` + + `\nThe server responded: "${res.body?.error ?? res.text}".\n` + + 'You may take a closer look at the logs. To see how to do so, check out this page: ' + + 'https://github.com/Chocobozzz/PeerTube/blob/develop/support/doc/development/tests.md#debug-server-logs'); + + (err as any).res = res + + throw err + } + + return res + }) +} + +function buildFields (req: request.Test, fields: { [ fieldName: string ]: any }, namespace?: string) { + if (!fields) return + + let formKey: string + + for (const key of Object.keys(fields)) { + if (namespace) formKey = `${namespace}[${key}]` + else formKey = key + + if (fields[key] === undefined) continue + + if (Array.isArray(fields[key]) && fields[key].length === 0) { + req.field(key, []) + continue + } + + if (fields[key] !== null && typeof fields[key] === 'object') { + buildFields(req, fields[key], formKey) + } else { + req.field(formKey, fields[key]) + } + } +} -- cgit v1.2.3