From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- server/server.ts | 376 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 376 insertions(+) create mode 100644 server/server.ts (limited to 'server/server.ts') diff --git a/server/server.ts b/server/server.ts new file mode 100644 index 000000000..7f9c68ad9 --- /dev/null +++ b/server/server.ts @@ -0,0 +1,376 @@ +import { registerOpentelemetryTracing } from '@server/lib/opentelemetry/tracing.js' +await registerOpentelemetryTracing() + +process.title = 'peertube' + +// ----------- Core checker ----------- +import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initializers/checker-before-init.js' + +// Do not use barrels because we don't want to load all modules here (we need to initialize database first) +import { CONFIG } from './server/initializers/config.js' +import { API_VERSION, WEBSERVER, loadLanguages } from './server/initializers/constants.js' +import { logger } from './server/helpers/logger.js' + +const missed = checkMissedConfig() +if (missed.length !== 0) { + logger.error('Your configuration files miss keys: ' + missed) + process.exit(-1) +} + +checkFFmpeg(CONFIG) + .catch(err => { + logger.error('Error in ffmpeg check.', { err }) + process.exit(-1) + }) + +try { + checkNodeVersion() +} catch (err) { + logger.error('Error in NodeJS check.', { err }) + process.exit(-1) +} + +import { checkConfig, checkActivityPubUrls, checkFFmpegVersion } from './server/initializers/checker-after-init.js' + +try { + checkConfig() +} catch (err) { + logger.error('Config error.', { err }) + process.exit(-1) +} + +// ----------- Database ----------- + +// Initialize database and models +import { initDatabaseModels, checkDatabaseConnectionOrDie } from './server/initializers/database.js' +checkDatabaseConnectionOrDie() + +import { migrate } from './server/initializers/migrator.js' +migrate() + .then(() => initDatabaseModels(false)) + .then(() => startApplication()) + .catch(err => { + logger.error('Cannot start application.', { err }) + process.exit(-1) + }) + +// ----------- Initialize ----------- +loadLanguages() + .catch(err => logger.error('Cannot load languages', { err })) + +// Express configuration +import express from 'express' +import morgan, { token } from 'morgan' +import cors from 'cors' +import cookieParser from 'cookie-parser' +import { frameguard } from 'helmet' +import { parse } from 'useragent' +import anonymize from 'ip-anonymize' +import { program as cli } from 'commander' + +const app = express().disable('x-powered-by') + +// Trust our proxy (IP forwarding...) +app.set('trust proxy', CONFIG.TRUST_PROXY) + +app.use((_req, res, next) => { + // OpenTelemetry + res.locals.requestStart = Date.now() + + if (CONFIG.SECURITY.POWERED_BY_HEADER.ENABLED === true) { + res.setHeader('x-powered-by', 'PeerTube') + } + + return next() +}) + +// Security middleware +import { baseCSP } from './server/middlewares/csp.js' + +if (CONFIG.CSP.ENABLED) { + app.use(baseCSP) +} + +if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) { + app.use(frameguard({ + action: 'deny' // we only allow it for /videos/embed, see server/controllers/client.ts + })) +} + +// ----------- PeerTube modules ----------- +import { installApplication } from './server/initializers/installer.js' +import { Emailer } from './server/lib/emailer.js' +import { JobQueue } from './server/lib/job-queue/index.js' +import { + activityPubRouter, + apiRouter, + miscRouter, + clientsRouter, + feedsRouter, + staticRouter, + wellKnownRouter, + lazyStaticRouter, + servicesRouter, + objectStorageProxyRouter, + pluginsRouter, + trackerRouter, + createWebsocketTrackerServer, + sitemapRouter, + downloadRouter +} from './server/controllers/index.js' +import { advertiseDoNotTrack } from './server/middlewares/dnt.js' +import { apiFailMiddleware } from './server/middlewares/error.js' +import { Redis } from './server/lib/redis.js' +import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler.js' +import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler.js' +import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler.js' +import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler.js' +import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler.js' +import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler.js' +import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances.js' +import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.js' +import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler.js' +import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler.js' +import { RunnerJobWatchDogScheduler } from './server/lib/schedulers/runner-job-watch-dog-scheduler.js' +import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto.js' +import { PeerTubeSocket } from './server/lib/peertube-socket.js' +import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls.js' +import { PluginsCheckScheduler } from './server/lib/schedulers/plugins-check-scheduler.js' +import { PeerTubeVersionCheckScheduler } from './server/lib/schedulers/peertube-version-check-scheduler.js' +import { Hooks } from './server/lib/plugins/hooks.js' +import { PluginManager } from './server/lib/plugins/plugin-manager.js' +import { LiveManager } from './server/lib/live/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { ServerConfigManager } from '@server/lib/server-config-manager.js' +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics.js' +import { ApplicationModel } from '@server/models/application/application.js' +import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler.js' + +// ----------- Command line ----------- + +cli + .option('--no-client', 'Start PeerTube without client interface') + .option('--no-plugins', 'Start PeerTube without plugins/themes enabled') + .option('--benchmark-startup', 'Automatically stop server when initialized') + .parse(process.argv) + +// ----------- App ----------- + +// Enable CORS for develop +if (isTestOrDevInstance()) { + app.use(cors({ + origin: '*', + exposedHeaders: 'Retry-After', + credentials: true + })) +} + +// For the logger +token('remote-addr', (req: express.Request) => { + if (CONFIG.LOG.ANONYMIZE_IP === true || req.get('DNT') === '1') { + return anonymize(req.ip, 16, 16) + } + + return req.ip +}) +token('user-agent', (req: express.Request) => { + if (req.get('DNT') === '1') { + return parse(req.get('user-agent')).family + } + + return req.get('user-agent') +}) +app.use(morgan('combined', { + stream: { + write: (str: string) => logger.info(str.trim(), { tags: [ 'http' ] }) + }, + skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping' +})) + +// Add .fail() helper to response +app.use(apiFailMiddleware) + +// For body requests +app.use(express.urlencoded({ extended: false })) +app.use(express.json({ + type: [ 'application/json', 'application/*+json' ], + limit: '500kb', + verify: (req: express.Request, res: express.Response, buf: Buffer) => { + const valid = isHTTPSignatureDigestValid(buf, req) + + if (valid !== true) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Invalid digest' + }) + } + } +})) + +// Cookies +app.use(cookieParser()) + +// W3C DNT Tracking Status +app.use(advertiseDoNotTrack) + +// ----------- Open Telemetry ----------- + +OpenTelemetryMetrics.Instance.init(app) + +// ----------- Views, routes and static files ----------- + +app.use('/api/' + API_VERSION, apiRouter) + +// Services (oembed...) +app.use('/services', servicesRouter) + +// Plugins & themes +app.use('/', pluginsRouter) + +app.use('/', activityPubRouter) +app.use('/', feedsRouter) +app.use('/', trackerRouter) +app.use('/', sitemapRouter) + +// Static files +app.use('/', staticRouter) +app.use('/', wellKnownRouter) +app.use('/', miscRouter) +app.use('/', downloadRouter) +app.use('/', lazyStaticRouter) +app.use('/', objectStorageProxyRouter) + +// Client files, last valid routes! +const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>() +if (cliOptions.client) app.use('/', clientsRouter) + +// ----------- Errors ----------- + +// Catch unmatched routes +app.use((_req, res: express.Response) => { + res.status(HttpStatusCode.NOT_FOUND_404).end() +}) + +// Catch thrown errors +app.use((err, _req, res: express.Response, _next) => { + // Format error to be logged + let error = 'Unknown error.' + if (err) { + error = err.stack || err.message || err + } + + // Handling Sequelize error traces + const sql = err?.parent ? err.parent.sql : undefined + + // Help us to debug SequelizeConnectionAcquireTimeoutError errors + const activeRequests = err?.name === 'SequelizeConnectionAcquireTimeoutError' && typeof (process as any)._getActiveRequests !== 'function' + ? (process as any)._getActiveRequests() + : undefined + + logger.error('Error in controller.', { err: error, sql, activeRequests }) + + return res.fail({ + status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: err.message, + type: err.name + }) +}) + +const { server, trackerServer } = createWebsocketTrackerServer(app) + +// ----------- Run ----------- + +async function startApplication () { + const port = CONFIG.LISTEN.PORT + const hostname = CONFIG.LISTEN.HOSTNAME + + await installApplication() + + // Check activity pub urls are valid + checkActivityPubUrls() + .catch(err => { + logger.error('Error in ActivityPub URLs checker.', { err }) + process.exit(-1) + }) + + checkFFmpegVersion() + .catch(err => logger.error('Cannot check ffmpeg version', { err })) + + Redis.Instance.init() + Emailer.Instance.init() + + await Promise.all([ + Emailer.Instance.checkConnection(), + JobQueue.Instance.init(), + ServerConfigManager.Instance.init() + ]) + + // Enable Schedulers + ActorFollowScheduler.Instance.enable() + UpdateVideosScheduler.Instance.enable() + YoutubeDlUpdateScheduler.Instance.enable() + VideosRedundancyScheduler.Instance.enable() + RemoveOldHistoryScheduler.Instance.enable() + RemoveOldViewsScheduler.Instance.enable() + PluginsCheckScheduler.Instance.enable() + PeerTubeVersionCheckScheduler.Instance.enable() + AutoFollowIndexInstances.Instance.enable() + RemoveDanglingResumableUploadsScheduler.Instance.enable() + VideoChannelSyncLatestScheduler.Instance.enable() + VideoViewsBufferScheduler.Instance.enable() + GeoIPUpdateScheduler.Instance.enable() + RunnerJobWatchDogScheduler.Instance.enable() + + OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer }) + + PluginManager.Instance.init(server) + // Before PeerTubeSocket init + PluginManager.Instance.registerWebSocketRouter() + + PeerTubeSocket.Instance.init(server) + VideoViewsManager.Instance.init() + + updateStreamingPlaylistsInfohashesIfNeeded() + .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err })) + + LiveManager.Instance.init() + if (CONFIG.LIVE.ENABLED) await LiveManager.Instance.run() + + // Make server listening + server.listen(port, hostname, async () => { + if (cliOptions.plugins) { + try { + await PluginManager.Instance.rebuildNativePluginsIfNeeded() + + await PluginManager.Instance.registerPluginsAndThemes() + } catch (err) { + logger.error('Cannot register plugins and themes.', { err }) + } + } + + ApplicationModel.updateNodeVersions() + .catch(err => logger.error('Cannot update node versions.', { err })) + + JobQueue.Instance.start() + .catch(err => { + logger.error('Cannot start job queue.', { err }) + process.exit(-1) + }) + + logger.info('HTTP server listening on %s:%d', hostname, port) + logger.info('Web server: %s', WEBSERVER.URL) + + Hooks.runAction('action:application.listening') + + if (cliOptions['benchmarkStartup']) process.exit(0) + }) + + process.on('exit', () => { + JobQueue.Instance.terminate() + .catch(err => logger.error('Cannot terminate job queue.', { err })) + }) + + process.on('SIGINT', () => process.exit(0)) +} -- cgit v1.2.3