server.ts

   1 // ----------- Node modules -----------
   2 import { registerOpentelemetryTracing } from './server/lib/opentelemetry/tracing'
   3 registerOpentelemetryTracing()
   4
   5 import express from 'express'
   6 import morgan, { token } from 'morgan'
   7 import cors from 'cors'
   8 import cookieParser from 'cookie-parser'
   9 import { frameguard } from 'helmet'
  10 import { parse } from 'useragent'
  11 import anonymize from 'ip-anonymize'
  12 import { program as cli } from 'commander'
  13
  14 process.title = 'peertube'
  15
  16 // Create our main app
  17 const app = express().disable('x-powered-by')
  18
  19 // ----------- Core checker -----------
  20 import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initializers/checker-before-init'
  21
  22 // Do not use barrels because we don't want to load all modules here (we need to initialize database first)
  23 import { CONFIG } from './server/initializers/config'
  24 import { API_VERSION, FILES_CACHE, WEBSERVER, loadLanguages } from './server/initializers/constants'
  25 import { logger } from './server/helpers/logger'
  26
  27 const missed = checkMissedConfig()
  28 if (missed.length !== 0) {
  29   logger.error('Your configuration files miss keys: ' + missed)
  30   process.exit(-1)
  31 }
  32
  33 checkFFmpeg(CONFIG)
  34   .catch(err => {
  35     logger.error('Error in ffmpeg check.', { err })
  36     process.exit(-1)
  37   })
  38
  39 try {
  40   checkNodeVersion()
  41 } catch (err) {
  42   logger.error('Error in NodeJS check.', { err })
  43   process.exit(-1)
  44 }
  45
  46 import { checkConfig, checkActivityPubUrls, checkFFmpegVersion } from './server/initializers/checker-after-init'
  47
  48 try {
  49   checkConfig()
  50 } catch (err) {
  51   logger.error('Config error.', { err })
  52   process.exit(-1)
  53 }
  54
  55 // Trust our proxy (IP forwarding...)
  56 app.set('trust proxy', CONFIG.TRUST_PROXY)
  57
  58 app.use((_req, res, next) => {
  59   // OpenTelemetry
  60   res.locals.requestStart = Date.now()
  61
  62   if (CONFIG.SECURITY.POWERED_BY_HEADER.ENABLED === true) {
  63     res.setHeader('x-powered-by', 'PeerTube')
  64   }
  65
  66   return next()
  67 })
  68
  69 // Security middleware
  70 import { baseCSP } from './server/middlewares/csp'
  71
  72 if (CONFIG.CSP.ENABLED) {
  73   app.use(baseCSP)
  74 }
  75
  76 if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) {
  77   app.use(frameguard({
  78     action: 'deny' // we only allow it for /videos/embed, see server/controllers/client.ts
  79   }))
  80 }
  81
  82 // ----------- Database -----------
  83
  84 // Initialize database and models
  85 import { initDatabaseModels, checkDatabaseConnectionOrDie } from './server/initializers/database'
  86 checkDatabaseConnectionOrDie()
  87
  88 import { migrate } from './server/initializers/migrator'
  89 migrate()
  90   .then(() => initDatabaseModels(false))
  91   .then(() => startApplication())
  92   .catch(err => {
  93     logger.error('Cannot start application.', { err })
  94     process.exit(-1)
  95   })
  96
  97 // ----------- Initialize -----------
  98 loadLanguages()
  99
 100 // ----------- PeerTube modules -----------
 101 import { installApplication } from './server/initializers/installer'
 102 import { Emailer } from './server/lib/emailer'
 103 import { JobQueue } from './server/lib/job-queue'
 104 import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
 105 import {
 106   activityPubRouter,
 107   apiRouter,
 108   miscRouter,
 109   clientsRouter,
 110   feedsRouter,
 111   staticRouter,
 112   wellKnownRouter,
 113   lazyStaticRouter,
 114   servicesRouter,
 115   objectStorageProxyRouter,
 116   pluginsRouter,
 117   trackerRouter,
 118   createWebsocketTrackerServer,
 119   botsRouter,
 120   downloadRouter
 121 } from './server/controllers'
 122 import { advertiseDoNotTrack } from './server/middlewares/dnt'
 123 import { apiFailMiddleware } from './server/middlewares/error'
 124 import { Redis } from './server/lib/redis'
 125 import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler'
 126 import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler'
 127 import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
 128 import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
 129 import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
 130 import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
 131 import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
 132 import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
 133 import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
 134 import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
 135 import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'
 136 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
 137 import { PeerTubeSocket } from './server/lib/peertube-socket'
 138 import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
 139 import { PluginsCheckScheduler } from './server/lib/schedulers/plugins-check-scheduler'
 140 import { PeerTubeVersionCheckScheduler } from './server/lib/schedulers/peertube-version-check-scheduler'
 141 import { Hooks } from './server/lib/plugins/hooks'
 142 import { PluginManager } from './server/lib/plugins/plugin-manager'
 143 import { LiveManager } from './server/lib/live'
 144 import { HttpStatusCode } from './shared/models/http/http-error-codes'
 145 import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
 146 import { ServerConfigManager } from '@server/lib/server-config-manager'
 147 import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 148 import { isTestOrDevInstance } from './server/helpers/core-utils'
 149 import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
 150 import { ApplicationModel } from '@server/models/application/application'
 151 import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
 152
 153 // ----------- Command line -----------
 154
 155 cli
 156   .option('--no-client', 'Start PeerTube without client interface')
 157   .option('--no-plugins', 'Start PeerTube without plugins/themes enabled')
 158   .option('--benchmark-startup', 'Automatically stop server when initialized')
 159   .parse(process.argv)
 160
 161 // ----------- App -----------
 162
 163 // Enable CORS for develop
 164 if (isTestOrDevInstance()) {
 165   app.use(cors({
 166     origin: '*',
 167     exposedHeaders: 'Retry-After',
 168     credentials: true
 169   }))
 170 }
 171
 172 // For the logger
 173 token('remote-addr', (req: express.Request) => {
 174   if (CONFIG.LOG.ANONYMIZE_IP === true || req.get('DNT') === '1') {
 175     return anonymize(req.ip, 16, 16)
 176   }
 177
 178   return req.ip
 179 })
 180 token('user-agent', (req: express.Request) => {
 181   if (req.get('DNT') === '1') {
 182     return parse(req.get('user-agent')).family
 183   }
 184
 185   return req.get('user-agent')
 186 })
 187 app.use(morgan('combined', {
 188   stream: {
 189     write: (str: string) => logger.info(str.trim(), { tags: [ 'http' ] })
 190   },
 191   skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping'
 192 }))
 193
 194 // Add .fail() helper to response
 195 app.use(apiFailMiddleware)
 196
 197 // For body requests
 198 app.use(express.urlencoded({ extended: false }))
 199 app.use(express.json({
 200   type: [ 'application/json', 'application/*+json' ],
 201   limit: '500kb',
 202   verify: (req: express.Request, res: express.Response, buf: Buffer) => {
 203     const valid = isHTTPSignatureDigestValid(buf, req)
 204
 205     if (valid !== true) {
 206       res.fail({
 207         status: HttpStatusCode.FORBIDDEN_403,
 208         message: 'Invalid digest'
 209       })
 210     }
 211   }
 212 }))
 213
 214 // Cookies
 215 app.use(cookieParser())
 216
 217 // W3C DNT Tracking Status
 218 app.use(advertiseDoNotTrack)
 219
 220 // ----------- Open Telemetry -----------
 221
 222 OpenTelemetryMetrics.Instance.init(app)
 223
 224 // ----------- Views, routes and static files -----------
 225
 226 // API
 227 const apiRoute = '/api/' + API_VERSION
 228 app.use(apiRoute, apiRouter)
 229
 230 // Services (oembed...)
 231 app.use('/services', servicesRouter)
 232
 233 // Plugins & themes
 234 app.use('/', pluginsRouter)
 235
 236 app.use('/', activityPubRouter)
 237 app.use('/', feedsRouter)
 238 app.use('/', trackerRouter)
 239 app.use('/', botsRouter)
 240
 241 // Static files
 242 app.use('/', staticRouter)
 243 app.use('/', wellKnownRouter)
 244 app.use('/', miscRouter)
 245 app.use('/', downloadRouter)
 246 app.use('/', lazyStaticRouter)
 247 app.use('/', objectStorageProxyRouter)
 248
 249 // Client files, last valid routes!
 250 const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>()
 251 if (cliOptions.client) app.use('/', clientsRouter)
 252
 253 // ----------- Errors -----------
 254
 255 // Catch unmatched routes
 256 app.use((_req, res: express.Response) => {
 257   res.status(HttpStatusCode.NOT_FOUND_404).end()
 258 })
 259
 260 // Catch thrown errors
 261 app.use((err, _req, res: express.Response, _next) => {
 262   // Format error to be logged
 263   let error = 'Unknown error.'
 264   if (err) {
 265     error = err.stack || err.message || err
 266   }
 267
 268   // Handling Sequelize error traces
 269   const sql = err?.parent ? err.parent.sql : undefined
 270
 271   // Help us to debug SequelizeConnectionAcquireTimeoutError errors
 272   const activeRequests = err?.name === 'SequelizeConnectionAcquireTimeoutError' && typeof (process as any)._getActiveRequests !== 'function'
 273     ? (process as any)._getActiveRequests()
 274     : undefined
 275
 276   logger.error('Error in controller.', { err: error, sql, activeRequests })
 277
 278   return res.fail({
 279     status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500,
 280     message: err.message,
 281     type: err.name
 282   })
 283 })
 284
 285 const { server, trackerServer } = createWebsocketTrackerServer(app)
 286
 287 // ----------- Run -----------
 288
 289 async function startApplication () {
 290   const port = CONFIG.LISTEN.PORT
 291   const hostname = CONFIG.LISTEN.HOSTNAME
 292
 293   await installApplication()
 294
 295   // Check activity pub urls are valid
 296   checkActivityPubUrls()
 297     .catch(err => {
 298       logger.error('Error in ActivityPub URLs checker.', { err })
 299       process.exit(-1)
 300     })
 301
 302   checkFFmpegVersion()
 303     .catch(err => logger.error('Cannot check ffmpeg version', { err }))
 304
 305   Redis.Instance.init()
 306   Emailer.Instance.init()
 307
 308   await Promise.all([
 309     Emailer.Instance.checkConnection(),
 310     JobQueue.Instance.init(),
 311     ServerConfigManager.Instance.init()
 312   ])
 313
 314   // Caches initializations
 315   VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
 316   VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
 317   VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
 318
 319   // Enable Schedulers
 320   ActorFollowScheduler.Instance.enable()
 321   RemoveOldJobsScheduler.Instance.enable()
 322   UpdateVideosScheduler.Instance.enable()
 323   YoutubeDlUpdateScheduler.Instance.enable()
 324   VideosRedundancyScheduler.Instance.enable()
 325   RemoveOldHistoryScheduler.Instance.enable()
 326   RemoveOldViewsScheduler.Instance.enable()
 327   PluginsCheckScheduler.Instance.enable()
 328   PeerTubeVersionCheckScheduler.Instance.enable()
 329   AutoFollowIndexInstances.Instance.enable()
 330   RemoveDanglingResumableUploadsScheduler.Instance.enable()
 331   VideoChannelSyncLatestScheduler.Instance.enable()
 332   VideoViewsBufferScheduler.Instance.enable()
 333   GeoIPUpdateScheduler.Instance.enable()
 334
 335   OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })
 336
 337   PluginManager.Instance.init(server)
 338   // Before PeerTubeSocket init
 339   PluginManager.Instance.registerWebSocketRouter()
 340
 341   PeerTubeSocket.Instance.init(server)
 342   VideoViewsManager.Instance.init()
 343
 344   updateStreamingPlaylistsInfohashesIfNeeded()
 345     .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
 346
 347   LiveManager.Instance.init()
 348   if (CONFIG.LIVE.ENABLED) await LiveManager.Instance.run()
 349
 350   // Make server listening
 351   server.listen(port, hostname, async () => {
 352     if (cliOptions.plugins) {
 353       try {
 354         await PluginManager.Instance.rebuildNativePluginsIfNeeded()
 355
 356         await PluginManager.Instance.registerPluginsAndThemes()
 357       } catch (err) {
 358         logger.error('Cannot register plugins and themes.', { err })
 359       }
 360     }
 361
 362     ApplicationModel.updateNodeVersions()
 363       .catch(err => logger.error('Cannot update node versions.', { err }))
 364
 365     JobQueue.Instance.start()
 366       .catch(err => {
 367         logger.error('Cannot start job queue.', { err })
 368         process.exit(-1)
 369       })
 370
 371     logger.info('HTTP server listening on %s:%d', hostname, port)
 372     logger.info('Web server: %s', WEBSERVER.URL)
 373
 374     Hooks.runAction('action:application.listening')
 375
 376     if (cliOptions['benchmarkStartup']) process.exit(0)
 377   })
 378
 379   process.on('exit', () => {
 380     JobQueue.Instance.terminate()
 381       .catch(err => logger.error('Cannot terminate job queue.', { err }))
 382   })
 383
 384   process.on('SIGINT', () => process.exit(0))
 385 }