server.ts

   1 // ----------- Node modules -----------
   2 import { registerOpentelemetryTracing } from './server/lib/opentelemetry/tracing'
   3 registerOpentelemetryTracing()
   4
   5 import express from 'express'
   6 import morgan, { token } from 'morgan'
   7 import cors from 'cors'
   8 import cookieParser from 'cookie-parser'
   9 import { frameguard } from 'helmet'
  10 import { parse } from 'useragent'
  11 import anonymize from 'ip-anonymize'
  12 import { program as cli } from 'commander'
  13
  14 process.title = 'peertube'
  15
  16 // Create our main app
  17 const app = express().disable('x-powered-by')
  18
  19 // ----------- Core checker -----------
  20 import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initializers/checker-before-init'
  21
  22 // Do not use barrels because we don't want to load all modules here (we need to initialize database first)
  23 import { CONFIG } from './server/initializers/config'
  24 import { API_VERSION, FILES_CACHE, WEBSERVER, loadLanguages } from './server/initializers/constants'
  25 import { logger } from './server/helpers/logger'
  26
  27 const missed = checkMissedConfig()
  28 if (missed.length !== 0) {
  29   logger.error('Your configuration files miss keys: ' + missed)
  30   process.exit(-1)
  31 }
  32
  33 checkFFmpeg(CONFIG)
  34   .catch(err => {
  35     logger.error('Error in ffmpeg check.', { err })
  36     process.exit(-1)
  37   })
  38
  39 try {
  40   checkNodeVersion()
  41 } catch (err) {
  42   logger.error('Error in NodeJS check.', { err })
  43   process.exit(-1)
  44 }
  45
  46 import { checkConfig, checkActivityPubUrls, checkFFmpegVersion } from './server/initializers/checker-after-init'
  47
  48 try {
  49   checkConfig()
  50 } catch (err) {
  51   logger.error('Config error.', { err })
  52   process.exit(-1)
  53 }
  54
  55 // Trust our proxy (IP forwarding...)
  56 app.set('trust proxy', CONFIG.TRUST_PROXY)
  57
  58 app.use((_req, res, next) => {
  59   res.locals.requestStart = Date.now()
  60
  61   return next()
  62 })
  63
  64 // Security middleware
  65 import { baseCSP } from './server/middlewares/csp'
  66
  67 if (CONFIG.CSP.ENABLED) {
  68   app.use(baseCSP)
  69 }
  70
  71 if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) {
  72   app.use(frameguard({
  73     action: 'deny' // we only allow it for /videos/embed, see server/controllers/client.ts
  74   }))
  75 }
  76
  77 // ----------- Database -----------
  78
  79 // Initialize database and models
  80 import { initDatabaseModels, checkDatabaseConnectionOrDie } from './server/initializers/database'
  81 checkDatabaseConnectionOrDie()
  82
  83 import { migrate } from './server/initializers/migrator'
  84 migrate()
  85   .then(() => initDatabaseModels(false))
  86   .then(() => startApplication())
  87   .catch(err => {
  88     logger.error('Cannot start application.', { err })
  89     process.exit(-1)
  90   })
  91
  92 // ----------- Initialize -----------
  93 loadLanguages()
  94
  95 // ----------- PeerTube modules -----------
  96 import { installApplication } from './server/initializers/installer'
  97 import { Emailer } from './server/lib/emailer'
  98 import { JobQueue } from './server/lib/job-queue'
  99 import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
 100 import {
 101   activityPubRouter,
 102   apiRouter,
 103   miscRouter,
 104   clientsRouter,
 105   feedsRouter,
 106   staticRouter,
 107   wellKnownRouter,
 108   lazyStaticRouter,
 109   servicesRouter,
 110   pluginsRouter,
 111   webfingerRouter,
 112   trackerRouter,
 113   createWebsocketTrackerServer,
 114   botsRouter,
 115   downloadRouter
 116 } from './server/controllers'
 117 import { advertiseDoNotTrack } from './server/middlewares/dnt'
 118 import { apiFailMiddleware } from './server/middlewares/error'
 119 import { Redis } from './server/lib/redis'
 120 import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler'
 121 import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler'
 122 import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
 123 import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
 124 import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
 125 import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
 126 import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
 127 import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
 128 import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
 129 import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
 130 import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'
 131 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
 132 import { PeerTubeSocket } from './server/lib/peertube-socket'
 133 import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
 134 import { PluginsCheckScheduler } from './server/lib/schedulers/plugins-check-scheduler'
 135 import { PeerTubeVersionCheckScheduler } from './server/lib/schedulers/peertube-version-check-scheduler'
 136 import { Hooks } from './server/lib/plugins/hooks'
 137 import { PluginManager } from './server/lib/plugins/plugin-manager'
 138 import { LiveManager } from './server/lib/live'
 139 import { HttpStatusCode } from './shared/models/http/http-error-codes'
 140 import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
 141 import { ServerConfigManager } from '@server/lib/server-config-manager'
 142 import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 143 import { isTestOrDevInstance } from './server/helpers/core-utils'
 144 import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
 145 import { ApplicationModel } from '@server/models/application/application'
 146 import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
 147
 148 // ----------- Command line -----------
 149
 150 cli
 151   .option('--no-client', 'Start PeerTube without client interface')
 152   .option('--no-plugins', 'Start PeerTube without plugins/themes enabled')
 153   .option('--benchmark-startup', 'Automatically stop server when initialized')
 154   .parse(process.argv)
 155
 156 // ----------- App -----------
 157
 158 // Enable CORS for develop
 159 if (isTestOrDevInstance()) {
 160   app.use(cors({
 161     origin: '*',
 162     exposedHeaders: 'Retry-After',
 163     credentials: true
 164   }))
 165 }
 166
 167 // For the logger
 168 token('remote-addr', (req: express.Request) => {
 169   if (CONFIG.LOG.ANONYMIZE_IP === true || req.get('DNT') === '1') {
 170     return anonymize(req.ip, 16, 16)
 171   }
 172
 173   return req.ip
 174 })
 175 token('user-agent', (req: express.Request) => {
 176   if (req.get('DNT') === '1') {
 177     return parse(req.get('user-agent')).family
 178   }
 179
 180   return req.get('user-agent')
 181 })
 182 app.use(morgan('combined', {
 183   stream: {
 184     write: (str: string) => logger.info(str.trim(), { tags: [ 'http' ] })
 185   },
 186   skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping'
 187 }))
 188
 189 // Add .fail() helper to response
 190 app.use(apiFailMiddleware)
 191
 192 // For body requests
 193 app.use(express.urlencoded({ extended: false }))
 194 app.use(express.json({
 195   type: [ 'application/json', 'application/*+json' ],
 196   limit: '500kb',
 197   verify: (req: express.Request, res: express.Response, buf: Buffer) => {
 198     const valid = isHTTPSignatureDigestValid(buf, req)
 199
 200     if (valid !== true) {
 201       res.fail({
 202         status: HttpStatusCode.FORBIDDEN_403,
 203         message: 'Invalid digest'
 204       })
 205     }
 206   }
 207 }))
 208
 209 // Cookies
 210 app.use(cookieParser())
 211
 212 // W3C DNT Tracking Status
 213 app.use(advertiseDoNotTrack)
 214
 215 // ----------- Open Telemetry -----------
 216
 217 OpenTelemetryMetrics.Instance.init(app)
 218
 219 // ----------- Views, routes and static files -----------
 220
 221 // API
 222 const apiRoute = '/api/' + API_VERSION
 223 app.use(apiRoute, apiRouter)
 224
 225 // Services (oembed...)
 226 app.use('/services', servicesRouter)
 227
 228 // Plugins & themes
 229 app.use('/', pluginsRouter)
 230
 231 app.use('/', activityPubRouter)
 232 app.use('/', feedsRouter)
 233 app.use('/', webfingerRouter)
 234 app.use('/', trackerRouter)
 235 app.use('/', botsRouter)
 236
 237 // Static files
 238 app.use('/', staticRouter)
 239 app.use('/', wellKnownRouter)
 240 app.use('/', miscRouter)
 241 app.use('/', downloadRouter)
 242 app.use('/', lazyStaticRouter)
 243
 244 // Client files, last valid routes!
 245 const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>()
 246 if (cliOptions.client) app.use('/', clientsRouter)
 247
 248 // ----------- Errors -----------
 249
 250 // Catch unmatched routes
 251 app.use((_req, res: express.Response) => {
 252   res.status(HttpStatusCode.NOT_FOUND_404).end()
 253 })
 254
 255 // Catch thrown errors
 256 app.use((err, _req, res: express.Response, _next) => {
 257   // Format error to be logged
 258   let error = 'Unknown error.'
 259   if (err) {
 260     error = err.stack || err.message || err
 261   }
 262
 263   // Handling Sequelize error traces
 264   const sql = err?.parent ? err.parent.sql : undefined
 265
 266   // Help us to debug SequelizeConnectionAcquireTimeoutError errors
 267   const activeRequests = err?.name === 'SequelizeConnectionAcquireTimeoutError' && typeof (process as any)._getActiveRequests !== 'function'
 268     ? (process as any)._getActiveRequests()
 269     : undefined
 270
 271   logger.error('Error in controller.', { err: error, sql, activeRequests })
 272
 273   return res.fail({
 274     status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500,
 275     message: err.message,
 276     type: err.name
 277   })
 278 })
 279
 280 const server = createWebsocketTrackerServer(app)
 281
 282 // ----------- Run -----------
 283
 284 async function startApplication () {
 285   const port = CONFIG.LISTEN.PORT
 286   const hostname = CONFIG.LISTEN.HOSTNAME
 287
 288   await installApplication()
 289
 290   // Check activity pub urls are valid
 291   checkActivityPubUrls()
 292     .catch(err => {
 293       logger.error('Error in ActivityPub URLs checker.', { err })
 294       process.exit(-1)
 295     })
 296
 297   checkFFmpegVersion()
 298     .catch(err => logger.error('Cannot check ffmpeg version', { err }))
 299
 300   Redis.Instance.init()
 301   Emailer.Instance.init()
 302
 303   await Promise.all([
 304     Emailer.Instance.checkConnection(),
 305     JobQueue.Instance.init(),
 306     ServerConfigManager.Instance.init()
 307   ])
 308
 309   // Caches initializations
 310   VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
 311   VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
 312   VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
 313
 314   // Enable Schedulers
 315   ActorFollowScheduler.Instance.enable()
 316   RemoveOldJobsScheduler.Instance.enable()
 317   UpdateVideosScheduler.Instance.enable()
 318   YoutubeDlUpdateScheduler.Instance.enable()
 319   VideosRedundancyScheduler.Instance.enable()
 320   RemoveOldHistoryScheduler.Instance.enable()
 321   RemoveOldViewsScheduler.Instance.enable()
 322   PluginsCheckScheduler.Instance.enable()
 323   PeerTubeVersionCheckScheduler.Instance.enable()
 324   AutoFollowIndexInstances.Instance.enable()
 325   RemoveDanglingResumableUploadsScheduler.Instance.enable()
 326   VideoChannelSyncLatestScheduler.Instance.enable()
 327   VideoViewsBufferScheduler.Instance.enable()
 328   GeoIPUpdateScheduler.Instance.enable()
 329   OpenTelemetryMetrics.Instance.registerMetrics()
 330
 331   PluginManager.Instance.init(server)
 332   // Before PeerTubeSocket init
 333   PluginManager.Instance.registerWebSocketRouter()
 334
 335   PeerTubeSocket.Instance.init(server)
 336   VideoViewsManager.Instance.init()
 337
 338   updateStreamingPlaylistsInfohashesIfNeeded()
 339     .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
 340
 341   LiveManager.Instance.init()
 342   if (CONFIG.LIVE.ENABLED) await LiveManager.Instance.run()
 343
 344   // Make server listening
 345   server.listen(port, hostname, async () => {
 346     if (cliOptions.plugins) {
 347       try {
 348         await PluginManager.Instance.rebuildNativePluginsIfNeeded()
 349
 350         await PluginManager.Instance.registerPluginsAndThemes()
 351       } catch (err) {
 352         logger.error('Cannot register plugins and themes.', { err })
 353       }
 354     }
 355
 356     ApplicationModel.updateNodeVersions()
 357       .catch(err => logger.error('Cannot update node versions.', { err }))
 358
 359     JobQueue.Instance.start()
 360       .catch(err => {
 361         logger.error('Cannot start job queue.', { err })
 362         process.exit(-1)
 363       })
 364
 365     logger.info('HTTP server listening on %s:%d', hostname, port)
 366     logger.info('Web server: %s', WEBSERVER.URL)
 367
 368     Hooks.runAction('action:application.listening')
 369
 370     if (cliOptions['benchmarkStartup']) process.exit(0)
 371   })
 372
 373   process.on('exit', () => {
 374     JobQueue.Instance.terminate()
 375       .catch(err => logger.error('Cannot terminate job queue.', { err }))
 376   })
 377
 378   process.on('SIGINT', () => process.exit(0))
 379 }