server.ts

   1 // ----------- Node modules -----------
   2 import { registerOpentelemetryTracing } from './server/lib/opentelemetry/tracing'
   3 registerOpentelemetryTracing()
   4
   5 import express from 'express'
   6 import morgan, { token } from 'morgan'
   7 import cors from 'cors'
   8 import cookieParser from 'cookie-parser'
   9 import { frameguard } from 'helmet'
  10 import { parse } from 'useragent'
  11 import anonymize from 'ip-anonymize'
  12 import { program as cli } from 'commander'
  13
  14 process.title = 'peertube'
  15
  16 // Create our main app
  17 const app = express().disable('x-powered-by')
  18
  19 // ----------- Core checker -----------
  20 import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initializers/checker-before-init'
  21
  22 // Do not use barrels because we don't want to load all modules here (we need to initialize database first)
  23 import { CONFIG } from './server/initializers/config'
  24 import { API_VERSION, FILES_CACHE, WEBSERVER, loadLanguages } from './server/initializers/constants'
  25 import { logger } from './server/helpers/logger'
  26
  27 const missed = checkMissedConfig()
  28 if (missed.length !== 0) {
  29   logger.error('Your configuration files miss keys: ' + missed)
  30   process.exit(-1)
  31 }
  32
  33 checkFFmpeg(CONFIG)
  34   .catch(err => {
  35     logger.error('Error in ffmpeg check.', { err })
  36     process.exit(-1)
  37   })
  38
  39 try {
  40   checkNodeVersion()
  41 } catch (err) {
  42   logger.error('Error in NodeJS check.', { err })
  43   process.exit(-1)
  44 }
  45
  46 import { checkConfig, checkActivityPubUrls, checkFFmpegVersion } from './server/initializers/checker-after-init'
  47
  48 try {
  49   checkConfig()
  50 } catch (err) {
  51   logger.error('Config error.', { err })
  52   process.exit(-1)
  53 }
  54
  55 // Trust our proxy (IP forwarding...)
  56 app.set('trust proxy', CONFIG.TRUST_PROXY)
  57
  58 app.use((_req, res, next) => {
  59   // OpenTelemetry
  60   res.locals.requestStart = Date.now()
  61
  62   if (CONFIG.SECURITY.POWERED_BY_HEADER.ENABLED === true) {
  63     res.setHeader('x-powered-by', 'PeerTube')
  64   }
  65
  66   return next()
  67 })
  68
  69 // Security middleware
  70 import { baseCSP } from './server/middlewares/csp'
  71
  72 if (CONFIG.CSP.ENABLED) {
  73   app.use(baseCSP)
  74 }
  75
  76 if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) {
  77   app.use(frameguard({
  78     action: 'deny' // we only allow it for /videos/embed, see server/controllers/client.ts
  79   }))
  80 }
  81
  82 // ----------- Database -----------
  83
  84 // Initialize database and models
  85 import { initDatabaseModels, checkDatabaseConnectionOrDie } from './server/initializers/database'
  86 checkDatabaseConnectionOrDie()
  87
  88 import { migrate } from './server/initializers/migrator'
  89 migrate()
  90   .then(() => initDatabaseModels(false))
  91   .then(() => startApplication())
  92   .catch(err => {
  93     logger.error('Cannot start application.', { err })
  94     process.exit(-1)
  95   })
  96
  97 // ----------- Initialize -----------
  98 loadLanguages()
  99
 100 // ----------- PeerTube modules -----------
 101 import { installApplication } from './server/initializers/installer'
 102 import { Emailer } from './server/lib/emailer'
 103 import { JobQueue } from './server/lib/job-queue'
 104 import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
 105 import {
 106   activityPubRouter,
 107   apiRouter,
 108   miscRouter,
 109   clientsRouter,
 110   feedsRouter,
 111   staticRouter,
 112   wellKnownRouter,
 113   lazyStaticRouter,
 114   servicesRouter,
 115   objectStorageProxyRouter,
 116   pluginsRouter,
 117   trackerRouter,
 118   createWebsocketTrackerServer,
 119   botsRouter,
 120   downloadRouter
 121 } from './server/controllers'
 122 import { advertiseDoNotTrack } from './server/middlewares/dnt'
 123 import { apiFailMiddleware } from './server/middlewares/error'
 124 import { Redis } from './server/lib/redis'
 125 import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler'
 126 import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler'
 127 import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
 128 import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
 129 import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
 130 import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
 131 import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
 132 import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
 133 import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
 134 import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
 135 import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'
 136 import { RunnerJobWatchDogScheduler } from './server/lib/schedulers/runner-job-watch-dog-scheduler'
 137 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
 138 import { PeerTubeSocket } from './server/lib/peertube-socket'
 139 import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
 140 import { PluginsCheckScheduler } from './server/lib/schedulers/plugins-check-scheduler'
 141 import { PeerTubeVersionCheckScheduler } from './server/lib/schedulers/peertube-version-check-scheduler'
 142 import { Hooks } from './server/lib/plugins/hooks'
 143 import { PluginManager } from './server/lib/plugins/plugin-manager'
 144 import { LiveManager } from './server/lib/live'
 145 import { HttpStatusCode } from './shared/models/http/http-error-codes'
 146 import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
 147 import { ServerConfigManager } from '@server/lib/server-config-manager'
 148 import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 149 import { isTestOrDevInstance } from './server/helpers/core-utils'
 150 import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
 151 import { ApplicationModel } from '@server/models/application/application'
 152 import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
 153
 154 // ----------- Command line -----------
 155
 156 cli
 157   .option('--no-client', 'Start PeerTube without client interface')
 158   .option('--no-plugins', 'Start PeerTube without plugins/themes enabled')
 159   .option('--benchmark-startup', 'Automatically stop server when initialized')
 160   .parse(process.argv)
 161
 162 // ----------- App -----------
 163
 164 // Enable CORS for develop
 165 if (isTestOrDevInstance()) {
 166   app.use(cors({
 167     origin: '*',
 168     exposedHeaders: 'Retry-After',
 169     credentials: true
 170   }))
 171 }
 172
 173 // For the logger
 174 token('remote-addr', (req: express.Request) => {
 175   if (CONFIG.LOG.ANONYMIZE_IP === true || req.get('DNT') === '1') {
 176     return anonymize(req.ip, 16, 16)
 177   }
 178
 179   return req.ip
 180 })
 181 token('user-agent', (req: express.Request) => {
 182   if (req.get('DNT') === '1') {
 183     return parse(req.get('user-agent')).family
 184   }
 185
 186   return req.get('user-agent')
 187 })
 188 app.use(morgan('combined', {
 189   stream: {
 190     write: (str: string) => logger.info(str.trim(), { tags: [ 'http' ] })
 191   },
 192   skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping'
 193 }))
 194
 195 // Add .fail() helper to response
 196 app.use(apiFailMiddleware)
 197
 198 // For body requests
 199 app.use(express.urlencoded({ extended: false }))
 200 app.use(express.json({
 201   type: [ 'application/json', 'application/*+json' ],
 202   limit: '500kb',
 203   verify: (req: express.Request, res: express.Response, buf: Buffer) => {
 204     const valid = isHTTPSignatureDigestValid(buf, req)
 205
 206     if (valid !== true) {
 207       res.fail({
 208         status: HttpStatusCode.FORBIDDEN_403,
 209         message: 'Invalid digest'
 210       })
 211     }
 212   }
 213 }))
 214
 215 // Cookies
 216 app.use(cookieParser())
 217
 218 // W3C DNT Tracking Status
 219 app.use(advertiseDoNotTrack)
 220
 221 // ----------- Open Telemetry -----------
 222
 223 OpenTelemetryMetrics.Instance.init(app)
 224
 225 // ----------- Views, routes and static files -----------
 226
 227 // API
 228 const apiRoute = '/api/' + API_VERSION
 229 app.use(apiRoute, apiRouter)
 230
 231 // Services (oembed...)
 232 app.use('/services', servicesRouter)
 233
 234 // Plugins & themes
 235 app.use('/', pluginsRouter)
 236
 237 app.use('/', activityPubRouter)
 238 app.use('/', feedsRouter)
 239 app.use('/', trackerRouter)
 240 app.use('/', botsRouter)
 241
 242 // Static files
 243 app.use('/', staticRouter)
 244 app.use('/', wellKnownRouter)
 245 app.use('/', miscRouter)
 246 app.use('/', downloadRouter)
 247 app.use('/', lazyStaticRouter)
 248 app.use('/', objectStorageProxyRouter)
 249
 250 // Client files, last valid routes!
 251 const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>()
 252 if (cliOptions.client) app.use('/', clientsRouter)
 253
 254 // ----------- Errors -----------
 255
 256 // Catch unmatched routes
 257 app.use((_req, res: express.Response) => {
 258   res.status(HttpStatusCode.NOT_FOUND_404).end()
 259 })
 260
 261 // Catch thrown errors
 262 app.use((err, _req, res: express.Response, _next) => {
 263   // Format error to be logged
 264   let error = 'Unknown error.'
 265   if (err) {
 266     error = err.stack || err.message || err
 267   }
 268
 269   // Handling Sequelize error traces
 270   const sql = err?.parent ? err.parent.sql : undefined
 271
 272   // Help us to debug SequelizeConnectionAcquireTimeoutError errors
 273   const activeRequests = err?.name === 'SequelizeConnectionAcquireTimeoutError' && typeof (process as any)._getActiveRequests !== 'function'
 274     ? (process as any)._getActiveRequests()
 275     : undefined
 276
 277   logger.error('Error in controller.', { err: error, sql, activeRequests })
 278
 279   return res.fail({
 280     status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500,
 281     message: err.message,
 282     type: err.name
 283   })
 284 })
 285
 286 const { server, trackerServer } = createWebsocketTrackerServer(app)
 287
 288 // ----------- Run -----------
 289
 290 async function startApplication () {
 291   const port = CONFIG.LISTEN.PORT
 292   const hostname = CONFIG.LISTEN.HOSTNAME
 293
 294   await installApplication()
 295
 296   // Check activity pub urls are valid
 297   checkActivityPubUrls()
 298     .catch(err => {
 299       logger.error('Error in ActivityPub URLs checker.', { err })
 300       process.exit(-1)
 301     })
 302
 303   checkFFmpegVersion()
 304     .catch(err => logger.error('Cannot check ffmpeg version', { err }))
 305
 306   Redis.Instance.init()
 307   Emailer.Instance.init()
 308
 309   await Promise.all([
 310     Emailer.Instance.checkConnection(),
 311     JobQueue.Instance.init(),
 312     ServerConfigManager.Instance.init()
 313   ])
 314
 315   // Caches initializations
 316   VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
 317   VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
 318   VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
 319
 320   // Enable Schedulers
 321   ActorFollowScheduler.Instance.enable()
 322   RemoveOldJobsScheduler.Instance.enable()
 323   UpdateVideosScheduler.Instance.enable()
 324   YoutubeDlUpdateScheduler.Instance.enable()
 325   VideosRedundancyScheduler.Instance.enable()
 326   RemoveOldHistoryScheduler.Instance.enable()
 327   RemoveOldViewsScheduler.Instance.enable()
 328   PluginsCheckScheduler.Instance.enable()
 329   PeerTubeVersionCheckScheduler.Instance.enable()
 330   AutoFollowIndexInstances.Instance.enable()
 331   RemoveDanglingResumableUploadsScheduler.Instance.enable()
 332   VideoChannelSyncLatestScheduler.Instance.enable()
 333   VideoViewsBufferScheduler.Instance.enable()
 334   GeoIPUpdateScheduler.Instance.enable()
 335   RunnerJobWatchDogScheduler.Instance.enable()
 336
 337   OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })
 338
 339   PluginManager.Instance.init(server)
 340   // Before PeerTubeSocket init
 341   PluginManager.Instance.registerWebSocketRouter()
 342
 343   PeerTubeSocket.Instance.init(server)
 344   VideoViewsManager.Instance.init()
 345
 346   updateStreamingPlaylistsInfohashesIfNeeded()
 347     .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
 348
 349   LiveManager.Instance.init()
 350   if (CONFIG.LIVE.ENABLED) await LiveManager.Instance.run()
 351
 352   // Make server listening
 353   server.listen(port, hostname, async () => {
 354     if (cliOptions.plugins) {
 355       try {
 356         await PluginManager.Instance.rebuildNativePluginsIfNeeded()
 357
 358         await PluginManager.Instance.registerPluginsAndThemes()
 359       } catch (err) {
 360         logger.error('Cannot register plugins and themes.', { err })
 361       }
 362     }
 363
 364     ApplicationModel.updateNodeVersions()
 365       .catch(err => logger.error('Cannot update node versions.', { err }))
 366
 367     JobQueue.Instance.start()
 368       .catch(err => {
 369         logger.error('Cannot start job queue.', { err })
 370         process.exit(-1)
 371       })
 372
 373     logger.info('HTTP server listening on %s:%d', hostname, port)
 374     logger.info('Web server: %s', WEBSERVER.URL)
 375
 376     Hooks.runAction('action:application.listening')
 377
 378     if (cliOptions['benchmarkStartup']) process.exit(0)
 379   })
 380
 381   process.on('exit', () => {
 382     JobQueue.Instance.terminate()
 383       .catch(err => logger.error('Cannot terminate job queue.', { err }))
 384   })
 385
 386   process.on('SIGINT', () => process.exit(0))
 387 }