server.ts

   1 // ----------- Node modules -----------
   2 import { registerOpentelemetryTracing } from './server/lib/opentelemetry/tracing'
   3 registerOpentelemetryTracing()
   4
   5 import express from 'express'
   6 import morgan, { token } from 'morgan'
   7 import cors from 'cors'
   8 import cookieParser from 'cookie-parser'
   9 import { frameguard } from 'helmet'
  10 import { parse } from 'useragent'
  11 import anonymize from 'ip-anonymize'
  12 import { program as cli } from 'commander'
  13
  14 process.title = 'peertube'
  15
  16 // Create our main app
  17 const app = express().disable('x-powered-by')
  18
  19 // ----------- Core checker -----------
  20 import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initializers/checker-before-init'
  21
  22 // Do not use barrels because we don't want to load all modules here (we need to initialize database first)
  23 import { CONFIG } from './server/initializers/config'
  24 import { API_VERSION, FILES_CACHE, WEBSERVER, loadLanguages } from './server/initializers/constants'
  25 import { logger } from './server/helpers/logger'
  26
  27 const missed = checkMissedConfig()
  28 if (missed.length !== 0) {
  29   logger.error('Your configuration files miss keys: ' + missed)
  30   process.exit(-1)
  31 }
  32
  33 checkFFmpeg(CONFIG)
  34   .catch(err => {
  35     logger.error('Error in ffmpeg check.', { err })
  36     process.exit(-1)
  37   })
  38
  39 try {
  40   checkNodeVersion()
  41 } catch (err) {
  42   logger.error('Error in NodeJS check.', { err })
  43   process.exit(-1)
  44 }
  45
  46 import { checkConfig, checkActivityPubUrls, checkFFmpegVersion } from './server/initializers/checker-after-init'
  47
  48 try {
  49   checkConfig()
  50 } catch (err) {
  51   logger.error('Config error.', { err })
  52   process.exit(-1)
  53 }
  54
  55 // Trust our proxy (IP forwarding...)
  56 app.set('trust proxy', CONFIG.TRUST_PROXY)
  57
  58 app.use((_req, res, next) => {
  59   res.locals.requestStart = Date.now()
  60
  61   return next()
  62 })
  63
  64 // Security middleware
  65 import { baseCSP } from './server/middlewares/csp'
  66
  67 if (CONFIG.CSP.ENABLED) {
  68   app.use(baseCSP)
  69 }
  70
  71 if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) {
  72   app.use(frameguard({
  73     action: 'deny' // we only allow it for /videos/embed, see server/controllers/client.ts
  74   }))
  75 }
  76
  77 // ----------- Database -----------
  78
  79 // Initialize database and models
  80 import { initDatabaseModels, checkDatabaseConnectionOrDie } from './server/initializers/database'
  81 checkDatabaseConnectionOrDie()
  82
  83 import { migrate } from './server/initializers/migrator'
  84 migrate()
  85   .then(() => initDatabaseModels(false))
  86   .then(() => startApplication())
  87   .catch(err => {
  88     logger.error('Cannot start application.', { err })
  89     process.exit(-1)
  90   })
  91
  92 // ----------- Initialize -----------
  93 loadLanguages()
  94
  95 // ----------- PeerTube modules -----------
  96 import { installApplication } from './server/initializers/installer'
  97 import { Emailer } from './server/lib/emailer'
  98 import { JobQueue } from './server/lib/job-queue'
  99 import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
 100 import {
 101   activityPubRouter,
 102   apiRouter,
 103   miscRouter,
 104   clientsRouter,
 105   feedsRouter,
 106   staticRouter,
 107   wellKnownRouter,
 108   lazyStaticRouter,
 109   servicesRouter,
 110   objectStorageProxyRouter,
 111   pluginsRouter,
 112   webfingerRouter,
 113   trackerRouter,
 114   createWebsocketTrackerServer,
 115   botsRouter,
 116   downloadRouter
 117 } from './server/controllers'
 118 import { advertiseDoNotTrack } from './server/middlewares/dnt'
 119 import { apiFailMiddleware } from './server/middlewares/error'
 120 import { Redis } from './server/lib/redis'
 121 import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler'
 122 import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler'
 123 import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
 124 import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
 125 import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
 126 import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
 127 import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
 128 import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
 129 import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
 130 import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
 131 import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'
 132 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
 133 import { PeerTubeSocket } from './server/lib/peertube-socket'
 134 import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
 135 import { PluginsCheckScheduler } from './server/lib/schedulers/plugins-check-scheduler'
 136 import { PeerTubeVersionCheckScheduler } from './server/lib/schedulers/peertube-version-check-scheduler'
 137 import { Hooks } from './server/lib/plugins/hooks'
 138 import { PluginManager } from './server/lib/plugins/plugin-manager'
 139 import { LiveManager } from './server/lib/live'
 140 import { HttpStatusCode } from './shared/models/http/http-error-codes'
 141 import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
 142 import { ServerConfigManager } from '@server/lib/server-config-manager'
 143 import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 144 import { isTestOrDevInstance } from './server/helpers/core-utils'
 145 import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
 146 import { ApplicationModel } from '@server/models/application/application'
 147 import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
 148
 149 // ----------- Command line -----------
 150
 151 cli
 152   .option('--no-client', 'Start PeerTube without client interface')
 153   .option('--no-plugins', 'Start PeerTube without plugins/themes enabled')
 154   .option('--benchmark-startup', 'Automatically stop server when initialized')
 155   .parse(process.argv)
 156
 157 // ----------- App -----------
 158
 159 // Enable CORS for develop
 160 if (isTestOrDevInstance()) {
 161   app.use(cors({
 162     origin: '*',
 163     exposedHeaders: 'Retry-After',
 164     credentials: true
 165   }))
 166 }
 167
 168 // For the logger
 169 token('remote-addr', (req: express.Request) => {
 170   if (CONFIG.LOG.ANONYMIZE_IP === true || req.get('DNT') === '1') {
 171     return anonymize(req.ip, 16, 16)
 172   }
 173
 174   return req.ip
 175 })
 176 token('user-agent', (req: express.Request) => {
 177   if (req.get('DNT') === '1') {
 178     return parse(req.get('user-agent')).family
 179   }
 180
 181   return req.get('user-agent')
 182 })
 183 app.use(morgan('combined', {
 184   stream: {
 185     write: (str: string) => logger.info(str.trim(), { tags: [ 'http' ] })
 186   },
 187   skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping'
 188 }))
 189
 190 // Add .fail() helper to response
 191 app.use(apiFailMiddleware)
 192
 193 // For body requests
 194 app.use(express.urlencoded({ extended: false }))
 195 app.use(express.json({
 196   type: [ 'application/json', 'application/*+json' ],
 197   limit: '500kb',
 198   verify: (req: express.Request, res: express.Response, buf: Buffer) => {
 199     const valid = isHTTPSignatureDigestValid(buf, req)
 200
 201     if (valid !== true) {
 202       res.fail({
 203         status: HttpStatusCode.FORBIDDEN_403,
 204         message: 'Invalid digest'
 205       })
 206     }
 207   }
 208 }))
 209
 210 // Cookies
 211 app.use(cookieParser())
 212
 213 // W3C DNT Tracking Status
 214 app.use(advertiseDoNotTrack)
 215
 216 // ----------- Open Telemetry -----------
 217
 218 OpenTelemetryMetrics.Instance.init(app)
 219
 220 // ----------- Views, routes and static files -----------
 221
 222 // API
 223 const apiRoute = '/api/' + API_VERSION
 224 app.use(apiRoute, apiRouter)
 225
 226 // Services (oembed...)
 227 app.use('/services', servicesRouter)
 228
 229 // Plugins & themes
 230 app.use('/', pluginsRouter)
 231
 232 app.use('/', activityPubRouter)
 233 app.use('/', feedsRouter)
 234 app.use('/', webfingerRouter)
 235 app.use('/', trackerRouter)
 236 app.use('/', botsRouter)
 237
 238 // Static files
 239 app.use('/', staticRouter)
 240 app.use('/', wellKnownRouter)
 241 app.use('/', miscRouter)
 242 app.use('/', downloadRouter)
 243 app.use('/', lazyStaticRouter)
 244 app.use('/', objectStorageProxyRouter)
 245
 246 // Client files, last valid routes!
 247 const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>()
 248 if (cliOptions.client) app.use('/', clientsRouter)
 249
 250 // ----------- Errors -----------
 251
 252 // Catch unmatched routes
 253 app.use((_req, res: express.Response) => {
 254   res.status(HttpStatusCode.NOT_FOUND_404).end()
 255 })
 256
 257 // Catch thrown errors
 258 app.use((err, _req, res: express.Response, _next) => {
 259   // Format error to be logged
 260   let error = 'Unknown error.'
 261   if (err) {
 262     error = err.stack || err.message || err
 263   }
 264
 265   // Handling Sequelize error traces
 266   const sql = err?.parent ? err.parent.sql : undefined
 267
 268   // Help us to debug SequelizeConnectionAcquireTimeoutError errors
 269   const activeRequests = err?.name === 'SequelizeConnectionAcquireTimeoutError' && typeof (process as any)._getActiveRequests !== 'function'
 270     ? (process as any)._getActiveRequests()
 271     : undefined
 272
 273   logger.error('Error in controller.', { err: error, sql, activeRequests })
 274
 275   return res.fail({
 276     status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500,
 277     message: err.message,
 278     type: err.name
 279   })
 280 })
 281
 282 const { server, trackerServer } = createWebsocketTrackerServer(app)
 283
 284 // ----------- Run -----------
 285
 286 async function startApplication () {
 287   const port = CONFIG.LISTEN.PORT
 288   const hostname = CONFIG.LISTEN.HOSTNAME
 289
 290   await installApplication()
 291
 292   // Check activity pub urls are valid
 293   checkActivityPubUrls()
 294     .catch(err => {
 295       logger.error('Error in ActivityPub URLs checker.', { err })
 296       process.exit(-1)
 297     })
 298
 299   checkFFmpegVersion()
 300     .catch(err => logger.error('Cannot check ffmpeg version', { err }))
 301
 302   Redis.Instance.init()
 303   Emailer.Instance.init()
 304
 305   await Promise.all([
 306     Emailer.Instance.checkConnection(),
 307     JobQueue.Instance.init(),
 308     ServerConfigManager.Instance.init()
 309   ])
 310
 311   // Caches initializations
 312   VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
 313   VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
 314   VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
 315
 316   // Enable Schedulers
 317   ActorFollowScheduler.Instance.enable()
 318   RemoveOldJobsScheduler.Instance.enable()
 319   UpdateVideosScheduler.Instance.enable()
 320   YoutubeDlUpdateScheduler.Instance.enable()
 321   VideosRedundancyScheduler.Instance.enable()
 322   RemoveOldHistoryScheduler.Instance.enable()
 323   RemoveOldViewsScheduler.Instance.enable()
 324   PluginsCheckScheduler.Instance.enable()
 325   PeerTubeVersionCheckScheduler.Instance.enable()
 326   AutoFollowIndexInstances.Instance.enable()
 327   RemoveDanglingResumableUploadsScheduler.Instance.enable()
 328   VideoChannelSyncLatestScheduler.Instance.enable()
 329   VideoViewsBufferScheduler.Instance.enable()
 330   GeoIPUpdateScheduler.Instance.enable()
 331
 332   OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })
 333
 334   PluginManager.Instance.init(server)
 335   // Before PeerTubeSocket init
 336   PluginManager.Instance.registerWebSocketRouter()
 337
 338   PeerTubeSocket.Instance.init(server)
 339   VideoViewsManager.Instance.init()
 340
 341   updateStreamingPlaylistsInfohashesIfNeeded()
 342     .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
 343
 344   LiveManager.Instance.init()
 345   if (CONFIG.LIVE.ENABLED) await LiveManager.Instance.run()
 346
 347   // Make server listening
 348   server.listen(port, hostname, async () => {
 349     if (cliOptions.plugins) {
 350       try {
 351         await PluginManager.Instance.rebuildNativePluginsIfNeeded()
 352
 353         await PluginManager.Instance.registerPluginsAndThemes()
 354       } catch (err) {
 355         logger.error('Cannot register plugins and themes.', { err })
 356       }
 357     }
 358
 359     ApplicationModel.updateNodeVersions()
 360       .catch(err => logger.error('Cannot update node versions.', { err }))
 361
 362     JobQueue.Instance.start()
 363       .catch(err => {
 364         logger.error('Cannot start job queue.', { err })
 365         process.exit(-1)
 366       })
 367
 368     logger.info('HTTP server listening on %s:%d', hostname, port)
 369     logger.info('Web server: %s', WEBSERVER.URL)
 370
 371     Hooks.runAction('action:application.listening')
 372
 373     if (cliOptions['benchmarkStartup']) process.exit(0)
 374   })
 375
 376   process.on('exit', () => {
 377     JobQueue.Instance.terminate()
 378       .catch(err => logger.error('Cannot terminate job queue.', { err }))
 379   })
 380
 381   process.on('SIGINT', () => process.exit(0))
 382 }