server/lib/live/live-manager.ts

   1 import { readdir, readFile } from 'fs-extra'
   2 import { createServer, Server } from 'net'
   3 import { join } from 'path'
   4 import { createServer as createServerTLS, Server as ServerTLS } from 'tls'
   5 import {
   6   computeResolutionsToTranscode,
   7   ffprobePromise,
   8   getLiveSegmentTime,
   9   getVideoStreamBitrate,
  10   getVideoStreamDimensionsInfo,
  11   getVideoStreamFPS,
  12   hasAudioStream
  13 } from '@server/helpers/ffmpeg'
  14 import { logger, loggerTagsFactory } from '@server/helpers/logger'
  15 import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
  16 import { VIDEO_LIVE } from '@server/initializers/constants'
  17 import { UserModel } from '@server/models/user/user'
  18 import { VideoModel } from '@server/models/video/video'
  19 import { VideoLiveModel } from '@server/models/video/video-live'
  20 import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
  21 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
  22 import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models'
  23 import { pick, wait } from '@shared/core-utils'
  24 import { LiveVideoError, VideoState } from '@shared/models'
  25 import { federateVideoIfNeeded } from '../activitypub/videos'
  26 import { JobQueue } from '../job-queue'
  27 import { getLiveReplayBaseDirectory } from '../paths'
  28 import { PeerTubeSocket } from '../peertube-socket'
  29 import { Hooks } from '../plugins/hooks'
  30 import { LiveQuotaStore } from './live-quota-store'
  31 import { cleanupAndDestroyPermanentLive } from './live-utils'
  32 import { MuxingSession } from './shared'
  33 import { sequelizeTypescript } from '@server/initializers/database'
  34 import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
  35
  36 const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
  37 const context = require('node-media-server/src/node_core_ctx')
  38 const nodeMediaServerLogger = require('node-media-server/src/node_core_logger')
  39
  40 // Disable node media server logs
  41 nodeMediaServerLogger.setLogType(0)
  42
  43 const config = {
  44   rtmp: {
  45     port: CONFIG.LIVE.RTMP.PORT,
  46     chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE,
  47     gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE,
  48     ping: VIDEO_LIVE.RTMP.PING,
  49     ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT
  50   }
  51 }
  52
  53 const lTags = loggerTagsFactory('live')
  54
  55 class LiveManager {
  56
  57   private static instance: LiveManager
  58
  59   private readonly muxingSessions = new Map<string, MuxingSession>()
  60   private readonly videoSessions = new Map<number, string>()
  61
  62   private rtmpServer: Server
  63   private rtmpsServer: ServerTLS
  64
  65   private running = false
  66
  67   private constructor () {
  68   }
  69
  70   init () {
  71     const events = this.getContext().nodeEvent
  72     events.on('postPublish', (sessionId: string, streamPath: string) => {
  73       logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) })
  74
  75       const splittedPath = streamPath.split('/')
  76       if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) {
  77         logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) })
  78         return this.abortSession(sessionId)
  79       }
  80
  81       const session = this.getContext().sessions.get(sessionId)
  82
  83       this.handleSession(sessionId, session.inputOriginUrl + streamPath, splittedPath[2])
  84         .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) }))
  85     })
  86
  87     events.on('donePublish', sessionId => {
  88       logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })
  89     })
  90
  91     registerConfigChangedHandler(() => {
  92       if (!this.running && CONFIG.LIVE.ENABLED === true) {
  93         this.run().catch(err => logger.error('Cannot run live server.', { err }))
  94         return
  95       }
  96
  97       if (this.running && CONFIG.LIVE.ENABLED === false) {
  98         this.stop()
  99       }
 100     })
 101
 102     // Cleanup broken lives, that were terminated by a server restart for example
 103     this.handleBrokenLives()
 104       .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() }))
 105   }
 106
 107   async run () {
 108     this.running = true
 109
 110     if (CONFIG.LIVE.RTMP.ENABLED) {
 111       logger.info('Running RTMP server on port %d', CONFIG.LIVE.RTMP.PORT, lTags())
 112
 113       this.rtmpServer = createServer(socket => {
 114         const session = new NodeRtmpSession(config, socket)
 115
 116         session.inputOriginUrl = 'rtmp://127.0.0.1:' + CONFIG.LIVE.RTMP.PORT
 117         session.run()
 118       })
 119
 120       this.rtmpServer.on('error', err => {
 121         logger.error('Cannot run RTMP server.', { err, ...lTags() })
 122       })
 123
 124       this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT, CONFIG.LIVE.RTMP.HOSTNAME)
 125     }
 126
 127     if (CONFIG.LIVE.RTMPS.ENABLED) {
 128       logger.info('Running RTMPS server on port %d', CONFIG.LIVE.RTMPS.PORT, lTags())
 129
 130       const [ key, cert ] = await Promise.all([
 131         readFile(CONFIG.LIVE.RTMPS.KEY_FILE),
 132         readFile(CONFIG.LIVE.RTMPS.CERT_FILE)
 133       ])
 134       const serverOptions = { key, cert }
 135
 136       this.rtmpsServer = createServerTLS(serverOptions, socket => {
 137         const session = new NodeRtmpSession(config, socket)
 138
 139         session.inputOriginUrl = 'rtmps://127.0.0.1:' + CONFIG.LIVE.RTMPS.PORT
 140         session.run()
 141       })
 142
 143       this.rtmpsServer.on('error', err => {
 144         logger.error('Cannot run RTMPS server.', { err, ...lTags() })
 145       })
 146
 147       this.rtmpsServer.listen(CONFIG.LIVE.RTMPS.PORT, CONFIG.LIVE.RTMPS.HOSTNAME)
 148     }
 149   }
 150
 151   stop () {
 152     this.running = false
 153
 154     if (this.rtmpServer) {
 155       logger.info('Stopping RTMP server.', lTags())
 156
 157       this.rtmpServer.close()
 158       this.rtmpServer = undefined
 159     }
 160
 161     if (this.rtmpsServer) {
 162       logger.info('Stopping RTMPS server.', lTags())
 163
 164       this.rtmpsServer.close()
 165       this.rtmpsServer = undefined
 166     }
 167
 168     // Sessions is an object
 169     this.getContext().sessions.forEach((session: any) => {
 170       if (session instanceof NodeRtmpSession) {
 171         session.stop()
 172       }
 173     })
 174   }
 175
 176   isRunning () {
 177     return !!this.rtmpServer
 178   }
 179
 180   stopSessionOf (videoId: number, error: LiveVideoError | null) {
 181     const sessionId = this.videoSessions.get(videoId)
 182     if (!sessionId) return
 183
 184     this.saveEndingSession(videoId, error)
 185       .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
 186
 187     this.videoSessions.delete(videoId)
 188     this.abortSession(sessionId)
 189   }
 190
 191   private getContext () {
 192     return context
 193   }
 194
 195   private abortSession (sessionId: string) {
 196     const session = this.getContext().sessions.get(sessionId)
 197     if (session) {
 198       session.stop()
 199       this.getContext().sessions.delete(sessionId)
 200     }
 201
 202     const muxingSession = this.muxingSessions.get(sessionId)
 203     if (muxingSession) {
 204       // Muxing session will fire and event so we correctly cleanup the session
 205       muxingSession.abort()
 206
 207       this.muxingSessions.delete(sessionId)
 208     }
 209   }
 210
 211   private async handleSession (sessionId: string, inputUrl: string, streamKey: string) {
 212     const videoLive = await VideoLiveModel.loadByStreamKey(streamKey)
 213     if (!videoLive) {
 214       logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId))
 215       return this.abortSession(sessionId)
 216     }
 217
 218     const video = videoLive.Video
 219     if (video.isBlacklisted()) {
 220       logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid))
 221       return this.abortSession(sessionId)
 222     }
 223
 224     // Cleanup old potential live (could happen with a permanent live)
 225     const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
 226     if (oldStreamingPlaylist) {
 227       if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid)
 228
 229       await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist)
 230     }
 231
 232     this.videoSessions.set(video.id, sessionId)
 233
 234     const now = Date.now()
 235     const probe = await ffprobePromise(inputUrl)
 236
 237     const [ { resolution, ratio }, fps, bitrate, hasAudio ] = await Promise.all([
 238       getVideoStreamDimensionsInfo(inputUrl, probe),
 239       getVideoStreamFPS(inputUrl, probe),
 240       getVideoStreamBitrate(inputUrl, probe),
 241       hasAudioStream(inputUrl, probe)
 242     ])
 243
 244     logger.info(
 245       '%s probing took %d ms (bitrate: %d, fps: %d, resolution: %d)',
 246       inputUrl, Date.now() - now, bitrate, fps, resolution, lTags(sessionId, video.uuid)
 247     )
 248
 249     const allResolutions = await Hooks.wrapObject(
 250       this.buildAllResolutionsToTranscode(resolution, hasAudio),
 251       'filter:transcoding.auto.resolutions-to-transcode.result',
 252       { video }
 253     )
 254
 255     logger.info(
 256       'Will mux/transcode live video of original resolution %d.', resolution,
 257       { allResolutions, ...lTags(sessionId, video.uuid) }
 258     )
 259
 260     return this.runMuxingSession({
 261       sessionId,
 262       videoLive,
 263
 264       inputUrl,
 265       fps,
 266       bitrate,
 267       ratio,
 268       allResolutions,
 269       hasAudio
 270     })
 271   }
 272
 273   private async runMuxingSession (options: {
 274     sessionId: string
 275     videoLive: MVideoLiveVideoWithSetting
 276
 277     inputUrl: string
 278     fps: number
 279     bitrate: number
 280     ratio: number
 281     allResolutions: number[]
 282     hasAudio: boolean
 283   }) {
 284     const { sessionId, videoLive } = options
 285     const videoUUID = videoLive.Video.uuid
 286     const localLTags = lTags(sessionId, videoUUID)
 287
 288     const liveSession = await this.saveStartingSession(videoLive)
 289
 290     const user = await UserModel.loadByLiveId(videoLive.id)
 291     LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id)
 292
 293     const muxingSession = new MuxingSession({
 294       context: this.getContext(),
 295       sessionId,
 296       videoLive,
 297       user,
 298
 299       ...pick(options, [ 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ])
 300     })
 301
 302     muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags))
 303
 304     muxingSession.on('bad-socket-health', ({ videoId }) => {
 305       logger.error(
 306         'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
 307         ' Stopping session of video %s.', videoUUID,
 308         localLTags
 309       )
 310
 311       this.stopSessionOf(videoId, LiveVideoError.BAD_SOCKET_HEALTH)
 312     })
 313
 314     muxingSession.on('duration-exceeded', ({ videoId }) => {
 315       logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags)
 316
 317       this.stopSessionOf(videoId, LiveVideoError.DURATION_EXCEEDED)
 318     })
 319
 320     muxingSession.on('quota-exceeded', ({ videoId }) => {
 321       logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags)
 322
 323       this.stopSessionOf(videoId, LiveVideoError.QUOTA_EXCEEDED)
 324     })
 325
 326     muxingSession.on('ffmpeg-error', ({ videoId }) => {
 327       this.stopSessionOf(videoId, LiveVideoError.FFMPEG_ERROR)
 328     })
 329
 330     muxingSession.on('ffmpeg-end', ({ videoId }) => {
 331       this.onMuxingFFmpegEnd(videoId, sessionId)
 332     })
 333
 334     muxingSession.on('after-cleanup', ({ videoId }) => {
 335       this.muxingSessions.delete(sessionId)
 336
 337       LiveQuotaStore.Instance.removeLive(user.id, videoLive.id)
 338
 339       muxingSession.destroy()
 340
 341       return this.onAfterMuxingCleanup({ videoId, liveSession })
 342         .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags }))
 343     })
 344
 345     this.muxingSessions.set(sessionId, muxingSession)
 346
 347     muxingSession.runMuxing()
 348       .catch(err => {
 349         logger.error('Cannot run muxing.', { err, ...localLTags })
 350         this.abortSession(sessionId)
 351       })
 352   }
 353
 354   private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) {
 355     const videoId = live.videoId
 356
 357     try {
 358       const video = await VideoModel.loadFull(videoId)
 359
 360       logger.info('Will publish and federate live %s.', video.url, localLTags)
 361
 362       video.state = VideoState.PUBLISHED
 363       video.publishedAt = new Date()
 364       await video.save()
 365
 366       live.Video = video
 367
 368       await wait(getLiveSegmentTime(live.latencyMode) * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION)
 369
 370       try {
 371         await federateVideoIfNeeded(video, false)
 372       } catch (err) {
 373         logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags })
 374       }
 375
 376       PeerTubeSocket.Instance.sendVideoLiveNewState(video)
 377     } catch (err) {
 378       logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags })
 379     }
 380   }
 381
 382   private onMuxingFFmpegEnd (videoId: number, sessionId: string) {
 383     this.videoSessions.delete(videoId)
 384
 385     this.saveEndingSession(videoId, null)
 386       .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
 387   }
 388
 389   private async onAfterMuxingCleanup (options: {
 390     videoId: number | string
 391     liveSession?: MVideoLiveSession
 392     cleanupNow?: boolean // Default false
 393   }) {
 394     const { videoId, liveSession: liveSessionArg, cleanupNow = false } = options
 395
 396     try {
 397       const fullVideo = await VideoModel.loadFull(videoId)
 398       if (!fullVideo) return
 399
 400       const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
 401
 402       const liveSession = liveSessionArg ?? await VideoLiveSessionModel.findLatestSessionOf(fullVideo.id)
 403
 404       // On server restart during a live
 405       if (!liveSession.endDate) {
 406         liveSession.endDate = new Date()
 407         await liveSession.save()
 408       }
 409
 410       JobQueue.Instance.createJobAsync({
 411         type: 'video-live-ending',
 412         payload: {
 413           videoId: fullVideo.id,
 414
 415           replayDirectory: live.saveReplay
 416             ? await this.findReplayDirectory(fullVideo)
 417             : undefined,
 418
 419           liveSessionId: liveSession.id,
 420           streamingPlaylistId: fullVideo.getHLSPlaylist()?.id,
 421
 422           publishedAt: fullVideo.publishedAt.toISOString()
 423         },
 424
 425         delay: cleanupNow
 426           ? 0
 427           : VIDEO_LIVE.CLEANUP_DELAY
 428       })
 429
 430       fullVideo.state = live.permanentLive
 431         ? VideoState.WAITING_FOR_LIVE
 432         : VideoState.LIVE_ENDED
 433
 434       await fullVideo.save()
 435
 436       PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo)
 437
 438       await federateVideoIfNeeded(fullVideo, false)
 439     } catch (err) {
 440       logger.error('Cannot save/federate new video state of live streaming of video %d.', videoId, { err, ...lTags(videoId + '') })
 441     }
 442   }
 443
 444   private async handleBrokenLives () {
 445     const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
 446
 447     for (const uuid of videoUUIDs) {
 448       await this.onAfterMuxingCleanup({ videoId: uuid, cleanupNow: true })
 449     }
 450   }
 451
 452   private async findReplayDirectory (video: MVideo) {
 453     const directory = getLiveReplayBaseDirectory(video)
 454     const files = await readdir(directory)
 455
 456     if (files.length === 0) return undefined
 457
 458     return join(directory, files.sort().reverse()[0])
 459   }
 460
 461   private buildAllResolutionsToTranscode (originResolution: number, hasAudio: boolean) {
 462     const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
 463
 464     const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
 465       ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false, hasAudio })
 466       : []
 467
 468     if (resolutionsEnabled.length === 0) {
 469       return [ originResolution ]
 470     }
 471
 472     return resolutionsEnabled
 473   }
 474
 475   private async saveStartingSession (videoLive: MVideoLiveVideoWithSetting) {
 476     const replaySettings = videoLive.saveReplay
 477       ? new VideoLiveReplaySettingModel({
 478         privacy: videoLive.ReplaySetting.privacy
 479       })
 480       : null
 481
 482     return sequelizeTypescript.transaction(async t => {
 483       if (videoLive.saveReplay) {
 484         await replaySettings.save({ transaction: t })
 485       }
 486
 487       return VideoLiveSessionModel.create({
 488         startDate: new Date(),
 489         liveVideoId: videoLive.videoId,
 490         saveReplay: videoLive.saveReplay,
 491         replaySettingId: videoLive.saveReplay ? replaySettings.id : null,
 492         endingProcessed: false
 493       }, { transaction: t })
 494     })
 495   }
 496
 497   private async saveEndingSession (videoId: number, error: LiveVideoError | null) {
 498     const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoId)
 499     if (!liveSession) return
 500
 501     liveSession.endDate = new Date()
 502     liveSession.error = error
 503
 504     return liveSession.save()
 505   }
 506
 507   static get Instance () {
 508     return this.instance || (this.instance = new this())
 509   }
 510 }
 511
 512 // ---------------------------------------------------------------------------
 513
 514 export {
 515   LiveManager
 516 }