1 import { readdir, readFile } from 'fs-extra'
2 import { createServer, Server } from 'net'
3 import { join } from 'path'
4 import { createServer as createServerTLS, Server as ServerTLS } from 'tls'
6 computeResolutionsToTranscode,
10 getVideoStreamDimensionsInfo,
13 } from '@server/helpers/ffmpeg'
14 import { logger, loggerTagsFactory } from '@server/helpers/logger'
15 import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
16 import { VIDEO_LIVE } from '@server/initializers/constants'
17 import { UserModel } from '@server/models/user/user'
18 import { VideoModel } from '@server/models/video/video'
19 import { VideoLiveModel } from '@server/models/video/video-live'
20 import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
21 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
22 import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models'
23 import { pick, wait } from '@shared/core-utils'
24 import { LiveVideoError, VideoState } from '@shared/models'
25 import { federateVideoIfNeeded } from '../activitypub/videos'
26 import { JobQueue } from '../job-queue'
27 import { getLiveReplayBaseDirectory } from '../paths'
28 import { PeerTubeSocket } from '../peertube-socket'
29 import { Hooks } from '../plugins/hooks'
30 import { LiveQuotaStore } from './live-quota-store'
31 import { cleanupAndDestroyPermanentLive } from './live-utils'
32 import { MuxingSession } from './shared'
33 import { sequelizeTypescript } from '@server/initializers/database'
34 import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
36 const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
37 const context = require('node-media-server/src/node_core_ctx')
38 const nodeMediaServerLogger = require('node-media-server/src/node_core_logger')
40 // Disable node media server logs
41 nodeMediaServerLogger.setLogType(0)
45 port: CONFIG.LIVE.RTMP.PORT,
46 chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE,
47 gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE,
48 ping: VIDEO_LIVE.RTMP.PING,
49 ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT
53 const lTags = loggerTagsFactory('live')
57 private static instance: LiveManager
59 private readonly muxingSessions = new Map<string, MuxingSession>()
60 private readonly videoSessions = new Map<number, string>()
62 private rtmpServer: Server
63 private rtmpsServer: ServerTLS
65 private running = false
67 private constructor () {
71 const events = this.getContext().nodeEvent
72 events.on('postPublish', (sessionId: string, streamPath: string) => {
73 logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) })
75 const splittedPath = streamPath.split('/')
76 if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) {
77 logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) })
78 return this.abortSession(sessionId)
81 const session = this.getContext().sessions.get(sessionId)
83 this.handleSession(sessionId, session.inputOriginUrl + streamPath, splittedPath[2])
84 .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) }))
87 events.on('donePublish', sessionId => {
88 logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })
91 registerConfigChangedHandler(() => {
92 if (!this.running && CONFIG.LIVE.ENABLED === true) {
93 this.run().catch(err => logger.error('Cannot run live server.', { err }))
97 if (this.running && CONFIG.LIVE.ENABLED === false) {
102 // Cleanup broken lives, that were terminated by a server restart for example
103 this.handleBrokenLives()
104 .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() }))
110 if (CONFIG.LIVE.RTMP.ENABLED) {
111 logger.info('Running RTMP server on port %d', CONFIG.LIVE.RTMP.PORT, lTags())
113 this.rtmpServer = createServer(socket => {
114 const session = new NodeRtmpSession(config, socket)
116 session.inputOriginUrl = 'rtmp://127.0.0.1:' + CONFIG.LIVE.RTMP.PORT
120 this.rtmpServer.on('error', err => {
121 logger.error('Cannot run RTMP server.', { err, ...lTags() })
124 this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT, CONFIG.LIVE.RTMP.HOSTNAME)
127 if (CONFIG.LIVE.RTMPS.ENABLED) {
128 logger.info('Running RTMPS server on port %d', CONFIG.LIVE.RTMPS.PORT, lTags())
130 const [ key, cert ] = await Promise.all([
131 readFile(CONFIG.LIVE.RTMPS.KEY_FILE),
132 readFile(CONFIG.LIVE.RTMPS.CERT_FILE)
134 const serverOptions = { key, cert }
136 this.rtmpsServer = createServerTLS(serverOptions, socket => {
137 const session = new NodeRtmpSession(config, socket)
139 session.inputOriginUrl = 'rtmps://127.0.0.1:' + CONFIG.LIVE.RTMPS.PORT
143 this.rtmpsServer.on('error', err => {
144 logger.error('Cannot run RTMPS server.', { err, ...lTags() })
147 this.rtmpsServer.listen(CONFIG.LIVE.RTMPS.PORT, CONFIG.LIVE.RTMPS.HOSTNAME)
154 if (this.rtmpServer) {
155 logger.info('Stopping RTMP server.', lTags())
157 this.rtmpServer.close()
158 this.rtmpServer = undefined
161 if (this.rtmpsServer) {
162 logger.info('Stopping RTMPS server.', lTags())
164 this.rtmpsServer.close()
165 this.rtmpsServer = undefined
168 // Sessions is an object
169 this.getContext().sessions.forEach((session: any) => {
170 if (session instanceof NodeRtmpSession) {
177 return !!this.rtmpServer
180 stopSessionOf (videoId: number, error: LiveVideoError | null) {
181 const sessionId = this.videoSessions.get(videoId)
182 if (!sessionId) return
184 this.saveEndingSession(videoId, error)
185 .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
187 this.videoSessions.delete(videoId)
188 this.abortSession(sessionId)
191 private getContext () {
195 private abortSession (sessionId: string) {
196 const session = this.getContext().sessions.get(sessionId)
199 this.getContext().sessions.delete(sessionId)
202 const muxingSession = this.muxingSessions.get(sessionId)
204 // Muxing session will fire and event so we correctly cleanup the session
205 muxingSession.abort()
207 this.muxingSessions.delete(sessionId)
211 private async handleSession (sessionId: string, inputUrl: string, streamKey: string) {
212 const videoLive = await VideoLiveModel.loadByStreamKey(streamKey)
214 logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId))
215 return this.abortSession(sessionId)
218 const video = videoLive.Video
219 if (video.isBlacklisted()) {
220 logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid))
221 return this.abortSession(sessionId)
224 // Cleanup old potential live (could happen with a permanent live)
225 const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
226 if (oldStreamingPlaylist) {
227 if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid)
229 await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist)
232 this.videoSessions.set(video.id, sessionId)
234 const now = Date.now()
235 const probe = await ffprobePromise(inputUrl)
237 const [ { resolution, ratio }, fps, bitrate, hasAudio ] = await Promise.all([
238 getVideoStreamDimensionsInfo(inputUrl, probe),
239 getVideoStreamFPS(inputUrl, probe),
240 getVideoStreamBitrate(inputUrl, probe),
241 hasAudioStream(inputUrl, probe)
245 '%s probing took %d ms (bitrate: %d, fps: %d, resolution: %d)',
246 inputUrl, Date.now() - now, bitrate, fps, resolution, lTags(sessionId, video.uuid)
249 const allResolutions = await Hooks.wrapObject(
250 this.buildAllResolutionsToTranscode(resolution, hasAudio),
251 'filter:transcoding.auto.resolutions-to-transcode.result',
256 'Will mux/transcode live video of original resolution %d.', resolution,
257 { allResolutions, ...lTags(sessionId, video.uuid) }
260 return this.runMuxingSession({
273 private async runMuxingSession (options: {
275 videoLive: MVideoLiveVideoWithSetting
281 allResolutions: number[]
284 const { sessionId, videoLive } = options
285 const videoUUID = videoLive.Video.uuid
286 const localLTags = lTags(sessionId, videoUUID)
288 const liveSession = await this.saveStartingSession(videoLive)
290 const user = await UserModel.loadByLiveId(videoLive.id)
291 LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id)
293 const muxingSession = new MuxingSession({
294 context: this.getContext(),
299 ...pick(options, [ 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ])
302 muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags))
304 muxingSession.on('bad-socket-health', ({ videoId }) => {
306 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
307 ' Stopping session of video %s.', videoUUID,
311 this.stopSessionOf(videoId, LiveVideoError.BAD_SOCKET_HEALTH)
314 muxingSession.on('duration-exceeded', ({ videoId }) => {
315 logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags)
317 this.stopSessionOf(videoId, LiveVideoError.DURATION_EXCEEDED)
320 muxingSession.on('quota-exceeded', ({ videoId }) => {
321 logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags)
323 this.stopSessionOf(videoId, LiveVideoError.QUOTA_EXCEEDED)
326 muxingSession.on('ffmpeg-error', ({ videoId }) => {
327 this.stopSessionOf(videoId, LiveVideoError.FFMPEG_ERROR)
330 muxingSession.on('ffmpeg-end', ({ videoId }) => {
331 this.onMuxingFFmpegEnd(videoId, sessionId)
334 muxingSession.on('after-cleanup', ({ videoId }) => {
335 this.muxingSessions.delete(sessionId)
337 LiveQuotaStore.Instance.removeLive(user.id, videoLive.id)
339 muxingSession.destroy()
341 return this.onAfterMuxingCleanup({ videoId, liveSession })
342 .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags }))
345 this.muxingSessions.set(sessionId, muxingSession)
347 muxingSession.runMuxing()
349 logger.error('Cannot run muxing.', { err, ...localLTags })
350 this.abortSession(sessionId)
354 private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) {
355 const videoId = live.videoId
358 const video = await VideoModel.loadFull(videoId)
360 logger.info('Will publish and federate live %s.', video.url, localLTags)
362 video.state = VideoState.PUBLISHED
363 video.publishedAt = new Date()
368 await wait(getLiveSegmentTime(live.latencyMode) * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION)
371 await federateVideoIfNeeded(video, false)
373 logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags })
376 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
378 logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags })
382 private onMuxingFFmpegEnd (videoId: number, sessionId: string) {
383 this.videoSessions.delete(videoId)
385 this.saveEndingSession(videoId, null)
386 .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
389 private async onAfterMuxingCleanup (options: {
390 videoId: number | string
391 liveSession?: MVideoLiveSession
392 cleanupNow?: boolean // Default false
394 const { videoId, liveSession: liveSessionArg, cleanupNow = false } = options
397 const fullVideo = await VideoModel.loadFull(videoId)
398 if (!fullVideo) return
400 const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
402 const liveSession = liveSessionArg ?? await VideoLiveSessionModel.findLatestSessionOf(fullVideo.id)
404 // On server restart during a live
405 if (!liveSession.endDate) {
406 liveSession.endDate = new Date()
407 await liveSession.save()
410 JobQueue.Instance.createJobAsync({
411 type: 'video-live-ending',
413 videoId: fullVideo.id,
415 replayDirectory: live.saveReplay
416 ? await this.findReplayDirectory(fullVideo)
419 liveSessionId: liveSession.id,
420 streamingPlaylistId: fullVideo.getHLSPlaylist()?.id,
422 publishedAt: fullVideo.publishedAt.toISOString()
427 : VIDEO_LIVE.CLEANUP_DELAY
430 fullVideo.state = live.permanentLive
431 ? VideoState.WAITING_FOR_LIVE
432 : VideoState.LIVE_ENDED
434 await fullVideo.save()
436 PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo)
438 await federateVideoIfNeeded(fullVideo, false)
440 logger.error('Cannot save/federate new video state of live streaming of video %d.', videoId, { err, ...lTags(videoId + '') })
444 private async handleBrokenLives () {
445 const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
447 for (const uuid of videoUUIDs) {
448 await this.onAfterMuxingCleanup({ videoId: uuid, cleanupNow: true })
452 private async findReplayDirectory (video: MVideo) {
453 const directory = getLiveReplayBaseDirectory(video)
454 const files = await readdir(directory)
456 if (files.length === 0) return undefined
458 return join(directory, files.sort().reverse()[0])
461 private buildAllResolutionsToTranscode (originResolution: number, hasAudio: boolean) {
462 const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
464 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
465 ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false, hasAudio })
468 if (resolutionsEnabled.length === 0) {
469 return [ originResolution ]
472 return resolutionsEnabled
475 private async saveStartingSession (videoLive: MVideoLiveVideoWithSetting) {
476 const replaySettings = videoLive.saveReplay
477 ? new VideoLiveReplaySettingModel({
478 privacy: videoLive.ReplaySetting.privacy
482 return sequelizeTypescript.transaction(async t => {
483 if (videoLive.saveReplay) {
484 await replaySettings.save({ transaction: t })
487 return VideoLiveSessionModel.create({
488 startDate: new Date(),
489 liveVideoId: videoLive.videoId,
490 saveReplay: videoLive.saveReplay,
491 replaySettingId: videoLive.saveReplay ? replaySettings.id : null,
492 endingProcessed: false
493 }, { transaction: t })
497 private async saveEndingSession (videoId: number, error: LiveVideoError | null) {
498 const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoId)
499 if (!liveSession) return
501 liveSession.endDate = new Date()
502 liveSession.error = error
504 return liveSession.save()
507 static get Instance () {
508 return this.instance || (this.instance = new this())
512 // ---------------------------------------------------------------------------