]>
Commit | Line | Data |
---|---|---|
8ebf2a5d | 1 | |
4ec52d04 | 2 | import { readdir, readFile } from 'fs-extra' |
8ebf2a5d | 3 | import { createServer, Server } from 'net' |
4ec52d04 | 4 | import { join } from 'path' |
df1db951 | 5 | import { createServer as createServerTLS, Server as ServerTLS } from 'tls' |
c826f34a | 6 | import { |
ad5db104 | 7 | computeLowerResolutionsToTranscode, |
c826f34a | 8 | ffprobePromise, |
f443a746 | 9 | getLiveSegmentTime, |
c729caf6 | 10 | getVideoStreamBitrate, |
f443a746 C |
11 | getVideoStreamDimensionsInfo, |
12 | getVideoStreamFPS | |
c729caf6 | 13 | } from '@server/helpers/ffmpeg' |
8ebf2a5d C |
14 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
15 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' | |
51353d9a | 16 | import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants' |
8ebf2a5d C |
17 | import { UserModel } from '@server/models/user/user' |
18 | import { VideoModel } from '@server/models/video/video' | |
19 | import { VideoLiveModel } from '@server/models/video/video-live' | |
20 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | |
764b1a14 | 21 | import { MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' |
4ec52d04 | 22 | import { wait } from '@shared/core-utils' |
8ebf2a5d C |
23 | import { VideoState, VideoStreamingPlaylistType } from '@shared/models' |
24 | import { federateVideoIfNeeded } from '../activitypub/videos' | |
25 | import { JobQueue } from '../job-queue' | |
4ec52d04 | 26 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' |
df1db951 | 27 | import { PeerTubeSocket } from '../peertube-socket' |
8ebf2a5d C |
28 | import { LiveQuotaStore } from './live-quota-store' |
29 | import { LiveSegmentShaStore } from './live-segment-sha-store' | |
30 | import { cleanupLive } from './live-utils' | |
31 | import { MuxingSession } from './shared' | |
32 | ||
7a397c7f C |
33 | const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') |
34 | const context = require('node-media-server/src/node_core_ctx') | |
35 | const nodeMediaServerLogger = require('node-media-server/src/node_core_logger') | |
8ebf2a5d C |
36 | |
37 | // Disable node media server logs | |
38 | nodeMediaServerLogger.setLogType(0) | |
39 | ||
40 | const config = { | |
41 | rtmp: { | |
42 | port: CONFIG.LIVE.RTMP.PORT, | |
43 | chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, | |
44 | gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, | |
45 | ping: VIDEO_LIVE.RTMP.PING, | |
46 | ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT | |
8ebf2a5d C |
47 | } |
48 | } | |
49 | ||
50 | const lTags = loggerTagsFactory('live') | |
51 | ||
52 | class LiveManager { | |
53 | ||
54 | private static instance: LiveManager | |
55 | ||
56 | private readonly muxingSessions = new Map<string, MuxingSession>() | |
57 | private readonly videoSessions = new Map<number, string>() | |
8ebf2a5d C |
58 | |
59 | private rtmpServer: Server | |
df1db951 C |
60 | private rtmpsServer: ServerTLS |
61 | ||
62 | private running = false | |
8ebf2a5d C |
63 | |
64 | private constructor () { | |
65 | } | |
66 | ||
67 | init () { | |
68 | const events = this.getContext().nodeEvent | |
69 | events.on('postPublish', (sessionId: string, streamPath: string) => { | |
70 | logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) }) | |
71 | ||
72 | const splittedPath = streamPath.split('/') | |
73 | if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { | |
74 | logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) }) | |
75 | return this.abortSession(sessionId) | |
76 | } | |
77 | ||
df1db951 C |
78 | const session = this.getContext().sessions.get(sessionId) |
79 | ||
80 | this.handleSession(sessionId, session.inputOriginUrl + streamPath, splittedPath[2]) | |
8ebf2a5d C |
81 | .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) })) |
82 | }) | |
83 | ||
84 | events.on('donePublish', sessionId => { | |
85 | logger.info('Live session ended.', { sessionId, ...lTags(sessionId) }) | |
86 | }) | |
87 | ||
88 | registerConfigChangedHandler(() => { | |
df1db951 C |
89 | if (!this.running && CONFIG.LIVE.ENABLED === true) { |
90 | this.run().catch(err => logger.error('Cannot run live server.', { err })) | |
8ebf2a5d C |
91 | return |
92 | } | |
93 | ||
df1db951 | 94 | if (this.running && CONFIG.LIVE.ENABLED === false) { |
8ebf2a5d C |
95 | this.stop() |
96 | } | |
97 | }) | |
98 | ||
99 | // Cleanup broken lives, that were terminated by a server restart for example | |
100 | this.handleBrokenLives() | |
101 | .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() })) | |
8ebf2a5d C |
102 | } |
103 | ||
df1db951 C |
104 | async run () { |
105 | this.running = true | |
8ebf2a5d | 106 | |
df1db951 C |
107 | if (CONFIG.LIVE.RTMP.ENABLED) { |
108 | logger.info('Running RTMP server on port %d', CONFIG.LIVE.RTMP.PORT, lTags()) | |
8ebf2a5d | 109 | |
df1db951 C |
110 | this.rtmpServer = createServer(socket => { |
111 | const session = new NodeRtmpSession(config, socket) | |
8ebf2a5d | 112 | |
df1db951 C |
113 | session.inputOriginUrl = 'rtmp://127.0.0.1:' + CONFIG.LIVE.RTMP.PORT |
114 | session.run() | |
115 | }) | |
116 | ||
117 | this.rtmpServer.on('error', err => { | |
118 | logger.error('Cannot run RTMP server.', { err, ...lTags() }) | |
119 | }) | |
120 | ||
121 | this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT) | |
122 | } | |
8ebf2a5d | 123 | |
df1db951 C |
124 | if (CONFIG.LIVE.RTMPS.ENABLED) { |
125 | logger.info('Running RTMPS server on port %d', CONFIG.LIVE.RTMPS.PORT, lTags()) | |
126 | ||
127 | const [ key, cert ] = await Promise.all([ | |
128 | readFile(CONFIG.LIVE.RTMPS.KEY_FILE), | |
129 | readFile(CONFIG.LIVE.RTMPS.CERT_FILE) | |
130 | ]) | |
131 | const serverOptions = { key, cert } | |
132 | ||
133 | this.rtmpsServer = createServerTLS(serverOptions, socket => { | |
134 | const session = new NodeRtmpSession(config, socket) | |
135 | ||
136 | session.inputOriginUrl = 'rtmps://127.0.0.1:' + CONFIG.LIVE.RTMPS.PORT | |
137 | session.run() | |
138 | }) | |
139 | ||
140 | this.rtmpsServer.on('error', err => { | |
141 | logger.error('Cannot run RTMPS server.', { err, ...lTags() }) | |
142 | }) | |
143 | ||
144 | this.rtmpsServer.listen(CONFIG.LIVE.RTMPS.PORT) | |
145 | } | |
8ebf2a5d C |
146 | } |
147 | ||
148 | stop () { | |
df1db951 C |
149 | this.running = false |
150 | ||
5037e0e4 C |
151 | if (this.rtmpServer) { |
152 | logger.info('Stopping RTMP server.', lTags()) | |
8ebf2a5d | 153 | |
5037e0e4 C |
154 | this.rtmpServer.close() |
155 | this.rtmpServer = undefined | |
156 | } | |
157 | ||
158 | if (this.rtmpsServer) { | |
159 | logger.info('Stopping RTMPS server.', lTags()) | |
160 | ||
161 | this.rtmpsServer.close() | |
162 | this.rtmpsServer = undefined | |
163 | } | |
8ebf2a5d C |
164 | |
165 | // Sessions is an object | |
166 | this.getContext().sessions.forEach((session: any) => { | |
167 | if (session instanceof NodeRtmpSession) { | |
168 | session.stop() | |
169 | } | |
170 | }) | |
171 | } | |
172 | ||
173 | isRunning () { | |
174 | return !!this.rtmpServer | |
175 | } | |
176 | ||
177 | stopSessionOf (videoId: number) { | |
178 | const sessionId = this.videoSessions.get(videoId) | |
179 | if (!sessionId) return | |
180 | ||
181 | this.videoSessions.delete(videoId) | |
182 | this.abortSession(sessionId) | |
183 | } | |
184 | ||
8ebf2a5d C |
185 | private getContext () { |
186 | return context | |
187 | } | |
188 | ||
189 | private abortSession (sessionId: string) { | |
190 | const session = this.getContext().sessions.get(sessionId) | |
191 | if (session) { | |
192 | session.stop() | |
193 | this.getContext().sessions.delete(sessionId) | |
194 | } | |
195 | ||
196 | const muxingSession = this.muxingSessions.get(sessionId) | |
609a4442 | 197 | if (muxingSession) { |
e466544f | 198 | // Muxing session will fire and event so we correctly cleanup the session |
609a4442 | 199 | muxingSession.abort() |
609a4442 C |
200 | |
201 | this.muxingSessions.delete(sessionId) | |
202 | } | |
8ebf2a5d C |
203 | } |
204 | ||
df1db951 | 205 | private async handleSession (sessionId: string, inputUrl: string, streamKey: string) { |
8ebf2a5d C |
206 | const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) |
207 | if (!videoLive) { | |
208 | logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId)) | |
209 | return this.abortSession(sessionId) | |
210 | } | |
211 | ||
212 | const video = videoLive.Video | |
213 | if (video.isBlacklisted()) { | |
214 | logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid)) | |
215 | return this.abortSession(sessionId) | |
216 | } | |
217 | ||
218 | // Cleanup old potential live files (could happen with a permanent live) | |
219 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) | |
220 | ||
221 | const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) | |
222 | if (oldStreamingPlaylist) { | |
223 | await cleanupLive(video, oldStreamingPlaylist) | |
224 | } | |
225 | ||
226 | this.videoSessions.set(video.id, sessionId) | |
227 | ||
c826f34a | 228 | const now = Date.now() |
df1db951 | 229 | const probe = await ffprobePromise(inputUrl) |
c826f34a | 230 | |
679c12e6 | 231 | const [ { resolution, ratio }, fps, bitrate ] = await Promise.all([ |
c729caf6 C |
232 | getVideoStreamDimensionsInfo(inputUrl, probe), |
233 | getVideoStreamFPS(inputUrl, probe), | |
234 | getVideoStreamBitrate(inputUrl, probe) | |
8ebf2a5d C |
235 | ]) |
236 | ||
c826f34a C |
237 | logger.info( |
238 | '%s probing took %d ms (bitrate: %d, fps: %d, resolution: %d)', | |
df1db951 | 239 | inputUrl, Date.now() - now, bitrate, fps, resolution, lTags(sessionId, video.uuid) |
c826f34a C |
240 | ) |
241 | ||
679c12e6 | 242 | const allResolutions = this.buildAllResolutionsToTranscode(resolution) |
8ebf2a5d C |
243 | |
244 | logger.info( | |
679c12e6 | 245 | 'Will mux/transcode live video of original resolution %d.', resolution, |
8ebf2a5d C |
246 | { allResolutions, ...lTags(sessionId, video.uuid) } |
247 | ) | |
248 | ||
249 | const streamingPlaylist = await this.createLivePlaylist(video, allResolutions) | |
250 | ||
251 | return this.runMuxingSession({ | |
252 | sessionId, | |
253 | videoLive, | |
254 | streamingPlaylist, | |
df1db951 | 255 | inputUrl, |
8ebf2a5d | 256 | fps, |
c826f34a | 257 | bitrate, |
679c12e6 | 258 | ratio, |
8ebf2a5d C |
259 | allResolutions |
260 | }) | |
261 | } | |
262 | ||
263 | private async runMuxingSession (options: { | |
264 | sessionId: string | |
265 | videoLive: MVideoLiveVideo | |
266 | streamingPlaylist: MStreamingPlaylistVideo | |
df1db951 | 267 | inputUrl: string |
8ebf2a5d | 268 | fps: number |
c826f34a | 269 | bitrate: number |
679c12e6 | 270 | ratio: number |
8ebf2a5d C |
271 | allResolutions: number[] |
272 | }) { | |
df1db951 | 273 | const { sessionId, videoLive, streamingPlaylist, allResolutions, fps, bitrate, ratio, inputUrl } = options |
8ebf2a5d C |
274 | const videoUUID = videoLive.Video.uuid |
275 | const localLTags = lTags(sessionId, videoUUID) | |
276 | ||
277 | const user = await UserModel.loadByLiveId(videoLive.id) | |
278 | LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id) | |
279 | ||
280 | const muxingSession = new MuxingSession({ | |
281 | context: this.getContext(), | |
282 | user, | |
283 | sessionId, | |
284 | videoLive, | |
285 | streamingPlaylist, | |
df1db951 | 286 | inputUrl, |
c826f34a | 287 | bitrate, |
679c12e6 | 288 | ratio, |
8ebf2a5d C |
289 | fps, |
290 | allResolutions | |
291 | }) | |
292 | ||
293 | muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) | |
294 | ||
295 | muxingSession.on('bad-socket-health', ({ videoId }) => { | |
296 | logger.error( | |
297 | 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + | |
298 | ' Stopping session of video %s.', videoUUID, | |
299 | localLTags | |
300 | ) | |
301 | ||
302 | this.stopSessionOf(videoId) | |
303 | }) | |
304 | ||
305 | muxingSession.on('duration-exceeded', ({ videoId }) => { | |
306 | logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) | |
307 | ||
308 | this.stopSessionOf(videoId) | |
309 | }) | |
310 | ||
311 | muxingSession.on('quota-exceeded', ({ videoId }) => { | |
312 | logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) | |
313 | ||
314 | this.stopSessionOf(videoId) | |
315 | }) | |
316 | ||
317 | muxingSession.on('ffmpeg-error', ({ sessionId }) => this.abortSession(sessionId)) | |
318 | muxingSession.on('ffmpeg-end', ({ videoId }) => { | |
319 | this.onMuxingFFmpegEnd(videoId) | |
320 | }) | |
321 | ||
322 | muxingSession.on('after-cleanup', ({ videoId }) => { | |
323 | this.muxingSessions.delete(sessionId) | |
324 | ||
609a4442 C |
325 | muxingSession.destroy() |
326 | ||
4ec52d04 | 327 | return this.onAfterMuxingCleanup({ videoId }) |
8ebf2a5d C |
328 | .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) |
329 | }) | |
330 | ||
331 | this.muxingSessions.set(sessionId, muxingSession) | |
332 | ||
333 | muxingSession.runMuxing() | |
334 | .catch(err => { | |
335 | logger.error('Cannot run muxing.', { err, ...localLTags }) | |
336 | this.abortSession(sessionId) | |
337 | }) | |
338 | } | |
339 | ||
340 | private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) { | |
341 | const videoId = live.videoId | |
342 | ||
343 | try { | |
344 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | |
345 | ||
346 | logger.info('Will publish and federate live %s.', video.url, localLTags) | |
347 | ||
348 | video.state = VideoState.PUBLISHED | |
7137377d | 349 | video.publishedAt = new Date() |
8ebf2a5d C |
350 | await video.save() |
351 | ||
352 | live.Video = video | |
353 | ||
4ec52d04 | 354 | await wait(getLiveSegmentTime(live.latencyMode) * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) |
8ebf2a5d | 355 | |
4ec52d04 C |
356 | try { |
357 | await federateVideoIfNeeded(video, false) | |
358 | } catch (err) { | |
359 | logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags }) | |
360 | } | |
361 | ||
362 | PeerTubeSocket.Instance.sendVideoLiveNewState(video) | |
8ebf2a5d C |
363 | } catch (err) { |
364 | logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags }) | |
365 | } | |
366 | } | |
367 | ||
368 | private onMuxingFFmpegEnd (videoId: number) { | |
8ebf2a5d C |
369 | this.videoSessions.delete(videoId) |
370 | } | |
371 | ||
4ec52d04 C |
372 | private async onAfterMuxingCleanup (options: { |
373 | videoId: number | string | |
374 | cleanupNow?: boolean // Default false | |
375 | }) { | |
376 | const { videoId, cleanupNow = false } = options | |
377 | ||
8ebf2a5d | 378 | try { |
4ec52d04 | 379 | const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) |
8ebf2a5d C |
380 | if (!fullVideo) return |
381 | ||
382 | const live = await VideoLiveModel.loadByVideoId(fullVideo.id) | |
383 | ||
4ec52d04 C |
384 | JobQueue.Instance.createJob({ |
385 | type: 'video-live-ending', | |
386 | payload: { | |
387 | videoId: fullVideo.id, | |
388 | replayDirectory: live.saveReplay | |
389 | ? await this.findReplayDirectory(fullVideo) | |
390 | : undefined, | |
391 | publishedAt: fullVideo.publishedAt.toISOString() | |
392 | } | |
393 | }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY }) | |
394 | ||
395 | fullVideo.state = live.permanentLive | |
396 | ? VideoState.WAITING_FOR_LIVE | |
397 | : VideoState.LIVE_ENDED | |
8ebf2a5d C |
398 | |
399 | await fullVideo.save() | |
400 | ||
401 | PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) | |
402 | ||
403 | await federateVideoIfNeeded(fullVideo, false) | |
404 | } catch (err) { | |
4ec52d04 | 405 | logger.error('Cannot save/federate new video state of live streaming of video %d.', videoId, { err, ...lTags(videoId + '') }) |
8ebf2a5d C |
406 | } |
407 | } | |
408 | ||
8ebf2a5d C |
409 | private async handleBrokenLives () { |
410 | const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() | |
411 | ||
412 | for (const uuid of videoUUIDs) { | |
4ec52d04 | 413 | await this.onAfterMuxingCleanup({ videoId: uuid, cleanupNow: true }) |
8ebf2a5d C |
414 | } |
415 | } | |
416 | ||
4ec52d04 C |
417 | private async findReplayDirectory (video: MVideo) { |
418 | const directory = getLiveReplayBaseDirectory(video) | |
419 | const files = await readdir(directory) | |
420 | ||
421 | if (files.length === 0) return undefined | |
422 | ||
423 | return join(directory, files.sort().reverse()[0]) | |
424 | } | |
425 | ||
8ebf2a5d C |
426 | private buildAllResolutionsToTranscode (originResolution: number) { |
427 | const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED | |
ad5db104 | 428 | ? computeLowerResolutionsToTranscode(originResolution, 'live') |
8ebf2a5d C |
429 | : [] |
430 | ||
431 | return resolutionsEnabled.concat([ originResolution ]) | |
432 | } | |
433 | ||
764b1a14 C |
434 | private async createLivePlaylist (video: MVideo, allResolutions: number[]): Promise<MStreamingPlaylistVideo> { |
435 | const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video) | |
8ebf2a5d | 436 | |
764b1a14 C |
437 | playlist.playlistFilename = generateHLSMasterPlaylistFilename(true) |
438 | playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true) | |
8ebf2a5d | 439 | |
764b1a14 C |
440 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION |
441 | playlist.type = VideoStreamingPlaylistType.HLS | |
442 | ||
443 | playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) | |
444 | ||
445 | return playlist.save() | |
8ebf2a5d C |
446 | } |
447 | ||
448 | static get Instance () { | |
449 | return this.instance || (this.instance = new this()) | |
450 | } | |
451 | } | |
452 | ||
453 | // --------------------------------------------------------------------------- | |
454 | ||
455 | export { | |
456 | LiveManager | |
457 | } |