diff options
Diffstat (limited to 'server/lib/live')
-rw-r--r-- | server/lib/live/index.ts | 4 | ||||
-rw-r--r-- | server/lib/live/live-manager.ts | 419 | ||||
-rw-r--r-- | server/lib/live/live-quota-store.ts | 48 | ||||
-rw-r--r-- | server/lib/live/live-segment-sha-store.ts | 64 | ||||
-rw-r--r-- | server/lib/live/live-utils.ts | 23 | ||||
-rw-r--r-- | server/lib/live/shared/index.ts | 1 | ||||
-rw-r--r-- | server/lib/live/shared/muxing-session.ts | 346 |
7 files changed, 905 insertions, 0 deletions
diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts new file mode 100644 index 000000000..8b46800da --- /dev/null +++ b/server/lib/live/index.ts | |||
@@ -0,0 +1,4 @@ | |||
1 | export * from './live-manager' | ||
2 | export * from './live-quota-store' | ||
3 | export * from './live-segment-sha-store' | ||
4 | export * from './live-utils' | ||
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts new file mode 100644 index 000000000..014cd3fcf --- /dev/null +++ b/server/lib/live/live-manager.ts | |||
@@ -0,0 +1,419 @@ | |||
1 | |||
2 | import { createServer, Server } from 'net' | ||
3 | import { isTestInstance } from '@server/helpers/core-utils' | ||
4 | import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' | ||
5 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
6 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' | ||
7 | import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants' | ||
8 | import { UserModel } from '@server/models/user/user' | ||
9 | import { VideoModel } from '@server/models/video/video' | ||
10 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
11 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
12 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' | ||
13 | import { VideoState, VideoStreamingPlaylistType } from '@shared/models' | ||
14 | import { federateVideoIfNeeded } from '../activitypub/videos' | ||
15 | import { JobQueue } from '../job-queue' | ||
16 | import { PeerTubeSocket } from '../peertube-socket' | ||
17 | import { LiveQuotaStore } from './live-quota-store' | ||
18 | import { LiveSegmentShaStore } from './live-segment-sha-store' | ||
19 | import { cleanupLive } from './live-utils' | ||
20 | import { MuxingSession } from './shared' | ||
21 | |||
22 | const NodeRtmpSession = require('node-media-server/node_rtmp_session') | ||
23 | const context = require('node-media-server/node_core_ctx') | ||
24 | const nodeMediaServerLogger = require('node-media-server/node_core_logger') | ||
25 | |||
26 | // Disable node media server logs | ||
27 | nodeMediaServerLogger.setLogType(0) | ||
28 | |||
29 | const config = { | ||
30 | rtmp: { | ||
31 | port: CONFIG.LIVE.RTMP.PORT, | ||
32 | chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, | ||
33 | gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, | ||
34 | ping: VIDEO_LIVE.RTMP.PING, | ||
35 | ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT | ||
36 | }, | ||
37 | transcoding: { | ||
38 | ffmpeg: 'ffmpeg' | ||
39 | } | ||
40 | } | ||
41 | |||
42 | const lTags = loggerTagsFactory('live') | ||
43 | |||
44 | class LiveManager { | ||
45 | |||
46 | private static instance: LiveManager | ||
47 | |||
48 | private readonly muxingSessions = new Map<string, MuxingSession>() | ||
49 | private readonly videoSessions = new Map<number, string>() | ||
50 | // Values are Date().getTime() | ||
51 | private readonly watchersPerVideo = new Map<number, number[]>() | ||
52 | |||
53 | private rtmpServer: Server | ||
54 | |||
55 | private constructor () { | ||
56 | } | ||
57 | |||
58 | init () { | ||
59 | const events = this.getContext().nodeEvent | ||
60 | events.on('postPublish', (sessionId: string, streamPath: string) => { | ||
61 | logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) }) | ||
62 | |||
63 | const splittedPath = streamPath.split('/') | ||
64 | if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { | ||
65 | logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) }) | ||
66 | return this.abortSession(sessionId) | ||
67 | } | ||
68 | |||
69 | this.handleSession(sessionId, streamPath, splittedPath[2]) | ||
70 | .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) })) | ||
71 | }) | ||
72 | |||
73 | events.on('donePublish', sessionId => { | ||
74 | logger.info('Live session ended.', { sessionId, ...lTags(sessionId) }) | ||
75 | }) | ||
76 | |||
77 | registerConfigChangedHandler(() => { | ||
78 | if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) { | ||
79 | this.run() | ||
80 | return | ||
81 | } | ||
82 | |||
83 | if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) { | ||
84 | this.stop() | ||
85 | } | ||
86 | }) | ||
87 | |||
88 | // Cleanup broken lives, that were terminated by a server restart for example | ||
89 | this.handleBrokenLives() | ||
90 | .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() })) | ||
91 | |||
92 | setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE) | ||
93 | } | ||
94 | |||
95 | run () { | ||
96 | logger.info('Running RTMP server on port %d', config.rtmp.port, lTags()) | ||
97 | |||
98 | this.rtmpServer = createServer(socket => { | ||
99 | const session = new NodeRtmpSession(config, socket) | ||
100 | |||
101 | session.run() | ||
102 | }) | ||
103 | |||
104 | this.rtmpServer.on('error', err => { | ||
105 | logger.error('Cannot run RTMP server.', { err, ...lTags() }) | ||
106 | }) | ||
107 | |||
108 | this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT) | ||
109 | } | ||
110 | |||
111 | stop () { | ||
112 | logger.info('Stopping RTMP server.', lTags()) | ||
113 | |||
114 | this.rtmpServer.close() | ||
115 | this.rtmpServer = undefined | ||
116 | |||
117 | // Sessions is an object | ||
118 | this.getContext().sessions.forEach((session: any) => { | ||
119 | if (session instanceof NodeRtmpSession) { | ||
120 | session.stop() | ||
121 | } | ||
122 | }) | ||
123 | } | ||
124 | |||
125 | isRunning () { | ||
126 | return !!this.rtmpServer | ||
127 | } | ||
128 | |||
129 | stopSessionOf (videoId: number) { | ||
130 | const sessionId = this.videoSessions.get(videoId) | ||
131 | if (!sessionId) return | ||
132 | |||
133 | this.videoSessions.delete(videoId) | ||
134 | this.abortSession(sessionId) | ||
135 | } | ||
136 | |||
137 | addViewTo (videoId: number) { | ||
138 | if (this.videoSessions.has(videoId) === false) return | ||
139 | |||
140 | let watchers = this.watchersPerVideo.get(videoId) | ||
141 | |||
142 | if (!watchers) { | ||
143 | watchers = [] | ||
144 | this.watchersPerVideo.set(videoId, watchers) | ||
145 | } | ||
146 | |||
147 | watchers.push(new Date().getTime()) | ||
148 | } | ||
149 | |||
150 | private getContext () { | ||
151 | return context | ||
152 | } | ||
153 | |||
154 | private abortSession (sessionId: string) { | ||
155 | const session = this.getContext().sessions.get(sessionId) | ||
156 | if (session) { | ||
157 | session.stop() | ||
158 | this.getContext().sessions.delete(sessionId) | ||
159 | } | ||
160 | |||
161 | const muxingSession = this.muxingSessions.get(sessionId) | ||
162 | if (muxingSession) { | ||
163 | // Muxing session will fire and event so we correctly cleanup the session | ||
164 | muxingSession.abort() | ||
165 | |||
166 | this.muxingSessions.delete(sessionId) | ||
167 | } | ||
168 | } | ||
169 | |||
170 | private async handleSession (sessionId: string, streamPath: string, streamKey: string) { | ||
171 | const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) | ||
172 | if (!videoLive) { | ||
173 | logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId)) | ||
174 | return this.abortSession(sessionId) | ||
175 | } | ||
176 | |||
177 | const video = videoLive.Video | ||
178 | if (video.isBlacklisted()) { | ||
179 | logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid)) | ||
180 | return this.abortSession(sessionId) | ||
181 | } | ||
182 | |||
183 | // Cleanup old potential live files (could happen with a permanent live) | ||
184 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) | ||
185 | |||
186 | const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) | ||
187 | if (oldStreamingPlaylist) { | ||
188 | await cleanupLive(video, oldStreamingPlaylist) | ||
189 | } | ||
190 | |||
191 | this.videoSessions.set(video.id, sessionId) | ||
192 | |||
193 | const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath | ||
194 | |||
195 | const [ { videoFileResolution }, fps ] = await Promise.all([ | ||
196 | getVideoFileResolution(rtmpUrl), | ||
197 | getVideoFileFPS(rtmpUrl) | ||
198 | ]) | ||
199 | |||
200 | const allResolutions = this.buildAllResolutionsToTranscode(videoFileResolution) | ||
201 | |||
202 | logger.info( | ||
203 | 'Will mux/transcode live video of original resolution %d.', videoFileResolution, | ||
204 | { allResolutions, ...lTags(sessionId, video.uuid) } | ||
205 | ) | ||
206 | |||
207 | const streamingPlaylist = await this.createLivePlaylist(video, allResolutions) | ||
208 | |||
209 | return this.runMuxingSession({ | ||
210 | sessionId, | ||
211 | videoLive, | ||
212 | streamingPlaylist, | ||
213 | rtmpUrl, | ||
214 | fps, | ||
215 | allResolutions | ||
216 | }) | ||
217 | } | ||
218 | |||
219 | private async runMuxingSession (options: { | ||
220 | sessionId: string | ||
221 | videoLive: MVideoLiveVideo | ||
222 | streamingPlaylist: MStreamingPlaylistVideo | ||
223 | rtmpUrl: string | ||
224 | fps: number | ||
225 | allResolutions: number[] | ||
226 | }) { | ||
227 | const { sessionId, videoLive, streamingPlaylist, allResolutions, fps, rtmpUrl } = options | ||
228 | const videoUUID = videoLive.Video.uuid | ||
229 | const localLTags = lTags(sessionId, videoUUID) | ||
230 | |||
231 | const user = await UserModel.loadByLiveId(videoLive.id) | ||
232 | LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id) | ||
233 | |||
234 | const muxingSession = new MuxingSession({ | ||
235 | context: this.getContext(), | ||
236 | user, | ||
237 | sessionId, | ||
238 | videoLive, | ||
239 | streamingPlaylist, | ||
240 | rtmpUrl, | ||
241 | fps, | ||
242 | allResolutions | ||
243 | }) | ||
244 | |||
245 | muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) | ||
246 | |||
247 | muxingSession.on('bad-socket-health', ({ videoId }) => { | ||
248 | logger.error( | ||
249 | 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + | ||
250 | ' Stopping session of video %s.', videoUUID, | ||
251 | localLTags | ||
252 | ) | ||
253 | |||
254 | this.stopSessionOf(videoId) | ||
255 | }) | ||
256 | |||
257 | muxingSession.on('duration-exceeded', ({ videoId }) => { | ||
258 | logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) | ||
259 | |||
260 | this.stopSessionOf(videoId) | ||
261 | }) | ||
262 | |||
263 | muxingSession.on('quota-exceeded', ({ videoId }) => { | ||
264 | logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) | ||
265 | |||
266 | this.stopSessionOf(videoId) | ||
267 | }) | ||
268 | |||
269 | muxingSession.on('ffmpeg-error', ({ sessionId }) => this.abortSession(sessionId)) | ||
270 | muxingSession.on('ffmpeg-end', ({ videoId }) => { | ||
271 | this.onMuxingFFmpegEnd(videoId) | ||
272 | }) | ||
273 | |||
274 | muxingSession.on('after-cleanup', ({ videoId }) => { | ||
275 | this.muxingSessions.delete(sessionId) | ||
276 | |||
277 | muxingSession.destroy() | ||
278 | |||
279 | return this.onAfterMuxingCleanup(videoId) | ||
280 | .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) | ||
281 | }) | ||
282 | |||
283 | this.muxingSessions.set(sessionId, muxingSession) | ||
284 | |||
285 | muxingSession.runMuxing() | ||
286 | .catch(err => { | ||
287 | logger.error('Cannot run muxing.', { err, ...localLTags }) | ||
288 | this.abortSession(sessionId) | ||
289 | }) | ||
290 | } | ||
291 | |||
292 | private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) { | ||
293 | const videoId = live.videoId | ||
294 | |||
295 | try { | ||
296 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
297 | |||
298 | logger.info('Will publish and federate live %s.', video.url, localLTags) | ||
299 | |||
300 | video.state = VideoState.PUBLISHED | ||
301 | await video.save() | ||
302 | |||
303 | live.Video = video | ||
304 | |||
305 | setTimeout(() => { | ||
306 | federateVideoIfNeeded(video, false) | ||
307 | .catch(err => logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags })) | ||
308 | |||
309 | PeerTubeSocket.Instance.sendVideoLiveNewState(video) | ||
310 | }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) | ||
311 | } catch (err) { | ||
312 | logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags }) | ||
313 | } | ||
314 | } | ||
315 | |||
316 | private onMuxingFFmpegEnd (videoId: number) { | ||
317 | this.watchersPerVideo.delete(videoId) | ||
318 | this.videoSessions.delete(videoId) | ||
319 | } | ||
320 | |||
321 | private async onAfterMuxingCleanup (videoUUID: string, cleanupNow = false) { | ||
322 | try { | ||
323 | const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoUUID) | ||
324 | if (!fullVideo) return | ||
325 | |||
326 | const live = await VideoLiveModel.loadByVideoId(fullVideo.id) | ||
327 | |||
328 | if (!live.permanentLive) { | ||
329 | JobQueue.Instance.createJob({ | ||
330 | type: 'video-live-ending', | ||
331 | payload: { | ||
332 | videoId: fullVideo.id | ||
333 | } | ||
334 | }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY }) | ||
335 | |||
336 | fullVideo.state = VideoState.LIVE_ENDED | ||
337 | } else { | ||
338 | fullVideo.state = VideoState.WAITING_FOR_LIVE | ||
339 | } | ||
340 | |||
341 | await fullVideo.save() | ||
342 | |||
343 | PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) | ||
344 | |||
345 | await federateVideoIfNeeded(fullVideo, false) | ||
346 | } catch (err) { | ||
347 | logger.error('Cannot save/federate new video state of live streaming of video %d.', videoUUID, { err, ...lTags(videoUUID) }) | ||
348 | } | ||
349 | } | ||
350 | |||
351 | private async updateLiveViews () { | ||
352 | if (!this.isRunning()) return | ||
353 | |||
354 | if (!isTestInstance()) logger.info('Updating live video views.', lTags()) | ||
355 | |||
356 | for (const videoId of this.watchersPerVideo.keys()) { | ||
357 | const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE | ||
358 | |||
359 | const watchers = this.watchersPerVideo.get(videoId) | ||
360 | |||
361 | const numWatchers = watchers.length | ||
362 | |||
363 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
364 | video.views = numWatchers | ||
365 | await video.save() | ||
366 | |||
367 | await federateVideoIfNeeded(video, false) | ||
368 | |||
369 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video) | ||
370 | |||
371 | // Only keep not expired watchers | ||
372 | const newWatchers = watchers.filter(w => w > notBefore) | ||
373 | this.watchersPerVideo.set(videoId, newWatchers) | ||
374 | |||
375 | logger.debug('New live video views for %s is %d.', video.url, numWatchers, lTags()) | ||
376 | } | ||
377 | } | ||
378 | |||
379 | private async handleBrokenLives () { | ||
380 | const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() | ||
381 | |||
382 | for (const uuid of videoUUIDs) { | ||
383 | await this.onAfterMuxingCleanup(uuid, true) | ||
384 | } | ||
385 | } | ||
386 | |||
387 | private buildAllResolutionsToTranscode (originResolution: number) { | ||
388 | const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED | ||
389 | ? computeResolutionsToTranscode(originResolution, 'live') | ||
390 | : [] | ||
391 | |||
392 | return resolutionsEnabled.concat([ originResolution ]) | ||
393 | } | ||
394 | |||
395 | private async createLivePlaylist (video: MVideo, allResolutions: number[]) { | ||
396 | const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) | ||
397 | const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ | ||
398 | videoId: video.id, | ||
399 | playlistUrl, | ||
400 | segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), | ||
401 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions), | ||
402 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
403 | |||
404 | type: VideoStreamingPlaylistType.HLS | ||
405 | }, { returning: true }) as [ MStreamingPlaylist, boolean ] | ||
406 | |||
407 | return Object.assign(videoStreamingPlaylist, { Video: video }) | ||
408 | } | ||
409 | |||
410 | static get Instance () { | ||
411 | return this.instance || (this.instance = new this()) | ||
412 | } | ||
413 | } | ||
414 | |||
415 | // --------------------------------------------------------------------------- | ||
416 | |||
417 | export { | ||
418 | LiveManager | ||
419 | } | ||
diff --git a/server/lib/live/live-quota-store.ts b/server/lib/live/live-quota-store.ts new file mode 100644 index 000000000..8ceccde98 --- /dev/null +++ b/server/lib/live/live-quota-store.ts | |||
@@ -0,0 +1,48 @@ | |||
1 | class LiveQuotaStore { | ||
2 | |||
3 | private static instance: LiveQuotaStore | ||
4 | |||
5 | private readonly livesPerUser = new Map<number, { liveId: number, size: number }[]>() | ||
6 | |||
7 | private constructor () { | ||
8 | } | ||
9 | |||
10 | addNewLive (userId: number, liveId: number) { | ||
11 | if (!this.livesPerUser.has(userId)) { | ||
12 | this.livesPerUser.set(userId, []) | ||
13 | } | ||
14 | |||
15 | const currentUserLive = { liveId, size: 0 } | ||
16 | const livesOfUser = this.livesPerUser.get(userId) | ||
17 | livesOfUser.push(currentUserLive) | ||
18 | } | ||
19 | |||
20 | removeLive (userId: number, liveId: number) { | ||
21 | const newLivesPerUser = this.livesPerUser.get(userId) | ||
22 | .filter(o => o.liveId !== liveId) | ||
23 | |||
24 | this.livesPerUser.set(userId, newLivesPerUser) | ||
25 | } | ||
26 | |||
27 | addQuotaTo (userId: number, liveId: number, size: number) { | ||
28 | const lives = this.livesPerUser.get(userId) | ||
29 | const live = lives.find(l => l.liveId === liveId) | ||
30 | |||
31 | live.size += size | ||
32 | } | ||
33 | |||
34 | getLiveQuotaOf (userId: number) { | ||
35 | const currentLives = this.livesPerUser.get(userId) | ||
36 | if (!currentLives) return 0 | ||
37 | |||
38 | return currentLives.reduce((sum, obj) => sum + obj.size, 0) | ||
39 | } | ||
40 | |||
41 | static get Instance () { | ||
42 | return this.instance || (this.instance = new this()) | ||
43 | } | ||
44 | } | ||
45 | |||
46 | export { | ||
47 | LiveQuotaStore | ||
48 | } | ||
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts new file mode 100644 index 000000000..4af6f3ebf --- /dev/null +++ b/server/lib/live/live-segment-sha-store.ts | |||
@@ -0,0 +1,64 @@ | |||
1 | import { basename } from 'path' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { buildSha256Segment } from '../hls' | ||
4 | |||
5 | const lTags = loggerTagsFactory('live') | ||
6 | |||
7 | class LiveSegmentShaStore { | ||
8 | |||
9 | private static instance: LiveSegmentShaStore | ||
10 | |||
11 | private readonly segmentsSha256 = new Map<string, Map<string, string>>() | ||
12 | |||
13 | private constructor () { | ||
14 | } | ||
15 | |||
16 | getSegmentsSha256 (videoUUID: string) { | ||
17 | return this.segmentsSha256.get(videoUUID) | ||
18 | } | ||
19 | |||
20 | async addSegmentSha (videoUUID: string, segmentPath: string) { | ||
21 | const segmentName = basename(segmentPath) | ||
22 | logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID)) | ||
23 | |||
24 | const shaResult = await buildSha256Segment(segmentPath) | ||
25 | |||
26 | if (!this.segmentsSha256.has(videoUUID)) { | ||
27 | this.segmentsSha256.set(videoUUID, new Map()) | ||
28 | } | ||
29 | |||
30 | const filesMap = this.segmentsSha256.get(videoUUID) | ||
31 | filesMap.set(segmentName, shaResult) | ||
32 | } | ||
33 | |||
34 | removeSegmentSha (videoUUID: string, segmentPath: string) { | ||
35 | const segmentName = basename(segmentPath) | ||
36 | |||
37 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID)) | ||
38 | |||
39 | const filesMap = this.segmentsSha256.get(videoUUID) | ||
40 | if (!filesMap) { | ||
41 | logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID)) | ||
42 | return | ||
43 | } | ||
44 | |||
45 | if (!filesMap.has(segmentName)) { | ||
46 | logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID)) | ||
47 | return | ||
48 | } | ||
49 | |||
50 | filesMap.delete(segmentName) | ||
51 | } | ||
52 | |||
53 | cleanupShaSegments (videoUUID: string) { | ||
54 | this.segmentsSha256.delete(videoUUID) | ||
55 | } | ||
56 | |||
57 | static get Instance () { | ||
58 | return this.instance || (this.instance = new this()) | ||
59 | } | ||
60 | } | ||
61 | |||
62 | export { | ||
63 | LiveSegmentShaStore | ||
64 | } | ||
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts new file mode 100644 index 000000000..e4526c7a5 --- /dev/null +++ b/server/lib/live/live-utils.ts | |||
@@ -0,0 +1,23 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { basename } from 'path' | ||
3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | ||
4 | import { getHLSDirectory } from '../video-paths' | ||
5 | |||
6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | ||
7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | ||
8 | |||
9 | return 'concat-' + num[1] + '.ts' | ||
10 | } | ||
11 | |||
12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | ||
13 | const hlsDirectory = getHLSDirectory(video) | ||
14 | |||
15 | await remove(hlsDirectory) | ||
16 | |||
17 | await streamingPlaylist.destroy() | ||
18 | } | ||
19 | |||
20 | export { | ||
21 | cleanupLive, | ||
22 | buildConcatenatedName | ||
23 | } | ||
diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts new file mode 100644 index 000000000..c4d1b59ec --- /dev/null +++ b/server/lib/live/shared/index.ts | |||
@@ -0,0 +1 @@ | |||
export * from './muxing-session' | |||
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts new file mode 100644 index 000000000..26467f060 --- /dev/null +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -0,0 +1,346 @@ | |||
1 | |||
2 | import * as Bluebird from 'bluebird' | ||
3 | import * as chokidar from 'chokidar' | ||
4 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
5 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' | ||
6 | import { basename, join } from 'path' | ||
7 | import { EventEmitter } from 'stream' | ||
8 | import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils' | ||
9 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | ||
10 | import { CONFIG } from '@server/initializers/config' | ||
11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | ||
12 | import { VideoFileModel } from '@server/models/video/video-file' | ||
13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | ||
14 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' | ||
15 | import { isAbleToUploadVideo } from '../../user' | ||
16 | import { getHLSDirectory } from '../../video-paths' | ||
17 | import { LiveQuotaStore } from '../live-quota-store' | ||
18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | ||
19 | import { buildConcatenatedName } from '../live-utils' | ||
20 | |||
21 | import memoizee = require('memoizee') | ||
22 | |||
23 | interface MuxingSessionEvents { | ||
24 | 'master-playlist-created': ({ videoId: number }) => void | ||
25 | |||
26 | 'bad-socket-health': ({ videoId: number }) => void | ||
27 | 'duration-exceeded': ({ videoId: number }) => void | ||
28 | 'quota-exceeded': ({ videoId: number }) => void | ||
29 | |||
30 | 'ffmpeg-end': ({ videoId: number }) => void | ||
31 | 'ffmpeg-error': ({ sessionId: string }) => void | ||
32 | |||
33 | 'after-cleanup': ({ videoId: number }) => void | ||
34 | } | ||
35 | |||
36 | declare interface MuxingSession { | ||
37 | on<U extends keyof MuxingSessionEvents>( | ||
38 | event: U, listener: MuxingSessionEvents[U] | ||
39 | ): this | ||
40 | |||
41 | emit<U extends keyof MuxingSessionEvents>( | ||
42 | event: U, ...args: Parameters<MuxingSessionEvents[U]> | ||
43 | ): boolean | ||
44 | } | ||
45 | |||
46 | class MuxingSession extends EventEmitter { | ||
47 | |||
48 | private ffmpegCommand: FfmpegCommand | ||
49 | |||
50 | private readonly context: any | ||
51 | private readonly user: MUserId | ||
52 | private readonly sessionId: string | ||
53 | private readonly videoLive: MVideoLiveVideo | ||
54 | private readonly streamingPlaylist: MStreamingPlaylistVideo | ||
55 | private readonly rtmpUrl: string | ||
56 | private readonly fps: number | ||
57 | private readonly allResolutions: number[] | ||
58 | |||
59 | private readonly videoId: number | ||
60 | private readonly videoUUID: string | ||
61 | private readonly saveReplay: boolean | ||
62 | |||
63 | private readonly lTags: LoggerTagsFn | ||
64 | |||
65 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} | ||
66 | |||
67 | private tsWatcher: chokidar.FSWatcher | ||
68 | private masterWatcher: chokidar.FSWatcher | ||
69 | |||
70 | private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { | ||
71 | return isAbleToUploadVideo(userId, 1000) | ||
72 | }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) | ||
73 | |||
74 | private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { | ||
75 | return this.hasClientSocketInBadHealth(sessionId) | ||
76 | }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) | ||
77 | |||
78 | constructor (options: { | ||
79 | context: any | ||
80 | user: MUserId | ||
81 | sessionId: string | ||
82 | videoLive: MVideoLiveVideo | ||
83 | streamingPlaylist: MStreamingPlaylistVideo | ||
84 | rtmpUrl: string | ||
85 | fps: number | ||
86 | allResolutions: number[] | ||
87 | }) { | ||
88 | super() | ||
89 | |||
90 | this.context = options.context | ||
91 | this.user = options.user | ||
92 | this.sessionId = options.sessionId | ||
93 | this.videoLive = options.videoLive | ||
94 | this.streamingPlaylist = options.streamingPlaylist | ||
95 | this.rtmpUrl = options.rtmpUrl | ||
96 | this.fps = options.fps | ||
97 | this.allResolutions = options.allResolutions | ||
98 | |||
99 | this.videoId = this.videoLive.Video.id | ||
100 | this.videoUUID = this.videoLive.Video.uuid | ||
101 | |||
102 | this.saveReplay = this.videoLive.saveReplay | ||
103 | |||
104 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) | ||
105 | } | ||
106 | |||
107 | async runMuxing () { | ||
108 | this.createFiles() | ||
109 | |||
110 | const outPath = await this.prepareDirectories() | ||
111 | |||
112 | this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED | ||
113 | ? await getLiveTranscodingCommand({ | ||
114 | rtmpUrl: this.rtmpUrl, | ||
115 | outPath, | ||
116 | resolutions: this.allResolutions, | ||
117 | fps: this.fps, | ||
118 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
119 | profile: CONFIG.LIVE.TRANSCODING.PROFILE | ||
120 | }) | ||
121 | : getLiveMuxingCommand(this.rtmpUrl, outPath) | ||
122 | |||
123 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags) | ||
124 | |||
125 | this.watchTSFiles(outPath) | ||
126 | this.watchMasterFile(outPath) | ||
127 | |||
128 | this.ffmpegCommand.on('error', (err, stdout, stderr) => { | ||
129 | this.onFFmpegError(err, stdout, stderr, outPath) | ||
130 | }) | ||
131 | |||
132 | this.ffmpegCommand.on('end', () => this.onFFmpegEnded(outPath)) | ||
133 | |||
134 | this.ffmpegCommand.run() | ||
135 | } | ||
136 | |||
137 | abort () { | ||
138 | if (!this.ffmpegCommand) return | ||
139 | |||
140 | this.ffmpegCommand.kill('SIGINT') | ||
141 | } | ||
142 | |||
143 | destroy () { | ||
144 | this.removeAllListeners() | ||
145 | this.isAbleToUploadVideoWithCache.clear() | ||
146 | this.hasClientSocketInBadHealthWithCache.clear() | ||
147 | } | ||
148 | |||
149 | private onFFmpegError (err: any, stdout: string, stderr: string, outPath: string) { | ||
150 | this.onFFmpegEnded(outPath) | ||
151 | |||
152 | // Don't care that we killed the ffmpeg process | ||
153 | if (err?.message?.includes('Exiting normally')) return | ||
154 | |||
155 | logger.error('Live transcoding error.', { err, stdout, stderr, ...this.lTags }) | ||
156 | |||
157 | this.emit('ffmpeg-error', ({ sessionId: this.sessionId })) | ||
158 | } | ||
159 | |||
160 | private onFFmpegEnded (outPath: string) { | ||
161 | logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.rtmpUrl, this.lTags) | ||
162 | |||
163 | setTimeout(() => { | ||
164 | // Wait latest segments generation, and close watchers | ||
165 | |||
166 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ]) | ||
167 | .then(() => { | ||
168 | // Process remaining segments hash | ||
169 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | ||
170 | this.processSegments(outPath, this.segmentsToProcessPerPlaylist[key]) | ||
171 | } | ||
172 | }) | ||
173 | .catch(err => { | ||
174 | logger.error( | ||
175 | 'Cannot close watchers of %s or process remaining hash segments.', outPath, | ||
176 | { err, ...this.lTags } | ||
177 | ) | ||
178 | }) | ||
179 | |||
180 | this.emit('after-cleanup', { videoId: this.videoId }) | ||
181 | }, 1000) | ||
182 | } | ||
183 | |||
184 | private watchMasterFile (outPath: string) { | ||
185 | this.masterWatcher = chokidar.watch(outPath + '/master.m3u8') | ||
186 | |||
187 | this.masterWatcher.on('add', async () => { | ||
188 | this.emit('master-playlist-created', { videoId: this.videoId }) | ||
189 | |||
190 | this.masterWatcher.close() | ||
191 | .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags })) | ||
192 | }) | ||
193 | } | ||
194 | |||
195 | private watchTSFiles (outPath: string) { | ||
196 | const startStreamDateTime = new Date().getTime() | ||
197 | |||
198 | this.tsWatcher = chokidar.watch(outPath + '/*.ts') | ||
199 | |||
200 | const playlistIdMatcher = /^([\d+])-/ | ||
201 | |||
202 | const addHandler = async segmentPath => { | ||
203 | logger.debug('Live add handler of %s.', segmentPath, this.lTags) | ||
204 | |||
205 | const playlistId = basename(segmentPath).match(playlistIdMatcher)[0] | ||
206 | |||
207 | const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || [] | ||
208 | this.processSegments(outPath, segmentsToProcess) | ||
209 | |||
210 | this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] | ||
211 | |||
212 | if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { | ||
213 | this.emit('bad-socket-health', { videoId: this.videoId }) | ||
214 | return | ||
215 | } | ||
216 | |||
217 | // Duration constraint check | ||
218 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { | ||
219 | this.emit('duration-exceeded', { videoId: this.videoId }) | ||
220 | return | ||
221 | } | ||
222 | |||
223 | // Check user quota if the user enabled replay saving | ||
224 | if (await this.isQuotaExceeded(segmentPath) === true) { | ||
225 | this.emit('quota-exceeded', { videoId: this.videoId }) | ||
226 | } | ||
227 | } | ||
228 | |||
229 | const deleteHandler = segmentPath => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath) | ||
230 | |||
231 | this.tsWatcher.on('add', p => addHandler(p)) | ||
232 | this.tsWatcher.on('unlink', p => deleteHandler(p)) | ||
233 | } | ||
234 | |||
235 | private async isQuotaExceeded (segmentPath: string) { | ||
236 | if (this.saveReplay !== true) return false | ||
237 | |||
238 | try { | ||
239 | const segmentStat = await stat(segmentPath) | ||
240 | |||
241 | LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.videoLive.id, segmentStat.size) | ||
242 | |||
243 | const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id) | ||
244 | |||
245 | return canUpload !== true | ||
246 | } catch (err) { | ||
247 | logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags }) | ||
248 | } | ||
249 | } | ||
250 | |||
251 | private createFiles () { | ||
252 | for (let i = 0; i < this.allResolutions.length; i++) { | ||
253 | const resolution = this.allResolutions[i] | ||
254 | |||
255 | const file = new VideoFileModel({ | ||
256 | resolution, | ||
257 | size: -1, | ||
258 | extname: '.ts', | ||
259 | infoHash: null, | ||
260 | fps: this.fps, | ||
261 | videoStreamingPlaylistId: this.streamingPlaylist.id | ||
262 | }) | ||
263 | |||
264 | VideoFileModel.customUpsert(file, 'streaming-playlist', null) | ||
265 | .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags })) | ||
266 | } | ||
267 | } | ||
268 | |||
269 | private async prepareDirectories () { | ||
270 | const outPath = getHLSDirectory(this.videoLive.Video) | ||
271 | await ensureDir(outPath) | ||
272 | |||
273 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) | ||
274 | |||
275 | if (this.videoLive.saveReplay === true) { | ||
276 | await ensureDir(replayDirectory) | ||
277 | } | ||
278 | |||
279 | return outPath | ||
280 | } | ||
281 | |||
282 | private isDurationConstraintValid (streamingStartTime: number) { | ||
283 | const maxDuration = CONFIG.LIVE.MAX_DURATION | ||
284 | // No limit | ||
285 | if (maxDuration < 0) return true | ||
286 | |||
287 | const now = new Date().getTime() | ||
288 | const max = streamingStartTime + maxDuration | ||
289 | |||
290 | return now <= max | ||
291 | } | ||
292 | |||
293 | private processSegments (hlsVideoPath: string, segmentPaths: string[]) { | ||
294 | Bluebird.mapSeries(segmentPaths, async previousSegment => { | ||
295 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | ||
296 | await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment) | ||
297 | |||
298 | if (this.saveReplay) { | ||
299 | await this.addSegmentToReplay(hlsVideoPath, previousSegment) | ||
300 | } | ||
301 | }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags })) | ||
302 | } | ||
303 | |||
304 | private hasClientSocketInBadHealth (sessionId: string) { | ||
305 | const rtmpSession = this.context.sessions.get(sessionId) | ||
306 | |||
307 | if (!rtmpSession) { | ||
308 | logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags) | ||
309 | return | ||
310 | } | ||
311 | |||
312 | for (const playerSessionId of rtmpSession.players) { | ||
313 | const playerSession = this.context.sessions.get(playerSessionId) | ||
314 | |||
315 | if (!playerSession) { | ||
316 | logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags) | ||
317 | continue | ||
318 | } | ||
319 | |||
320 | if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { | ||
321 | return true | ||
322 | } | ||
323 | } | ||
324 | |||
325 | return false | ||
326 | } | ||
327 | |||
328 | private async addSegmentToReplay (hlsVideoPath: string, segmentPath: string) { | ||
329 | const segmentName = basename(segmentPath) | ||
330 | const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, buildConcatenatedName(segmentName)) | ||
331 | |||
332 | try { | ||
333 | const data = await readFile(segmentPath) | ||
334 | |||
335 | await appendFile(dest, data) | ||
336 | } catch (err) { | ||
337 | logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags }) | ||
338 | } | ||
339 | } | ||
340 | } | ||
341 | |||
342 | // --------------------------------------------------------------------------- | ||
343 | |||
344 | export { | ||
345 | MuxingSession | ||
346 | } | ||