diff options
author | Chocobozzz <me@florianbigard.com> | 2021-06-16 15:14:41 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2021-06-17 08:47:03 +0200 |
commit | 8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d (patch) | |
tree | eee3ef972943be629519c2dc97928878dbd2d732 /server/lib/live | |
parent | fd6584844b070cfc283902aba26f2853473e83d1 (diff) | |
download | PeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.tar.gz PeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.tar.zst PeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.zip |
Refactor live manager
Diffstat (limited to 'server/lib/live')
-rw-r--r-- | server/lib/live/index.ts | 4 | ||||
-rw-r--r-- | server/lib/live/live-manager.ts | 412 | ||||
-rw-r--r-- | server/lib/live/live-quota-store.ts | 48 | ||||
-rw-r--r-- | server/lib/live/live-segment-sha-store.ts | 64 | ||||
-rw-r--r-- | server/lib/live/live-utils.ts | 23 | ||||
-rw-r--r-- | server/lib/live/shared/index.ts | 1 | ||||
-rw-r--r-- | server/lib/live/shared/muxing-session.ts | 341 |
7 files changed, 893 insertions, 0 deletions
diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts new file mode 100644 index 000000000..8b46800da --- /dev/null +++ b/server/lib/live/index.ts | |||
@@ -0,0 +1,4 @@ | |||
1 | export * from './live-manager' | ||
2 | export * from './live-quota-store' | ||
3 | export * from './live-segment-sha-store' | ||
4 | export * from './live-utils' | ||
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts new file mode 100644 index 000000000..d7199cc89 --- /dev/null +++ b/server/lib/live/live-manager.ts | |||
@@ -0,0 +1,412 @@ | |||
1 | |||
2 | import { createServer, Server } from 'net' | ||
3 | import { isTestInstance } from '@server/helpers/core-utils' | ||
4 | import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' | ||
5 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
6 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' | ||
7 | import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants' | ||
8 | import { UserModel } from '@server/models/user/user' | ||
9 | import { VideoModel } from '@server/models/video/video' | ||
10 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
11 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
12 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' | ||
13 | import { VideoState, VideoStreamingPlaylistType } from '@shared/models' | ||
14 | import { federateVideoIfNeeded } from '../activitypub/videos' | ||
15 | import { JobQueue } from '../job-queue' | ||
16 | import { PeerTubeSocket } from '../peertube-socket' | ||
17 | import { LiveQuotaStore } from './live-quota-store' | ||
18 | import { LiveSegmentShaStore } from './live-segment-sha-store' | ||
19 | import { cleanupLive } from './live-utils' | ||
20 | import { MuxingSession } from './shared' | ||
21 | |||
22 | const NodeRtmpSession = require('node-media-server/node_rtmp_session') | ||
23 | const context = require('node-media-server/node_core_ctx') | ||
24 | const nodeMediaServerLogger = require('node-media-server/node_core_logger') | ||
25 | |||
26 | // Disable node media server logs | ||
27 | nodeMediaServerLogger.setLogType(0) | ||
28 | |||
29 | const config = { | ||
30 | rtmp: { | ||
31 | port: CONFIG.LIVE.RTMP.PORT, | ||
32 | chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, | ||
33 | gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, | ||
34 | ping: VIDEO_LIVE.RTMP.PING, | ||
35 | ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT | ||
36 | }, | ||
37 | transcoding: { | ||
38 | ffmpeg: 'ffmpeg' | ||
39 | } | ||
40 | } | ||
41 | |||
42 | const lTags = loggerTagsFactory('live') | ||
43 | |||
44 | class LiveManager { | ||
45 | |||
46 | private static instance: LiveManager | ||
47 | |||
48 | private readonly muxingSessions = new Map<string, MuxingSession>() | ||
49 | private readonly videoSessions = new Map<number, string>() | ||
50 | // Values are Date().getTime() | ||
51 | private readonly watchersPerVideo = new Map<number, number[]>() | ||
52 | |||
53 | private rtmpServer: Server | ||
54 | |||
55 | private constructor () { | ||
56 | } | ||
57 | |||
58 | init () { | ||
59 | const events = this.getContext().nodeEvent | ||
60 | events.on('postPublish', (sessionId: string, streamPath: string) => { | ||
61 | logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) }) | ||
62 | |||
63 | const splittedPath = streamPath.split('/') | ||
64 | if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { | ||
65 | logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) }) | ||
66 | return this.abortSession(sessionId) | ||
67 | } | ||
68 | |||
69 | this.handleSession(sessionId, streamPath, splittedPath[2]) | ||
70 | .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) })) | ||
71 | }) | ||
72 | |||
73 | events.on('donePublish', sessionId => { | ||
74 | logger.info('Live session ended.', { sessionId, ...lTags(sessionId) }) | ||
75 | }) | ||
76 | |||
77 | registerConfigChangedHandler(() => { | ||
78 | if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) { | ||
79 | this.run() | ||
80 | return | ||
81 | } | ||
82 | |||
83 | if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) { | ||
84 | this.stop() | ||
85 | } | ||
86 | }) | ||
87 | |||
88 | // Cleanup broken lives, that were terminated by a server restart for example | ||
89 | this.handleBrokenLives() | ||
90 | .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() })) | ||
91 | |||
92 | setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE) | ||
93 | } | ||
94 | |||
95 | run () { | ||
96 | logger.info('Running RTMP server on port %d', config.rtmp.port, lTags()) | ||
97 | |||
98 | this.rtmpServer = createServer(socket => { | ||
99 | const session = new NodeRtmpSession(config, socket) | ||
100 | |||
101 | session.run() | ||
102 | }) | ||
103 | |||
104 | this.rtmpServer.on('error', err => { | ||
105 | logger.error('Cannot run RTMP server.', { err, ...lTags() }) | ||
106 | }) | ||
107 | |||
108 | this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT) | ||
109 | } | ||
110 | |||
111 | stop () { | ||
112 | logger.info('Stopping RTMP server.', lTags()) | ||
113 | |||
114 | this.rtmpServer.close() | ||
115 | this.rtmpServer = undefined | ||
116 | |||
117 | // Sessions is an object | ||
118 | this.getContext().sessions.forEach((session: any) => { | ||
119 | if (session instanceof NodeRtmpSession) { | ||
120 | session.stop() | ||
121 | } | ||
122 | }) | ||
123 | } | ||
124 | |||
125 | isRunning () { | ||
126 | return !!this.rtmpServer | ||
127 | } | ||
128 | |||
129 | stopSessionOf (videoId: number) { | ||
130 | const sessionId = this.videoSessions.get(videoId) | ||
131 | if (!sessionId) return | ||
132 | |||
133 | this.videoSessions.delete(videoId) | ||
134 | this.abortSession(sessionId) | ||
135 | } | ||
136 | |||
137 | addViewTo (videoId: number) { | ||
138 | if (this.videoSessions.has(videoId) === false) return | ||
139 | |||
140 | let watchers = this.watchersPerVideo.get(videoId) | ||
141 | |||
142 | if (!watchers) { | ||
143 | watchers = [] | ||
144 | this.watchersPerVideo.set(videoId, watchers) | ||
145 | } | ||
146 | |||
147 | watchers.push(new Date().getTime()) | ||
148 | } | ||
149 | |||
150 | private getContext () { | ||
151 | return context | ||
152 | } | ||
153 | |||
154 | private abortSession (sessionId: string) { | ||
155 | const session = this.getContext().sessions.get(sessionId) | ||
156 | if (session) { | ||
157 | session.stop() | ||
158 | this.getContext().sessions.delete(sessionId) | ||
159 | } | ||
160 | |||
161 | const muxingSession = this.muxingSessions.get(sessionId) | ||
162 | if (muxingSession) muxingSession.abort() | ||
163 | } | ||
164 | |||
165 | private async handleSession (sessionId: string, streamPath: string, streamKey: string) { | ||
166 | const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) | ||
167 | if (!videoLive) { | ||
168 | logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId)) | ||
169 | return this.abortSession(sessionId) | ||
170 | } | ||
171 | |||
172 | const video = videoLive.Video | ||
173 | if (video.isBlacklisted()) { | ||
174 | logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid)) | ||
175 | return this.abortSession(sessionId) | ||
176 | } | ||
177 | |||
178 | // Cleanup old potential live files (could happen with a permanent live) | ||
179 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) | ||
180 | |||
181 | const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) | ||
182 | if (oldStreamingPlaylist) { | ||
183 | await cleanupLive(video, oldStreamingPlaylist) | ||
184 | } | ||
185 | |||
186 | this.videoSessions.set(video.id, sessionId) | ||
187 | |||
188 | const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath | ||
189 | |||
190 | const [ { videoFileResolution }, fps ] = await Promise.all([ | ||
191 | getVideoFileResolution(rtmpUrl), | ||
192 | getVideoFileFPS(rtmpUrl) | ||
193 | ]) | ||
194 | |||
195 | const allResolutions = this.buildAllResolutionsToTranscode(videoFileResolution) | ||
196 | |||
197 | logger.info( | ||
198 | 'Will mux/transcode live video of original resolution %d.', videoFileResolution, | ||
199 | { allResolutions, ...lTags(sessionId, video.uuid) } | ||
200 | ) | ||
201 | |||
202 | const streamingPlaylist = await this.createLivePlaylist(video, allResolutions) | ||
203 | |||
204 | return this.runMuxingSession({ | ||
205 | sessionId, | ||
206 | videoLive, | ||
207 | streamingPlaylist, | ||
208 | rtmpUrl, | ||
209 | fps, | ||
210 | allResolutions | ||
211 | }) | ||
212 | } | ||
213 | |||
214 | private async runMuxingSession (options: { | ||
215 | sessionId: string | ||
216 | videoLive: MVideoLiveVideo | ||
217 | streamingPlaylist: MStreamingPlaylistVideo | ||
218 | rtmpUrl: string | ||
219 | fps: number | ||
220 | allResolutions: number[] | ||
221 | }) { | ||
222 | const { sessionId, videoLive, streamingPlaylist, allResolutions, fps, rtmpUrl } = options | ||
223 | const videoUUID = videoLive.Video.uuid | ||
224 | const localLTags = lTags(sessionId, videoUUID) | ||
225 | |||
226 | const user = await UserModel.loadByLiveId(videoLive.id) | ||
227 | LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id) | ||
228 | |||
229 | const muxingSession = new MuxingSession({ | ||
230 | context: this.getContext(), | ||
231 | user, | ||
232 | sessionId, | ||
233 | videoLive, | ||
234 | streamingPlaylist, | ||
235 | rtmpUrl, | ||
236 | fps, | ||
237 | allResolutions | ||
238 | }) | ||
239 | |||
240 | muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) | ||
241 | |||
242 | muxingSession.on('bad-socket-health', ({ videoId }) => { | ||
243 | logger.error( | ||
244 | 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + | ||
245 | ' Stopping session of video %s.', videoUUID, | ||
246 | localLTags | ||
247 | ) | ||
248 | |||
249 | this.stopSessionOf(videoId) | ||
250 | }) | ||
251 | |||
252 | muxingSession.on('duration-exceeded', ({ videoId }) => { | ||
253 | logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) | ||
254 | |||
255 | this.stopSessionOf(videoId) | ||
256 | }) | ||
257 | |||
258 | muxingSession.on('quota-exceeded', ({ videoId }) => { | ||
259 | logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) | ||
260 | |||
261 | this.stopSessionOf(videoId) | ||
262 | }) | ||
263 | |||
264 | muxingSession.on('ffmpeg-error', ({ sessionId }) => this.abortSession(sessionId)) | ||
265 | muxingSession.on('ffmpeg-end', ({ videoId }) => { | ||
266 | this.onMuxingFFmpegEnd(videoId) | ||
267 | }) | ||
268 | |||
269 | muxingSession.on('after-cleanup', ({ videoId }) => { | ||
270 | this.muxingSessions.delete(sessionId) | ||
271 | |||
272 | return this.onAfterMuxingCleanup(videoId) | ||
273 | .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) | ||
274 | }) | ||
275 | |||
276 | this.muxingSessions.set(sessionId, muxingSession) | ||
277 | |||
278 | muxingSession.runMuxing() | ||
279 | .catch(err => { | ||
280 | logger.error('Cannot run muxing.', { err, ...localLTags }) | ||
281 | this.abortSession(sessionId) | ||
282 | }) | ||
283 | } | ||
284 | |||
285 | private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) { | ||
286 | const videoId = live.videoId | ||
287 | |||
288 | try { | ||
289 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
290 | |||
291 | logger.info('Will publish and federate live %s.', video.url, localLTags) | ||
292 | |||
293 | video.state = VideoState.PUBLISHED | ||
294 | await video.save() | ||
295 | |||
296 | live.Video = video | ||
297 | |||
298 | setTimeout(() => { | ||
299 | federateVideoIfNeeded(video, false) | ||
300 | .catch(err => logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags })) | ||
301 | |||
302 | PeerTubeSocket.Instance.sendVideoLiveNewState(video) | ||
303 | }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) | ||
304 | } catch (err) { | ||
305 | logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags }) | ||
306 | } | ||
307 | } | ||
308 | |||
309 | private onMuxingFFmpegEnd (videoId: number) { | ||
310 | this.watchersPerVideo.delete(videoId) | ||
311 | this.videoSessions.delete(videoId) | ||
312 | } | ||
313 | |||
314 | private async onAfterMuxingCleanup (videoUUID: string, cleanupNow = false) { | ||
315 | try { | ||
316 | const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoUUID) | ||
317 | if (!fullVideo) return | ||
318 | |||
319 | const live = await VideoLiveModel.loadByVideoId(fullVideo.id) | ||
320 | |||
321 | if (!live.permanentLive) { | ||
322 | JobQueue.Instance.createJob({ | ||
323 | type: 'video-live-ending', | ||
324 | payload: { | ||
325 | videoId: fullVideo.id | ||
326 | } | ||
327 | }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY }) | ||
328 | |||
329 | fullVideo.state = VideoState.LIVE_ENDED | ||
330 | } else { | ||
331 | fullVideo.state = VideoState.WAITING_FOR_LIVE | ||
332 | } | ||
333 | |||
334 | await fullVideo.save() | ||
335 | |||
336 | PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) | ||
337 | |||
338 | await federateVideoIfNeeded(fullVideo, false) | ||
339 | } catch (err) { | ||
340 | logger.error('Cannot save/federate new video state of live streaming of video %d.', videoUUID, { err, ...lTags(videoUUID) }) | ||
341 | } | ||
342 | } | ||
343 | |||
344 | private async updateLiveViews () { | ||
345 | if (!this.isRunning()) return | ||
346 | |||
347 | if (!isTestInstance()) logger.info('Updating live video views.', lTags()) | ||
348 | |||
349 | for (const videoId of this.watchersPerVideo.keys()) { | ||
350 | const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE | ||
351 | |||
352 | const watchers = this.watchersPerVideo.get(videoId) | ||
353 | |||
354 | const numWatchers = watchers.length | ||
355 | |||
356 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
357 | video.views = numWatchers | ||
358 | await video.save() | ||
359 | |||
360 | await federateVideoIfNeeded(video, false) | ||
361 | |||
362 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video) | ||
363 | |||
364 | // Only keep not expired watchers | ||
365 | const newWatchers = watchers.filter(w => w > notBefore) | ||
366 | this.watchersPerVideo.set(videoId, newWatchers) | ||
367 | |||
368 | logger.debug('New live video views for %s is %d.', video.url, numWatchers, lTags()) | ||
369 | } | ||
370 | } | ||
371 | |||
372 | private async handleBrokenLives () { | ||
373 | const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() | ||
374 | |||
375 | for (const uuid of videoUUIDs) { | ||
376 | await this.onAfterMuxingCleanup(uuid, true) | ||
377 | } | ||
378 | } | ||
379 | |||
380 | private buildAllResolutionsToTranscode (originResolution: number) { | ||
381 | const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED | ||
382 | ? computeResolutionsToTranscode(originResolution, 'live') | ||
383 | : [] | ||
384 | |||
385 | return resolutionsEnabled.concat([ originResolution ]) | ||
386 | } | ||
387 | |||
388 | private async createLivePlaylist (video: MVideo, allResolutions: number[]) { | ||
389 | const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) | ||
390 | const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ | ||
391 | videoId: video.id, | ||
392 | playlistUrl, | ||
393 | segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), | ||
394 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions), | ||
395 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
396 | |||
397 | type: VideoStreamingPlaylistType.HLS | ||
398 | }, { returning: true }) as [ MStreamingPlaylist, boolean ] | ||
399 | |||
400 | return Object.assign(videoStreamingPlaylist, { Video: video }) | ||
401 | } | ||
402 | |||
403 | static get Instance () { | ||
404 | return this.instance || (this.instance = new this()) | ||
405 | } | ||
406 | } | ||
407 | |||
408 | // --------------------------------------------------------------------------- | ||
409 | |||
410 | export { | ||
411 | LiveManager | ||
412 | } | ||
diff --git a/server/lib/live/live-quota-store.ts b/server/lib/live/live-quota-store.ts new file mode 100644 index 000000000..8ceccde98 --- /dev/null +++ b/server/lib/live/live-quota-store.ts | |||
@@ -0,0 +1,48 @@ | |||
1 | class LiveQuotaStore { | ||
2 | |||
3 | private static instance: LiveQuotaStore | ||
4 | |||
5 | private readonly livesPerUser = new Map<number, { liveId: number, size: number }[]>() | ||
6 | |||
7 | private constructor () { | ||
8 | } | ||
9 | |||
10 | addNewLive (userId: number, liveId: number) { | ||
11 | if (!this.livesPerUser.has(userId)) { | ||
12 | this.livesPerUser.set(userId, []) | ||
13 | } | ||
14 | |||
15 | const currentUserLive = { liveId, size: 0 } | ||
16 | const livesOfUser = this.livesPerUser.get(userId) | ||
17 | livesOfUser.push(currentUserLive) | ||
18 | } | ||
19 | |||
20 | removeLive (userId: number, liveId: number) { | ||
21 | const newLivesPerUser = this.livesPerUser.get(userId) | ||
22 | .filter(o => o.liveId !== liveId) | ||
23 | |||
24 | this.livesPerUser.set(userId, newLivesPerUser) | ||
25 | } | ||
26 | |||
27 | addQuotaTo (userId: number, liveId: number, size: number) { | ||
28 | const lives = this.livesPerUser.get(userId) | ||
29 | const live = lives.find(l => l.liveId === liveId) | ||
30 | |||
31 | live.size += size | ||
32 | } | ||
33 | |||
34 | getLiveQuotaOf (userId: number) { | ||
35 | const currentLives = this.livesPerUser.get(userId) | ||
36 | if (!currentLives) return 0 | ||
37 | |||
38 | return currentLives.reduce((sum, obj) => sum + obj.size, 0) | ||
39 | } | ||
40 | |||
41 | static get Instance () { | ||
42 | return this.instance || (this.instance = new this()) | ||
43 | } | ||
44 | } | ||
45 | |||
46 | export { | ||
47 | LiveQuotaStore | ||
48 | } | ||
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts new file mode 100644 index 000000000..4af6f3ebf --- /dev/null +++ b/server/lib/live/live-segment-sha-store.ts | |||
@@ -0,0 +1,64 @@ | |||
1 | import { basename } from 'path' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { buildSha256Segment } from '../hls' | ||
4 | |||
5 | const lTags = loggerTagsFactory('live') | ||
6 | |||
7 | class LiveSegmentShaStore { | ||
8 | |||
9 | private static instance: LiveSegmentShaStore | ||
10 | |||
11 | private readonly segmentsSha256 = new Map<string, Map<string, string>>() | ||
12 | |||
13 | private constructor () { | ||
14 | } | ||
15 | |||
16 | getSegmentsSha256 (videoUUID: string) { | ||
17 | return this.segmentsSha256.get(videoUUID) | ||
18 | } | ||
19 | |||
20 | async addSegmentSha (videoUUID: string, segmentPath: string) { | ||
21 | const segmentName = basename(segmentPath) | ||
22 | logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID)) | ||
23 | |||
24 | const shaResult = await buildSha256Segment(segmentPath) | ||
25 | |||
26 | if (!this.segmentsSha256.has(videoUUID)) { | ||
27 | this.segmentsSha256.set(videoUUID, new Map()) | ||
28 | } | ||
29 | |||
30 | const filesMap = this.segmentsSha256.get(videoUUID) | ||
31 | filesMap.set(segmentName, shaResult) | ||
32 | } | ||
33 | |||
34 | removeSegmentSha (videoUUID: string, segmentPath: string) { | ||
35 | const segmentName = basename(segmentPath) | ||
36 | |||
37 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID)) | ||
38 | |||
39 | const filesMap = this.segmentsSha256.get(videoUUID) | ||
40 | if (!filesMap) { | ||
41 | logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID)) | ||
42 | return | ||
43 | } | ||
44 | |||
45 | if (!filesMap.has(segmentName)) { | ||
46 | logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID)) | ||
47 | return | ||
48 | } | ||
49 | |||
50 | filesMap.delete(segmentName) | ||
51 | } | ||
52 | |||
53 | cleanupShaSegments (videoUUID: string) { | ||
54 | this.segmentsSha256.delete(videoUUID) | ||
55 | } | ||
56 | |||
57 | static get Instance () { | ||
58 | return this.instance || (this.instance = new this()) | ||
59 | } | ||
60 | } | ||
61 | |||
62 | export { | ||
63 | LiveSegmentShaStore | ||
64 | } | ||
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts new file mode 100644 index 000000000..e4526c7a5 --- /dev/null +++ b/server/lib/live/live-utils.ts | |||
@@ -0,0 +1,23 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { basename } from 'path' | ||
3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | ||
4 | import { getHLSDirectory } from '../video-paths' | ||
5 | |||
6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | ||
7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | ||
8 | |||
9 | return 'concat-' + num[1] + '.ts' | ||
10 | } | ||
11 | |||
12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | ||
13 | const hlsDirectory = getHLSDirectory(video) | ||
14 | |||
15 | await remove(hlsDirectory) | ||
16 | |||
17 | await streamingPlaylist.destroy() | ||
18 | } | ||
19 | |||
20 | export { | ||
21 | cleanupLive, | ||
22 | buildConcatenatedName | ||
23 | } | ||
diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts new file mode 100644 index 000000000..c4d1b59ec --- /dev/null +++ b/server/lib/live/shared/index.ts | |||
@@ -0,0 +1 @@ | |||
export * from './muxing-session' | |||
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts new file mode 100644 index 000000000..96f6c2c89 --- /dev/null +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -0,0 +1,341 @@ | |||
1 | |||
2 | import * as Bluebird from 'bluebird' | ||
3 | import * as chokidar from 'chokidar' | ||
4 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
5 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' | ||
6 | import { basename, join } from 'path' | ||
7 | import { EventEmitter } from 'stream' | ||
8 | import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils' | ||
9 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | ||
10 | import { CONFIG } from '@server/initializers/config' | ||
11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | ||
12 | import { VideoFileModel } from '@server/models/video/video-file' | ||
13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | ||
14 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' | ||
15 | import { isAbleToUploadVideo } from '../../user' | ||
16 | import { getHLSDirectory } from '../../video-paths' | ||
17 | import { LiveQuotaStore } from '../live-quota-store' | ||
18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | ||
19 | import { buildConcatenatedName } from '../live-utils' | ||
20 | |||
21 | import memoizee = require('memoizee') | ||
22 | |||
23 | interface MuxingSessionEvents { | ||
24 | 'master-playlist-created': ({ videoId: number }) => void | ||
25 | |||
26 | 'bad-socket-health': ({ videoId: number }) => void | ||
27 | 'duration-exceeded': ({ videoId: number }) => void | ||
28 | 'quota-exceeded': ({ videoId: number }) => void | ||
29 | |||
30 | 'ffmpeg-end': ({ videoId: number }) => void | ||
31 | 'ffmpeg-error': ({ sessionId: string }) => void | ||
32 | |||
33 | 'after-cleanup': ({ videoId: number }) => void | ||
34 | } | ||
35 | |||
36 | declare interface MuxingSession { | ||
37 | on<U extends keyof MuxingSessionEvents>( | ||
38 | event: U, listener: MuxingSessionEvents[U] | ||
39 | ): this | ||
40 | |||
41 | emit<U extends keyof MuxingSessionEvents>( | ||
42 | event: U, ...args: Parameters<MuxingSessionEvents[U]> | ||
43 | ): boolean | ||
44 | } | ||
45 | |||
46 | class MuxingSession extends EventEmitter { | ||
47 | |||
48 | private ffmpegCommand: FfmpegCommand | ||
49 | |||
50 | private readonly context: any | ||
51 | private readonly user: MUserId | ||
52 | private readonly sessionId: string | ||
53 | private readonly videoLive: MVideoLiveVideo | ||
54 | private readonly streamingPlaylist: MStreamingPlaylistVideo | ||
55 | private readonly rtmpUrl: string | ||
56 | private readonly fps: number | ||
57 | private readonly allResolutions: number[] | ||
58 | |||
59 | private readonly videoId: number | ||
60 | private readonly videoUUID: string | ||
61 | private readonly saveReplay: boolean | ||
62 | |||
63 | private readonly lTags: LoggerTagsFn | ||
64 | |||
65 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} | ||
66 | |||
67 | private tsWatcher: chokidar.FSWatcher | ||
68 | private masterWatcher: chokidar.FSWatcher | ||
69 | |||
70 | private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { | ||
71 | return isAbleToUploadVideo(userId, 1000) | ||
72 | }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) | ||
73 | |||
74 | private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { | ||
75 | return this.hasClientSocketInBadHealth(sessionId) | ||
76 | }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) | ||
77 | |||
78 | constructor (options: { | ||
79 | context: any | ||
80 | user: MUserId | ||
81 | sessionId: string | ||
82 | videoLive: MVideoLiveVideo | ||
83 | streamingPlaylist: MStreamingPlaylistVideo | ||
84 | rtmpUrl: string | ||
85 | fps: number | ||
86 | allResolutions: number[] | ||
87 | }) { | ||
88 | super() | ||
89 | |||
90 | this.context = options.context | ||
91 | this.user = options.user | ||
92 | this.sessionId = options.sessionId | ||
93 | this.videoLive = options.videoLive | ||
94 | this.streamingPlaylist = options.streamingPlaylist | ||
95 | this.rtmpUrl = options.rtmpUrl | ||
96 | this.fps = options.fps | ||
97 | this.allResolutions = options.allResolutions | ||
98 | |||
99 | this.videoId = this.videoLive.Video.id | ||
100 | this.videoUUID = this.videoLive.Video.uuid | ||
101 | |||
102 | this.saveReplay = this.videoLive.saveReplay | ||
103 | |||
104 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) | ||
105 | } | ||
106 | |||
107 | async runMuxing () { | ||
108 | this.createFiles() | ||
109 | |||
110 | const outPath = await this.prepareDirectories() | ||
111 | |||
112 | this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED | ||
113 | ? await getLiveTranscodingCommand({ | ||
114 | rtmpUrl: this.rtmpUrl, | ||
115 | outPath, | ||
116 | resolutions: this.allResolutions, | ||
117 | fps: this.fps, | ||
118 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
119 | profile: CONFIG.LIVE.TRANSCODING.PROFILE | ||
120 | }) | ||
121 | : getLiveMuxingCommand(this.rtmpUrl, outPath) | ||
122 | |||
123 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags) | ||
124 | |||
125 | this.watchTSFiles(outPath) | ||
126 | this.watchMasterFile(outPath) | ||
127 | |||
128 | this.ffmpegCommand.on('error', (err, stdout, stderr) => { | ||
129 | this.onFFmpegError(err, stdout, stderr, outPath) | ||
130 | }) | ||
131 | |||
132 | this.ffmpegCommand.on('end', () => this.onFFmpegEnded(outPath)) | ||
133 | |||
134 | this.ffmpegCommand.run() | ||
135 | } | ||
136 | |||
137 | abort () { | ||
138 | if (!this.ffmpegCommand) return false | ||
139 | |||
140 | this.ffmpegCommand.kill('SIGINT') | ||
141 | return true | ||
142 | } | ||
143 | |||
144 | private onFFmpegError (err: any, stdout: string, stderr: string, outPath: string) { | ||
145 | this.onFFmpegEnded(outPath) | ||
146 | |||
147 | // Don't care that we killed the ffmpeg process | ||
148 | if (err?.message?.includes('Exiting normally')) return | ||
149 | |||
150 | logger.error('Live transcoding error.', { err, stdout, stderr, ...this.lTags }) | ||
151 | |||
152 | this.emit('ffmpeg-error', ({ sessionId: this.sessionId })) | ||
153 | } | ||
154 | |||
155 | private onFFmpegEnded (outPath: string) { | ||
156 | logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.rtmpUrl, this.lTags) | ||
157 | |||
158 | setTimeout(() => { | ||
159 | // Wait latest segments generation, and close watchers | ||
160 | |||
161 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ]) | ||
162 | .then(() => { | ||
163 | // Process remaining segments hash | ||
164 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | ||
165 | this.processSegments(outPath, this.segmentsToProcessPerPlaylist[key]) | ||
166 | } | ||
167 | }) | ||
168 | .catch(err => { | ||
169 | logger.error( | ||
170 | 'Cannot close watchers of %s or process remaining hash segments.', outPath, | ||
171 | { err, ...this.lTags } | ||
172 | ) | ||
173 | }) | ||
174 | |||
175 | this.emit('after-cleanup', { videoId: this.videoId }) | ||
176 | }, 1000) | ||
177 | } | ||
178 | |||
179 | private watchMasterFile (outPath: string) { | ||
180 | this.masterWatcher = chokidar.watch(outPath + '/master.m3u8') | ||
181 | |||
182 | this.masterWatcher.on('add', async () => { | ||
183 | this.emit('master-playlist-created', { videoId: this.videoId }) | ||
184 | |||
185 | this.masterWatcher.close() | ||
186 | .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags })) | ||
187 | }) | ||
188 | } | ||
189 | |||
190 | private watchTSFiles (outPath: string) { | ||
191 | const startStreamDateTime = new Date().getTime() | ||
192 | |||
193 | this.tsWatcher = chokidar.watch(outPath + '/*.ts') | ||
194 | |||
195 | const playlistIdMatcher = /^([\d+])-/ | ||
196 | |||
197 | const addHandler = async segmentPath => { | ||
198 | logger.debug('Live add handler of %s.', segmentPath, this.lTags) | ||
199 | |||
200 | const playlistId = basename(segmentPath).match(playlistIdMatcher)[0] | ||
201 | |||
202 | const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || [] | ||
203 | this.processSegments(outPath, segmentsToProcess) | ||
204 | |||
205 | this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] | ||
206 | |||
207 | if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { | ||
208 | this.emit('bad-socket-health', { videoId: this.videoId }) | ||
209 | return | ||
210 | } | ||
211 | |||
212 | // Duration constraint check | ||
213 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { | ||
214 | this.emit('duration-exceeded', { videoId: this.videoId }) | ||
215 | return | ||
216 | } | ||
217 | |||
218 | // Check user quota if the user enabled replay saving | ||
219 | if (await this.isQuotaExceeded(segmentPath) === true) { | ||
220 | this.emit('quota-exceeded', { videoId: this.videoId }) | ||
221 | } | ||
222 | } | ||
223 | |||
224 | const deleteHandler = segmentPath => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath) | ||
225 | |||
226 | this.tsWatcher.on('add', p => addHandler(p)) | ||
227 | this.tsWatcher.on('unlink', p => deleteHandler(p)) | ||
228 | } | ||
229 | |||
230 | private async isQuotaExceeded (segmentPath: string) { | ||
231 | if (this.saveReplay !== true) return false | ||
232 | |||
233 | try { | ||
234 | const segmentStat = await stat(segmentPath) | ||
235 | |||
236 | LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.videoLive.id, segmentStat.size) | ||
237 | |||
238 | const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id) | ||
239 | |||
240 | return canUpload !== true | ||
241 | } catch (err) { | ||
242 | logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags }) | ||
243 | } | ||
244 | } | ||
245 | |||
246 | private createFiles () { | ||
247 | for (let i = 0; i < this.allResolutions.length; i++) { | ||
248 | const resolution = this.allResolutions[i] | ||
249 | |||
250 | const file = new VideoFileModel({ | ||
251 | resolution, | ||
252 | size: -1, | ||
253 | extname: '.ts', | ||
254 | infoHash: null, | ||
255 | fps: this.fps, | ||
256 | videoStreamingPlaylistId: this.streamingPlaylist.id | ||
257 | }) | ||
258 | |||
259 | VideoFileModel.customUpsert(file, 'streaming-playlist', null) | ||
260 | .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags })) | ||
261 | } | ||
262 | } | ||
263 | |||
264 | private async prepareDirectories () { | ||
265 | const outPath = getHLSDirectory(this.videoLive.Video) | ||
266 | await ensureDir(outPath) | ||
267 | |||
268 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) | ||
269 | |||
270 | if (this.videoLive.saveReplay === true) { | ||
271 | await ensureDir(replayDirectory) | ||
272 | } | ||
273 | |||
274 | return outPath | ||
275 | } | ||
276 | |||
277 | private isDurationConstraintValid (streamingStartTime: number) { | ||
278 | const maxDuration = CONFIG.LIVE.MAX_DURATION | ||
279 | // No limit | ||
280 | if (maxDuration < 0) return true | ||
281 | |||
282 | const now = new Date().getTime() | ||
283 | const max = streamingStartTime + maxDuration | ||
284 | |||
285 | return now <= max | ||
286 | } | ||
287 | |||
288 | private processSegments (hlsVideoPath: string, segmentPaths: string[]) { | ||
289 | Bluebird.mapSeries(segmentPaths, async previousSegment => { | ||
290 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | ||
291 | await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment) | ||
292 | |||
293 | if (this.saveReplay) { | ||
294 | await this.addSegmentToReplay(hlsVideoPath, previousSegment) | ||
295 | } | ||
296 | }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags })) | ||
297 | } | ||
298 | |||
299 | private hasClientSocketInBadHealth (sessionId: string) { | ||
300 | const rtmpSession = this.context.sessions.get(sessionId) | ||
301 | |||
302 | if (!rtmpSession) { | ||
303 | logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags) | ||
304 | return | ||
305 | } | ||
306 | |||
307 | for (const playerSessionId of rtmpSession.players) { | ||
308 | const playerSession = this.context.sessions.get(playerSessionId) | ||
309 | |||
310 | if (!playerSession) { | ||
311 | logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags) | ||
312 | continue | ||
313 | } | ||
314 | |||
315 | if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { | ||
316 | return true | ||
317 | } | ||
318 | } | ||
319 | |||
320 | return false | ||
321 | } | ||
322 | |||
323 | private async addSegmentToReplay (hlsVideoPath: string, segmentPath: string) { | ||
324 | const segmentName = basename(segmentPath) | ||
325 | const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, buildConcatenatedName(segmentName)) | ||
326 | |||
327 | try { | ||
328 | const data = await readFile(segmentPath) | ||
329 | |||
330 | await appendFile(dest, data) | ||
331 | } catch (err) { | ||
332 | logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags }) | ||
333 | } | ||
334 | } | ||
335 | } | ||
336 | |||
337 | // --------------------------------------------------------------------------- | ||
338 | |||
339 | export { | ||
340 | MuxingSession | ||
341 | } | ||