aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-06-16 15:14:41 +0200
committerChocobozzz <me@florianbigard.com>2021-06-17 08:47:03 +0200
commit8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d (patch)
treeeee3ef972943be629519c2dc97928878dbd2d732 /server/lib
parentfd6584844b070cfc283902aba26f2853473e83d1 (diff)
downloadPeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.tar.gz
PeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.tar.zst
PeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.zip
Refactor live manager
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/process/process-view.ts2
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts19
-rw-r--r--server/lib/live-manager.ts633
-rw-r--r--server/lib/live/index.ts4
-rw-r--r--server/lib/live/live-manager.ts412
-rw-r--r--server/lib/live/live-quota-store.ts48
-rw-r--r--server/lib/live/live-segment-sha-store.ts64
-rw-r--r--server/lib/live/live-utils.ts23
-rw-r--r--server/lib/live/shared/index.ts1
-rw-r--r--server/lib/live/shared/muxing-session.ts341
-rw-r--r--server/lib/user.ts12
-rw-r--r--server/lib/video-blacklist.ts2
12 files changed, 906 insertions, 655 deletions
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
index 0a0231a3a..5593ee257 100644
--- a/server/lib/activitypub/process/process-view.ts
+++ b/server/lib/activitypub/process/process-view.ts
@@ -4,7 +4,7 @@ import { Redis } from '../../redis'
4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' 4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub'
5import { APProcessorOptions } from '../../../types/activitypub-processor.model' 5import { APProcessorOptions } from '../../../types/activitypub-processor.model'
6import { MActorSignature } from '../../../types/models' 6import { MActorSignature } from '../../../types/models'
7import { LiveManager } from '@server/lib/live-manager' 7import { LiveManager } from '@server/lib/live/live-manager'
8 8
9async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) { 9async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) {
10 const { activity, byActor } = options 10 const { activity, byActor } = options
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 517b90abc..9eba41bf8 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -3,7 +3,7 @@ import { pathExists, readdir, remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { VIDEO_LIVE } from '@server/initializers/constants' 5import { VIDEO_LIVE } from '@server/initializers/constants'
6import { LiveManager } from '@server/lib/live-manager' 6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live'
7import { generateVideoMiniature } from '@server/lib/thumbnail' 7import { generateVideoMiniature } from '@server/lib/thumbnail'
8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' 8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
9import { publishAndFederateIfNeeded } from '@server/lib/video' 9import { publishAndFederateIfNeeded } from '@server/lib/video'
@@ -12,7 +12,7 @@ import { VideoModel } from '@server/models/video/video'
12import { VideoFileModel } from '@server/models/video/video-file' 12import { VideoFileModel } from '@server/models/video/video-file'
13import { VideoLiveModel } from '@server/models/video/video-live' 13import { VideoLiveModel } from '@server/models/video/video-live'
14import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 14import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
15import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models' 15import { MVideo, MVideoLive } from '@server/types/models'
16import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 16import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
17import { logger } from '../../../helpers/logger' 17import { logger } from '../../../helpers/logger'
18 18
@@ -37,7 +37,7 @@ async function processVideoLiveEnding (job: Bull.Job) {
37 return 37 return
38 } 38 }
39 39
40 LiveManager.Instance.cleanupShaSegments(video.uuid) 40 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid)
41 41
42 if (live.saveReplay !== true) { 42 if (live.saveReplay !== true) {
43 return cleanupLive(video, streamingPlaylist) 43 return cleanupLive(video, streamingPlaylist)
@@ -46,19 +46,10 @@ async function processVideoLiveEnding (job: Bull.Job) {
46 return saveLive(video, live) 46 return saveLive(video, live)
47} 47}
48 48
49async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
50 const hlsDirectory = getHLSDirectory(video)
51
52 await remove(hlsDirectory)
53
54 await streamingPlaylist.destroy()
55}
56
57// --------------------------------------------------------------------------- 49// ---------------------------------------------------------------------------
58 50
59export { 51export {
60 processVideoLiveEnding, 52 processVideoLiveEnding
61 cleanupLive
62} 53}
63 54
64// --------------------------------------------------------------------------- 55// ---------------------------------------------------------------------------
@@ -94,7 +85,7 @@ async function saveLive (video: MVideo, live: MVideoLive) {
94 let durationDone = false 85 let durationDone = false
95 86
96 for (const playlistFile of playlistFiles) { 87 for (const playlistFile of playlistFiles) {
97 const concatenatedTsFile = LiveManager.Instance.buildConcatenatedName(playlistFile) 88 const concatenatedTsFile = buildConcatenatedName(playlistFile)
98 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) 89 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
99 90
100 const probe = await ffprobePromise(concatenatedTsFilePath) 91 const probe = await ffprobePromise(concatenatedTsFilePath)
diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts
deleted file mode 100644
index 563ba2578..000000000
--- a/server/lib/live-manager.ts
+++ /dev/null
@@ -1,633 +0,0 @@
1
2import * as Bluebird from 'bluebird'
3import * as chokidar from 'chokidar'
4import { FfmpegCommand } from 'fluent-ffmpeg'
5import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
6import { createServer, Server } from 'net'
7import { basename, join } from 'path'
8import { isTestInstance } from '@server/helpers/core-utils'
9import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils'
10import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
11import { logger, loggerTagsFactory } from '@server/helpers/logger'
12import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
13import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
14import { UserModel } from '@server/models/user/user'
15import { VideoModel } from '@server/models/video/video'
16import { VideoFileModel } from '@server/models/video/video-file'
17import { VideoLiveModel } from '@server/models/video/video-live'
18import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
19import { MStreamingPlaylist, MStreamingPlaylistVideo, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models'
20import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
21import { federateVideoIfNeeded } from './activitypub/videos'
22import { buildSha256Segment } from './hls'
23import { JobQueue } from './job-queue'
24import { cleanupLive } from './job-queue/handlers/video-live-ending'
25import { PeerTubeSocket } from './peertube-socket'
26import { VideoTranscodingProfilesManager } from './transcoding/video-transcoding-profiles'
27import { isAbleToUploadVideo } from './user'
28import { getHLSDirectory } from './video-paths'
29
30import memoizee = require('memoizee')
31const NodeRtmpSession = require('node-media-server/node_rtmp_session')
32const context = require('node-media-server/node_core_ctx')
33const nodeMediaServerLogger = require('node-media-server/node_core_logger')
34
35// Disable node media server logs
36nodeMediaServerLogger.setLogType(0)
37
38const config = {
39 rtmp: {
40 port: CONFIG.LIVE.RTMP.PORT,
41 chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE,
42 gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE,
43 ping: VIDEO_LIVE.RTMP.PING,
44 ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT
45 },
46 transcoding: {
47 ffmpeg: 'ffmpeg'
48 }
49}
50
51const lTags = loggerTagsFactory('live')
52
53class LiveManager {
54
55 private static instance: LiveManager
56
57 private readonly transSessions = new Map<string, FfmpegCommand>()
58 private readonly videoSessions = new Map<number, string>()
59 // Values are Date().getTime()
60 private readonly watchersPerVideo = new Map<number, number[]>()
61 private readonly segmentsSha256 = new Map<string, Map<string, string>>()
62 private readonly livesPerUser = new Map<number, { liveId: number, videoId: number, size: number }[]>()
63
64 private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
65 return isAbleToUploadVideo(userId, 1000)
66 }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
67
68 private readonly hasClientSocketsInBadHealthWithCache = memoizee((sessionId: string) => {
69 return this.hasClientSocketsInBadHealth(sessionId)
70 }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH })
71
72 private rtmpServer: Server
73
74 private constructor () {
75 }
76
77 init () {
78 const events = this.getContext().nodeEvent
79 events.on('postPublish', (sessionId: string, streamPath: string) => {
80 logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) })
81
82 const splittedPath = streamPath.split('/')
83 if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) {
84 logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) })
85 return this.abortSession(sessionId)
86 }
87
88 this.handleSession(sessionId, streamPath, splittedPath[2])
89 .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) }))
90 })
91
92 events.on('donePublish', sessionId => {
93 logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })
94 })
95
96 registerConfigChangedHandler(() => {
97 if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) {
98 this.run()
99 return
100 }
101
102 if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) {
103 this.stop()
104 }
105 })
106
107 // Cleanup broken lives, that were terminated by a server restart for example
108 this.handleBrokenLives()
109 .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() }))
110
111 setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE)
112 }
113
114 run () {
115 logger.info('Running RTMP server on port %d', config.rtmp.port, lTags())
116
117 this.rtmpServer = createServer(socket => {
118 const session = new NodeRtmpSession(config, socket)
119
120 session.run()
121 })
122
123 this.rtmpServer.on('error', err => {
124 logger.error('Cannot run RTMP server.', { err, ...lTags() })
125 })
126
127 this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT)
128 }
129
130 stop () {
131 logger.info('Stopping RTMP server.', lTags())
132
133 this.rtmpServer.close()
134 this.rtmpServer = undefined
135
136 // Sessions is an object
137 this.getContext().sessions.forEach((session: any) => {
138 if (session instanceof NodeRtmpSession) {
139 session.stop()
140 }
141 })
142 }
143
144 isRunning () {
145 return !!this.rtmpServer
146 }
147
148 getSegmentsSha256 (videoUUID: string) {
149 return this.segmentsSha256.get(videoUUID)
150 }
151
152 stopSessionOf (videoId: number) {
153 const sessionId = this.videoSessions.get(videoId)
154 if (!sessionId) return
155
156 this.videoSessions.delete(videoId)
157 this.abortSession(sessionId)
158 }
159
160 getLiveQuotaUsedByUser (userId: number) {
161 const currentLives = this.livesPerUser.get(userId)
162 if (!currentLives) return 0
163
164 return currentLives.reduce((sum, obj) => sum + obj.size, 0)
165 }
166
167 addViewTo (videoId: number) {
168 if (this.videoSessions.has(videoId) === false) return
169
170 let watchers = this.watchersPerVideo.get(videoId)
171
172 if (!watchers) {
173 watchers = []
174 this.watchersPerVideo.set(videoId, watchers)
175 }
176
177 watchers.push(new Date().getTime())
178 }
179
180 cleanupShaSegments (videoUUID: string) {
181 this.segmentsSha256.delete(videoUUID)
182 }
183
184 addSegmentToReplay (hlsVideoPath: string, segmentPath: string) {
185 const segmentName = basename(segmentPath)
186 const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, this.buildConcatenatedName(segmentName))
187
188 return readFile(segmentPath)
189 .then(data => appendFile(dest, data))
190 .catch(err => logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...lTags() }))
191 }
192
193 buildConcatenatedName (segmentOrPlaylistPath: string) {
194 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
195
196 return 'concat-' + num[1] + '.ts'
197 }
198
199 private processSegments (hlsVideoPath: string, videoUUID: string, videoLive: MVideoLive, segmentPaths: string[]) {
200 Bluebird.mapSeries(segmentPaths, async previousSegment => {
201 // Add sha hash of previous segments, because ffmpeg should have finished generating them
202 await this.addSegmentSha(videoUUID, previousSegment)
203
204 if (videoLive.saveReplay) {
205 await this.addSegmentToReplay(hlsVideoPath, previousSegment)
206 }
207 }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...lTags(videoUUID) }))
208 }
209
210 private getContext () {
211 return context
212 }
213
214 private abortSession (id: string) {
215 const session = this.getContext().sessions.get(id)
216 if (session) {
217 session.stop()
218 this.getContext().sessions.delete(id)
219 }
220
221 const transSession = this.transSessions.get(id)
222 if (transSession) {
223 transSession.kill('SIGINT')
224 this.transSessions.delete(id)
225 }
226 }
227
228 private async handleSession (sessionId: string, streamPath: string, streamKey: string) {
229 const videoLive = await VideoLiveModel.loadByStreamKey(streamKey)
230 if (!videoLive) {
231 logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId))
232 return this.abortSession(sessionId)
233 }
234
235 const video = videoLive.Video
236 if (video.isBlacklisted()) {
237 logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid))
238 return this.abortSession(sessionId)
239 }
240
241 // Cleanup old potential live files (could happen with a permanent live)
242 this.cleanupShaSegments(video.uuid)
243
244 const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
245 if (oldStreamingPlaylist) {
246 await cleanupLive(video, oldStreamingPlaylist)
247 }
248
249 this.videoSessions.set(video.id, sessionId)
250
251 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
252
253 const session = this.getContext().sessions.get(sessionId)
254 const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath
255
256 const [ resolutionResult, fps ] = await Promise.all([
257 getVideoFileResolution(rtmpUrl),
258 getVideoFileFPS(rtmpUrl)
259 ])
260
261 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
262 ? computeResolutionsToTranscode(resolutionResult.videoFileResolution, 'live')
263 : []
264
265 const allResolutions = resolutionsEnabled.concat([ session.videoHeight ])
266
267 logger.info(
268 'Will mux/transcode live video of original resolution %d.', session.videoHeight,
269 { allResolutions, ...lTags(sessionId, video.uuid) }
270 )
271
272 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
273 videoId: video.id,
274 playlistUrl,
275 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
276 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
277 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
278
279 type: VideoStreamingPlaylistType.HLS
280 }, { returning: true }) as [ MStreamingPlaylist, boolean ]
281
282 return this.runMuxing({
283 sessionId,
284 videoLive,
285 playlist: Object.assign(videoStreamingPlaylist, { Video: video }),
286 rtmpUrl,
287 fps,
288 allResolutions
289 })
290 }
291
292 private async runMuxing (options: {
293 sessionId: string
294 videoLive: MVideoLiveVideo
295 playlist: MStreamingPlaylistVideo
296 rtmpUrl: string
297 fps: number
298 allResolutions: number[]
299 }) {
300 const { sessionId, videoLive, playlist, allResolutions, fps, rtmpUrl } = options
301 const startStreamDateTime = new Date().getTime()
302
303 const user = await UserModel.loadByLiveId(videoLive.id)
304 if (!this.livesPerUser.has(user.id)) {
305 this.livesPerUser.set(user.id, [])
306 }
307
308 const currentUserLive = { liveId: videoLive.id, videoId: videoLive.videoId, size: 0 }
309 const livesOfUser = this.livesPerUser.get(user.id)
310 livesOfUser.push(currentUserLive)
311
312 for (let i = 0; i < allResolutions.length; i++) {
313 const resolution = allResolutions[i]
314
315 const file = new VideoFileModel({
316 resolution,
317 size: -1,
318 extname: '.ts',
319 infoHash: null,
320 fps,
321 videoStreamingPlaylistId: playlist.id
322 })
323
324 VideoFileModel.customUpsert(file, 'streaming-playlist', null)
325 .catch(err => logger.error('Cannot create file for live streaming.', { err, ...lTags(sessionId, videoLive.Video.uuid) }))
326 }
327
328 const outPath = getHLSDirectory(videoLive.Video)
329 await ensureDir(outPath)
330
331 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
332
333 if (videoLive.saveReplay === true) {
334 await ensureDir(replayDirectory)
335 }
336
337 const videoUUID = videoLive.Video.uuid
338
339 const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED
340 ? await getLiveTranscodingCommand({
341 rtmpUrl,
342 outPath,
343 resolutions: allResolutions,
344 fps,
345 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
346 profile: CONFIG.LIVE.TRANSCODING.PROFILE
347 })
348 : getLiveMuxingCommand(rtmpUrl, outPath)
349
350 logger.info('Running live muxing/transcoding for %s.', videoUUID, lTags(sessionId, videoUUID))
351 this.transSessions.set(sessionId, ffmpegExec)
352
353 const tsWatcher = chokidar.watch(outPath + '/*.ts')
354
355 const segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
356 const playlistIdMatcher = /^([\d+])-/
357
358 const addHandler = segmentPath => {
359 logger.debug('Live add handler of %s.', segmentPath, lTags(sessionId, videoUUID))
360
361 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
362
363 const segmentsToProcess = segmentsToProcessPerPlaylist[playlistId] || []
364 this.processSegments(outPath, videoUUID, videoLive, segmentsToProcess)
365
366 segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
367
368 if (this.hasClientSocketsInBadHealthWithCache(sessionId)) {
369 logger.error(
370 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
371 ' Stopping session of video %s.', videoUUID,
372 lTags(sessionId, videoUUID)
373 )
374
375 this.stopSessionOf(videoLive.videoId)
376 return
377 }
378
379 // Duration constraint check
380 if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
381 logger.info('Stopping session of %s: max duration exceeded.', videoUUID, lTags(sessionId, videoUUID))
382
383 this.stopSessionOf(videoLive.videoId)
384 return
385 }
386
387 // Check user quota if the user enabled replay saving
388 if (videoLive.saveReplay === true) {
389 stat(segmentPath)
390 .then(segmentStat => {
391 currentUserLive.size += segmentStat.size
392 })
393 .then(() => this.isQuotaConstraintValid(user, videoLive))
394 .then(quotaValid => {
395 if (quotaValid !== true) {
396 logger.info('Stopping session of %s: user quota exceeded.', videoUUID, lTags(sessionId, videoUUID))
397
398 this.stopSessionOf(videoLive.videoId)
399 }
400 })
401 .catch(err => logger.error('Cannot stat %s or check quota of %d.', segmentPath, user.id, { err, ...lTags(sessionId, videoUUID) }))
402 }
403 }
404
405 const deleteHandler = segmentPath => this.removeSegmentSha(videoUUID, segmentPath)
406
407 tsWatcher.on('add', p => addHandler(p))
408 tsWatcher.on('unlink', p => deleteHandler(p))
409
410 const masterWatcher = chokidar.watch(outPath + '/master.m3u8')
411 masterWatcher.on('add', async () => {
412 try {
413 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoLive.videoId)
414
415 video.state = VideoState.PUBLISHED
416 await video.save()
417 videoLive.Video = video
418
419 setTimeout(() => {
420 federateVideoIfNeeded(video, false)
421 .catch(err => logger.error('Cannot federate live video %s.', video.url, { err, ...lTags(sessionId, videoUUID) }))
422
423 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
424 }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION)
425
426 } catch (err) {
427 logger.error('Cannot save/federate live video %d.', videoLive.videoId, { err, ...lTags(sessionId, videoUUID) })
428 } finally {
429 masterWatcher.close()
430 .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...lTags(sessionId, videoUUID) }))
431 }
432 })
433
434 const onFFmpegEnded = () => {
435 logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', rtmpUrl, lTags(sessionId, videoUUID))
436
437 this.transSessions.delete(sessionId)
438
439 this.watchersPerVideo.delete(videoLive.videoId)
440 this.videoSessions.delete(videoLive.videoId)
441
442 const newLivesPerUser = this.livesPerUser.get(user.id)
443 .filter(o => o.liveId !== videoLive.id)
444 this.livesPerUser.set(user.id, newLivesPerUser)
445
446 setTimeout(() => {
447 // Wait latest segments generation, and close watchers
448
449 Promise.all([ tsWatcher.close(), masterWatcher.close() ])
450 .then(() => {
451 // Process remaining segments hash
452 for (const key of Object.keys(segmentsToProcessPerPlaylist)) {
453 this.processSegments(outPath, videoUUID, videoLive, segmentsToProcessPerPlaylist[key])
454 }
455 })
456 .catch(err => {
457 logger.error(
458 'Cannot close watchers of %s or process remaining hash segments.', outPath,
459 { err, ...lTags(sessionId, videoUUID) }
460 )
461 })
462
463 this.onEndTransmuxing(videoLive.Video.id)
464 .catch(err => logger.error('Error in closed transmuxing.', { err, ...lTags(sessionId, videoUUID) }))
465 }, 1000)
466 }
467
468 ffmpegExec.on('error', (err, stdout, stderr) => {
469 onFFmpegEnded()
470
471 // Don't care that we killed the ffmpeg process
472 if (err?.message?.includes('Exiting normally')) return
473
474 logger.error('Live transcoding error.', { err, stdout, stderr, ...lTags(sessionId, videoUUID) })
475
476 this.abortSession(sessionId)
477 })
478
479 ffmpegExec.on('end', () => onFFmpegEnded())
480
481 ffmpegExec.run()
482 }
483
484 private async onEndTransmuxing (videoUUID: string, cleanupNow = false) {
485 try {
486 const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoUUID)
487 if (!fullVideo) return
488
489 const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
490
491 if (!live.permanentLive) {
492 JobQueue.Instance.createJob({
493 type: 'video-live-ending',
494 payload: {
495 videoId: fullVideo.id
496 }
497 }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY })
498
499 fullVideo.state = VideoState.LIVE_ENDED
500 } else {
501 fullVideo.state = VideoState.WAITING_FOR_LIVE
502 }
503
504 await fullVideo.save()
505
506 PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo)
507
508 await federateVideoIfNeeded(fullVideo, false)
509 } catch (err) {
510 logger.error('Cannot save/federate new video state of live streaming of video %d.', videoUUID, { err, ...lTags(videoUUID) })
511 }
512 }
513
514 private async addSegmentSha (videoUUID: string, segmentPath: string) {
515 const segmentName = basename(segmentPath)
516 logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID))
517
518 const shaResult = await buildSha256Segment(segmentPath)
519
520 if (!this.segmentsSha256.has(videoUUID)) {
521 this.segmentsSha256.set(videoUUID, new Map())
522 }
523
524 const filesMap = this.segmentsSha256.get(videoUUID)
525 filesMap.set(segmentName, shaResult)
526 }
527
528 private removeSegmentSha (videoUUID: string, segmentPath: string) {
529 const segmentName = basename(segmentPath)
530
531 logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID))
532
533 const filesMap = this.segmentsSha256.get(videoUUID)
534 if (!filesMap) {
535 logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID))
536 return
537 }
538
539 if (!filesMap.has(segmentName)) {
540 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID))
541 return
542 }
543
544 filesMap.delete(segmentName)
545 }
546
547 private isDurationConstraintValid (streamingStartTime: number) {
548 const maxDuration = CONFIG.LIVE.MAX_DURATION
549 // No limit
550 if (maxDuration < 0) return true
551
552 const now = new Date().getTime()
553 const max = streamingStartTime + maxDuration
554
555 return now <= max
556 }
557
558 private hasClientSocketsInBadHealth (sessionId: string) {
559 const rtmpSession = this.getContext().sessions.get(sessionId)
560
561 if (!rtmpSession) {
562 logger.warn('Cannot get session %s to check players socket health.', sessionId, lTags(sessionId))
563 return
564 }
565
566 for (const playerSessionId of rtmpSession.players) {
567 const playerSession = this.getContext().sessions.get(playerSessionId)
568
569 if (!playerSession) {
570 logger.error('Cannot get player session %s to check socket health.', playerSession, lTags(sessionId))
571 continue
572 }
573
574 if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) {
575 return true
576 }
577 }
578
579 return false
580 }
581
582 private async isQuotaConstraintValid (user: MUserId, live: MVideoLive) {
583 if (live.saveReplay !== true) return true
584
585 return this.isAbleToUploadVideoWithCache(user.id)
586 }
587
588 private async updateLiveViews () {
589 if (!this.isRunning()) return
590
591 if (!isTestInstance()) logger.info('Updating live video views.', lTags())
592
593 for (const videoId of this.watchersPerVideo.keys()) {
594 const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE
595
596 const watchers = this.watchersPerVideo.get(videoId)
597
598 const numWatchers = watchers.length
599
600 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
601 video.views = numWatchers
602 await video.save()
603
604 await federateVideoIfNeeded(video, false)
605
606 PeerTubeSocket.Instance.sendVideoViewsUpdate(video)
607
608 // Only keep not expired watchers
609 const newWatchers = watchers.filter(w => w > notBefore)
610 this.watchersPerVideo.set(videoId, newWatchers)
611
612 logger.debug('New live video views for %s is %d.', video.url, numWatchers, lTags())
613 }
614 }
615
616 private async handleBrokenLives () {
617 const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
618
619 for (const uuid of videoUUIDs) {
620 await this.onEndTransmuxing(uuid, true)
621 }
622 }
623
624 static get Instance () {
625 return this.instance || (this.instance = new this())
626 }
627}
628
629// ---------------------------------------------------------------------------
630
631export {
632 LiveManager
633}
diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts
new file mode 100644
index 000000000..8b46800da
--- /dev/null
+++ b/server/lib/live/index.ts
@@ -0,0 +1,4 @@
1export * from './live-manager'
2export * from './live-quota-store'
3export * from './live-segment-sha-store'
4export * from './live-utils'
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
new file mode 100644
index 000000000..d7199cc89
--- /dev/null
+++ b/server/lib/live/live-manager.ts
@@ -0,0 +1,412 @@
1
2import { createServer, Server } from 'net'
3import { isTestInstance } from '@server/helpers/core-utils'
4import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { logger, loggerTagsFactory } from '@server/helpers/logger'
6import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
8import { UserModel } from '@server/models/user/user'
9import { VideoModel } from '@server/models/video/video'
10import { VideoLiveModel } from '@server/models/video/video-live'
11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
12import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
13import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
14import { federateVideoIfNeeded } from '../activitypub/videos'
15import { JobQueue } from '../job-queue'
16import { PeerTubeSocket } from '../peertube-socket'
17import { LiveQuotaStore } from './live-quota-store'
18import { LiveSegmentShaStore } from './live-segment-sha-store'
19import { cleanupLive } from './live-utils'
20import { MuxingSession } from './shared'
21
22const NodeRtmpSession = require('node-media-server/node_rtmp_session')
23const context = require('node-media-server/node_core_ctx')
24const nodeMediaServerLogger = require('node-media-server/node_core_logger')
25
26// Disable node media server logs
27nodeMediaServerLogger.setLogType(0)
28
29const config = {
30 rtmp: {
31 port: CONFIG.LIVE.RTMP.PORT,
32 chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE,
33 gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE,
34 ping: VIDEO_LIVE.RTMP.PING,
35 ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT
36 },
37 transcoding: {
38 ffmpeg: 'ffmpeg'
39 }
40}
41
42const lTags = loggerTagsFactory('live')
43
44class LiveManager {
45
46 private static instance: LiveManager
47
48 private readonly muxingSessions = new Map<string, MuxingSession>()
49 private readonly videoSessions = new Map<number, string>()
50 // Values are Date().getTime()
51 private readonly watchersPerVideo = new Map<number, number[]>()
52
53 private rtmpServer: Server
54
55 private constructor () {
56 }
57
58 init () {
59 const events = this.getContext().nodeEvent
60 events.on('postPublish', (sessionId: string, streamPath: string) => {
61 logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) })
62
63 const splittedPath = streamPath.split('/')
64 if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) {
65 logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) })
66 return this.abortSession(sessionId)
67 }
68
69 this.handleSession(sessionId, streamPath, splittedPath[2])
70 .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) }))
71 })
72
73 events.on('donePublish', sessionId => {
74 logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })
75 })
76
77 registerConfigChangedHandler(() => {
78 if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) {
79 this.run()
80 return
81 }
82
83 if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) {
84 this.stop()
85 }
86 })
87
88 // Cleanup broken lives, that were terminated by a server restart for example
89 this.handleBrokenLives()
90 .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() }))
91
92 setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE)
93 }
94
95 run () {
96 logger.info('Running RTMP server on port %d', config.rtmp.port, lTags())
97
98 this.rtmpServer = createServer(socket => {
99 const session = new NodeRtmpSession(config, socket)
100
101 session.run()
102 })
103
104 this.rtmpServer.on('error', err => {
105 logger.error('Cannot run RTMP server.', { err, ...lTags() })
106 })
107
108 this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT)
109 }
110
111 stop () {
112 logger.info('Stopping RTMP server.', lTags())
113
114 this.rtmpServer.close()
115 this.rtmpServer = undefined
116
117 // Sessions is an object
118 this.getContext().sessions.forEach((session: any) => {
119 if (session instanceof NodeRtmpSession) {
120 session.stop()
121 }
122 })
123 }
124
125 isRunning () {
126 return !!this.rtmpServer
127 }
128
129 stopSessionOf (videoId: number) {
130 const sessionId = this.videoSessions.get(videoId)
131 if (!sessionId) return
132
133 this.videoSessions.delete(videoId)
134 this.abortSession(sessionId)
135 }
136
137 addViewTo (videoId: number) {
138 if (this.videoSessions.has(videoId) === false) return
139
140 let watchers = this.watchersPerVideo.get(videoId)
141
142 if (!watchers) {
143 watchers = []
144 this.watchersPerVideo.set(videoId, watchers)
145 }
146
147 watchers.push(new Date().getTime())
148 }
149
150 private getContext () {
151 return context
152 }
153
154 private abortSession (sessionId: string) {
155 const session = this.getContext().sessions.get(sessionId)
156 if (session) {
157 session.stop()
158 this.getContext().sessions.delete(sessionId)
159 }
160
161 const muxingSession = this.muxingSessions.get(sessionId)
162 if (muxingSession) muxingSession.abort()
163 }
164
165 private async handleSession (sessionId: string, streamPath: string, streamKey: string) {
166 const videoLive = await VideoLiveModel.loadByStreamKey(streamKey)
167 if (!videoLive) {
168 logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId))
169 return this.abortSession(sessionId)
170 }
171
172 const video = videoLive.Video
173 if (video.isBlacklisted()) {
174 logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid))
175 return this.abortSession(sessionId)
176 }
177
178 // Cleanup old potential live files (could happen with a permanent live)
179 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid)
180
181 const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
182 if (oldStreamingPlaylist) {
183 await cleanupLive(video, oldStreamingPlaylist)
184 }
185
186 this.videoSessions.set(video.id, sessionId)
187
188 const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath
189
190 const [ { videoFileResolution }, fps ] = await Promise.all([
191 getVideoFileResolution(rtmpUrl),
192 getVideoFileFPS(rtmpUrl)
193 ])
194
195 const allResolutions = this.buildAllResolutionsToTranscode(videoFileResolution)
196
197 logger.info(
198 'Will mux/transcode live video of original resolution %d.', videoFileResolution,
199 { allResolutions, ...lTags(sessionId, video.uuid) }
200 )
201
202 const streamingPlaylist = await this.createLivePlaylist(video, allResolutions)
203
204 return this.runMuxingSession({
205 sessionId,
206 videoLive,
207 streamingPlaylist,
208 rtmpUrl,
209 fps,
210 allResolutions
211 })
212 }
213
214 private async runMuxingSession (options: {
215 sessionId: string
216 videoLive: MVideoLiveVideo
217 streamingPlaylist: MStreamingPlaylistVideo
218 rtmpUrl: string
219 fps: number
220 allResolutions: number[]
221 }) {
222 const { sessionId, videoLive, streamingPlaylist, allResolutions, fps, rtmpUrl } = options
223 const videoUUID = videoLive.Video.uuid
224 const localLTags = lTags(sessionId, videoUUID)
225
226 const user = await UserModel.loadByLiveId(videoLive.id)
227 LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id)
228
229 const muxingSession = new MuxingSession({
230 context: this.getContext(),
231 user,
232 sessionId,
233 videoLive,
234 streamingPlaylist,
235 rtmpUrl,
236 fps,
237 allResolutions
238 })
239
240 muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags))
241
242 muxingSession.on('bad-socket-health', ({ videoId }) => {
243 logger.error(
244 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
245 ' Stopping session of video %s.', videoUUID,
246 localLTags
247 )
248
249 this.stopSessionOf(videoId)
250 })
251
252 muxingSession.on('duration-exceeded', ({ videoId }) => {
253 logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags)
254
255 this.stopSessionOf(videoId)
256 })
257
258 muxingSession.on('quota-exceeded', ({ videoId }) => {
259 logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags)
260
261 this.stopSessionOf(videoId)
262 })
263
264 muxingSession.on('ffmpeg-error', ({ sessionId }) => this.abortSession(sessionId))
265 muxingSession.on('ffmpeg-end', ({ videoId }) => {
266 this.onMuxingFFmpegEnd(videoId)
267 })
268
269 muxingSession.on('after-cleanup', ({ videoId }) => {
270 this.muxingSessions.delete(sessionId)
271
272 return this.onAfterMuxingCleanup(videoId)
273 .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags }))
274 })
275
276 this.muxingSessions.set(sessionId, muxingSession)
277
278 muxingSession.runMuxing()
279 .catch(err => {
280 logger.error('Cannot run muxing.', { err, ...localLTags })
281 this.abortSession(sessionId)
282 })
283 }
284
285 private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) {
286 const videoId = live.videoId
287
288 try {
289 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
290
291 logger.info('Will publish and federate live %s.', video.url, localLTags)
292
293 video.state = VideoState.PUBLISHED
294 await video.save()
295
296 live.Video = video
297
298 setTimeout(() => {
299 federateVideoIfNeeded(video, false)
300 .catch(err => logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags }))
301
302 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
303 }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION)
304 } catch (err) {
305 logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags })
306 }
307 }
308
309 private onMuxingFFmpegEnd (videoId: number) {
310 this.watchersPerVideo.delete(videoId)
311 this.videoSessions.delete(videoId)
312 }
313
314 private async onAfterMuxingCleanup (videoUUID: string, cleanupNow = false) {
315 try {
316 const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoUUID)
317 if (!fullVideo) return
318
319 const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
320
321 if (!live.permanentLive) {
322 JobQueue.Instance.createJob({
323 type: 'video-live-ending',
324 payload: {
325 videoId: fullVideo.id
326 }
327 }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY })
328
329 fullVideo.state = VideoState.LIVE_ENDED
330 } else {
331 fullVideo.state = VideoState.WAITING_FOR_LIVE
332 }
333
334 await fullVideo.save()
335
336 PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo)
337
338 await federateVideoIfNeeded(fullVideo, false)
339 } catch (err) {
340 logger.error('Cannot save/federate new video state of live streaming of video %d.', videoUUID, { err, ...lTags(videoUUID) })
341 }
342 }
343
344 private async updateLiveViews () {
345 if (!this.isRunning()) return
346
347 if (!isTestInstance()) logger.info('Updating live video views.', lTags())
348
349 for (const videoId of this.watchersPerVideo.keys()) {
350 const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE
351
352 const watchers = this.watchersPerVideo.get(videoId)
353
354 const numWatchers = watchers.length
355
356 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
357 video.views = numWatchers
358 await video.save()
359
360 await federateVideoIfNeeded(video, false)
361
362 PeerTubeSocket.Instance.sendVideoViewsUpdate(video)
363
364 // Only keep not expired watchers
365 const newWatchers = watchers.filter(w => w > notBefore)
366 this.watchersPerVideo.set(videoId, newWatchers)
367
368 logger.debug('New live video views for %s is %d.', video.url, numWatchers, lTags())
369 }
370 }
371
372 private async handleBrokenLives () {
373 const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
374
375 for (const uuid of videoUUIDs) {
376 await this.onAfterMuxingCleanup(uuid, true)
377 }
378 }
379
380 private buildAllResolutionsToTranscode (originResolution: number) {
381 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
382 ? computeResolutionsToTranscode(originResolution, 'live')
383 : []
384
385 return resolutionsEnabled.concat([ originResolution ])
386 }
387
388 private async createLivePlaylist (video: MVideo, allResolutions: number[]) {
389 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
390 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
391 videoId: video.id,
392 playlistUrl,
393 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
394 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
395 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
396
397 type: VideoStreamingPlaylistType.HLS
398 }, { returning: true }) as [ MStreamingPlaylist, boolean ]
399
400 return Object.assign(videoStreamingPlaylist, { Video: video })
401 }
402
403 static get Instance () {
404 return this.instance || (this.instance = new this())
405 }
406}
407
408// ---------------------------------------------------------------------------
409
410export {
411 LiveManager
412}
diff --git a/server/lib/live/live-quota-store.ts b/server/lib/live/live-quota-store.ts
new file mode 100644
index 000000000..8ceccde98
--- /dev/null
+++ b/server/lib/live/live-quota-store.ts
@@ -0,0 +1,48 @@
1class LiveQuotaStore {
2
3 private static instance: LiveQuotaStore
4
5 private readonly livesPerUser = new Map<number, { liveId: number, size: number }[]>()
6
7 private constructor () {
8 }
9
10 addNewLive (userId: number, liveId: number) {
11 if (!this.livesPerUser.has(userId)) {
12 this.livesPerUser.set(userId, [])
13 }
14
15 const currentUserLive = { liveId, size: 0 }
16 const livesOfUser = this.livesPerUser.get(userId)
17 livesOfUser.push(currentUserLive)
18 }
19
20 removeLive (userId: number, liveId: number) {
21 const newLivesPerUser = this.livesPerUser.get(userId)
22 .filter(o => o.liveId !== liveId)
23
24 this.livesPerUser.set(userId, newLivesPerUser)
25 }
26
27 addQuotaTo (userId: number, liveId: number, size: number) {
28 const lives = this.livesPerUser.get(userId)
29 const live = lives.find(l => l.liveId === liveId)
30
31 live.size += size
32 }
33
34 getLiveQuotaOf (userId: number) {
35 const currentLives = this.livesPerUser.get(userId)
36 if (!currentLives) return 0
37
38 return currentLives.reduce((sum, obj) => sum + obj.size, 0)
39 }
40
41 static get Instance () {
42 return this.instance || (this.instance = new this())
43 }
44}
45
46export {
47 LiveQuotaStore
48}
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts
new file mode 100644
index 000000000..4af6f3ebf
--- /dev/null
+++ b/server/lib/live/live-segment-sha-store.ts
@@ -0,0 +1,64 @@
1import { basename } from 'path'
2import { logger, loggerTagsFactory } from '@server/helpers/logger'
3import { buildSha256Segment } from '../hls'
4
5const lTags = loggerTagsFactory('live')
6
7class LiveSegmentShaStore {
8
9 private static instance: LiveSegmentShaStore
10
11 private readonly segmentsSha256 = new Map<string, Map<string, string>>()
12
13 private constructor () {
14 }
15
16 getSegmentsSha256 (videoUUID: string) {
17 return this.segmentsSha256.get(videoUUID)
18 }
19
20 async addSegmentSha (videoUUID: string, segmentPath: string) {
21 const segmentName = basename(segmentPath)
22 logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID))
23
24 const shaResult = await buildSha256Segment(segmentPath)
25
26 if (!this.segmentsSha256.has(videoUUID)) {
27 this.segmentsSha256.set(videoUUID, new Map())
28 }
29
30 const filesMap = this.segmentsSha256.get(videoUUID)
31 filesMap.set(segmentName, shaResult)
32 }
33
34 removeSegmentSha (videoUUID: string, segmentPath: string) {
35 const segmentName = basename(segmentPath)
36
37 logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID))
38
39 const filesMap = this.segmentsSha256.get(videoUUID)
40 if (!filesMap) {
41 logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID))
42 return
43 }
44
45 if (!filesMap.has(segmentName)) {
46 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID))
47 return
48 }
49
50 filesMap.delete(segmentName)
51 }
52
53 cleanupShaSegments (videoUUID: string) {
54 this.segmentsSha256.delete(videoUUID)
55 }
56
57 static get Instance () {
58 return this.instance || (this.instance = new this())
59 }
60}
61
62export {
63 LiveSegmentShaStore
64}
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
new file mode 100644
index 000000000..e4526c7a5
--- /dev/null
+++ b/server/lib/live/live-utils.ts
@@ -0,0 +1,23 @@
1import { remove } from 'fs-extra'
2import { basename } from 'path'
3import { MStreamingPlaylist, MVideo } from '@server/types/models'
4import { getHLSDirectory } from '../video-paths'
5
6function buildConcatenatedName (segmentOrPlaylistPath: string) {
7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
8
9 return 'concat-' + num[1] + '.ts'
10}
11
12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
13 const hlsDirectory = getHLSDirectory(video)
14
15 await remove(hlsDirectory)
16
17 await streamingPlaylist.destroy()
18}
19
20export {
21 cleanupLive,
22 buildConcatenatedName
23}
diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts
new file mode 100644
index 000000000..c4d1b59ec
--- /dev/null
+++ b/server/lib/live/shared/index.ts
@@ -0,0 +1 @@
export * from './muxing-session'
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
new file mode 100644
index 000000000..96f6c2c89
--- /dev/null
+++ b/server/lib/live/shared/muxing-session.ts
@@ -0,0 +1,341 @@
1
2import * as Bluebird from 'bluebird'
3import * as chokidar from 'chokidar'
4import { FfmpegCommand } from 'fluent-ffmpeg'
5import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
6import { basename, join } from 'path'
7import { EventEmitter } from 'stream'
8import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils'
9import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
10import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
12import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
14import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles'
15import { isAbleToUploadVideo } from '../../user'
16import { getHLSDirectory } from '../../video-paths'
17import { LiveQuotaStore } from '../live-quota-store'
18import { LiveSegmentShaStore } from '../live-segment-sha-store'
19import { buildConcatenatedName } from '../live-utils'
20
21import memoizee = require('memoizee')
22
23interface MuxingSessionEvents {
24 'master-playlist-created': ({ videoId: number }) => void
25
26 'bad-socket-health': ({ videoId: number }) => void
27 'duration-exceeded': ({ videoId: number }) => void
28 'quota-exceeded': ({ videoId: number }) => void
29
30 'ffmpeg-end': ({ videoId: number }) => void
31 'ffmpeg-error': ({ sessionId: string }) => void
32
33 'after-cleanup': ({ videoId: number }) => void
34}
35
36declare interface MuxingSession {
37 on<U extends keyof MuxingSessionEvents>(
38 event: U, listener: MuxingSessionEvents[U]
39 ): this
40
41 emit<U extends keyof MuxingSessionEvents>(
42 event: U, ...args: Parameters<MuxingSessionEvents[U]>
43 ): boolean
44}
45
46class MuxingSession extends EventEmitter {
47
48 private ffmpegCommand: FfmpegCommand
49
50 private readonly context: any
51 private readonly user: MUserId
52 private readonly sessionId: string
53 private readonly videoLive: MVideoLiveVideo
54 private readonly streamingPlaylist: MStreamingPlaylistVideo
55 private readonly rtmpUrl: string
56 private readonly fps: number
57 private readonly allResolutions: number[]
58
59 private readonly videoId: number
60 private readonly videoUUID: string
61 private readonly saveReplay: boolean
62
63 private readonly lTags: LoggerTagsFn
64
65 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
66
67 private tsWatcher: chokidar.FSWatcher
68 private masterWatcher: chokidar.FSWatcher
69
70 private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
71 return isAbleToUploadVideo(userId, 1000)
72 }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
73
74 private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => {
75 return this.hasClientSocketInBadHealth(sessionId)
76 }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH })
77
78 constructor (options: {
79 context: any
80 user: MUserId
81 sessionId: string
82 videoLive: MVideoLiveVideo
83 streamingPlaylist: MStreamingPlaylistVideo
84 rtmpUrl: string
85 fps: number
86 allResolutions: number[]
87 }) {
88 super()
89
90 this.context = options.context
91 this.user = options.user
92 this.sessionId = options.sessionId
93 this.videoLive = options.videoLive
94 this.streamingPlaylist = options.streamingPlaylist
95 this.rtmpUrl = options.rtmpUrl
96 this.fps = options.fps
97 this.allResolutions = options.allResolutions
98
99 this.videoId = this.videoLive.Video.id
100 this.videoUUID = this.videoLive.Video.uuid
101
102 this.saveReplay = this.videoLive.saveReplay
103
104 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID)
105 }
106
107 async runMuxing () {
108 this.createFiles()
109
110 const outPath = await this.prepareDirectories()
111
112 this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
113 ? await getLiveTranscodingCommand({
114 rtmpUrl: this.rtmpUrl,
115 outPath,
116 resolutions: this.allResolutions,
117 fps: this.fps,
118 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
119 profile: CONFIG.LIVE.TRANSCODING.PROFILE
120 })
121 : getLiveMuxingCommand(this.rtmpUrl, outPath)
122
123 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags)
124
125 this.watchTSFiles(outPath)
126 this.watchMasterFile(outPath)
127
128 this.ffmpegCommand.on('error', (err, stdout, stderr) => {
129 this.onFFmpegError(err, stdout, stderr, outPath)
130 })
131
132 this.ffmpegCommand.on('end', () => this.onFFmpegEnded(outPath))
133
134 this.ffmpegCommand.run()
135 }
136
137 abort () {
138 if (!this.ffmpegCommand) return false
139
140 this.ffmpegCommand.kill('SIGINT')
141 return true
142 }
143
144 private onFFmpegError (err: any, stdout: string, stderr: string, outPath: string) {
145 this.onFFmpegEnded(outPath)
146
147 // Don't care that we killed the ffmpeg process
148 if (err?.message?.includes('Exiting normally')) return
149
150 logger.error('Live transcoding error.', { err, stdout, stderr, ...this.lTags })
151
152 this.emit('ffmpeg-error', ({ sessionId: this.sessionId }))
153 }
154
155 private onFFmpegEnded (outPath: string) {
156 logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.rtmpUrl, this.lTags)
157
158 setTimeout(() => {
159 // Wait latest segments generation, and close watchers
160
161 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ])
162 .then(() => {
163 // Process remaining segments hash
164 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
165 this.processSegments(outPath, this.segmentsToProcessPerPlaylist[key])
166 }
167 })
168 .catch(err => {
169 logger.error(
170 'Cannot close watchers of %s or process remaining hash segments.', outPath,
171 { err, ...this.lTags }
172 )
173 })
174
175 this.emit('after-cleanup', { videoId: this.videoId })
176 }, 1000)
177 }
178
179 private watchMasterFile (outPath: string) {
180 this.masterWatcher = chokidar.watch(outPath + '/master.m3u8')
181
182 this.masterWatcher.on('add', async () => {
183 this.emit('master-playlist-created', { videoId: this.videoId })
184
185 this.masterWatcher.close()
186 .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags }))
187 })
188 }
189
190 private watchTSFiles (outPath: string) {
191 const startStreamDateTime = new Date().getTime()
192
193 this.tsWatcher = chokidar.watch(outPath + '/*.ts')
194
195 const playlistIdMatcher = /^([\d+])-/
196
197 const addHandler = async segmentPath => {
198 logger.debug('Live add handler of %s.', segmentPath, this.lTags)
199
200 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
201
202 const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || []
203 this.processSegments(outPath, segmentsToProcess)
204
205 this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
206
207 if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) {
208 this.emit('bad-socket-health', { videoId: this.videoId })
209 return
210 }
211
212 // Duration constraint check
213 if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
214 this.emit('duration-exceeded', { videoId: this.videoId })
215 return
216 }
217
218 // Check user quota if the user enabled replay saving
219 if (await this.isQuotaExceeded(segmentPath) === true) {
220 this.emit('quota-exceeded', { videoId: this.videoId })
221 }
222 }
223
224 const deleteHandler = segmentPath => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath)
225
226 this.tsWatcher.on('add', p => addHandler(p))
227 this.tsWatcher.on('unlink', p => deleteHandler(p))
228 }
229
230 private async isQuotaExceeded (segmentPath: string) {
231 if (this.saveReplay !== true) return false
232
233 try {
234 const segmentStat = await stat(segmentPath)
235
236 LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.videoLive.id, segmentStat.size)
237
238 const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id)
239
240 return canUpload !== true
241 } catch (err) {
242 logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags })
243 }
244 }
245
246 private createFiles () {
247 for (let i = 0; i < this.allResolutions.length; i++) {
248 const resolution = this.allResolutions[i]
249
250 const file = new VideoFileModel({
251 resolution,
252 size: -1,
253 extname: '.ts',
254 infoHash: null,
255 fps: this.fps,
256 videoStreamingPlaylistId: this.streamingPlaylist.id
257 })
258
259 VideoFileModel.customUpsert(file, 'streaming-playlist', null)
260 .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags }))
261 }
262 }
263
264 private async prepareDirectories () {
265 const outPath = getHLSDirectory(this.videoLive.Video)
266 await ensureDir(outPath)
267
268 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
269
270 if (this.videoLive.saveReplay === true) {
271 await ensureDir(replayDirectory)
272 }
273
274 return outPath
275 }
276
277 private isDurationConstraintValid (streamingStartTime: number) {
278 const maxDuration = CONFIG.LIVE.MAX_DURATION
279 // No limit
280 if (maxDuration < 0) return true
281
282 const now = new Date().getTime()
283 const max = streamingStartTime + maxDuration
284
285 return now <= max
286 }
287
288 private processSegments (hlsVideoPath: string, segmentPaths: string[]) {
289 Bluebird.mapSeries(segmentPaths, async previousSegment => {
290 // Add sha hash of previous segments, because ffmpeg should have finished generating them
291 await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment)
292
293 if (this.saveReplay) {
294 await this.addSegmentToReplay(hlsVideoPath, previousSegment)
295 }
296 }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags }))
297 }
298
299 private hasClientSocketInBadHealth (sessionId: string) {
300 const rtmpSession = this.context.sessions.get(sessionId)
301
302 if (!rtmpSession) {
303 logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags)
304 return
305 }
306
307 for (const playerSessionId of rtmpSession.players) {
308 const playerSession = this.context.sessions.get(playerSessionId)
309
310 if (!playerSession) {
311 logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags)
312 continue
313 }
314
315 if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) {
316 return true
317 }
318 }
319
320 return false
321 }
322
323 private async addSegmentToReplay (hlsVideoPath: string, segmentPath: string) {
324 const segmentName = basename(segmentPath)
325 const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, buildConcatenatedName(segmentName))
326
327 try {
328 const data = await readFile(segmentPath)
329
330 await appendFile(dest, data)
331 } catch (err) {
332 logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags })
333 }
334 }
335}
336
337// ---------------------------------------------------------------------------
338
339export {
340 MuxingSession
341}
diff --git a/server/lib/user.ts b/server/lib/user.ts
index 91a682a7e..8820e8243 100644
--- a/server/lib/user.ts
+++ b/server/lib/user.ts
@@ -14,7 +14,7 @@ import { MUser, MUserDefault, MUserId } from '../types/models/user'
14import { generateAndSaveActorKeys } from './activitypub/actors' 14import { generateAndSaveActorKeys } from './activitypub/actors'
15import { getLocalAccountActivityPubUrl } from './activitypub/url' 15import { getLocalAccountActivityPubUrl } from './activitypub/url'
16import { Emailer } from './emailer' 16import { Emailer } from './emailer'
17import { LiveManager } from './live-manager' 17import { LiveQuotaStore } from './live/live-quota-store'
18import { buildActorInstance } from './local-actor' 18import { buildActorInstance } from './local-actor'
19import { Redis } from './redis' 19import { Redis } from './redis'
20import { createLocalVideoChannel } from './video-channel' 20import { createLocalVideoChannel } from './video-channel'
@@ -129,7 +129,7 @@ async function getOriginalVideoFileTotalFromUser (user: MUserId) {
129 129
130 const base = await UserModel.getTotalRawQuery(query, user.id) 130 const base = await UserModel.getTotalRawQuery(query, user.id)
131 131
132 return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) 132 return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id)
133} 133}
134 134
135// Returns cumulative size of all video files uploaded in the last 24 hours. 135// Returns cumulative size of all video files uploaded in the last 24 hours.
@@ -143,10 +143,10 @@ async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) {
143 143
144 const base = await UserModel.getTotalRawQuery(query, user.id) 144 const base = await UserModel.getTotalRawQuery(query, user.id)
145 145
146 return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) 146 return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id)
147} 147}
148 148
149async function isAbleToUploadVideo (userId: number, size: number) { 149async function isAbleToUploadVideo (userId: number, newVideoSize: number) {
150 const user = await UserModel.loadById(userId) 150 const user = await UserModel.loadById(userId)
151 151
152 if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) 152 if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true)
@@ -156,8 +156,8 @@ async function isAbleToUploadVideo (userId: number, size: number) {
156 getOriginalVideoFileTotalDailyFromUser(user) 156 getOriginalVideoFileTotalDailyFromUser(user)
157 ]) 157 ])
158 158
159 const uploadedTotal = size + totalBytes 159 const uploadedTotal = newVideoSize + totalBytes
160 const uploadedDaily = size + totalBytesDaily 160 const uploadedDaily = newVideoSize + totalBytesDaily
161 161
162 if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota 162 if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota
163 if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily 163 if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts
index 37c43c3b0..0984c0d7a 100644
--- a/server/lib/video-blacklist.ts
+++ b/server/lib/video-blacklist.ts
@@ -16,7 +16,7 @@ import { CONFIG } from '../initializers/config'
16import { VideoBlacklistModel } from '../models/video/video-blacklist' 16import { VideoBlacklistModel } from '../models/video/video-blacklist'
17import { sendDeleteVideo } from './activitypub/send' 17import { sendDeleteVideo } from './activitypub/send'
18import { federateVideoIfNeeded } from './activitypub/videos' 18import { federateVideoIfNeeded } from './activitypub/videos'
19import { LiveManager } from './live-manager' 19import { LiveManager } from './live/live-manager'
20import { Notifier } from './notifier' 20import { Notifier } from './notifier'
21import { Hooks } from './plugins/hooks' 21import { Hooks } from './plugins/hooks'
22 22
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 18:43:38 +0000