aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-06-16 15:14:41 +0200
committerChocobozzz <me@florianbigard.com>2021-06-17 08:47:03 +0200
commit8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d (patch)
treeeee3ef972943be629519c2dc97928878dbd2d732 /server
parentfd6584844b070cfc283902aba26f2853473e83d1 (diff)
downloadPeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.tar.gz
PeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.tar.zst
PeerTube-8ebf2a5d5d126e6ef9b89109124adf2a5e9e293d.zip
Refactor live manager
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/index.ts2
-rw-r--r--server/controllers/live.ts4
-rw-r--r--server/lib/activitypub/process/process-view.ts2
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts19
-rw-r--r--server/lib/live-manager.ts633
-rw-r--r--server/lib/live/index.ts4
-rw-r--r--server/lib/live/live-manager.ts412
-rw-r--r--server/lib/live/live-quota-store.ts48
-rw-r--r--server/lib/live/live-segment-sha-store.ts64
-rw-r--r--server/lib/live/live-utils.ts23
-rw-r--r--server/lib/live/shared/index.ts1
-rw-r--r--server/lib/live/shared/muxing-session.ts341
-rw-r--r--server/lib/user.ts12
-rw-r--r--server/lib/video-blacklist.ts2
-rw-r--r--server/models/video/video.ts2
-rw-r--r--server/tests/api/live/index.ts2
-rw-r--r--server/tests/api/live/live-socket-messages.ts196
-rw-r--r--server/tests/api/live/live-views.ts130
-rw-r--r--server/tests/api/live/live.ts231
19 files changed, 1244 insertions, 884 deletions
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 5fdb7d5bc..74b100e59 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -1,7 +1,7 @@
1import * as express from 'express' 1import * as express from 'express'
2import toInt from 'validator/lib/toInt' 2import toInt from 'validator/lib/toInt'
3import { doJSONRequest } from '@server/helpers/requests' 3import { doJSONRequest } from '@server/helpers/requests'
4import { LiveManager } from '@server/lib/live-manager' 4import { LiveManager } from '@server/lib/live'
5import { openapiOperationDoc } from '@server/middlewares/doc' 5import { openapiOperationDoc } from '@server/middlewares/doc'
6import { getServerActor } from '@server/models/application/application' 6import { getServerActor } from '@server/models/application/application'
7import { MVideoAccountLight } from '@server/types/models' 7import { MVideoAccountLight } from '@server/types/models'
diff --git a/server/controllers/live.ts b/server/controllers/live.ts
index cfb4741b7..f2686fb23 100644
--- a/server/controllers/live.ts
+++ b/server/controllers/live.ts
@@ -1,7 +1,7 @@
1import * as cors from 'cors' 1import * as cors from 'cors'
2import * as express from 'express' 2import * as express from 'express'
3import { mapToJSON } from '@server/helpers/core-utils' 3import { mapToJSON } from '@server/helpers/core-utils'
4import { LiveManager } from '@server/lib/live-manager' 4import { LiveSegmentShaStore } from '@server/lib/live'
5import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes' 5import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
6 6
7const liveRouter = express.Router() 7const liveRouter = express.Router()
@@ -22,7 +22,7 @@ export {
22function getSegmentsSha256 (req: express.Request, res: express.Response) { 22function getSegmentsSha256 (req: express.Request, res: express.Response) {
23 const videoUUID = req.params.videoUUID 23 const videoUUID = req.params.videoUUID
24 24
25 const result = LiveManager.Instance.getSegmentsSha256(videoUUID) 25 const result = LiveSegmentShaStore.Instance.getSegmentsSha256(videoUUID)
26 26
27 if (!result) { 27 if (!result) {
28 return res.status(HttpStatusCode.NOT_FOUND_404).end() 28 return res.status(HttpStatusCode.NOT_FOUND_404).end()
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
index 0a0231a3a..5593ee257 100644
--- a/server/lib/activitypub/process/process-view.ts
+++ b/server/lib/activitypub/process/process-view.ts
@@ -4,7 +4,7 @@ import { Redis } from '../../redis'
4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' 4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub'
5import { APProcessorOptions } from '../../../types/activitypub-processor.model' 5import { APProcessorOptions } from '../../../types/activitypub-processor.model'
6import { MActorSignature } from '../../../types/models' 6import { MActorSignature } from '../../../types/models'
7import { LiveManager } from '@server/lib/live-manager' 7import { LiveManager } from '@server/lib/live/live-manager'
8 8
9async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) { 9async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) {
10 const { activity, byActor } = options 10 const { activity, byActor } = options
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 517b90abc..9eba41bf8 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -3,7 +3,7 @@ import { pathExists, readdir, remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { VIDEO_LIVE } from '@server/initializers/constants' 5import { VIDEO_LIVE } from '@server/initializers/constants'
6import { LiveManager } from '@server/lib/live-manager' 6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live'
7import { generateVideoMiniature } from '@server/lib/thumbnail' 7import { generateVideoMiniature } from '@server/lib/thumbnail'
8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' 8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
9import { publishAndFederateIfNeeded } from '@server/lib/video' 9import { publishAndFederateIfNeeded } from '@server/lib/video'
@@ -12,7 +12,7 @@ import { VideoModel } from '@server/models/video/video'
12import { VideoFileModel } from '@server/models/video/video-file' 12import { VideoFileModel } from '@server/models/video/video-file'
13import { VideoLiveModel } from '@server/models/video/video-live' 13import { VideoLiveModel } from '@server/models/video/video-live'
14import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 14import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
15import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models' 15import { MVideo, MVideoLive } from '@server/types/models'
16import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 16import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
17import { logger } from '../../../helpers/logger' 17import { logger } from '../../../helpers/logger'
18 18
@@ -37,7 +37,7 @@ async function processVideoLiveEnding (job: Bull.Job) {
37 return 37 return
38 } 38 }
39 39
40 LiveManager.Instance.cleanupShaSegments(video.uuid) 40 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid)
41 41
42 if (live.saveReplay !== true) { 42 if (live.saveReplay !== true) {
43 return cleanupLive(video, streamingPlaylist) 43 return cleanupLive(video, streamingPlaylist)
@@ -46,19 +46,10 @@ async function processVideoLiveEnding (job: Bull.Job) {
46 return saveLive(video, live) 46 return saveLive(video, live)
47} 47}
48 48
49async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
50 const hlsDirectory = getHLSDirectory(video)
51
52 await remove(hlsDirectory)
53
54 await streamingPlaylist.destroy()
55}
56
57// --------------------------------------------------------------------------- 49// ---------------------------------------------------------------------------
58 50
59export { 51export {
60 processVideoLiveEnding, 52 processVideoLiveEnding
61 cleanupLive
62} 53}
63 54
64// --------------------------------------------------------------------------- 55// ---------------------------------------------------------------------------
@@ -94,7 +85,7 @@ async function saveLive (video: MVideo, live: MVideoLive) {
94 let durationDone = false 85 let durationDone = false
95 86
96 for (const playlistFile of playlistFiles) { 87 for (const playlistFile of playlistFiles) {
97 const concatenatedTsFile = LiveManager.Instance.buildConcatenatedName(playlistFile) 88 const concatenatedTsFile = buildConcatenatedName(playlistFile)
98 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) 89 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
99 90
100 const probe = await ffprobePromise(concatenatedTsFilePath) 91 const probe = await ffprobePromise(concatenatedTsFilePath)
diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts
deleted file mode 100644
index 563ba2578..000000000
--- a/server/lib/live-manager.ts
+++ /dev/null
@@ -1,633 +0,0 @@
1
2import * as Bluebird from 'bluebird'
3import * as chokidar from 'chokidar'
4import { FfmpegCommand } from 'fluent-ffmpeg'
5import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
6import { createServer, Server } from 'net'
7import { basename, join } from 'path'
8import { isTestInstance } from '@server/helpers/core-utils'
9import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils'
10import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
11import { logger, loggerTagsFactory } from '@server/helpers/logger'
12import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
13import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
14import { UserModel } from '@server/models/user/user'
15import { VideoModel } from '@server/models/video/video'
16import { VideoFileModel } from '@server/models/video/video-file'
17import { VideoLiveModel } from '@server/models/video/video-live'
18import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
19import { MStreamingPlaylist, MStreamingPlaylistVideo, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models'
20import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
21import { federateVideoIfNeeded } from './activitypub/videos'
22import { buildSha256Segment } from './hls'
23import { JobQueue } from './job-queue'
24import { cleanupLive } from './job-queue/handlers/video-live-ending'
25import { PeerTubeSocket } from './peertube-socket'
26import { VideoTranscodingProfilesManager } from './transcoding/video-transcoding-profiles'
27import { isAbleToUploadVideo } from './user'
28import { getHLSDirectory } from './video-paths'
29
30import memoizee = require('memoizee')
31const NodeRtmpSession = require('node-media-server/node_rtmp_session')
32const context = require('node-media-server/node_core_ctx')
33const nodeMediaServerLogger = require('node-media-server/node_core_logger')
34
35// Disable node media server logs
36nodeMediaServerLogger.setLogType(0)
37
38const config = {
39 rtmp: {
40 port: CONFIG.LIVE.RTMP.PORT,
41 chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE,
42 gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE,
43 ping: VIDEO_LIVE.RTMP.PING,
44 ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT
45 },
46 transcoding: {
47 ffmpeg: 'ffmpeg'
48 }
49}
50
51const lTags = loggerTagsFactory('live')
52
53class LiveManager {
54
55 private static instance: LiveManager
56
57 private readonly transSessions = new Map<string, FfmpegCommand>()
58 private readonly videoSessions = new Map<number, string>()
59 // Values are Date().getTime()
60 private readonly watchersPerVideo = new Map<number, number[]>()
61 private readonly segmentsSha256 = new Map<string, Map<string, string>>()
62 private readonly livesPerUser = new Map<number, { liveId: number, videoId: number, size: number }[]>()
63
64 private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
65 return isAbleToUploadVideo(userId, 1000)
66 }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
67
68 private readonly hasClientSocketsInBadHealthWithCache = memoizee((sessionId: string) => {
69 return this.hasClientSocketsInBadHealth(sessionId)
70 }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH })
71
72 private rtmpServer: Server
73
74 private constructor () {
75 }
76
77 init () {
78 const events = this.getContext().nodeEvent
79 events.on('postPublish', (sessionId: string, streamPath: string) => {
80 logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) })
81
82 const splittedPath = streamPath.split('/')
83 if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) {
84 logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) })
85 return this.abortSession(sessionId)
86 }
87
88 this.handleSession(sessionId, streamPath, splittedPath[2])
89 .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) }))
90 })
91
92 events.on('donePublish', sessionId => {
93 logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })
94 })
95
96 registerConfigChangedHandler(() => {
97 if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) {
98 this.run()
99 return
100 }
101
102 if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) {
103 this.stop()
104 }
105 })
106
107 // Cleanup broken lives, that were terminated by a server restart for example
108 this.handleBrokenLives()
109 .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() }))
110
111 setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE)
112 }
113
114 run () {
115 logger.info('Running RTMP server on port %d', config.rtmp.port, lTags())
116
117 this.rtmpServer = createServer(socket => {
118 const session = new NodeRtmpSession(config, socket)
119
120 session.run()
121 })
122
123 this.rtmpServer.on('error', err => {
124 logger.error('Cannot run RTMP server.', { err, ...lTags() })
125 })
126
127 this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT)
128 }
129
130 stop () {
131 logger.info('Stopping RTMP server.', lTags())
132
133 this.rtmpServer.close()
134 this.rtmpServer = undefined
135
136 // Sessions is an object
137 this.getContext().sessions.forEach((session: any) => {
138 if (session instanceof NodeRtmpSession) {
139 session.stop()
140 }
141 })
142 }
143
144 isRunning () {
145 return !!this.rtmpServer
146 }
147
148 getSegmentsSha256 (videoUUID: string) {
149 return this.segmentsSha256.get(videoUUID)
150 }
151
152 stopSessionOf (videoId: number) {
153 const sessionId = this.videoSessions.get(videoId)
154 if (!sessionId) return
155
156 this.videoSessions.delete(videoId)
157 this.abortSession(sessionId)
158 }
159
160 getLiveQuotaUsedByUser (userId: number) {
161 const currentLives = this.livesPerUser.get(userId)
162 if (!currentLives) return 0
163
164 return currentLives.reduce((sum, obj) => sum + obj.size, 0)
165 }
166
167 addViewTo (videoId: number) {
168 if (this.videoSessions.has(videoId) === false) return
169
170 let watchers = this.watchersPerVideo.get(videoId)
171
172 if (!watchers) {
173 watchers = []
174 this.watchersPerVideo.set(videoId, watchers)
175 }
176
177 watchers.push(new Date().getTime())
178 }
179
180 cleanupShaSegments (videoUUID: string) {
181 this.segmentsSha256.delete(videoUUID)
182 }
183
184 addSegmentToReplay (hlsVideoPath: string, segmentPath: string) {
185 const segmentName = basename(segmentPath)
186 const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, this.buildConcatenatedName(segmentName))
187
188 return readFile(segmentPath)
189 .then(data => appendFile(dest, data))
190 .catch(err => logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...lTags() }))
191 }
192
193 buildConcatenatedName (segmentOrPlaylistPath: string) {
194 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
195
196 return 'concat-' + num[1] + '.ts'
197 }
198
199 private processSegments (hlsVideoPath: string, videoUUID: string, videoLive: MVideoLive, segmentPaths: string[]) {
200 Bluebird.mapSeries(segmentPaths, async previousSegment => {
201 // Add sha hash of previous segments, because ffmpeg should have finished generating them
202 await this.addSegmentSha(videoUUID, previousSegment)
203
204 if (videoLive.saveReplay) {
205 await this.addSegmentToReplay(hlsVideoPath, previousSegment)
206 }
207 }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...lTags(videoUUID) }))
208 }
209
210 private getContext () {
211 return context
212 }
213
214 private abortSession (id: string) {
215 const session = this.getContext().sessions.get(id)
216 if (session) {
217 session.stop()
218 this.getContext().sessions.delete(id)
219 }
220
221 const transSession = this.transSessions.get(id)
222 if (transSession) {
223 transSession.kill('SIGINT')
224 this.transSessions.delete(id)
225 }
226 }
227
228 private async handleSession (sessionId: string, streamPath: string, streamKey: string) {
229 const videoLive = await VideoLiveModel.loadByStreamKey(streamKey)
230 if (!videoLive) {
231 logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId))
232 return this.abortSession(sessionId)
233 }
234
235 const video = videoLive.Video
236 if (video.isBlacklisted()) {
237 logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid))
238 return this.abortSession(sessionId)
239 }
240
241 // Cleanup old potential live files (could happen with a permanent live)
242 this.cleanupShaSegments(video.uuid)
243
244 const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
245 if (oldStreamingPlaylist) {
246 await cleanupLive(video, oldStreamingPlaylist)
247 }
248
249 this.videoSessions.set(video.id, sessionId)
250
251 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
252
253 const session = this.getContext().sessions.get(sessionId)
254 const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath
255
256 const [ resolutionResult, fps ] = await Promise.all([
257 getVideoFileResolution(rtmpUrl),
258 getVideoFileFPS(rtmpUrl)
259 ])
260
261 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
262 ? computeResolutionsToTranscode(resolutionResult.videoFileResolution, 'live')
263 : []
264
265 const allResolutions = resolutionsEnabled.concat([ session.videoHeight ])
266
267 logger.info(
268 'Will mux/transcode live video of original resolution %d.', session.videoHeight,
269 { allResolutions, ...lTags(sessionId, video.uuid) }
270 )
271
272 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
273 videoId: video.id,
274 playlistUrl,
275 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
276 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
277 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
278
279 type: VideoStreamingPlaylistType.HLS
280 }, { returning: true }) as [ MStreamingPlaylist, boolean ]
281
282 return this.runMuxing({
283 sessionId,
284 videoLive,
285 playlist: Object.assign(videoStreamingPlaylist, { Video: video }),
286 rtmpUrl,
287 fps,
288 allResolutions
289 })
290 }
291
292 private async runMuxing (options: {
293 sessionId: string
294 videoLive: MVideoLiveVideo
295 playlist: MStreamingPlaylistVideo
296 rtmpUrl: string
297 fps: number
298 allResolutions: number[]
299 }) {
300 const { sessionId, videoLive, playlist, allResolutions, fps, rtmpUrl } = options
301 const startStreamDateTime = new Date().getTime()
302
303 const user = await UserModel.loadByLiveId(videoLive.id)
304 if (!this.livesPerUser.has(user.id)) {
305 this.livesPerUser.set(user.id, [])
306 }
307
308 const currentUserLive = { liveId: videoLive.id, videoId: videoLive.videoId, size: 0 }
309 const livesOfUser = this.livesPerUser.get(user.id)
310 livesOfUser.push(currentUserLive)
311
312 for (let i = 0; i < allResolutions.length; i++) {
313 const resolution = allResolutions[i]
314
315 const file = new VideoFileModel({
316 resolution,
317 size: -1,
318 extname: '.ts',
319 infoHash: null,
320 fps,
321 videoStreamingPlaylistId: playlist.id
322 })
323
324 VideoFileModel.customUpsert(file, 'streaming-playlist', null)
325 .catch(err => logger.error('Cannot create file for live streaming.', { err, ...lTags(sessionId, videoLive.Video.uuid) }))
326 }
327
328 const outPath = getHLSDirectory(videoLive.Video)
329 await ensureDir(outPath)
330
331 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
332
333 if (videoLive.saveReplay === true) {
334 await ensureDir(replayDirectory)
335 }
336
337 const videoUUID = videoLive.Video.uuid
338
339 const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED
340 ? await getLiveTranscodingCommand({
341 rtmpUrl,
342 outPath,
343 resolutions: allResolutions,
344 fps,
345 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
346 profile: CONFIG.LIVE.TRANSCODING.PROFILE
347 })
348 : getLiveMuxingCommand(rtmpUrl, outPath)
349
350 logger.info('Running live muxing/transcoding for %s.', videoUUID, lTags(sessionId, videoUUID))
351 this.transSessions.set(sessionId, ffmpegExec)
352
353 const tsWatcher = chokidar.watch(outPath + '/*.ts')
354
355 const segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
356 const playlistIdMatcher = /^([\d+])-/
357
358 const addHandler = segmentPath => {
359 logger.debug('Live add handler of %s.', segmentPath, lTags(sessionId, videoUUID))
360
361 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
362
363 const segmentsToProcess = segmentsToProcessPerPlaylist[playlistId] || []
364 this.processSegments(outPath, videoUUID, videoLive, segmentsToProcess)
365
366 segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
367
368 if (this.hasClientSocketsInBadHealthWithCache(sessionId)) {
369 logger.error(
370 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
371 ' Stopping session of video %s.', videoUUID,
372 lTags(sessionId, videoUUID)
373 )
374
375 this.stopSessionOf(videoLive.videoId)
376 return
377 }
378
379 // Duration constraint check
380 if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
381 logger.info('Stopping session of %s: max duration exceeded.', videoUUID, lTags(sessionId, videoUUID))
382
383 this.stopSessionOf(videoLive.videoId)
384 return
385 }
386
387 // Check user quota if the user enabled replay saving
388 if (videoLive.saveReplay === true) {
389 stat(segmentPath)
390 .then(segmentStat => {
391 currentUserLive.size += segmentStat.size
392 })
393 .then(() => this.isQuotaConstraintValid(user, videoLive))
394 .then(quotaValid => {
395 if (quotaValid !== true) {
396 logger.info('Stopping session of %s: user quota exceeded.', videoUUID, lTags(sessionId, videoUUID))
397
398 this.stopSessionOf(videoLive.videoId)
399 }
400 })
401 .catch(err => logger.error('Cannot stat %s or check quota of %d.', segmentPath, user.id, { err, ...lTags(sessionId, videoUUID) }))
402 }
403 }
404
405 const deleteHandler = segmentPath => this.removeSegmentSha(videoUUID, segmentPath)
406
407 tsWatcher.on('add', p => addHandler(p))
408 tsWatcher.on('unlink', p => deleteHandler(p))
409
410 const masterWatcher = chokidar.watch(outPath + '/master.m3u8')
411 masterWatcher.on('add', async () => {
412 try {
413 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoLive.videoId)
414
415 video.state = VideoState.PUBLISHED
416 await video.save()
417 videoLive.Video = video
418
419 setTimeout(() => {
420 federateVideoIfNeeded(video, false)
421 .catch(err => logger.error('Cannot federate live video %s.', video.url, { err, ...lTags(sessionId, videoUUID) }))
422
423 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
424 }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION)
425
426 } catch (err) {
427 logger.error('Cannot save/federate live video %d.', videoLive.videoId, { err, ...lTags(sessionId, videoUUID) })
428 } finally {
429 masterWatcher.close()
430 .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...lTags(sessionId, videoUUID) }))
431 }
432 })
433
434 const onFFmpegEnded = () => {
435 logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', rtmpUrl, lTags(sessionId, videoUUID))
436
437 this.transSessions.delete(sessionId)
438
439 this.watchersPerVideo.delete(videoLive.videoId)
440 this.videoSessions.delete(videoLive.videoId)
441
442 const newLivesPerUser = this.livesPerUser.get(user.id)
443 .filter(o => o.liveId !== videoLive.id)
444 this.livesPerUser.set(user.id, newLivesPerUser)
445
446 setTimeout(() => {
447 // Wait latest segments generation, and close watchers
448
449 Promise.all([ tsWatcher.close(), masterWatcher.close() ])
450 .then(() => {
451 // Process remaining segments hash
452 for (const key of Object.keys(segmentsToProcessPerPlaylist)) {
453 this.processSegments(outPath, videoUUID, videoLive, segmentsToProcessPerPlaylist[key])
454 }
455 })
456 .catch(err => {
457 logger.error(
458 'Cannot close watchers of %s or process remaining hash segments.', outPath,
459 { err, ...lTags(sessionId, videoUUID) }
460 )
461 })
462
463 this.onEndTransmuxing(videoLive.Video.id)
464 .catch(err => logger.error('Error in closed transmuxing.', { err, ...lTags(sessionId, videoUUID) }))
465 }, 1000)
466 }
467
468 ffmpegExec.on('error', (err, stdout, stderr) => {
469 onFFmpegEnded()
470
471 // Don't care that we killed the ffmpeg process
472 if (err?.message?.includes('Exiting normally')) return
473
474 logger.error('Live transcoding error.', { err, stdout, stderr, ...lTags(sessionId, videoUUID) })
475
476 this.abortSession(sessionId)
477 })
478
479 ffmpegExec.on('end', () => onFFmpegEnded())
480
481 ffmpegExec.run()
482 }
483
484 private async onEndTransmuxing (videoUUID: string, cleanupNow = false) {
485 try {
486 const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoUUID)
487 if (!fullVideo) return
488
489 const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
490
491 if (!live.permanentLive) {
492 JobQueue.Instance.createJob({
493 type: 'video-live-ending',
494 payload: {
495 videoId: fullVideo.id
496 }
497 }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY })
498
499 fullVideo.state = VideoState.LIVE_ENDED
500 } else {
501 fullVideo.state = VideoState.WAITING_FOR_LIVE
502 }
503
504 await fullVideo.save()
505
506 PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo)
507
508 await federateVideoIfNeeded(fullVideo, false)
509 } catch (err) {
510 logger.error('Cannot save/federate new video state of live streaming of video %d.', videoUUID, { err, ...lTags(videoUUID) })
511 }
512 }
513
514 private async addSegmentSha (videoUUID: string, segmentPath: string) {
515 const segmentName = basename(segmentPath)
516 logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID))
517
518 const shaResult = await buildSha256Segment(segmentPath)
519
520 if (!this.segmentsSha256.has(videoUUID)) {
521 this.segmentsSha256.set(videoUUID, new Map())
522 }
523
524 const filesMap = this.segmentsSha256.get(videoUUID)
525 filesMap.set(segmentName, shaResult)
526 }
527
528 private removeSegmentSha (videoUUID: string, segmentPath: string) {
529 const segmentName = basename(segmentPath)
530
531 logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID))
532
533 const filesMap = this.segmentsSha256.get(videoUUID)
534 if (!filesMap) {
535 logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID))
536 return
537 }
538
539 if (!filesMap.has(segmentName)) {
540 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID))
541 return
542 }
543
544 filesMap.delete(segmentName)
545 }
546
547 private isDurationConstraintValid (streamingStartTime: number) {
548 const maxDuration = CONFIG.LIVE.MAX_DURATION
549 // No limit
550 if (maxDuration < 0) return true
551
552 const now = new Date().getTime()
553 const max = streamingStartTime + maxDuration
554
555 return now <= max
556 }
557
558 private hasClientSocketsInBadHealth (sessionId: string) {
559 const rtmpSession = this.getContext().sessions.get(sessionId)
560
561 if (!rtmpSession) {
562 logger.warn('Cannot get session %s to check players socket health.', sessionId, lTags(sessionId))
563 return
564 }
565
566 for (const playerSessionId of rtmpSession.players) {
567 const playerSession = this.getContext().sessions.get(playerSessionId)
568
569 if (!playerSession) {
570 logger.error('Cannot get player session %s to check socket health.', playerSession, lTags(sessionId))
571 continue
572 }
573
574 if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) {
575 return true
576 }
577 }
578
579 return false
580 }
581
582 private async isQuotaConstraintValid (user: MUserId, live: MVideoLive) {
583 if (live.saveReplay !== true) return true
584
585 return this.isAbleToUploadVideoWithCache(user.id)
586 }
587
588 private async updateLiveViews () {
589 if (!this.isRunning()) return
590
591 if (!isTestInstance()) logger.info('Updating live video views.', lTags())
592
593 for (const videoId of this.watchersPerVideo.keys()) {
594 const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE
595
596 const watchers = this.watchersPerVideo.get(videoId)
597
598 const numWatchers = watchers.length
599
600 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
601 video.views = numWatchers
602 await video.save()
603
604 await federateVideoIfNeeded(video, false)
605
606 PeerTubeSocket.Instance.sendVideoViewsUpdate(video)
607
608 // Only keep not expired watchers
609 const newWatchers = watchers.filter(w => w > notBefore)
610 this.watchersPerVideo.set(videoId, newWatchers)
611
612 logger.debug('New live video views for %s is %d.', video.url, numWatchers, lTags())
613 }
614 }
615
616 private async handleBrokenLives () {
617 const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
618
619 for (const uuid of videoUUIDs) {
620 await this.onEndTransmuxing(uuid, true)
621 }
622 }
623
624 static get Instance () {
625 return this.instance || (this.instance = new this())
626 }
627}
628
629// ---------------------------------------------------------------------------
630
631export {
632 LiveManager
633}
diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts
new file mode 100644
index 000000000..8b46800da
--- /dev/null
+++ b/server/lib/live/index.ts
@@ -0,0 +1,4 @@
1export * from './live-manager'
2export * from './live-quota-store'
3export * from './live-segment-sha-store'
4export * from './live-utils'
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
new file mode 100644
index 000000000..d7199cc89
--- /dev/null
+++ b/server/lib/live/live-manager.ts
@@ -0,0 +1,412 @@
1
2import { createServer, Server } from 'net'
3import { isTestInstance } from '@server/helpers/core-utils'
4import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { logger, loggerTagsFactory } from '@server/helpers/logger'
6import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
8import { UserModel } from '@server/models/user/user'
9import { VideoModel } from '@server/models/video/video'
10import { VideoLiveModel } from '@server/models/video/video-live'
11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
12import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
13import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
14import { federateVideoIfNeeded } from '../activitypub/videos'
15import { JobQueue } from '../job-queue'
16import { PeerTubeSocket } from '../peertube-socket'
17import { LiveQuotaStore } from './live-quota-store'
18import { LiveSegmentShaStore } from './live-segment-sha-store'
19import { cleanupLive } from './live-utils'
20import { MuxingSession } from './shared'
21
22const NodeRtmpSession = require('node-media-server/node_rtmp_session')
23const context = require('node-media-server/node_core_ctx')
24const nodeMediaServerLogger = require('node-media-server/node_core_logger')
25
26// Disable node media server logs
27nodeMediaServerLogger.setLogType(0)
28
29const config = {
30 rtmp: {
31 port: CONFIG.LIVE.RTMP.PORT,
32 chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE,
33 gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE,
34 ping: VIDEO_LIVE.RTMP.PING,
35 ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT
36 },
37 transcoding: {
38 ffmpeg: 'ffmpeg'
39 }
40}
41
42const lTags = loggerTagsFactory('live')
43
44class LiveManager {
45
46 private static instance: LiveManager
47
48 private readonly muxingSessions = new Map<string, MuxingSession>()
49 private readonly videoSessions = new Map<number, string>()
50 // Values are Date().getTime()
51 private readonly watchersPerVideo = new Map<number, number[]>()
52
53 private rtmpServer: Server
54
55 private constructor () {
56 }
57
58 init () {
59 const events = this.getContext().nodeEvent
60 events.on('postPublish', (sessionId: string, streamPath: string) => {
61 logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) })
62
63 const splittedPath = streamPath.split('/')
64 if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) {
65 logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) })
66 return this.abortSession(sessionId)
67 }
68
69 this.handleSession(sessionId, streamPath, splittedPath[2])
70 .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) }))
71 })
72
73 events.on('donePublish', sessionId => {
74 logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })
75 })
76
77 registerConfigChangedHandler(() => {
78 if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) {
79 this.run()
80 return
81 }
82
83 if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) {
84 this.stop()
85 }
86 })
87
88 // Cleanup broken lives, that were terminated by a server restart for example
89 this.handleBrokenLives()
90 .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() }))
91
92 setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE)
93 }
94
95 run () {
96 logger.info('Running RTMP server on port %d', config.rtmp.port, lTags())
97
98 this.rtmpServer = createServer(socket => {
99 const session = new NodeRtmpSession(config, socket)
100
101 session.run()
102 })
103
104 this.rtmpServer.on('error', err => {
105 logger.error('Cannot run RTMP server.', { err, ...lTags() })
106 })
107
108 this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT)
109 }
110
111 stop () {
112 logger.info('Stopping RTMP server.', lTags())
113
114 this.rtmpServer.close()
115 this.rtmpServer = undefined
116
117 // Sessions is an object
118 this.getContext().sessions.forEach((session: any) => {
119 if (session instanceof NodeRtmpSession) {
120 session.stop()
121 }
122 })
123 }
124
125 isRunning () {
126 return !!this.rtmpServer
127 }
128
129 stopSessionOf (videoId: number) {
130 const sessionId = this.videoSessions.get(videoId)
131 if (!sessionId) return
132
133 this.videoSessions.delete(videoId)
134 this.abortSession(sessionId)
135 }
136
137 addViewTo (videoId: number) {
138 if (this.videoSessions.has(videoId) === false) return
139
140 let watchers = this.watchersPerVideo.get(videoId)
141
142 if (!watchers) {
143 watchers = []
144 this.watchersPerVideo.set(videoId, watchers)
145 }
146
147 watchers.push(new Date().getTime())
148 }
149
150 private getContext () {
151 return context
152 }
153
154 private abortSession (sessionId: string) {
155 const session = this.getContext().sessions.get(sessionId)
156 if (session) {
157 session.stop()
158 this.getContext().sessions.delete(sessionId)
159 }
160
161 const muxingSession = this.muxingSessions.get(sessionId)
162 if (muxingSession) muxingSession.abort()
163 }
164
165 private async handleSession (sessionId: string, streamPath: string, streamKey: string) {
166 const videoLive = await VideoLiveModel.loadByStreamKey(streamKey)
167 if (!videoLive) {
168 logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId))
169 return this.abortSession(sessionId)
170 }
171
172 const video = videoLive.Video
173 if (video.isBlacklisted()) {
174 logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid))
175 return this.abortSession(sessionId)
176 }
177
178 // Cleanup old potential live files (could happen with a permanent live)
179 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid)
180
181 const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
182 if (oldStreamingPlaylist) {
183 await cleanupLive(video, oldStreamingPlaylist)
184 }
185
186 this.videoSessions.set(video.id, sessionId)
187
188 const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath
189
190 const [ { videoFileResolution }, fps ] = await Promise.all([
191 getVideoFileResolution(rtmpUrl),
192 getVideoFileFPS(rtmpUrl)
193 ])
194
195 const allResolutions = this.buildAllResolutionsToTranscode(videoFileResolution)
196
197 logger.info(
198 'Will mux/transcode live video of original resolution %d.', videoFileResolution,
199 { allResolutions, ...lTags(sessionId, video.uuid) }
200 )
201
202 const streamingPlaylist = await this.createLivePlaylist(video, allResolutions)
203
204 return this.runMuxingSession({
205 sessionId,
206 videoLive,
207 streamingPlaylist,
208 rtmpUrl,
209 fps,
210 allResolutions
211 })
212 }
213
214 private async runMuxingSession (options: {
215 sessionId: string
216 videoLive: MVideoLiveVideo
217 streamingPlaylist: MStreamingPlaylistVideo
218 rtmpUrl: string
219 fps: number
220 allResolutions: number[]
221 }) {
222 const { sessionId, videoLive, streamingPlaylist, allResolutions, fps, rtmpUrl } = options
223 const videoUUID = videoLive.Video.uuid
224 const localLTags = lTags(sessionId, videoUUID)
225
226 const user = await UserModel.loadByLiveId(videoLive.id)
227 LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id)
228
229 const muxingSession = new MuxingSession({
230 context: this.getContext(),
231 user,
232 sessionId,
233 videoLive,
234 streamingPlaylist,
235 rtmpUrl,
236 fps,
237 allResolutions
238 })
239
240 muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags))
241
242 muxingSession.on('bad-socket-health', ({ videoId }) => {
243 logger.error(
244 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
245 ' Stopping session of video %s.', videoUUID,
246 localLTags
247 )
248
249 this.stopSessionOf(videoId)
250 })
251
252 muxingSession.on('duration-exceeded', ({ videoId }) => {
253 logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags)
254
255 this.stopSessionOf(videoId)
256 })
257
258 muxingSession.on('quota-exceeded', ({ videoId }) => {
259 logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags)
260
261 this.stopSessionOf(videoId)
262 })
263
264 muxingSession.on('ffmpeg-error', ({ sessionId }) => this.abortSession(sessionId))
265 muxingSession.on('ffmpeg-end', ({ videoId }) => {
266 this.onMuxingFFmpegEnd(videoId)
267 })
268
269 muxingSession.on('after-cleanup', ({ videoId }) => {
270 this.muxingSessions.delete(sessionId)
271
272 return this.onAfterMuxingCleanup(videoId)
273 .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags }))
274 })
275
276 this.muxingSessions.set(sessionId, muxingSession)
277
278 muxingSession.runMuxing()
279 .catch(err => {
280 logger.error('Cannot run muxing.', { err, ...localLTags })
281 this.abortSession(sessionId)
282 })
283 }
284
285 private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) {
286 const videoId = live.videoId
287
288 try {
289 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
290
291 logger.info('Will publish and federate live %s.', video.url, localLTags)
292
293 video.state = VideoState.PUBLISHED
294 await video.save()
295
296 live.Video = video
297
298 setTimeout(() => {
299 federateVideoIfNeeded(video, false)
300 .catch(err => logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags }))
301
302 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
303 }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION)
304 } catch (err) {
305 logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags })
306 }
307 }
308
309 private onMuxingFFmpegEnd (videoId: number) {
310 this.watchersPerVideo.delete(videoId)
311 this.videoSessions.delete(videoId)
312 }
313
314 private async onAfterMuxingCleanup (videoUUID: string, cleanupNow = false) {
315 try {
316 const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoUUID)
317 if (!fullVideo) return
318
319 const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
320
321 if (!live.permanentLive) {
322 JobQueue.Instance.createJob({
323 type: 'video-live-ending',
324 payload: {
325 videoId: fullVideo.id
326 }
327 }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY })
328
329 fullVideo.state = VideoState.LIVE_ENDED
330 } else {
331 fullVideo.state = VideoState.WAITING_FOR_LIVE
332 }
333
334 await fullVideo.save()
335
336 PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo)
337
338 await federateVideoIfNeeded(fullVideo, false)
339 } catch (err) {
340 logger.error('Cannot save/federate new video state of live streaming of video %d.', videoUUID, { err, ...lTags(videoUUID) })
341 }
342 }
343
344 private async updateLiveViews () {
345 if (!this.isRunning()) return
346
347 if (!isTestInstance()) logger.info('Updating live video views.', lTags())
348
349 for (const videoId of this.watchersPerVideo.keys()) {
350 const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE
351
352 const watchers = this.watchersPerVideo.get(videoId)
353
354 const numWatchers = watchers.length
355
356 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
357 video.views = numWatchers
358 await video.save()
359
360 await federateVideoIfNeeded(video, false)
361
362 PeerTubeSocket.Instance.sendVideoViewsUpdate(video)
363
364 // Only keep not expired watchers
365 const newWatchers = watchers.filter(w => w > notBefore)
366 this.watchersPerVideo.set(videoId, newWatchers)
367
368 logger.debug('New live video views for %s is %d.', video.url, numWatchers, lTags())
369 }
370 }
371
372 private async handleBrokenLives () {
373 const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
374
375 for (const uuid of videoUUIDs) {
376 await this.onAfterMuxingCleanup(uuid, true)
377 }
378 }
379
380 private buildAllResolutionsToTranscode (originResolution: number) {
381 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
382 ? computeResolutionsToTranscode(originResolution, 'live')
383 : []
384
385 return resolutionsEnabled.concat([ originResolution ])
386 }
387
388 private async createLivePlaylist (video: MVideo, allResolutions: number[]) {
389 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
390 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
391 videoId: video.id,
392 playlistUrl,
393 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
394 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
395 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
396
397 type: VideoStreamingPlaylistType.HLS
398 }, { returning: true }) as [ MStreamingPlaylist, boolean ]
399
400 return Object.assign(videoStreamingPlaylist, { Video: video })
401 }
402
403 static get Instance () {
404 return this.instance || (this.instance = new this())
405 }
406}
407
408// ---------------------------------------------------------------------------
409
410export {
411 LiveManager
412}
diff --git a/server/lib/live/live-quota-store.ts b/server/lib/live/live-quota-store.ts
new file mode 100644
index 000000000..8ceccde98
--- /dev/null
+++ b/server/lib/live/live-quota-store.ts
@@ -0,0 +1,48 @@
1class LiveQuotaStore {
2
3 private static instance: LiveQuotaStore
4
5 private readonly livesPerUser = new Map<number, { liveId: number, size: number }[]>()
6
7 private constructor () {
8 }
9
10 addNewLive (userId: number, liveId: number) {
11 if (!this.livesPerUser.has(userId)) {
12 this.livesPerUser.set(userId, [])
13 }
14
15 const currentUserLive = { liveId, size: 0 }
16 const livesOfUser = this.livesPerUser.get(userId)
17 livesOfUser.push(currentUserLive)
18 }
19
20 removeLive (userId: number, liveId: number) {
21 const newLivesPerUser = this.livesPerUser.get(userId)
22 .filter(o => o.liveId !== liveId)
23
24 this.livesPerUser.set(userId, newLivesPerUser)
25 }
26
27 addQuotaTo (userId: number, liveId: number, size: number) {
28 const lives = this.livesPerUser.get(userId)
29 const live = lives.find(l => l.liveId === liveId)
30
31 live.size += size
32 }
33
34 getLiveQuotaOf (userId: number) {
35 const currentLives = this.livesPerUser.get(userId)
36 if (!currentLives) return 0
37
38 return currentLives.reduce((sum, obj) => sum + obj.size, 0)
39 }
40
41 static get Instance () {
42 return this.instance || (this.instance = new this())
43 }
44}
45
46export {
47 LiveQuotaStore
48}
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts
new file mode 100644
index 000000000..4af6f3ebf
--- /dev/null
+++ b/server/lib/live/live-segment-sha-store.ts
@@ -0,0 +1,64 @@
1import { basename } from 'path'
2import { logger, loggerTagsFactory } from '@server/helpers/logger'
3import { buildSha256Segment } from '../hls'
4
5const lTags = loggerTagsFactory('live')
6
7class LiveSegmentShaStore {
8
9 private static instance: LiveSegmentShaStore
10
11 private readonly segmentsSha256 = new Map<string, Map<string, string>>()
12
13 private constructor () {
14 }
15
16 getSegmentsSha256 (videoUUID: string) {
17 return this.segmentsSha256.get(videoUUID)
18 }
19
20 async addSegmentSha (videoUUID: string, segmentPath: string) {
21 const segmentName = basename(segmentPath)
22 logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID))
23
24 const shaResult = await buildSha256Segment(segmentPath)
25
26 if (!this.segmentsSha256.has(videoUUID)) {
27 this.segmentsSha256.set(videoUUID, new Map())
28 }
29
30 const filesMap = this.segmentsSha256.get(videoUUID)
31 filesMap.set(segmentName, shaResult)
32 }
33
34 removeSegmentSha (videoUUID: string, segmentPath: string) {
35 const segmentName = basename(segmentPath)
36
37 logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID))
38
39 const filesMap = this.segmentsSha256.get(videoUUID)
40 if (!filesMap) {
41 logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID))
42 return
43 }
44
45 if (!filesMap.has(segmentName)) {
46 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID))
47 return
48 }
49
50 filesMap.delete(segmentName)
51 }
52
53 cleanupShaSegments (videoUUID: string) {
54 this.segmentsSha256.delete(videoUUID)
55 }
56
57 static get Instance () {
58 return this.instance || (this.instance = new this())
59 }
60}
61
62export {
63 LiveSegmentShaStore
64}
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
new file mode 100644
index 000000000..e4526c7a5
--- /dev/null
+++ b/server/lib/live/live-utils.ts
@@ -0,0 +1,23 @@
1import { remove } from 'fs-extra'
2import { basename } from 'path'
3import { MStreamingPlaylist, MVideo } from '@server/types/models'
4import { getHLSDirectory } from '../video-paths'
5
6function buildConcatenatedName (segmentOrPlaylistPath: string) {
7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
8
9 return 'concat-' + num[1] + '.ts'
10}
11
12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
13 const hlsDirectory = getHLSDirectory(video)
14
15 await remove(hlsDirectory)
16
17 await streamingPlaylist.destroy()
18}
19
20export {
21 cleanupLive,
22 buildConcatenatedName
23}
diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts
new file mode 100644
index 000000000..c4d1b59ec
--- /dev/null
+++ b/server/lib/live/shared/index.ts
@@ -0,0 +1 @@
export * from './muxing-session'
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
new file mode 100644
index 000000000..96f6c2c89
--- /dev/null
+++ b/server/lib/live/shared/muxing-session.ts
@@ -0,0 +1,341 @@
1
2import * as Bluebird from 'bluebird'
3import * as chokidar from 'chokidar'
4import { FfmpegCommand } from 'fluent-ffmpeg'
5import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
6import { basename, join } from 'path'
7import { EventEmitter } from 'stream'
8import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils'
9import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
10import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
12import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
14import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles'
15import { isAbleToUploadVideo } from '../../user'
16import { getHLSDirectory } from '../../video-paths'
17import { LiveQuotaStore } from '../live-quota-store'
18import { LiveSegmentShaStore } from '../live-segment-sha-store'
19import { buildConcatenatedName } from '../live-utils'
20
21import memoizee = require('memoizee')
22
23interface MuxingSessionEvents {
24 'master-playlist-created': ({ videoId: number }) => void
25
26 'bad-socket-health': ({ videoId: number }) => void
27 'duration-exceeded': ({ videoId: number }) => void
28 'quota-exceeded': ({ videoId: number }) => void
29
30 'ffmpeg-end': ({ videoId: number }) => void
31 'ffmpeg-error': ({ sessionId: string }) => void
32
33 'after-cleanup': ({ videoId: number }) => void
34}
35
36declare interface MuxingSession {
37 on<U extends keyof MuxingSessionEvents>(
38 event: U, listener: MuxingSessionEvents[U]
39 ): this
40
41 emit<U extends keyof MuxingSessionEvents>(
42 event: U, ...args: Parameters<MuxingSessionEvents[U]>
43 ): boolean
44}
45
46class MuxingSession extends EventEmitter {
47
48 private ffmpegCommand: FfmpegCommand
49
50 private readonly context: any
51 private readonly user: MUserId
52 private readonly sessionId: string
53 private readonly videoLive: MVideoLiveVideo
54 private readonly streamingPlaylist: MStreamingPlaylistVideo
55 private readonly rtmpUrl: string
56 private readonly fps: number
57 private readonly allResolutions: number[]
58
59 private readonly videoId: number
60 private readonly videoUUID: string
61 private readonly saveReplay: boolean
62
63 private readonly lTags: LoggerTagsFn
64
65 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
66
67 private tsWatcher: chokidar.FSWatcher
68 private masterWatcher: chokidar.FSWatcher
69
70 private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
71 return isAbleToUploadVideo(userId, 1000)
72 }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
73
74 private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => {
75 return this.hasClientSocketInBadHealth(sessionId)
76 }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH })
77
78 constructor (options: {
79 context: any
80 user: MUserId
81 sessionId: string
82 videoLive: MVideoLiveVideo
83 streamingPlaylist: MStreamingPlaylistVideo
84 rtmpUrl: string
85 fps: number
86 allResolutions: number[]
87 }) {
88 super()
89
90 this.context = options.context
91 this.user = options.user
92 this.sessionId = options.sessionId
93 this.videoLive = options.videoLive
94 this.streamingPlaylist = options.streamingPlaylist
95 this.rtmpUrl = options.rtmpUrl
96 this.fps = options.fps
97 this.allResolutions = options.allResolutions
98
99 this.videoId = this.videoLive.Video.id
100 this.videoUUID = this.videoLive.Video.uuid
101
102 this.saveReplay = this.videoLive.saveReplay
103
104 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID)
105 }
106
107 async runMuxing () {
108 this.createFiles()
109
110 const outPath = await this.prepareDirectories()
111
112 this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
113 ? await getLiveTranscodingCommand({
114 rtmpUrl: this.rtmpUrl,
115 outPath,
116 resolutions: this.allResolutions,
117 fps: this.fps,
118 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
119 profile: CONFIG.LIVE.TRANSCODING.PROFILE
120 })
121 : getLiveMuxingCommand(this.rtmpUrl, outPath)
122
123 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags)
124
125 this.watchTSFiles(outPath)
126 this.watchMasterFile(outPath)
127
128 this.ffmpegCommand.on('error', (err, stdout, stderr) => {
129 this.onFFmpegError(err, stdout, stderr, outPath)
130 })
131
132 this.ffmpegCommand.on('end', () => this.onFFmpegEnded(outPath))
133
134 this.ffmpegCommand.run()
135 }
136
137 abort () {
138 if (!this.ffmpegCommand) return false
139
140 this.ffmpegCommand.kill('SIGINT')
141 return true
142 }
143
144 private onFFmpegError (err: any, stdout: string, stderr: string, outPath: string) {
145 this.onFFmpegEnded(outPath)
146
147 // Don't care that we killed the ffmpeg process
148 if (err?.message?.includes('Exiting normally')) return
149
150 logger.error('Live transcoding error.', { err, stdout, stderr, ...this.lTags })
151
152 this.emit('ffmpeg-error', ({ sessionId: this.sessionId }))
153 }
154
155 private onFFmpegEnded (outPath: string) {
156 logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.rtmpUrl, this.lTags)
157
158 setTimeout(() => {
159 // Wait latest segments generation, and close watchers
160
161 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ])
162 .then(() => {
163 // Process remaining segments hash
164 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
165 this.processSegments(outPath, this.segmentsToProcessPerPlaylist[key])
166 }
167 })
168 .catch(err => {
169 logger.error(
170 'Cannot close watchers of %s or process remaining hash segments.', outPath,
171 { err, ...this.lTags }
172 )
173 })
174
175 this.emit('after-cleanup', { videoId: this.videoId })
176 }, 1000)
177 }
178
179 private watchMasterFile (outPath: string) {
180 this.masterWatcher = chokidar.watch(outPath + '/master.m3u8')
181
182 this.masterWatcher.on('add', async () => {
183 this.emit('master-playlist-created', { videoId: this.videoId })
184
185 this.masterWatcher.close()
186 .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags }))
187 })
188 }
189
190 private watchTSFiles (outPath: string) {
191 const startStreamDateTime = new Date().getTime()
192
193 this.tsWatcher = chokidar.watch(outPath + '/*.ts')
194
195 const playlistIdMatcher = /^([\d+])-/
196
197 const addHandler = async segmentPath => {
198 logger.debug('Live add handler of %s.', segmentPath, this.lTags)
199
200 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
201
202 const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || []
203 this.processSegments(outPath, segmentsToProcess)
204
205 this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
206
207 if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) {
208 this.emit('bad-socket-health', { videoId: this.videoId })
209 return
210 }
211
212 // Duration constraint check
213 if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
214 this.emit('duration-exceeded', { videoId: this.videoId })
215 return
216 }
217
218 // Check user quota if the user enabled replay saving
219 if (await this.isQuotaExceeded(segmentPath) === true) {
220 this.emit('quota-exceeded', { videoId: this.videoId })
221 }
222 }
223
224 const deleteHandler = segmentPath => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath)
225
226 this.tsWatcher.on('add', p => addHandler(p))
227 this.tsWatcher.on('unlink', p => deleteHandler(p))
228 }
229
230 private async isQuotaExceeded (segmentPath: string) {
231 if (this.saveReplay !== true) return false
232
233 try {
234 const segmentStat = await stat(segmentPath)
235
236 LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.videoLive.id, segmentStat.size)
237
238 const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id)
239
240 return canUpload !== true
241 } catch (err) {
242 logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags })
243 }
244 }
245
246 private createFiles () {
247 for (let i = 0; i < this.allResolutions.length; i++) {
248 const resolution = this.allResolutions[i]
249
250 const file = new VideoFileModel({
251 resolution,
252 size: -1,
253 extname: '.ts',
254 infoHash: null,
255 fps: this.fps,
256 videoStreamingPlaylistId: this.streamingPlaylist.id
257 })
258
259 VideoFileModel.customUpsert(file, 'streaming-playlist', null)
260 .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags }))
261 }
262 }
263
264 private async prepareDirectories () {
265 const outPath = getHLSDirectory(this.videoLive.Video)
266 await ensureDir(outPath)
267
268 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
269
270 if (this.videoLive.saveReplay === true) {
271 await ensureDir(replayDirectory)
272 }
273
274 return outPath
275 }
276
277 private isDurationConstraintValid (streamingStartTime: number) {
278 const maxDuration = CONFIG.LIVE.MAX_DURATION
279 // No limit
280 if (maxDuration < 0) return true
281
282 const now = new Date().getTime()
283 const max = streamingStartTime + maxDuration
284
285 return now <= max
286 }
287
288 private processSegments (hlsVideoPath: string, segmentPaths: string[]) {
289 Bluebird.mapSeries(segmentPaths, async previousSegment => {
290 // Add sha hash of previous segments, because ffmpeg should have finished generating them
291 await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment)
292
293 if (this.saveReplay) {
294 await this.addSegmentToReplay(hlsVideoPath, previousSegment)
295 }
296 }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags }))
297 }
298
299 private hasClientSocketInBadHealth (sessionId: string) {
300 const rtmpSession = this.context.sessions.get(sessionId)
301
302 if (!rtmpSession) {
303 logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags)
304 return
305 }
306
307 for (const playerSessionId of rtmpSession.players) {
308 const playerSession = this.context.sessions.get(playerSessionId)
309
310 if (!playerSession) {
311 logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags)
312 continue
313 }
314
315 if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) {
316 return true
317 }
318 }
319
320 return false
321 }
322
323 private async addSegmentToReplay (hlsVideoPath: string, segmentPath: string) {
324 const segmentName = basename(segmentPath)
325 const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, buildConcatenatedName(segmentName))
326
327 try {
328 const data = await readFile(segmentPath)
329
330 await appendFile(dest, data)
331 } catch (err) {
332 logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags })
333 }
334 }
335}
336
337// ---------------------------------------------------------------------------
338
339export {
340 MuxingSession
341}
diff --git a/server/lib/user.ts b/server/lib/user.ts
index 91a682a7e..8820e8243 100644
--- a/server/lib/user.ts
+++ b/server/lib/user.ts
@@ -14,7 +14,7 @@ import { MUser, MUserDefault, MUserId } from '../types/models/user'
14import { generateAndSaveActorKeys } from './activitypub/actors' 14import { generateAndSaveActorKeys } from './activitypub/actors'
15import { getLocalAccountActivityPubUrl } from './activitypub/url' 15import { getLocalAccountActivityPubUrl } from './activitypub/url'
16import { Emailer } from './emailer' 16import { Emailer } from './emailer'
17import { LiveManager } from './live-manager' 17import { LiveQuotaStore } from './live/live-quota-store'
18import { buildActorInstance } from './local-actor' 18import { buildActorInstance } from './local-actor'
19import { Redis } from './redis' 19import { Redis } from './redis'
20import { createLocalVideoChannel } from './video-channel' 20import { createLocalVideoChannel } from './video-channel'
@@ -129,7 +129,7 @@ async function getOriginalVideoFileTotalFromUser (user: MUserId) {
129 129
130 const base = await UserModel.getTotalRawQuery(query, user.id) 130 const base = await UserModel.getTotalRawQuery(query, user.id)
131 131
132 return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) 132 return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id)
133} 133}
134 134
135// Returns cumulative size of all video files uploaded in the last 24 hours. 135// Returns cumulative size of all video files uploaded in the last 24 hours.
@@ -143,10 +143,10 @@ async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) {
143 143
144 const base = await UserModel.getTotalRawQuery(query, user.id) 144 const base = await UserModel.getTotalRawQuery(query, user.id)
145 145
146 return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) 146 return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id)
147} 147}
148 148
149async function isAbleToUploadVideo (userId: number, size: number) { 149async function isAbleToUploadVideo (userId: number, newVideoSize: number) {
150 const user = await UserModel.loadById(userId) 150 const user = await UserModel.loadById(userId)
151 151
152 if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) 152 if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true)
@@ -156,8 +156,8 @@ async function isAbleToUploadVideo (userId: number, size: number) {
156 getOriginalVideoFileTotalDailyFromUser(user) 156 getOriginalVideoFileTotalDailyFromUser(user)
157 ]) 157 ])
158 158
159 const uploadedTotal = size + totalBytes 159 const uploadedTotal = newVideoSize + totalBytes
160 const uploadedDaily = size + totalBytesDaily 160 const uploadedDaily = newVideoSize + totalBytesDaily
161 161
162 if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota 162 if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota
163 if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily 163 if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts
index 37c43c3b0..0984c0d7a 100644
--- a/server/lib/video-blacklist.ts
+++ b/server/lib/video-blacklist.ts
@@ -16,7 +16,7 @@ import { CONFIG } from '../initializers/config'
16import { VideoBlacklistModel } from '../models/video/video-blacklist' 16import { VideoBlacklistModel } from '../models/video/video-blacklist'
17import { sendDeleteVideo } from './activitypub/send' 17import { sendDeleteVideo } from './activitypub/send'
18import { federateVideoIfNeeded } from './activitypub/videos' 18import { federateVideoIfNeeded } from './activitypub/videos'
19import { LiveManager } from './live-manager' 19import { LiveManager } from './live/live-manager'
20import { Notifier } from './notifier' 20import { Notifier } from './notifier'
21import { Hooks } from './plugins/hooks' 21import { Hooks } from './plugins/hooks'
22 22
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index ca839b0d4..d2efc2553 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -27,7 +27,7 @@ import {
27import { setAsUpdated } from '@server/helpers/database-utils' 27import { setAsUpdated } from '@server/helpers/database-utils'
28import { buildNSFWFilter } from '@server/helpers/express-utils' 28import { buildNSFWFilter } from '@server/helpers/express-utils'
29import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' 29import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
30import { LiveManager } from '@server/lib/live-manager' 30import { LiveManager } from '@server/lib/live/live-manager'
31import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths' 31import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths'
32import { getServerActor } from '@server/models/application/application' 32import { getServerActor } from '@server/models/application/application'
33import { ModelCache } from '@server/models/model-cache' 33import { ModelCache } from '@server/models/model-cache'
diff --git a/server/tests/api/live/index.ts b/server/tests/api/live/index.ts
index c733f564e..e6bcef49f 100644
--- a/server/tests/api/live/index.ts
+++ b/server/tests/api/live/index.ts
@@ -1,4 +1,6 @@
1import './live-constraints' 1import './live-constraints'
2import './live-socket-messages'
2import './live-permanent' 3import './live-permanent'
3import './live-save-replay' 4import './live-save-replay'
5import './live-views'
4import './live' 6import './live'
diff --git a/server/tests/api/live/live-socket-messages.ts b/server/tests/api/live/live-socket-messages.ts
new file mode 100644
index 000000000..e00909ade
--- /dev/null
+++ b/server/tests/api/live/live-socket-messages.ts
@@ -0,0 +1,196 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { getLiveNotificationSocket } from '@shared/extra-utils/socket/socket-io'
6import { VideoPrivacy, VideoState } from '@shared/models'
7import {
8 cleanupTests,
9 createLive,
10 doubleFollow,
11 flushAndRunMultipleServers,
12 getVideoIdFromUUID,
13 sendRTMPStreamInVideo,
14 ServerInfo,
15 setAccessTokensToServers,
16 setDefaultVideoChannel,
17 stopFfmpeg,
18 updateCustomSubConfig,
19 viewVideo,
20 wait,
21 waitJobs,
22 waitUntilLiveEnded,
23 waitUntilLivePublishedOnAllServers
24} from '../../../../shared/extra-utils'
25
26const expect = chai.expect
27
28describe('Test live', function () {
29 let servers: ServerInfo[] = []
30
31 before(async function () {
32 this.timeout(120000)
33
34 servers = await flushAndRunMultipleServers(2)
35
36 // Get the access tokens
37 await setAccessTokensToServers(servers)
38 await setDefaultVideoChannel(servers)
39
40 await updateCustomSubConfig(servers[0].url, servers[0].accessToken, {
41 live: {
42 enabled: true,
43 allowReplay: true,
44 transcoding: {
45 enabled: false
46 }
47 }
48 })
49
50 // Server 1 and server 2 follow each other
51 await doubleFollow(servers[0], servers[1])
52 })
53
54 describe('Live socket messages', function () {
55
56 async function createLiveWrapper () {
57 const liveAttributes = {
58 name: 'live video',
59 channelId: servers[0].videoChannel.id,
60 privacy: VideoPrivacy.PUBLIC
61 }
62
63 const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
64 return res.body.video.uuid
65 }
66
67 it('Should correctly send a message when the live starts and ends', async function () {
68 this.timeout(60000)
69
70 const localStateChanges: VideoState[] = []
71 const remoteStateChanges: VideoState[] = []
72
73 const liveVideoUUID = await createLiveWrapper()
74 await waitJobs(servers)
75
76 {
77 const videoId = await getVideoIdFromUUID(servers[0].url, liveVideoUUID)
78
79 const localSocket = getLiveNotificationSocket(servers[0].url)
80 localSocket.on('state-change', data => localStateChanges.push(data.state))
81 localSocket.emit('subscribe', { videoId })
82 }
83
84 {
85 const videoId = await getVideoIdFromUUID(servers[1].url, liveVideoUUID)
86
87 const remoteSocket = getLiveNotificationSocket(servers[1].url)
88 remoteSocket.on('state-change', data => remoteStateChanges.push(data.state))
89 remoteSocket.emit('subscribe', { videoId })
90 }
91
92 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
93
94 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
95 await waitJobs(servers)
96
97 for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
98 expect(stateChanges).to.have.length.at.least(1)
99 expect(stateChanges[stateChanges.length - 1]).to.equal(VideoState.PUBLISHED)
100 }
101
102 await stopFfmpeg(command)
103
104 for (const server of servers) {
105 await waitUntilLiveEnded(server.url, server.accessToken, liveVideoUUID)
106 }
107 await waitJobs(servers)
108
109 for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
110 expect(stateChanges).to.have.length.at.least(2)
111 expect(stateChanges[stateChanges.length - 1]).to.equal(VideoState.LIVE_ENDED)
112 }
113 })
114
115 it('Should correctly send views change notification', async function () {
116 this.timeout(60000)
117
118 let localLastVideoViews = 0
119 let remoteLastVideoViews = 0
120
121 const liveVideoUUID = await createLiveWrapper()
122 await waitJobs(servers)
123
124 {
125 const videoId = await getVideoIdFromUUID(servers[0].url, liveVideoUUID)
126
127 const localSocket = getLiveNotificationSocket(servers[0].url)
128 localSocket.on('views-change', data => { localLastVideoViews = data.views })
129 localSocket.emit('subscribe', { videoId })
130 }
131
132 {
133 const videoId = await getVideoIdFromUUID(servers[1].url, liveVideoUUID)
134
135 const remoteSocket = getLiveNotificationSocket(servers[1].url)
136 remoteSocket.on('views-change', data => { remoteLastVideoViews = data.views })
137 remoteSocket.emit('subscribe', { videoId })
138 }
139
140 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
141
142 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
143 await waitJobs(servers)
144
145 expect(localLastVideoViews).to.equal(0)
146 expect(remoteLastVideoViews).to.equal(0)
147
148 await viewVideo(servers[0].url, liveVideoUUID)
149 await viewVideo(servers[1].url, liveVideoUUID)
150
151 await waitJobs(servers)
152 await wait(5000)
153 await waitJobs(servers)
154
155 expect(localLastVideoViews).to.equal(2)
156 expect(remoteLastVideoViews).to.equal(2)
157
158 await stopFfmpeg(command)
159 })
160
161 it('Should not receive a notification after unsubscribe', async function () {
162 this.timeout(120000)
163
164 const stateChanges: VideoState[] = []
165
166 const liveVideoUUID = await createLiveWrapper()
167 await waitJobs(servers)
168
169 const videoId = await getVideoIdFromUUID(servers[0].url, liveVideoUUID)
170
171 const socket = getLiveNotificationSocket(servers[0].url)
172 socket.on('state-change', data => stateChanges.push(data.state))
173 socket.emit('subscribe', { videoId })
174
175 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
176
177 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
178 await waitJobs(servers)
179
180 // Notifier waits before sending a notification
181 await wait(10000)
182
183 expect(stateChanges).to.have.lengthOf(1)
184 socket.emit('unsubscribe', { videoId })
185
186 await stopFfmpeg(command)
187 await waitJobs(servers)
188
189 expect(stateChanges).to.have.lengthOf(1)
190 })
191 })
192
193 after(async function () {
194 await cleanupTests(servers)
195 })
196})
diff --git a/server/tests/api/live/live-views.ts b/server/tests/api/live/live-views.ts
new file mode 100644
index 000000000..a44d21ffa
--- /dev/null
+++ b/server/tests/api/live/live-views.ts
@@ -0,0 +1,130 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { FfmpegCommand } from 'fluent-ffmpeg'
6import { VideoDetails, VideoPrivacy } from '@shared/models'
7import {
8 cleanupTests,
9 createLive,
10 doubleFollow,
11 flushAndRunMultipleServers,
12 getVideo,
13 sendRTMPStreamInVideo,
14 ServerInfo,
15 setAccessTokensToServers,
16 setDefaultVideoChannel,
17 stopFfmpeg,
18 updateCustomSubConfig,
19 viewVideo,
20 wait,
21 waitJobs,
22 waitUntilLivePublishedOnAllServers
23} from '../../../../shared/extra-utils'
24
25const expect = chai.expect
26
27describe('Test live', function () {
28 let servers: ServerInfo[] = []
29
30 before(async function () {
31 this.timeout(120000)
32
33 servers = await flushAndRunMultipleServers(2)
34
35 // Get the access tokens
36 await setAccessTokensToServers(servers)
37 await setDefaultVideoChannel(servers)
38
39 await updateCustomSubConfig(servers[0].url, servers[0].accessToken, {
40 live: {
41 enabled: true,
42 allowReplay: true,
43 transcoding: {
44 enabled: false
45 }
46 }
47 })
48
49 // Server 1 and server 2 follow each other
50 await doubleFollow(servers[0], servers[1])
51 })
52
53 describe('Live views', function () {
54 let liveVideoId: string
55 let command: FfmpegCommand
56
57 async function countViews (expected: number) {
58 for (const server of servers) {
59 const res = await getVideo(server.url, liveVideoId)
60 const video: VideoDetails = res.body
61
62 expect(video.views).to.equal(expected)
63 }
64 }
65
66 before(async function () {
67 this.timeout(30000)
68
69 const liveAttributes = {
70 name: 'live video',
71 channelId: servers[0].videoChannel.id,
72 privacy: VideoPrivacy.PUBLIC
73 }
74
75 const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
76 liveVideoId = res.body.video.uuid
77
78 command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
79 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
80 await waitJobs(servers)
81 })
82
83 it('Should display no views for a live', async function () {
84 await countViews(0)
85 })
86
87 it('Should view a live twice and display 1 view', async function () {
88 this.timeout(30000)
89
90 await viewVideo(servers[0].url, liveVideoId)
91 await viewVideo(servers[0].url, liveVideoId)
92
93 await wait(7000)
94
95 await waitJobs(servers)
96
97 await countViews(1)
98 })
99
100 it('Should wait and display 0 views', async function () {
101 this.timeout(30000)
102
103 await wait(12000)
104 await waitJobs(servers)
105
106 await countViews(0)
107 })
108
109 it('Should view a live on a remote and on local and display 2 views', async function () {
110 this.timeout(30000)
111
112 await viewVideo(servers[0].url, liveVideoId)
113 await viewVideo(servers[1].url, liveVideoId)
114 await viewVideo(servers[1].url, liveVideoId)
115
116 await wait(7000)
117 await waitJobs(servers)
118
119 await countViews(2)
120 })
121
122 after(async function () {
123 await stopFfmpeg(command)
124 })
125 })
126
127 after(async function () {
128 await cleanupTests(servers)
129 })
130})
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index 57fb58150..50397924e 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -2,10 +2,8 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { FfmpegCommand } from 'fluent-ffmpeg'
6import { join } from 'path' 5import { join } from 'path'
7import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils' 6import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
8import { getLiveNotificationSocket } from '@shared/extra-utils/socket/socket-io'
9import { LiveVideo, LiveVideoCreate, Video, VideoDetails, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models' 7import { LiveVideo, LiveVideoCreate, Video, VideoDetails, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models'
10import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' 8import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
11import { 9import {
@@ -22,7 +20,6 @@ import {
22 getMyVideosWithFilter, 20 getMyVideosWithFilter,
23 getPlaylist, 21 getPlaylist,
24 getVideo, 22 getVideo,
25 getVideoIdFromUUID,
26 getVideosList, 23 getVideosList,
27 getVideosWithFilters, 24 getVideosWithFilters,
28 killallServers, 25 killallServers,
@@ -40,11 +37,11 @@ import {
40 updateCustomSubConfig, 37 updateCustomSubConfig,
41 updateLive, 38 updateLive,
42 uploadVideoAndGetId, 39 uploadVideoAndGetId,
43 viewVideo,
44 wait, 40 wait,
45 waitJobs, 41 waitJobs,
46 waitUntilLiveEnded, 42 waitUntilLiveEnded,
47 waitUntilLivePublished, 43 waitUntilLivePublished,
44 waitUntilLivePublishedOnAllServers,
48 waitUntilLiveSegmentGeneration 45 waitUntilLiveSegmentGeneration
49} from '../../../../shared/extra-utils' 46} from '../../../../shared/extra-utils'
50 47
@@ -53,12 +50,6 @@ const expect = chai.expect
53describe('Test live', function () { 50describe('Test live', function () {
54 let servers: ServerInfo[] = [] 51 let servers: ServerInfo[] = []
55 52
56 async function waitUntilLivePublishedOnAllServers (videoId: string) {
57 for (const server of servers) {
58 await waitUntilLivePublished(server.url, server.accessToken, videoId)
59 }
60 }
61
62 before(async function () { 53 before(async function () {
63 this.timeout(120000) 54 this.timeout(120000)
64 55
@@ -247,7 +238,7 @@ describe('Test live', function () {
247 liveVideoId = resLive.body.video.uuid 238 liveVideoId = resLive.body.video.uuid
248 239
249 command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId) 240 command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
250 await waitUntilLivePublishedOnAllServers(liveVideoId) 241 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
251 await waitJobs(servers) 242 await waitJobs(servers)
252 }) 243 })
253 244
@@ -461,7 +452,7 @@ describe('Test live', function () {
461 liveVideoId = await createLiveWrapper(false) 452 liveVideoId = await createLiveWrapper(false)
462 453
463 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId) 454 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
464 await waitUntilLivePublishedOnAllServers(liveVideoId) 455 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
465 await waitJobs(servers) 456 await waitJobs(servers)
466 457
467 await testVideoResolutions(liveVideoId, [ 720 ]) 458 await testVideoResolutions(liveVideoId, [ 720 ])
@@ -477,7 +468,7 @@ describe('Test live', function () {
477 liveVideoId = await createLiveWrapper(false) 468 liveVideoId = await createLiveWrapper(false)
478 469
479 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId) 470 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
480 await waitUntilLivePublishedOnAllServers(liveVideoId) 471 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
481 await waitJobs(servers) 472 await waitJobs(servers)
482 473
483 await testVideoResolutions(liveVideoId, resolutions) 474 await testVideoResolutions(liveVideoId, resolutions)
@@ -494,7 +485,7 @@ describe('Test live', function () {
494 liveVideoId = await createLiveWrapper(true) 485 liveVideoId = await createLiveWrapper(true)
495 486
496 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId, 'video_short2.webm') 487 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId, 'video_short2.webm')
497 await waitUntilLivePublishedOnAllServers(liveVideoId) 488 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
498 await waitJobs(servers) 489 await waitJobs(servers)
499 490
500 await testVideoResolutions(liveVideoId, resolutions) 491 await testVideoResolutions(liveVideoId, resolutions)
@@ -504,7 +495,7 @@ describe('Test live', function () {
504 495
505 await waitJobs(servers) 496 await waitJobs(servers)
506 497
507 await waitUntilLivePublishedOnAllServers(liveVideoId) 498 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
508 499
509 const bitrateLimits = { 500 const bitrateLimits = {
510 720: 5000 * 1000, // 60FPS 501 720: 5000 * 1000, // 60FPS
@@ -559,216 +550,6 @@ describe('Test live', function () {
559 }) 550 })
560 }) 551 })
561 552
562 describe('Live views', function () {
563 let liveVideoId: string
564 let command: FfmpegCommand
565
566 async function countViews (expected: number) {
567 for (const server of servers) {
568 const res = await getVideo(server.url, liveVideoId)
569 const video: VideoDetails = res.body
570
571 expect(video.views).to.equal(expected)
572 }
573 }
574
575 before(async function () {
576 this.timeout(30000)
577
578 const liveAttributes = {
579 name: 'live video',
580 channelId: servers[0].videoChannel.id,
581 privacy: VideoPrivacy.PUBLIC
582 }
583
584 const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
585 liveVideoId = res.body.video.uuid
586
587 command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
588 await waitUntilLivePublishedOnAllServers(liveVideoId)
589 await waitJobs(servers)
590 })
591
592 it('Should display no views for a live', async function () {
593 await countViews(0)
594 })
595
596 it('Should view a live twice and display 1 view', async function () {
597 this.timeout(30000)
598
599 await viewVideo(servers[0].url, liveVideoId)
600 await viewVideo(servers[0].url, liveVideoId)
601
602 await wait(7000)
603
604 await waitJobs(servers)
605
606 await countViews(1)
607 })
608
609 it('Should wait and display 0 views', async function () {
610 this.timeout(30000)
611
612 await wait(7000)
613 await waitJobs(servers)
614
615 await countViews(0)
616 })
617
618 it('Should view a live on a remote and on local and display 2 views', async function () {
619 this.timeout(30000)
620
621 await viewVideo(servers[0].url, liveVideoId)
622 await viewVideo(servers[1].url, liveVideoId)
623 await viewVideo(servers[1].url, liveVideoId)
624
625 await wait(7000)
626 await waitJobs(servers)
627
628 await countViews(2)
629 })
630
631 after(async function () {
632 await stopFfmpeg(command)
633 })
634 })
635
636 describe('Live socket messages', function () {
637
638 async function createLiveWrapper () {
639 const liveAttributes = {
640 name: 'live video',
641 channelId: servers[0].videoChannel.id,
642 privacy: VideoPrivacy.PUBLIC
643 }
644
645 const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
646 return res.body.video.uuid
647 }
648
649 it('Should correctly send a message when the live starts and ends', async function () {
650 this.timeout(60000)
651
652 const localStateChanges: VideoState[] = []
653 const remoteStateChanges: VideoState[] = []
654
655 const liveVideoUUID = await createLiveWrapper()
656 await waitJobs(servers)
657
658 {
659 const videoId = await getVideoIdFromUUID(servers[0].url, liveVideoUUID)
660
661 const localSocket = getLiveNotificationSocket(servers[0].url)
662 localSocket.on('state-change', data => localStateChanges.push(data.state))
663 localSocket.emit('subscribe', { videoId })
664 }
665
666 {
667 const videoId = await getVideoIdFromUUID(servers[1].url, liveVideoUUID)
668
669 const remoteSocket = getLiveNotificationSocket(servers[1].url)
670 remoteSocket.on('state-change', data => remoteStateChanges.push(data.state))
671 remoteSocket.emit('subscribe', { videoId })
672 }
673
674 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
675
676 await waitUntilLivePublishedOnAllServers(liveVideoUUID)
677 await waitJobs(servers)
678
679 for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
680 expect(stateChanges).to.have.length.at.least(1)
681 expect(stateChanges[stateChanges.length - 1]).to.equal(VideoState.PUBLISHED)
682 }
683
684 await stopFfmpeg(command)
685
686 for (const server of servers) {
687 await waitUntilLiveEnded(server.url, server.accessToken, liveVideoUUID)
688 }
689 await waitJobs(servers)
690
691 for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
692 expect(stateChanges).to.have.length.at.least(2)
693 expect(stateChanges[stateChanges.length - 1]).to.equal(VideoState.LIVE_ENDED)
694 }
695 })
696
697 it('Should correctly send views change notification', async function () {
698 this.timeout(60000)
699
700 let localLastVideoViews = 0
701 let remoteLastVideoViews = 0
702
703 const liveVideoUUID = await createLiveWrapper()
704 await waitJobs(servers)
705
706 {
707 const videoId = await getVideoIdFromUUID(servers[0].url, liveVideoUUID)
708
709 const localSocket = getLiveNotificationSocket(servers[0].url)
710 localSocket.on('views-change', data => { localLastVideoViews = data.views })
711 localSocket.emit('subscribe', { videoId })
712 }
713
714 {
715 const videoId = await getVideoIdFromUUID(servers[1].url, liveVideoUUID)
716
717 const remoteSocket = getLiveNotificationSocket(servers[1].url)
718 remoteSocket.on('views-change', data => { remoteLastVideoViews = data.views })
719 remoteSocket.emit('subscribe', { videoId })
720 }
721
722 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
723
724 await waitUntilLivePublishedOnAllServers(liveVideoUUID)
725 await waitJobs(servers)
726
727 expect(localLastVideoViews).to.equal(0)
728 expect(remoteLastVideoViews).to.equal(0)
729
730 await viewVideo(servers[0].url, liveVideoUUID)
731 await viewVideo(servers[1].url, liveVideoUUID)
732
733 await waitJobs(servers)
734 await wait(5000)
735 await waitJobs(servers)
736
737 expect(localLastVideoViews).to.equal(2)
738 expect(remoteLastVideoViews).to.equal(2)
739
740 await stopFfmpeg(command)
741 })
742
743 it('Should not receive a notification after unsubscribe', async function () {
744 this.timeout(60000)
745
746 const stateChanges: VideoState[] = []
747
748 const liveVideoUUID = await createLiveWrapper()
749 await waitJobs(servers)
750
751 const videoId = await getVideoIdFromUUID(servers[0].url, liveVideoUUID)
752
753 const socket = getLiveNotificationSocket(servers[0].url)
754 socket.on('state-change', data => stateChanges.push(data.state))
755 socket.emit('subscribe', { videoId })
756
757 const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
758
759 await waitUntilLivePublishedOnAllServers(liveVideoUUID)
760 await waitJobs(servers)
761
762 expect(stateChanges).to.have.lengthOf(1)
763 socket.emit('unsubscribe', { videoId })
764
765 await stopFfmpeg(command)
766 await waitJobs(servers)
767
768 expect(stateChanges).to.have.lengthOf(1)
769 })
770 })
771
772 describe('After a server restart', function () { 553 describe('After a server restart', function () {
773 let liveVideoId: string 554 let liveVideoId: string
774 let liveVideoReplayId: string 555 let liveVideoReplayId: string
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-03 13:22:05 +0000