aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-11-10 14:15:59 +0100
committerChocobozzz <me@florianbigard.com>2020-11-10 14:15:59 +0100
commit210856a7be4631540791bad027fb3ef0f7a51f14 (patch)
treedb187b29a37fb1cfba1c7f97efb8a145b1f1a149
parent52a350a15c34aa13bffaeedeb39de37cb0534fde (diff)
downloadPeerTube-210856a7be4631540791bad027fb3ef0f7a51f14.tar.gz
PeerTube-210856a7be4631540791bad027fb3ef0f7a51f14.tar.zst
PeerTube-210856a7be4631540791bad027fb3ef0f7a51f14.zip
Try to fix live segments check
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts10
-rw-r--r--client/src/assets/player/p2p-media-loader/segment-validator.ts15
-rw-r--r--client/src/root-helpers/utils.ts9
-rw-r--r--server/lib/live-manager.ts60
4 files changed, 49 insertions, 45 deletions
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
index de1cf46b1..d442df0e3 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
+++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
@@ -95,7 +95,7 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A
95 ] 95 ]
96 96
97 this.liveMaxDurationOptions = [ 97 this.liveMaxDurationOptions = [
98 { value: 0, label: $localize`No limit` }, 98 { value: null, label: $localize`No limit` },
99 { value: 1000 * 3600, label: $localize`1 hour` }, 99 { value: 1000 * 3600, label: $localize`1 hour` },
100 { value: 1000 * 3600 * 3, label: $localize`3 hours` }, 100 { value: 1000 * 3600 * 3, label: $localize`3 hours` },
101 { value: 1000 * 3600 * 5, label: $localize`5 hours` }, 101 { value: 1000 * 3600 * 5, label: $localize`5 hours` },
@@ -328,7 +328,13 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A
328 } 328 }
329 329
330 async formValidated () { 330 async formValidated () {
331 this.configService.updateCustomConfig(this.form.getRawValue()) 331 const value: CustomConfig = this.form.getRawValue()
332
333 // Transform "null" to null
334 const maxDuration = value.live.maxDuration as any
335 if (maxDuration === 'null') value.live.maxDuration = null
336
337 this.configService.updateCustomConfig(value)
332 .subscribe( 338 .subscribe(
333 res => { 339 res => {
334 this.customConfig = res 340 this.customConfig = res
diff --git a/client/src/assets/player/p2p-media-loader/segment-validator.ts b/client/src/assets/player/p2p-media-loader/segment-validator.ts
index fa6e6df1d..9add35184 100644
--- a/client/src/assets/player/p2p-media-loader/segment-validator.ts
+++ b/client/src/assets/player/p2p-media-loader/segment-validator.ts
@@ -1,27 +1,32 @@
1import { wait } from '@root-helpers/utils'
1import { Segment } from 'p2p-media-loader-core' 2import { Segment } from 'p2p-media-loader-core'
2import { basename } from 'path' 3import { basename } from 'path'
3 4
4type SegmentsJSON = { [filename: string]: string | { [byterange: string]: string } } 5type SegmentsJSON = { [filename: string]: string | { [byterange: string]: string } }
5 6
7const maxRetries = 3
8
6function segmentValidatorFactory (segmentsSha256Url: string) { 9function segmentValidatorFactory (segmentsSha256Url: string) {
7 let segmentsJSON = fetchSha256Segments(segmentsSha256Url) 10 let segmentsJSON = fetchSha256Segments(segmentsSha256Url)
8 const regex = /bytes=(\d+)-(\d+)/ 11 const regex = /bytes=(\d+)-(\d+)/
9 12
10 return async function segmentValidator (segment: Segment, canRefetchSegmentHashes = true) { 13 return async function segmentValidator (segment: Segment, retry = 1) {
11 const filename = basename(segment.url) 14 const filename = basename(segment.url)
12 15
13 const segmentValue = (await segmentsJSON)[filename] 16 const segmentValue = (await segmentsJSON)[filename]
14 17
15 if (!segmentValue && !canRefetchSegmentHashes) { 18 if (!segmentValue && retry > maxRetries) {
16 throw new Error(`Unknown segment name ${filename} in segment validator`) 19 throw new Error(`Unknown segment name ${filename} in segment validator`)
17 } 20 }
18 21
19 if (!segmentValue) { 22 if (!segmentValue) {
20 console.log('Refetching sha segments.') 23 await wait(1000)
24
25 console.log('Refetching sha segments for %s.', filename)
21 26
22 // Refetch
23 segmentsJSON = fetchSha256Segments(segmentsSha256Url) 27 segmentsJSON = fetchSha256Segments(segmentsSha256Url)
24 segmentValidator(segment, false) 28 await segmentValidator(segment, retry + 1)
29
25 return 30 return
26 } 31 }
27 32
diff --git a/client/src/root-helpers/utils.ts b/client/src/root-helpers/utils.ts
index 6df151ad9..de4e08bf5 100644
--- a/client/src/root-helpers/utils.ts
+++ b/client/src/root-helpers/utils.ts
@@ -44,7 +44,14 @@ function importModule (path: string) {
44 }) 44 })
45} 45}
46 46
47function wait (ms: number) {
48 return new Promise(res => {
49 setTimeout(() => res(), ms)
50 })
51}
52
47export { 53export {
48 importModule, 54 importModule,
49 objectToUrlEncoded 55 objectToUrlEncoded,
56 wait
50} 57}
diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts
index e85998686..31753619c 100644
--- a/server/lib/live-manager.ts
+++ b/server/lib/live-manager.ts
@@ -1,5 +1,4 @@
1 1
2import { AsyncQueue, queue } from 'async'
3import * as chokidar from 'chokidar' 2import * as chokidar from 'chokidar'
4import { FfmpegCommand } from 'fluent-ffmpeg' 3import { FfmpegCommand } from 'fluent-ffmpeg'
5import { ensureDir, stat } from 'fs-extra' 4import { ensureDir, stat } from 'fs-extra'
@@ -50,12 +49,6 @@ const config = {
50 } 49 }
51} 50}
52 51
53type SegmentSha256QueueParam = {
54 operation: 'update' | 'delete'
55 videoUUID: string
56 segmentPath: string
57}
58
59class LiveManager { 52class LiveManager {
60 53
61 private static instance: LiveManager 54 private static instance: LiveManager
@@ -71,7 +64,6 @@ class LiveManager {
71 return isAbleToUploadVideo(userId, 1000) 64 return isAbleToUploadVideo(userId, 1000)
72 }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) 65 }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
73 66
74 private segmentsSha256Queue: AsyncQueue<SegmentSha256QueueParam>
75 private rtmpServer: any 67 private rtmpServer: any
76 68
77 private constructor () { 69 private constructor () {
@@ -96,18 +88,6 @@ class LiveManager {
96 logger.info('Live session ended.', { sessionId }) 88 logger.info('Live session ended.', { sessionId })
97 }) 89 })
98 90
99 this.segmentsSha256Queue = queue<SegmentSha256QueueParam, Error>((options, cb) => {
100 const promise = options.operation === 'update'
101 ? this.addSegmentSha(options)
102 : Promise.resolve(this.removeSegmentSha(options))
103
104 promise.then(() => cb())
105 .catch(err => {
106 logger.error('Cannot update/remove sha segment %s.', options.segmentPath, { err })
107 cb()
108 })
109 })
110
111 registerConfigChangedHandler(() => { 91 registerConfigChangedHandler(() => {
112 if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) { 92 if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) {
113 this.run() 93 this.run()
@@ -294,11 +274,18 @@ class LiveManager {
294 274
295 const tsWatcher = chokidar.watch(outPath + '/*.ts') 275 const tsWatcher = chokidar.watch(outPath + '/*.ts')
296 276
297 const updateSegment = segmentPath => this.segmentsSha256Queue.push({ operation: 'update', segmentPath, videoUUID }) 277 let segmentsToProcess: string[] = []
298 278
299 const addHandler = segmentPath => { 279 const addHandler = segmentPath => {
300 updateSegment(segmentPath) 280 // Add sha hash of previous segments, because ffmpeg should have finished generating them
281 for (const previousSegment of segmentsToProcess) {
282 this.addSegmentSha(videoUUID, previousSegment)
283 .catch(err => logger.error('Cannot add sha segment of video %s -> %s.', videoUUID, previousSegment, { err }))
284 }
285
286 segmentsToProcess = [ segmentPath ]
301 287
288 // Duration constraint check
302 if (this.isDurationConstraintValid(startStreamDateTime) !== true) { 289 if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
303 logger.info('Stopping session of %s: max duration exceeded.', videoUUID) 290 logger.info('Stopping session of %s: max duration exceeded.', videoUUID)
304 291
@@ -323,10 +310,9 @@ class LiveManager {
323 } 310 }
324 } 311 }
325 312
326 const deleteHandler = segmentPath => this.segmentsSha256Queue.push({ operation: 'delete', segmentPath, videoUUID }) 313 const deleteHandler = segmentPath => this.removeSegmentSha(videoUUID, segmentPath)
327 314
328 tsWatcher.on('add', p => addHandler(p)) 315 tsWatcher.on('add', p => addHandler(p))
329 tsWatcher.on('change', p => updateSegment(p))
330 tsWatcher.on('unlink', p => deleteHandler(p)) 316 tsWatcher.on('unlink', p => deleteHandler(p))
331 317
332 const masterWatcher = chokidar.watch(outPath + '/master.m3u8') 318 const masterWatcher = chokidar.watch(outPath + '/master.m3u8')
@@ -399,33 +385,33 @@ class LiveManager {
399 } 385 }
400 } 386 }
401 387
402 private async addSegmentSha (options: SegmentSha256QueueParam) { 388 private async addSegmentSha (videoUUID: string, segmentPath: string) {
403 const segmentName = basename(options.segmentPath) 389 const segmentName = basename(segmentPath)
404 logger.debug('Updating live sha segment %s.', options.segmentPath) 390 logger.debug('Adding live sha segment %s.', segmentPath)
405 391
406 const shaResult = await buildSha256Segment(options.segmentPath) 392 const shaResult = await buildSha256Segment(segmentPath)
407 393
408 if (!this.segmentsSha256.has(options.videoUUID)) { 394 if (!this.segmentsSha256.has(videoUUID)) {
409 this.segmentsSha256.set(options.videoUUID, new Map()) 395 this.segmentsSha256.set(videoUUID, new Map())
410 } 396 }
411 397
412 const filesMap = this.segmentsSha256.get(options.videoUUID) 398 const filesMap = this.segmentsSha256.get(videoUUID)
413 filesMap.set(segmentName, shaResult) 399 filesMap.set(segmentName, shaResult)
414 } 400 }
415 401
416 private removeSegmentSha (options: SegmentSha256QueueParam) { 402 private removeSegmentSha (videoUUID: string, segmentPath: string) {
417 const segmentName = basename(options.segmentPath) 403 const segmentName = basename(segmentPath)
418 404
419 logger.debug('Removing live sha segment %s.', options.segmentPath) 405 logger.debug('Removing live sha segment %s.', segmentPath)
420 406
421 const filesMap = this.segmentsSha256.get(options.videoUUID) 407 const filesMap = this.segmentsSha256.get(videoUUID)
422 if (!filesMap) { 408 if (!filesMap) {
423 logger.warn('Unknown files map to remove sha for %s.', options.videoUUID) 409 logger.warn('Unknown files map to remove sha for %s.', videoUUID)
424 return 410 return
425 } 411 }
426 412
427 if (!filesMap.has(segmentName)) { 413 if (!filesMap.has(segmentName)) {
428 logger.warn('Unknown segment in files map for video %s and segment %s.', options.videoUUID, options.segmentPath) 414 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath)
429 return 415 return
430 } 416 }
431 417