aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-08-06 15:25:45 +0200
committerChocobozzz <me@florianbigard.com>2021-08-06 15:25:45 +0200
commit18998c45c001869a883ec7a2d286d8170f768381 (patch)
tree83cd925fc3ba14f5dabf66e24c26adfee061f21e
parenta1c63fe1a2159c5796e37a63c2e4acd96dc557ca (diff)
downloadPeerTube-18998c45c001869a883ec7a2d286d8170f768381.tar.gz
PeerTube-18998c45c001869a883ec7a2d286d8170f768381.tar.zst
PeerTube-18998c45c001869a883ec7a2d286d8170f768381.zip
Correctly handle large HLS files for redundancy
-rw-r--r--server/lib/hls.ts13
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts15
2 files changed, 16 insertions, 12 deletions
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 32b02bc26..0e77ab9fa 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,4 +1,4 @@
1import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' 1import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra'
2import { flatten, uniq } from 'lodash' 2import { flatten, uniq } from 'lodash'
3import { basename, dirname, join } from 'path' 3import { basename, dirname, join } from 'path'
4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' 4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
@@ -108,8 +108,9 @@ async function buildSha256Segment (segmentPath: string) {
108 return sha256(buf) 108 return sha256(buf)
109} 109}
110 110
111function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { 111function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
112 let timer 112 let timer
113 let remainingBodyKBLimit = bodyKBLimit
113 114
114 logger.info('Importing HLS playlist %s', playlistUrl) 115 logger.info('Importing HLS playlist %s', playlistUrl)
115 116
@@ -136,8 +137,12 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
136 for (const fileUrl of fileUrls) { 137 for (const fileUrl of fileUrls) {
137 const destPath = join(tmpDirectory, basename(fileUrl)) 138 const destPath = join(tmpDirectory, basename(fileUrl))
138 139
139 const bodyKBLimit = 10 * 1000 * 1000 // 10GB 140 await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit })
140 await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit }) 141
142 const { size } = await stat(destPath)
143 remainingBodyKBLimit -= (size / 1000)
144
145 logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
141 } 146 }
142 147
143 clearTimeout(timer) 148 clearTimeout(timer)
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 103ab1fab..137ae53a0 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -4,9 +4,7 @@ import { getServerActor } from '@server/models/application/application'
4import { TrackerModel } from '@server/models/server/tracker' 4import { TrackerModel } from '@server/models/server/tracker'
5import { VideoModel } from '@server/models/video/video' 5import { VideoModel } from '@server/models/video/video'
6import { 6import {
7 MStreamingPlaylist,
8 MStreamingPlaylistFiles, 7 MStreamingPlaylistFiles,
9 MStreamingPlaylistVideo,
10 MVideoAccountLight, 8 MVideoAccountLight,
11 MVideoFile, 9 MVideoFile,
12 MVideoFileVideo, 10 MVideoFileVideo,
@@ -249,7 +247,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
249 private async createStreamingPlaylistRedundancy ( 247 private async createStreamingPlaylistRedundancy (
250 redundancy: VideosRedundancyStrategy, 248 redundancy: VideosRedundancyStrategy,
251 video: MVideoAccountLight, 249 video: MVideoAccountLight,
252 playlistArg: MStreamingPlaylist 250 playlistArg: MStreamingPlaylistFiles
253 ) { 251 ) {
254 let strategy = 'manual' 252 let strategy = 'manual'
255 let expiresOn: Date = null 253 let expiresOn: Date = null
@@ -259,16 +257,17 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
259 expiresOn = this.buildNewExpiration(redundancy.minLifetime) 257 expiresOn = this.buildNewExpiration(redundancy.minLifetime)
260 } 258 }
261 259
262 const playlist = playlistArg as MStreamingPlaylistVideo 260 const playlist = Object.assign(playlistArg, { Video: video })
263 playlist.Video = video
264
265 const serverActor = await getServerActor() 261 const serverActor = await getServerActor()
266 262
267 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) 263 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
268 264
269 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 265 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
270 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) 266 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
271 await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) 267
268 const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000
269 const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance
270 await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB)
272 271
273 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ 272 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
274 expiresOn, 273 expiresOn,
@@ -334,7 +333,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
334 return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` 333 return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
335 } 334 }
336 335
337 private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { 336 private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number {
338 const fileReducer = (previous: number, current: MVideoFile) => previous + current.size 337 const fileReducer = (previous: number, current: MVideoFile) => previous + current.size
339 338
340 let allFiles = files 339 let allFiles = files