diff options
author | Chocobozzz <me@florianbigard.com> | 2021-08-06 15:25:45 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2021-08-06 15:25:45 +0200 |
commit | 18998c45c001869a883ec7a2d286d8170f768381 (patch) | |
tree | 83cd925fc3ba14f5dabf66e24c26adfee061f21e /server/lib | |
parent | a1c63fe1a2159c5796e37a63c2e4acd96dc557ca (diff) | |
download | PeerTube-18998c45c001869a883ec7a2d286d8170f768381.tar.gz PeerTube-18998c45c001869a883ec7a2d286d8170f768381.tar.zst PeerTube-18998c45c001869a883ec7a2d286d8170f768381.zip |
Correctly handle large HLS files for redundancy
Diffstat (limited to 'server/lib')
-rw-r--r-- | server/lib/hls.ts | 13 | ||||
-rw-r--r-- | server/lib/schedulers/videos-redundancy-scheduler.ts | 15 |
2 files changed, 16 insertions, 12 deletions
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 32b02bc26..0e77ab9fa 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' | 1 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra' |
2 | import { flatten, uniq } from 'lodash' | 2 | import { flatten, uniq } from 'lodash' |
3 | import { basename, dirname, join } from 'path' | 3 | import { basename, dirname, join } from 'path' |
4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' | 4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' |
@@ -108,8 +108,9 @@ async function buildSha256Segment (segmentPath: string) { | |||
108 | return sha256(buf) | 108 | return sha256(buf) |
109 | } | 109 | } |
110 | 110 | ||
111 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { | 111 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { |
112 | let timer | 112 | let timer |
113 | let remainingBodyKBLimit = bodyKBLimit | ||
113 | 114 | ||
114 | logger.info('Importing HLS playlist %s', playlistUrl) | 115 | logger.info('Importing HLS playlist %s', playlistUrl) |
115 | 116 | ||
@@ -136,8 +137,12 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, | |||
136 | for (const fileUrl of fileUrls) { | 137 | for (const fileUrl of fileUrls) { |
137 | const destPath = join(tmpDirectory, basename(fileUrl)) | 138 | const destPath = join(tmpDirectory, basename(fileUrl)) |
138 | 139 | ||
139 | const bodyKBLimit = 10 * 1000 * 1000 // 10GB | 140 | await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit }) |
140 | await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit }) | 141 | |
142 | const { size } = await stat(destPath) | ||
143 | remainingBodyKBLimit -= (size / 1000) | ||
144 | |||
145 | logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) | ||
141 | } | 146 | } |
142 | 147 | ||
143 | clearTimeout(timer) | 148 | clearTimeout(timer) |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 103ab1fab..137ae53a0 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -4,9 +4,7 @@ import { getServerActor } from '@server/models/application/application' | |||
4 | import { TrackerModel } from '@server/models/server/tracker' | 4 | import { TrackerModel } from '@server/models/server/tracker' |
5 | import { VideoModel } from '@server/models/video/video' | 5 | import { VideoModel } from '@server/models/video/video' |
6 | import { | 6 | import { |
7 | MStreamingPlaylist, | ||
8 | MStreamingPlaylistFiles, | 7 | MStreamingPlaylistFiles, |
9 | MStreamingPlaylistVideo, | ||
10 | MVideoAccountLight, | 8 | MVideoAccountLight, |
11 | MVideoFile, | 9 | MVideoFile, |
12 | MVideoFileVideo, | 10 | MVideoFileVideo, |
@@ -249,7 +247,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
249 | private async createStreamingPlaylistRedundancy ( | 247 | private async createStreamingPlaylistRedundancy ( |
250 | redundancy: VideosRedundancyStrategy, | 248 | redundancy: VideosRedundancyStrategy, |
251 | video: MVideoAccountLight, | 249 | video: MVideoAccountLight, |
252 | playlistArg: MStreamingPlaylist | 250 | playlistArg: MStreamingPlaylistFiles |
253 | ) { | 251 | ) { |
254 | let strategy = 'manual' | 252 | let strategy = 'manual' |
255 | let expiresOn: Date = null | 253 | let expiresOn: Date = null |
@@ -259,16 +257,17 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
259 | expiresOn = this.buildNewExpiration(redundancy.minLifetime) | 257 | expiresOn = this.buildNewExpiration(redundancy.minLifetime) |
260 | } | 258 | } |
261 | 259 | ||
262 | const playlist = playlistArg as MStreamingPlaylistVideo | 260 | const playlist = Object.assign(playlistArg, { Video: video }) |
263 | playlist.Video = video | ||
264 | |||
265 | const serverActor = await getServerActor() | 261 | const serverActor = await getServerActor() |
266 | 262 | ||
267 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) | 263 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) |
268 | 264 | ||
269 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | 265 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) |
270 | const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) | 266 | const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) |
271 | await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) | 267 | |
268 | const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 | ||
269 | const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance | ||
270 | await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB) | ||
272 | 271 | ||
273 | const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ | 272 | const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ |
274 | expiresOn, | 273 | expiresOn, |
@@ -334,7 +333,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
334 | return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` | 333 | return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` |
335 | } | 334 | } |
336 | 335 | ||
337 | private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { | 336 | private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number { |
338 | const fileReducer = (previous: number, current: MVideoFile) => previous + current.size | 337 | const fileReducer = (previous: number, current: MVideoFile) => previous + current.size |
339 | 338 | ||
340 | let allFiles = files | 339 | let allFiles = files |