aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/hls.ts
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib/hls.ts')
-rw-r--r--server/lib/hls.ts285
1 files changed, 0 insertions, 285 deletions
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
deleted file mode 100644
index 19044d7c2..000000000
--- a/server/lib/hls.ts
+++ /dev/null
@@ -1,285 +0,0 @@
1import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
2import { flatten } from 'lodash'
3import PQueue from 'p-queue'
4import { basename, dirname, join } from 'path'
5import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
6import { uniqify, uuidRegex } from '@shared/core-utils'
7import { sha256 } from '@shared/extra-utils'
8import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg'
9import { VideoStorage } from '@shared/models'
10import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg'
11import { logger, loggerTagsFactory } from '../helpers/logger'
12import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
13import { generateRandomString } from '../helpers/utils'
14import { CONFIG } from '../initializers/config'
15import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants'
16import { sequelizeTypescript } from '../initializers/database'
17import { VideoFileModel } from '../models/video/video-file'
18import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
19import { storeHLSFileFromFilename } from './object-storage'
20import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths'
21import { VideoPathManager } from './video-path-manager'
22
23const lTags = loggerTagsFactory('hls')
24
25async function updateStreamingPlaylistsInfohashesIfNeeded () {
26 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
27
28 // Use separate SQL queries, because we could have many videos to update
29 for (const playlist of playlistsToUpdate) {
30 await sequelizeTypescript.transaction(async t => {
31 const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
32
33 playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
34 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
35
36 await playlist.save({ transaction: t })
37 })
38 }
39}
40
41async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) {
42 try {
43 let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist)
44 playlistWithFiles = await updateSha256VODSegments(video, playlist)
45
46 // Refresh playlist, operations can take some time
47 playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id)
48 playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
49 await playlistWithFiles.save()
50
51 video.setHLSPlaylist(playlistWithFiles)
52 } catch (err) {
53 logger.warn('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err })
54 }
55}
56
57// ---------------------------------------------------------------------------
58
59// Avoid concurrency issues when updating streaming playlist files
60const playlistFilesQueue = new PQueue({ concurrency: 1 })
61
62function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
63 return playlistFilesQueue.add(async () => {
64 const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
65
66 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
67
68 for (const file of playlist.VideoFiles) {
69 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
70
71 await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
72 const size = await getVideoStreamDimensionsInfo(videoFilePath)
73
74 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
75 const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}`
76
77 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
78 if (file.fps) line += ',FRAME-RATE=' + file.fps
79
80 const codecs = await Promise.all([
81 getVideoStreamCodec(videoFilePath),
82 getAudioStreamCodec(videoFilePath)
83 ])
84
85 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
86
87 masterPlaylists.push(line)
88 masterPlaylists.push(playlistFilename)
89 })
90 }
91
92 if (playlist.playlistFilename) {
93 await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename)
94 }
95 playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
96
97 const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
98 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
99
100 logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid))
101
102 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
103 playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
104 await remove(masterPlaylistPath)
105 }
106
107 return playlist.save()
108 })
109}
110
111// ---------------------------------------------------------------------------
112
113function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
114 return playlistFilesQueue.add(async () => {
115 const json: { [filename: string]: { [range: string]: string } } = {}
116
117 const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
118
119 // For all the resolutions available for this video
120 for (const file of playlist.VideoFiles) {
121 const rangeHashes: { [range: string]: string } = {}
122 const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
123
124 await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
125
126 return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
127 const playlistContent = await readFile(resolutionPlaylistPath)
128 const ranges = getRangesFromPlaylist(playlistContent.toString())
129
130 const fd = await open(videoPath, 'r')
131 for (const range of ranges) {
132 const buf = Buffer.alloc(range.length)
133 await read(fd, buf, 0, range.length, range.offset)
134
135 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
136 }
137 await close(fd)
138
139 const videoFilename = file.filename
140 json[videoFilename] = rangeHashes
141 })
142 })
143 }
144
145 if (playlist.segmentsSha256Filename) {
146 await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename)
147 }
148 playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
149
150 const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
151 await outputJSON(outputPath, json)
152
153 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
154 playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
155 await remove(outputPath)
156 }
157
158 return playlist.save()
159 })
160}
161
162// ---------------------------------------------------------------------------
163
164async function buildSha256Segment (segmentPath: string) {
165 const buf = await readFile(segmentPath)
166 return sha256(buf)
167}
168
169function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
170 let timer
171 let remainingBodyKBLimit = bodyKBLimit
172
173 logger.info('Importing HLS playlist %s', playlistUrl)
174
175 return new Promise<void>(async (res, rej) => {
176 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
177
178 await ensureDir(tmpDirectory)
179
180 timer = setTimeout(() => {
181 deleteTmpDirectory(tmpDirectory)
182
183 return rej(new Error('HLS download timeout.'))
184 }, timeout)
185
186 try {
187 // Fetch master playlist
188 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
189
190 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
191 const fileUrls = uniqify(flatten(await Promise.all(subRequests)))
192
193 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
194
195 for (const fileUrl of fileUrls) {
196 const destPath = join(tmpDirectory, basename(fileUrl))
197
198 await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY })
199
200 const { size } = await stat(destPath)
201 remainingBodyKBLimit -= (size / 1000)
202
203 logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
204 }
205
206 clearTimeout(timer)
207
208 await move(tmpDirectory, destinationDir, { overwrite: true })
209
210 return res()
211 } catch (err) {
212 deleteTmpDirectory(tmpDirectory)
213
214 return rej(err)
215 }
216 })
217
218 function deleteTmpDirectory (directory: string) {
219 remove(directory)
220 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
221 }
222
223 async function fetchUniqUrls (playlistUrl: string) {
224 const { body } = await doRequest(playlistUrl)
225
226 if (!body) return []
227
228 const urls = body.split('\n')
229 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
230 .map(url => {
231 if (url.startsWith('http://') || url.startsWith('https://')) return url
232
233 return `${dirname(playlistUrl)}/${url}`
234 })
235
236 return uniqify(urls)
237 }
238}
239
240// ---------------------------------------------------------------------------
241
242async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) {
243 const content = await readFile(playlistPath, 'utf8')
244
245 const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename)
246
247 await writeFile(playlistPath, newContent, 'utf8')
248}
249
250// ---------------------------------------------------------------------------
251
252function injectQueryToPlaylistUrls (content: string, queryString: string) {
253 return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString)
254}
255
256// ---------------------------------------------------------------------------
257
258export {
259 updateMasterHLSPlaylist,
260 updateSha256VODSegments,
261 buildSha256Segment,
262 downloadPlaylistSegments,
263 updateStreamingPlaylistsInfohashesIfNeeded,
264 updatePlaylistAfterFileChange,
265 injectQueryToPlaylistUrls,
266 renameVideoFileInPlaylist
267}
268
269// ---------------------------------------------------------------------------
270
271function getRangesFromPlaylist (playlistContent: string) {
272 const ranges: { offset: number, length: number }[] = []
273 const lines = playlistContent.split('\n')
274 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
275
276 for (const line of lines) {
277 const captured = regex.exec(line)
278
279 if (captured) {
280 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
281 }
282 }
283
284 return ranges
285}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-26 21:47:33 +0000