]>
Commit | Line | Data |
---|---|---|
0305db28 | 1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' |
690bb8f9 | 2 | import { flatten } from 'lodash' |
1bb4c9ab | 3 | import PQueue from 'p-queue' |
daf6e480 | 4 | import { basename, dirname, join } from 'path' |
1bb4c9ab | 5 | import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' |
0c9668f7 | 6 | import { uniqify, uuidRegex } from '@shared/core-utils' |
f304a158 | 7 | import { sha256 } from '@shared/extra-utils' |
0c9668f7 | 8 | import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' |
a2caee9f | 9 | import { VideoStorage } from '@shared/models' |
0c9668f7 | 10 | import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' |
09209296 | 11 | import { logger } from '../helpers/logger' |
4c280004 C |
12 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
13 | import { generateRandomString } from '../helpers/utils' | |
6dd9de95 | 14 | import { CONFIG } from '../initializers/config' |
7b0c61e7 | 15 | import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants' |
74dc3bca | 16 | import { sequelizeTypescript } from '../initializers/database' |
daf6e480 C |
17 | import { VideoFileModel } from '../models/video/video-file' |
18 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | |
cfd57d2c | 19 | import { storeHLSFileFromFilename } from './object-storage' |
1bb4c9ab | 20 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' |
0305db28 | 21 | import { VideoPathManager } from './video-path-manager' |
ae9bbed4 C |
22 | |
23 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | |
24 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | |
25 | ||
26 | // Use separate SQL queries, because we could have many videos to update | |
27 | for (const playlist of playlistsToUpdate) { | |
28 | await sequelizeTypescript.transaction(async t => { | |
29 | const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) | |
30 | ||
764b1a14 | 31 | playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) |
0e9c48c2 | 32 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION |
764b1a14 | 33 | |
ae9bbed4 C |
34 | await playlist.save({ transaction: t }) |
35 | }) | |
36 | } | |
37 | } | |
09209296 | 38 | |
1bb4c9ab | 39 | async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { |
51335c72 C |
40 | try { |
41 | let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) | |
42 | playlistWithFiles = await updateSha256VODSegments(video, playlist) | |
43 | ||
44 | // Refresh playlist, operations can take some time | |
45 | playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) | |
46 | playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) | |
47 | await playlistWithFiles.save() | |
48 | ||
49 | video.setHLSPlaylist(playlistWithFiles) | |
50 | } catch (err) { | |
51 | logger.info('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err }) | |
52 | } | |
1bb4c9ab | 53 | } |
09209296 | 54 | |
1bb4c9ab | 55 | // --------------------------------------------------------------------------- |
09209296 | 56 | |
1bb4c9ab C |
57 | // Avoid concurrency issues when updating streaming playlist files |
58 | const playlistFilesQueue = new PQueue({ concurrency: 1 }) | |
09209296 | 59 | |
1bb4c9ab C |
60 | function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> { |
61 | return playlistFilesQueue.add(async () => { | |
62 | const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) | |
d7b1c7b4 | 63 | |
1bb4c9ab | 64 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] |
52201311 | 65 | |
1bb4c9ab C |
66 | for (const file of playlist.VideoFiles) { |
67 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | |
68 | ||
69 | await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { | |
70 | const size = await getVideoStreamDimensionsInfo(videoFilePath) | |
71 | ||
72 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | |
73 | const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` | |
74 | ||
75 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | |
76 | if (file.fps) line += ',FRAME-RATE=' + file.fps | |
77 | ||
78 | const codecs = await Promise.all([ | |
79 | getVideoStreamCodec(videoFilePath), | |
80 | getAudioStreamCodec(videoFilePath) | |
81 | ]) | |
09209296 | 82 | |
1bb4c9ab C |
83 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` |
84 | ||
85 | masterPlaylists.push(line) | |
86 | masterPlaylists.push(playlistFilename) | |
87 | }) | |
88 | } | |
89 | ||
90 | if (playlist.playlistFilename) { | |
91 | await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) | |
92 | } | |
93 | playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) | |
94 | ||
95 | const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) | |
a2caee9f C |
96 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') |
97 | ||
98 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | |
cfd57d2c | 99 | playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) |
1bb4c9ab | 100 | await remove(masterPlaylistPath) |
a2caee9f | 101 | } |
1bb4c9ab C |
102 | |
103 | return playlist.save() | |
0305db28 | 104 | }) |
09209296 C |
105 | } |
106 | ||
1bb4c9ab C |
107 | // --------------------------------------------------------------------------- |
108 | ||
b37d80e3 | 109 | function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> { |
1bb4c9ab C |
110 | return playlistFilesQueue.add(async () => { |
111 | const json: { [filename: string]: { [range: string]: string } } = {} | |
4c280004 | 112 | |
1bb4c9ab | 113 | const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) |
4c280004 | 114 | |
1bb4c9ab C |
115 | // For all the resolutions available for this video |
116 | for (const file of playlist.VideoFiles) { | |
117 | const rangeHashes: { [range: string]: string } = {} | |
118 | const fileWithPlaylist = file.withVideoOrPlaylist(playlist) | |
09209296 | 119 | |
1bb4c9ab | 120 | await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { |
09209296 | 121 | |
1bb4c9ab C |
122 | return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { |
123 | const playlistContent = await readFile(resolutionPlaylistPath) | |
124 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | |
09209296 | 125 | |
1bb4c9ab C |
126 | const fd = await open(videoPath, 'r') |
127 | for (const range of ranges) { | |
128 | const buf = Buffer.alloc(range.length) | |
129 | await read(fd, buf, 0, range.length, range.offset) | |
4c280004 | 130 | |
1bb4c9ab C |
131 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) |
132 | } | |
133 | await close(fd) | |
134 | ||
135 | const videoFilename = file.filename | |
136 | json[videoFilename] = rangeHashes | |
137 | }) | |
0305db28 | 138 | }) |
1bb4c9ab | 139 | } |
09209296 | 140 | |
1bb4c9ab C |
141 | if (playlist.segmentsSha256Filename) { |
142 | await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) | |
143 | } | |
144 | playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) | |
a2caee9f | 145 | |
1bb4c9ab C |
146 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) |
147 | await outputJSON(outputPath, json) | |
148 | ||
149 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | |
cfd57d2c | 150 | playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) |
1bb4c9ab C |
151 | await remove(outputPath) |
152 | } | |
153 | ||
154 | return playlist.save() | |
155 | }) | |
09209296 C |
156 | } |
157 | ||
1bb4c9ab C |
158 | // --------------------------------------------------------------------------- |
159 | ||
c6c0fa6c C |
160 | async function buildSha256Segment (segmentPath: string) { |
161 | const buf = await readFile(segmentPath) | |
162 | return sha256(buf) | |
163 | } | |
164 | ||
18998c45 | 165 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { |
4c280004 | 166 | let timer |
18998c45 | 167 | let remainingBodyKBLimit = bodyKBLimit |
09209296 | 168 | |
4c280004 | 169 | logger.info('Importing HLS playlist %s', playlistUrl) |
09209296 | 170 | |
ba5a8d89 | 171 | return new Promise<void>(async (res, rej) => { |
4c280004 | 172 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) |
09209296 | 173 | |
4c280004 | 174 | await ensureDir(tmpDirectory) |
09209296 C |
175 | |
176 | timer = setTimeout(() => { | |
4c280004 | 177 | deleteTmpDirectory(tmpDirectory) |
09209296 C |
178 | |
179 | return rej(new Error('HLS download timeout.')) | |
180 | }, timeout) | |
181 | ||
4c280004 C |
182 | try { |
183 | // Fetch master playlist | |
184 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | |
185 | ||
186 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | |
690bb8f9 | 187 | const fileUrls = uniqify(flatten(await Promise.all(subRequests))) |
4c280004 C |
188 | |
189 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | |
190 | ||
191 | for (const fileUrl of fileUrls) { | |
192 | const destPath = join(tmpDirectory, basename(fileUrl)) | |
193 | ||
a5ee023c | 194 | await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) |
18998c45 C |
195 | |
196 | const { size } = await stat(destPath) | |
197 | remainingBodyKBLimit -= (size / 1000) | |
198 | ||
199 | logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) | |
4c280004 C |
200 | } |
201 | ||
202 | clearTimeout(timer) | |
203 | ||
204 | await move(tmpDirectory, destinationDir, { overwrite: true }) | |
205 | ||
206 | return res() | |
207 | } catch (err) { | |
208 | deleteTmpDirectory(tmpDirectory) | |
209 | ||
210 | return rej(err) | |
09209296 C |
211 | } |
212 | }) | |
4c280004 C |
213 | |
214 | function deleteTmpDirectory (directory: string) { | |
215 | remove(directory) | |
216 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | |
217 | } | |
218 | ||
219 | async function fetchUniqUrls (playlistUrl: string) { | |
db4b15f2 | 220 | const { body } = await doRequest(playlistUrl) |
4c280004 C |
221 | |
222 | if (!body) return [] | |
223 | ||
224 | const urls = body.split('\n') | |
225 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | |
226 | .map(url => { | |
227 | if (url.startsWith('http://') || url.startsWith('https://')) return url | |
228 | ||
229 | return `${dirname(playlistUrl)}/${url}` | |
230 | }) | |
231 | ||
690bb8f9 | 232 | return uniqify(urls) |
4c280004 | 233 | } |
09209296 C |
234 | } |
235 | ||
236 | // --------------------------------------------------------------------------- | |
237 | ||
0c9668f7 C |
238 | async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) { |
239 | const content = await readFile(playlistPath, 'utf8') | |
240 | ||
241 | const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename) | |
242 | ||
243 | await writeFile(playlistPath, newContent, 'utf8') | |
244 | } | |
245 | ||
246 | // --------------------------------------------------------------------------- | |
247 | ||
71e3e879 C |
248 | function injectQueryToPlaylistUrls (content: string, queryString: string) { |
249 | return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) | |
250 | } | |
251 | ||
252 | // --------------------------------------------------------------------------- | |
253 | ||
09209296 C |
254 | export { |
255 | updateMasterHLSPlaylist, | |
c6c0fa6c C |
256 | updateSha256VODSegments, |
257 | buildSha256Segment, | |
ae9bbed4 | 258 | downloadPlaylistSegments, |
1bb4c9ab | 259 | updateStreamingPlaylistsInfohashesIfNeeded, |
71e3e879 | 260 | updatePlaylistAfterFileChange, |
0c9668f7 C |
261 | injectQueryToPlaylistUrls, |
262 | renameVideoFileInPlaylist | |
09209296 C |
263 | } |
264 | ||
265 | // --------------------------------------------------------------------------- | |
b5b68755 C |
266 | |
267 | function getRangesFromPlaylist (playlistContent: string) { | |
268 | const ranges: { offset: number, length: number }[] = [] | |
269 | const lines = playlistContent.split('\n') | |
270 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | |
271 | ||
272 | for (const line of lines) { | |
273 | const captured = regex.exec(line) | |
274 | ||
275 | if (captured) { | |
276 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | |
277 | } | |
278 | } | |
279 | ||
280 | return ranges | |
281 | } |