]>
Commit | Line | Data |
---|---|---|
0305db28 | 1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' |
daf6e480 C |
2 | import { flatten, uniq } from 'lodash' |
3 | import { basename, dirname, join } from 'path' | |
e1ab52d7 | 4 | import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' |
f304a158 | 5 | import { sha256 } from '@shared/extra-utils' |
a2caee9f | 6 | import { VideoStorage } from '@shared/models' |
c729caf6 | 7 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg' |
09209296 | 8 | import { logger } from '../helpers/logger' |
4c280004 C |
9 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
10 | import { generateRandomString } from '../helpers/utils' | |
6dd9de95 | 11 | import { CONFIG } from '../initializers/config' |
7b0c61e7 | 12 | import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants' |
74dc3bca | 13 | import { sequelizeTypescript } from '../initializers/database' |
daf6e480 C |
14 | import { VideoFileModel } from '../models/video/video-file' |
15 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | |
a2caee9f | 16 | import { storeHLSFile } from './object-storage' |
0305db28 JB |
17 | import { getHlsResolutionPlaylistFilename } from './paths' |
18 | import { VideoPathManager } from './video-path-manager' | |
ae9bbed4 C |
19 | |
20 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | |
21 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | |
22 | ||
23 | // Use separate SQL queries, because we could have many videos to update | |
24 | for (const playlist of playlistsToUpdate) { | |
25 | await sequelizeTypescript.transaction(async t => { | |
26 | const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) | |
27 | ||
764b1a14 | 28 | playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) |
0e9c48c2 | 29 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION |
764b1a14 | 30 | |
ae9bbed4 C |
31 | await playlist.save({ transaction: t }) |
32 | }) | |
33 | } | |
34 | } | |
09209296 | 35 | |
e1ab52d7 | 36 | async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) { |
09209296 | 37 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] |
09209296 | 38 | |
764b1a14 C |
39 | for (const file of playlist.VideoFiles) { |
40 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | |
83903cb6 | 41 | |
ad5db104 | 42 | await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { |
c729caf6 | 43 | const size = await getVideoStreamDimensionsInfo(videoFilePath) |
09209296 | 44 | |
0305db28 | 45 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) |
c729caf6 | 46 | const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` |
09209296 | 47 | |
0305db28 JB |
48 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` |
49 | if (file.fps) line += ',FRAME-RATE=' + file.fps | |
09209296 | 50 | |
0305db28 JB |
51 | const codecs = await Promise.all([ |
52 | getVideoStreamCodec(videoFilePath), | |
53 | getAudioStreamCodec(videoFilePath) | |
54 | ]) | |
d7b1c7b4 | 55 | |
0305db28 | 56 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` |
52201311 | 57 | |
0305db28 JB |
58 | masterPlaylists.push(line) |
59 | masterPlaylists.push(playlistFilename) | |
60 | }) | |
09209296 C |
61 | } |
62 | ||
a2caee9f C |
63 | await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => { |
64 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | |
65 | ||
66 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | |
67 | await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath) | |
68 | } | |
0305db28 | 69 | }) |
09209296 C |
70 | } |
71 | ||
e1ab52d7 | 72 | async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) { |
4c280004 C |
73 | const json: { [filename: string]: { [range: string]: string } } = {} |
74 | ||
4c280004 | 75 | // For all the resolutions available for this video |
764b1a14 | 76 | for (const file of playlist.VideoFiles) { |
4c280004 | 77 | const rangeHashes: { [range: string]: string } = {} |
ad5db104 | 78 | const fileWithPlaylist = file.withVideoOrPlaylist(playlist) |
4c280004 | 79 | |
ad5db104 | 80 | await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { |
09209296 | 81 | |
ad5db104 | 82 | return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { |
0305db28 JB |
83 | const playlistContent = await readFile(resolutionPlaylistPath) |
84 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | |
09209296 | 85 | |
0305db28 JB |
86 | const fd = await open(videoPath, 'r') |
87 | for (const range of ranges) { | |
88 | const buf = Buffer.alloc(range.length) | |
89 | await read(fd, buf, 0, range.length, range.offset) | |
09209296 | 90 | |
0305db28 JB |
91 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) |
92 | } | |
93 | await close(fd) | |
4c280004 | 94 | |
0305db28 JB |
95 | const videoFilename = file.filename |
96 | json[videoFilename] = rangeHashes | |
97 | }) | |
98 | }) | |
09209296 C |
99 | } |
100 | ||
0305db28 | 101 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) |
09209296 | 102 | await outputJSON(outputPath, json) |
a2caee9f C |
103 | |
104 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | |
105 | await storeHLSFile(playlist, playlist.segmentsSha256Filename) | |
106 | await remove(outputPath) | |
107 | } | |
09209296 C |
108 | } |
109 | ||
c6c0fa6c C |
110 | async function buildSha256Segment (segmentPath: string) { |
111 | const buf = await readFile(segmentPath) | |
112 | return sha256(buf) | |
113 | } | |
114 | ||
18998c45 | 115 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { |
4c280004 | 116 | let timer |
18998c45 | 117 | let remainingBodyKBLimit = bodyKBLimit |
09209296 | 118 | |
4c280004 | 119 | logger.info('Importing HLS playlist %s', playlistUrl) |
09209296 | 120 | |
ba5a8d89 | 121 | return new Promise<void>(async (res, rej) => { |
4c280004 | 122 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) |
09209296 | 123 | |
4c280004 | 124 | await ensureDir(tmpDirectory) |
09209296 C |
125 | |
126 | timer = setTimeout(() => { | |
4c280004 | 127 | deleteTmpDirectory(tmpDirectory) |
09209296 C |
128 | |
129 | return rej(new Error('HLS download timeout.')) | |
130 | }, timeout) | |
131 | ||
4c280004 C |
132 | try { |
133 | // Fetch master playlist | |
134 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | |
135 | ||
136 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | |
137 | const fileUrls = uniq(flatten(await Promise.all(subRequests))) | |
138 | ||
139 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | |
140 | ||
141 | for (const fileUrl of fileUrls) { | |
142 | const destPath = join(tmpDirectory, basename(fileUrl)) | |
143 | ||
a5ee023c | 144 | await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) |
18998c45 C |
145 | |
146 | const { size } = await stat(destPath) | |
147 | remainingBodyKBLimit -= (size / 1000) | |
148 | ||
149 | logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) | |
4c280004 C |
150 | } |
151 | ||
152 | clearTimeout(timer) | |
153 | ||
154 | await move(tmpDirectory, destinationDir, { overwrite: true }) | |
155 | ||
156 | return res() | |
157 | } catch (err) { | |
158 | deleteTmpDirectory(tmpDirectory) | |
159 | ||
160 | return rej(err) | |
09209296 C |
161 | } |
162 | }) | |
4c280004 C |
163 | |
164 | function deleteTmpDirectory (directory: string) { | |
165 | remove(directory) | |
166 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | |
167 | } | |
168 | ||
169 | async function fetchUniqUrls (playlistUrl: string) { | |
db4b15f2 | 170 | const { body } = await doRequest(playlistUrl) |
4c280004 C |
171 | |
172 | if (!body) return [] | |
173 | ||
174 | const urls = body.split('\n') | |
175 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | |
176 | .map(url => { | |
177 | if (url.startsWith('http://') || url.startsWith('https://')) return url | |
178 | ||
179 | return `${dirname(playlistUrl)}/${url}` | |
180 | }) | |
181 | ||
182 | return uniq(urls) | |
183 | } | |
09209296 C |
184 | } |
185 | ||
186 | // --------------------------------------------------------------------------- | |
187 | ||
188 | export { | |
189 | updateMasterHLSPlaylist, | |
c6c0fa6c C |
190 | updateSha256VODSegments, |
191 | buildSha256Segment, | |
ae9bbed4 C |
192 | downloadPlaylistSegments, |
193 | updateStreamingPlaylistsInfohashesIfNeeded | |
09209296 C |
194 | } |
195 | ||
196 | // --------------------------------------------------------------------------- | |
b5b68755 C |
197 | |
198 | function getRangesFromPlaylist (playlistContent: string) { | |
199 | const ranges: { offset: number, length: number }[] = [] | |
200 | const lines = playlistContent.split('\n') | |
201 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | |
202 | ||
203 | for (const line of lines) { | |
204 | const captured = regex.exec(line) | |
205 | ||
206 | if (captured) { | |
207 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | |
208 | } | |
209 | } | |
210 | ||
211 | return ranges | |
212 | } |