]>
Commit | Line | Data |
---|---|---|
4c280004 | 1 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' |
daf6e480 C |
2 | import { flatten, uniq } from 'lodash' |
3 | import { basename, dirname, join } from 'path' | |
764b1a14 | 4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' |
09209296 | 5 | import { sha256 } from '../helpers/core-utils' |
daf6e480 | 6 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' |
09209296 | 7 | import { logger } from '../helpers/logger' |
4c280004 C |
8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
9 | import { generateRandomString } from '../helpers/utils' | |
6dd9de95 | 10 | import { CONFIG } from '../initializers/config' |
daf6e480 | 11 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' |
74dc3bca | 12 | import { sequelizeTypescript } from '../initializers/database' |
daf6e480 C |
13 | import { VideoFileModel } from '../models/video/video-file' |
14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | |
764b1a14 | 15 | import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' |
ae9bbed4 C |
16 | |
17 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | |
18 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | |
19 | ||
20 | // Use separate SQL queries, because we could have many videos to update | |
21 | for (const playlist of playlistsToUpdate) { | |
22 | await sequelizeTypescript.transaction(async t => { | |
23 | const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) | |
24 | ||
764b1a14 | 25 | playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) |
0e9c48c2 | 26 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION |
764b1a14 | 27 | |
ae9bbed4 C |
28 | await playlist.save({ transaction: t }) |
29 | }) | |
30 | } | |
31 | } | |
09209296 | 32 | |
764b1a14 | 33 | async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { |
9c6ca37f | 34 | const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) |
764b1a14 | 35 | |
09209296 | 36 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] |
09209296 | 37 | |
764b1a14 C |
38 | const masterPlaylistPath = join(directory, playlist.playlistFilename) |
39 | ||
40 | for (const file of playlist.VideoFiles) { | |
41 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | |
83903cb6 | 42 | |
09209296 | 43 | // If we did not generated a playlist for this resolution, skip |
83903cb6 | 44 | const filePlaylistPath = join(directory, playlistFilename) |
09209296 C |
45 | if (await pathExists(filePlaylistPath) === false) continue |
46 | ||
764b1a14 | 47 | const videoFilePath = getVideoFilePath(playlist, file) |
09209296 | 48 | |
52201311 | 49 | const size = await getVideoStreamSize(videoFilePath) |
09209296 C |
50 | |
51 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | |
52 | const resolution = `RESOLUTION=${size.width}x${size.height}` | |
53 | ||
54 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | |
55 | if (file.fps) line += ',FRAME-RATE=' + file.fps | |
56 | ||
66f77f63 | 57 | const codecs = await Promise.all([ |
58 | getVideoStreamCodec(videoFilePath), | |
59 | getAudioStreamCodec(videoFilePath) | |
60 | ]) | |
d7b1c7b4 | 61 | |
66f77f63 | 62 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` |
52201311 | 63 | |
09209296 | 64 | masterPlaylists.push(line) |
83903cb6 | 65 | masterPlaylists.push(playlistFilename) |
09209296 C |
66 | } |
67 | ||
68 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | |
69 | } | |
70 | ||
764b1a14 | 71 | async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { |
4c280004 C |
72 | const json: { [filename: string]: { [range: string]: string } } = {} |
73 | ||
9c6ca37f | 74 | const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) |
4c280004 C |
75 | |
76 | // For all the resolutions available for this video | |
764b1a14 | 77 | for (const file of playlist.VideoFiles) { |
4c280004 C |
78 | const rangeHashes: { [range: string]: string } = {} |
79 | ||
764b1a14 C |
80 | const videoPath = getVideoFilePath(playlist, file) |
81 | const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename)) | |
09209296 | 82 | |
4c280004 | 83 | // Maybe the playlist is not generated for this resolution yet |
764b1a14 | 84 | if (!await pathExists(resolutionPlaylistPath)) continue |
09209296 | 85 | |
764b1a14 | 86 | const playlistContent = await readFile(resolutionPlaylistPath) |
4c280004 | 87 | const ranges = getRangesFromPlaylist(playlistContent.toString()) |
09209296 | 88 | |
4c280004 C |
89 | const fd = await open(videoPath, 'r') |
90 | for (const range of ranges) { | |
91 | const buf = Buffer.alloc(range.length) | |
92 | await read(fd, buf, 0, range.length, range.offset) | |
93 | ||
94 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | |
95 | } | |
96 | await close(fd) | |
97 | ||
90a8bd30 | 98 | const videoFilename = file.filename |
4c280004 | 99 | json[videoFilename] = rangeHashes |
09209296 C |
100 | } |
101 | ||
764b1a14 | 102 | const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) |
09209296 C |
103 | await outputJSON(outputPath, json) |
104 | } | |
105 | ||
c6c0fa6c C |
106 | async function buildSha256Segment (segmentPath: string) { |
107 | const buf = await readFile(segmentPath) | |
108 | return sha256(buf) | |
109 | } | |
110 | ||
4c280004 C |
111 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { |
112 | let timer | |
09209296 | 113 | |
4c280004 | 114 | logger.info('Importing HLS playlist %s', playlistUrl) |
09209296 | 115 | |
ba5a8d89 | 116 | return new Promise<void>(async (res, rej) => { |
4c280004 | 117 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) |
09209296 | 118 | |
4c280004 | 119 | await ensureDir(tmpDirectory) |
09209296 C |
120 | |
121 | timer = setTimeout(() => { | |
4c280004 | 122 | deleteTmpDirectory(tmpDirectory) |
09209296 C |
123 | |
124 | return rej(new Error('HLS download timeout.')) | |
125 | }, timeout) | |
126 | ||
4c280004 C |
127 | try { |
128 | // Fetch master playlist | |
129 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | |
130 | ||
131 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | |
132 | const fileUrls = uniq(flatten(await Promise.all(subRequests))) | |
133 | ||
134 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | |
135 | ||
136 | for (const fileUrl of fileUrls) { | |
137 | const destPath = join(tmpDirectory, basename(fileUrl)) | |
138 | ||
bfe2ef6b | 139 | const bodyKBLimit = 10 * 1000 * 1000 // 10GB |
db4b15f2 | 140 | await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit }) |
4c280004 C |
141 | } |
142 | ||
143 | clearTimeout(timer) | |
144 | ||
145 | await move(tmpDirectory, destinationDir, { overwrite: true }) | |
146 | ||
147 | return res() | |
148 | } catch (err) { | |
149 | deleteTmpDirectory(tmpDirectory) | |
150 | ||
151 | return rej(err) | |
09209296 C |
152 | } |
153 | }) | |
4c280004 C |
154 | |
155 | function deleteTmpDirectory (directory: string) { | |
156 | remove(directory) | |
157 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | |
158 | } | |
159 | ||
160 | async function fetchUniqUrls (playlistUrl: string) { | |
db4b15f2 | 161 | const { body } = await doRequest(playlistUrl) |
4c280004 C |
162 | |
163 | if (!body) return [] | |
164 | ||
165 | const urls = body.split('\n') | |
166 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | |
167 | .map(url => { | |
168 | if (url.startsWith('http://') || url.startsWith('https://')) return url | |
169 | ||
170 | return `${dirname(playlistUrl)}/${url}` | |
171 | }) | |
172 | ||
173 | return uniq(urls) | |
174 | } | |
09209296 C |
175 | } |
176 | ||
177 | // --------------------------------------------------------------------------- | |
178 | ||
179 | export { | |
180 | updateMasterHLSPlaylist, | |
c6c0fa6c C |
181 | updateSha256VODSegments, |
182 | buildSha256Segment, | |
ae9bbed4 C |
183 | downloadPlaylistSegments, |
184 | updateStreamingPlaylistsInfohashesIfNeeded | |
09209296 C |
185 | } |
186 | ||
187 | // --------------------------------------------------------------------------- | |
b5b68755 C |
188 | |
189 | function getRangesFromPlaylist (playlistContent: string) { | |
190 | const ranges: { offset: number, length: number }[] = [] | |
191 | const lines = playlistContent.split('\n') | |
192 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | |
193 | ||
194 | for (const line of lines) { | |
195 | const captured = regex.exec(line) | |
196 | ||
197 | if (captured) { | |
198 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | |
199 | } | |
200 | } | |
201 | ||
202 | return ranges | |
203 | } |