]>
Commit | Line | Data |
---|---|---|
1 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' | |
2 | import { flatten, uniq } from 'lodash' | |
3 | import { basename, dirname, join } from 'path' | |
4 | import { MVideoWithFile } from '@server/types/models' | |
5 | import { sha256 } from '../helpers/core-utils' | |
6 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' | |
7 | import { logger } from '../helpers/logger' | |
8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | |
9 | import { generateRandomString } from '../helpers/utils' | |
10 | import { CONFIG } from '../initializers/config' | |
11 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' | |
12 | import { sequelizeTypescript } from '../initializers/database' | |
13 | import { VideoFileModel } from '../models/video/video-file' | |
14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | |
15 | import { getVideoFilePath } from './video-paths' | |
16 | ||
17 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | |
18 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | |
19 | ||
20 | // Use separate SQL queries, because we could have many videos to update | |
21 | for (const playlist of playlistsToUpdate) { | |
22 | await sequelizeTypescript.transaction(async t => { | |
23 | const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) | |
24 | ||
25 | playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) | |
26 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION | |
27 | await playlist.save({ transaction: t }) | |
28 | }) | |
29 | } | |
30 | } | |
31 | ||
32 | async function updateMasterHLSPlaylist (video: MVideoWithFile) { | |
33 | const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | |
34 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | |
35 | const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) | |
36 | const streamingPlaylist = video.getHLSPlaylist() | |
37 | ||
38 | for (const file of streamingPlaylist.VideoFiles) { | |
39 | // If we did not generated a playlist for this resolution, skip | |
40 | const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | |
41 | if (await pathExists(filePlaylistPath) === false) continue | |
42 | ||
43 | const videoFilePath = getVideoFilePath(streamingPlaylist, file) | |
44 | ||
45 | const size = await getVideoStreamSize(videoFilePath) | |
46 | ||
47 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | |
48 | const resolution = `RESOLUTION=${size.width}x${size.height}` | |
49 | ||
50 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | |
51 | if (file.fps) line += ',FRAME-RATE=' + file.fps | |
52 | ||
53 | const videoCodec = await getVideoStreamCodec(videoFilePath) | |
54 | line += `,CODECS="${videoCodec}` | |
55 | ||
56 | const audioCodec = await getAudioStreamCodec(videoFilePath) | |
57 | if (audioCodec) line += `,${audioCodec}` | |
58 | ||
59 | line += '"' | |
60 | ||
61 | masterPlaylists.push(line) | |
62 | masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | |
63 | } | |
64 | ||
65 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | |
66 | } | |
67 | ||
68 | async function updateSha256VODSegments (video: MVideoWithFile) { | |
69 | const json: { [filename: string]: { [range: string]: string } } = {} | |
70 | ||
71 | const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | |
72 | const hlsPlaylist = video.getHLSPlaylist() | |
73 | ||
74 | // For all the resolutions available for this video | |
75 | for (const file of hlsPlaylist.VideoFiles) { | |
76 | const rangeHashes: { [range: string]: string } = {} | |
77 | ||
78 | const videoPath = getVideoFilePath(hlsPlaylist, file) | |
79 | const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | |
80 | ||
81 | // Maybe the playlist is not generated for this resolution yet | |
82 | if (!await pathExists(playlistPath)) continue | |
83 | ||
84 | const playlistContent = await readFile(playlistPath) | |
85 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | |
86 | ||
87 | const fd = await open(videoPath, 'r') | |
88 | for (const range of ranges) { | |
89 | const buf = Buffer.alloc(range.length) | |
90 | await read(fd, buf, 0, range.length, range.offset) | |
91 | ||
92 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | |
93 | } | |
94 | await close(fd) | |
95 | ||
96 | const videoFilename = file.filename | |
97 | json[videoFilename] = rangeHashes | |
98 | } | |
99 | ||
100 | const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) | |
101 | await outputJSON(outputPath, json) | |
102 | } | |
103 | ||
104 | async function buildSha256Segment (segmentPath: string) { | |
105 | const buf = await readFile(segmentPath) | |
106 | return sha256(buf) | |
107 | } | |
108 | ||
109 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { | |
110 | let timer | |
111 | ||
112 | logger.info('Importing HLS playlist %s', playlistUrl) | |
113 | ||
114 | return new Promise<void>(async (res, rej) => { | |
115 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) | |
116 | ||
117 | await ensureDir(tmpDirectory) | |
118 | ||
119 | timer = setTimeout(() => { | |
120 | deleteTmpDirectory(tmpDirectory) | |
121 | ||
122 | return rej(new Error('HLS download timeout.')) | |
123 | }, timeout) | |
124 | ||
125 | try { | |
126 | // Fetch master playlist | |
127 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | |
128 | ||
129 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | |
130 | const fileUrls = uniq(flatten(await Promise.all(subRequests))) | |
131 | ||
132 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | |
133 | ||
134 | for (const fileUrl of fileUrls) { | |
135 | const destPath = join(tmpDirectory, basename(fileUrl)) | |
136 | ||
137 | const bodyKBLimit = 10 * 1000 * 1000 // 10GB | |
138 | await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit }) | |
139 | } | |
140 | ||
141 | clearTimeout(timer) | |
142 | ||
143 | await move(tmpDirectory, destinationDir, { overwrite: true }) | |
144 | ||
145 | return res() | |
146 | } catch (err) { | |
147 | deleteTmpDirectory(tmpDirectory) | |
148 | ||
149 | return rej(err) | |
150 | } | |
151 | }) | |
152 | ||
153 | function deleteTmpDirectory (directory: string) { | |
154 | remove(directory) | |
155 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | |
156 | } | |
157 | ||
158 | async function fetchUniqUrls (playlistUrl: string) { | |
159 | const { body } = await doRequest(playlistUrl) | |
160 | ||
161 | if (!body) return [] | |
162 | ||
163 | const urls = body.split('\n') | |
164 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | |
165 | .map(url => { | |
166 | if (url.startsWith('http://') || url.startsWith('https://')) return url | |
167 | ||
168 | return `${dirname(playlistUrl)}/${url}` | |
169 | }) | |
170 | ||
171 | return uniq(urls) | |
172 | } | |
173 | } | |
174 | ||
175 | // --------------------------------------------------------------------------- | |
176 | ||
177 | export { | |
178 | updateMasterHLSPlaylist, | |
179 | updateSha256VODSegments, | |
180 | buildSha256Segment, | |
181 | downloadPlaylistSegments, | |
182 | updateStreamingPlaylistsInfohashesIfNeeded | |
183 | } | |
184 | ||
185 | // --------------------------------------------------------------------------- | |
186 | ||
187 | function getRangesFromPlaylist (playlistContent: string) { | |
188 | const ranges: { offset: number, length: number }[] = [] | |
189 | const lines = playlistContent.split('\n') | |
190 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | |
191 | ||
192 | for (const line of lines) { | |
193 | const captured = regex.exec(line) | |
194 | ||
195 | if (captured) { | |
196 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | |
197 | } | |
198 | } | |
199 | ||
200 | return ranges | |
201 | } |