aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actor.ts4
-rw-r--r--server/lib/hls.ts136
-rw-r--r--server/lib/video-transcoding.ts5
3 files changed, 101 insertions, 44 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 8215840da..a3f379b76 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -355,10 +355,10 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
355 355
356 logger.info('Fetching remote actor %s.', actorUrl) 356 logger.info('Fetching remote actor %s.', actorUrl)
357 357
358 const requestResult = await doRequest(options) 358 const requestResult = await doRequest<ActivityPubActor>(options)
359 normalizeActor(requestResult.body) 359 normalizeActor(requestResult.body)
360 360
361 const actorJSON: ActivityPubActor = requestResult.body 361 const actorJSON = requestResult.body
362 if (isActorObjectValid(actorJSON) === false) { 362 if (isActorObjectValid(actorJSON) === false) {
363 logger.debug('Remote actor JSON is not valid.', { actorJSON }) 363 logger.debug('Remote actor JSON is not valid.', { actorJSON })
364 return { result: undefined, statusCode: requestResult.response.statusCode } 364 return { result: undefined, statusCode: requestResult.response.statusCode }
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 10db6c3c3..3575981f4 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,13 +1,14 @@
1import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
2import { basename, dirname, join } from 'path' 2import { basename, join, dirname } from 'path'
3import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' 3import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
4import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' 4import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
5import { getVideoFileSize } from '../helpers/ffmpeg-utils' 5import { getVideoFileSize } from '../helpers/ffmpeg-utils'
6import { sha256 } from '../helpers/core-utils' 6import { sha256 } from '../helpers/core-utils'
7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
8import HLSDownloader from 'hlsdownloader'
9import { logger } from '../helpers/logger' 8import { logger } from '../helpers/logger'
10import { parse } from 'url' 9import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
10import { generateRandomString } from '../helpers/utils'
11import { flatten, uniq } from 'lodash'
11 12
12async function updateMasterHLSPlaylist (video: VideoModel) { 13async function updateMasterHLSPlaylist (video: VideoModel) {
13 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) 14 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
@@ -37,66 +38,119 @@ async function updateMasterHLSPlaylist (video: VideoModel) {
37} 38}
38 39
39async function updateSha256Segments (video: VideoModel) { 40async function updateSha256Segments (video: VideoModel) {
40 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) 41 const json: { [filename: string]: { [range: string]: string } } = {}
41 const files = await readdir(directory) 42
42 const json: { [filename: string]: string} = {} 43 const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
44
45 // For all the resolutions available for this video
46 for (const file of video.VideoFiles) {
47 const rangeHashes: { [range: string]: string } = {}
48
49 const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution))
50 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
43 51
44 for (const file of files) { 52 // Maybe the playlist is not generated for this resolution yet
45 if (file.endsWith('.ts') === false) continue 53 if (!await pathExists(playlistPath)) continue
46 54
47 const buffer = await readFile(join(directory, file)) 55 const playlistContent = await readFile(playlistPath)
48 const filename = basename(file) 56 const ranges = getRangesFromPlaylist(playlistContent.toString())
49 57
50 json[filename] = sha256(buffer) 58 const fd = await open(videoPath, 'r')
59 for (const range of ranges) {
60 const buf = Buffer.alloc(range.length)
61 await read(fd, buf, 0, range.length, range.offset)
62
63 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
64 }
65 await close(fd)
66
67 const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)
68 json[videoFilename] = rangeHashes
51 } 69 }
52 70
53 const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) 71 const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
54 await outputJSON(outputPath, json) 72 await outputJSON(outputPath, json)
55} 73}
56 74
57function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { 75function getRangesFromPlaylist (playlistContent: string) {
58 let timer 76 const ranges: { offset: number, length: number }[] = []
77 const lines = playlistContent.split('\n')
78 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
59 79
60 logger.info('Importing HLS playlist %s', playlistUrl) 80 for (const line of lines) {
81 const captured = regex.exec(line)
61 82
62 const params = { 83 if (captured) {
63 playlistURL: playlistUrl, 84 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
64 destination: CONFIG.STORAGE.TMP_DIR 85 }
65 } 86 }
66 const downloader = new HLSDownloader(params)
67
68 const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname))
69 87
70 return new Promise<string>(async (res, rej) => { 88 return ranges
71 downloader.startDownload(err => { 89}
72 clearTimeout(timer)
73 90
74 if (err) { 91function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
75 deleteTmpDirectory(hlsDestinationDir) 92 let timer
76 93
77 return rej(err) 94 logger.info('Importing HLS playlist %s', playlistUrl)
78 }
79 95
80 move(hlsDestinationDir, destinationDir, { overwrite: true }) 96 return new Promise<string>(async (res, rej) => {
81 .then(() => res()) 97 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
82 .catch(err => {
83 deleteTmpDirectory(hlsDestinationDir)
84 98
85 return rej(err) 99 await ensureDir(tmpDirectory)
86 })
87 })
88 100
89 timer = setTimeout(() => { 101 timer = setTimeout(() => {
90 deleteTmpDirectory(hlsDestinationDir) 102 deleteTmpDirectory(tmpDirectory)
91 103
92 return rej(new Error('HLS download timeout.')) 104 return rej(new Error('HLS download timeout.'))
93 }, timeout) 105 }, timeout)
94 106
95 function deleteTmpDirectory (directory: string) { 107 try {
96 remove(directory) 108 // Fetch master playlist
97 .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) 109 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
110
111 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
112 const fileUrls = uniq(flatten(await Promise.all(subRequests)))
113
114 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
115
116 for (const fileUrl of fileUrls) {
117 const destPath = join(tmpDirectory, basename(fileUrl))
118
119 await doRequestAndSaveToFile({ uri: fileUrl }, destPath)
120 }
121
122 clearTimeout(timer)
123
124 await move(tmpDirectory, destinationDir, { overwrite: true })
125
126 return res()
127 } catch (err) {
128 deleteTmpDirectory(tmpDirectory)
129
130 return rej(err)
98 } 131 }
99 }) 132 })
133
134 function deleteTmpDirectory (directory: string) {
135 remove(directory)
136 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
137 }
138
139 async function fetchUniqUrls (playlistUrl: string) {
140 const { body } = await doRequest<string>({ uri: playlistUrl })
141
142 if (!body) return []
143
144 const urls = body.split('\n')
145 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
146 .map(url => {
147 if (url.startsWith('http://') || url.startsWith('https://')) return url
148
149 return `${dirname(playlistUrl)}/${url}`
150 })
151
152 return uniq(urls)
153 }
100} 154}
101 155
102// --------------------------------------------------------------------------- 156// ---------------------------------------------------------------------------
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 608badfef..086b860a2 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -100,7 +100,10 @@ async function generateHlsPlaylist (video: VideoModel, resolution: VideoResoluti
100 outputPath, 100 outputPath,
101 resolution, 101 resolution,
102 isPortraitMode, 102 isPortraitMode,
103 generateHlsPlaylist: true 103
104 hlsPlaylist: {
105 videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution)
106 }
104 } 107 }
105 108
106 await transcode(transcodeOptions) 109 await transcode(transcodeOptions)