aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-02-07 15:08:19 +0100
committerChocobozzz <chocobozzz@cpy.re>2019-02-11 09:13:02 +0100
commit4c280004ce62bf11ddb091854c28f1e1d54a54d6 (patch)
tree1899fff4ef18f8663a865997d5d06119b2149319
parent6ec0b75beb9c8bcd84e178912319913b91830da2 (diff)
downloadPeerTube-4c280004ce62bf11ddb091854c28f1e1d54a54d6.tar.gz
PeerTube-4c280004ce62bf11ddb091854c28f1e1d54a54d6.tar.zst
PeerTube-4c280004ce62bf11ddb091854c28f1e1d54a54d6.zip
Use a single file instead of segments for HLS
-rw-r--r--client/src/assets/player/p2p-media-loader/segment-validator.ts15
-rw-r--r--package.json1
-rwxr-xr-xscripts/generate-code-contributors.ts2
-rw-r--r--server/helpers/ffmpeg-utils.ts12
-rw-r--r--server/helpers/requests.ts2
-rw-r--r--server/helpers/utils.ts1
-rw-r--r--server/lib/activitypub/actor.ts4
-rw-r--r--server/lib/hls.ts136
-rw-r--r--server/lib/video-transcoding.ts5
-rw-r--r--server/models/video/video-streaming-playlist.ts4
-rw-r--r--server/tests/api/redundancy/redundancy.ts22
-rw-r--r--server/tests/api/videos/video-hls.ts16
-rw-r--r--shared/models/activitypub/activitypub-ordered-collection.ts5
-rw-r--r--shared/utils/requests/requests.ts8
-rw-r--r--shared/utils/videos/video-playlists.ts36
-rw-r--r--yarn.lock64
16 files changed, 188 insertions, 145 deletions
diff --git a/client/src/assets/player/p2p-media-loader/segment-validator.ts b/client/src/assets/player/p2p-media-loader/segment-validator.ts
index 8f4922daa..72c32f9e0 100644
--- a/client/src/assets/player/p2p-media-loader/segment-validator.ts
+++ b/client/src/assets/player/p2p-media-loader/segment-validator.ts
@@ -3,18 +3,25 @@ import { basename } from 'path'
3 3
4function segmentValidatorFactory (segmentsSha256Url: string) { 4function segmentValidatorFactory (segmentsSha256Url: string) {
5 const segmentsJSON = fetchSha256Segments(segmentsSha256Url) 5 const segmentsJSON = fetchSha256Segments(segmentsSha256Url)
6 const regex = /bytes=(\d+)-(\d+)/
6 7
7 return async function segmentValidator (segment: Segment) { 8 return async function segmentValidator (segment: Segment) {
8 const segmentName = basename(segment.url) 9 const filename = basename(segment.url)
10 const captured = regex.exec(segment.range)
9 11
10 const hashShouldBe = (await segmentsJSON)[segmentName] 12 const range = captured[1] + '-' + captured[2]
13
14 const hashShouldBe = (await segmentsJSON)[filename][range]
11 if (hashShouldBe === undefined) { 15 if (hashShouldBe === undefined) {
12 throw new Error(`Unknown segment name ${segmentName} in segment validator`) 16 throw new Error(`Unknown segment name ${filename}/${range} in segment validator`)
13 } 17 }
14 18
15 const calculatedSha = bufferToEx(await sha256(segment.data)) 19 const calculatedSha = bufferToEx(await sha256(segment.data))
16 if (calculatedSha !== hashShouldBe) { 20 if (calculatedSha !== hashShouldBe) {
17 throw new Error(`Hashes does not correspond for segment ${segmentName} (expected: ${hashShouldBe} instead of ${calculatedSha})`) 21 throw new Error(
22 `Hashes does not correspond for segment ${filename}/${range}` +
23 `(expected: ${hashShouldBe} instead of ${calculatedSha})`
24 )
18 } 25 }
19 } 26 }
20} 27}
diff --git a/package.json b/package.json
index c8c9e64ae..0cf39c7ee 100644
--- a/package.json
+++ b/package.json
@@ -117,7 +117,6 @@
117 "fluent-ffmpeg": "^2.1.0", 117 "fluent-ffmpeg": "^2.1.0",
118 "fs-extra": "^7.0.0", 118 "fs-extra": "^7.0.0",
119 "helmet": "^3.12.1", 119 "helmet": "^3.12.1",
120 "hlsdownloader": "https://github.com/Chocobozzz/hlsdownloader#build",
121 "http-signature": "^1.2.0", 120 "http-signature": "^1.2.0",
122 "ip-anonymize": "^0.0.6", 121 "ip-anonymize": "^0.0.6",
123 "ipaddr.js": "1.8.1", 122 "ipaddr.js": "1.8.1",
diff --git a/scripts/generate-code-contributors.ts b/scripts/generate-code-contributors.ts
index 9824bc2f5..96110307a 100755
--- a/scripts/generate-code-contributors.ts
+++ b/scripts/generate-code-contributors.ts
@@ -41,7 +41,7 @@ async function run () {
41} 41}
42 42
43function get (url: string, headers: any = {}) { 43function get (url: string, headers: any = {}) {
44 return doRequest({ 44 return doRequest<any>({
45 uri: url, 45 uri: url,
46 json: true, 46 json: true,
47 headers: Object.assign(headers, { 47 headers: Object.assign(headers, {
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 5ad8ed48e..133b1b03b 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -122,7 +122,9 @@ type TranscodeOptions = {
122 resolution: VideoResolution 122 resolution: VideoResolution
123 isPortraitMode?: boolean 123 isPortraitMode?: boolean
124 124
125 generateHlsPlaylist?: boolean 125 hlsPlaylist?: {
126 videoFilename: string
127 }
126} 128}
127 129
128function transcode (options: TranscodeOptions) { 130function transcode (options: TranscodeOptions) {
@@ -161,14 +163,16 @@ function transcode (options: TranscodeOptions) {
161 command = command.withFPS(fps) 163 command = command.withFPS(fps)
162 } 164 }
163 165
164 if (options.generateHlsPlaylist) { 166 if (options.hlsPlaylist) {
165 const segmentFilename = `${dirname(options.outputPath)}/${options.resolution}_%03d.ts` 167 const videoPath = `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
166 168
167 command = command.outputOption('-hls_time 4') 169 command = command.outputOption('-hls_time 4')
168 .outputOption('-hls_list_size 0') 170 .outputOption('-hls_list_size 0')
169 .outputOption('-hls_playlist_type vod') 171 .outputOption('-hls_playlist_type vod')
170 .outputOption('-hls_segment_filename ' + segmentFilename) 172 .outputOption('-hls_segment_filename ' + videoPath)
173 .outputOption('-hls_segment_type fmp4')
171 .outputOption('-f hls') 174 .outputOption('-f hls')
175 .outputOption('-hls_flags single_file')
172 } 176 }
173 177
174 command 178 command
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index 3fc776f1a..5c6dc5e19 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -7,7 +7,7 @@ import { join } from 'path'
7 7
8function doRequest <T> ( 8function doRequest <T> (
9 requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean } 9 requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }
10): Bluebird<{ response: request.RequestResponse, body: any }> { 10): Bluebird<{ response: request.RequestResponse, body: T }> {
11 if (requestOptions.activityPub === true) { 11 if (requestOptions.activityPub === true) {
12 if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {} 12 if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {}
13 requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER 13 requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 3c3406e38..cb0e823c5 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -7,7 +7,6 @@ import { join } from 'path'
7import { Instance as ParseTorrent } from 'parse-torrent' 7import { Instance as ParseTorrent } from 'parse-torrent'
8import { remove } from 'fs-extra' 8import { remove } from 'fs-extra'
9import * as memoizee from 'memoizee' 9import * as memoizee from 'memoizee'
10import { isArray } from './custom-validators/misc'
11 10
12function deleteFileAsync (path: string) { 11function deleteFileAsync (path: string) {
13 remove(path) 12 remove(path)
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 8215840da..a3f379b76 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -355,10 +355,10 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
355 355
356 logger.info('Fetching remote actor %s.', actorUrl) 356 logger.info('Fetching remote actor %s.', actorUrl)
357 357
358 const requestResult = await doRequest(options) 358 const requestResult = await doRequest<ActivityPubActor>(options)
359 normalizeActor(requestResult.body) 359 normalizeActor(requestResult.body)
360 360
361 const actorJSON: ActivityPubActor = requestResult.body 361 const actorJSON = requestResult.body
362 if (isActorObjectValid(actorJSON) === false) { 362 if (isActorObjectValid(actorJSON) === false) {
363 logger.debug('Remote actor JSON is not valid.', { actorJSON }) 363 logger.debug('Remote actor JSON is not valid.', { actorJSON })
364 return { result: undefined, statusCode: requestResult.response.statusCode } 364 return { result: undefined, statusCode: requestResult.response.statusCode }
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 10db6c3c3..3575981f4 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,13 +1,14 @@
1import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
2import { basename, dirname, join } from 'path' 2import { basename, join, dirname } from 'path'
3import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' 3import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
4import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' 4import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
5import { getVideoFileSize } from '../helpers/ffmpeg-utils' 5import { getVideoFileSize } from '../helpers/ffmpeg-utils'
6import { sha256 } from '../helpers/core-utils' 6import { sha256 } from '../helpers/core-utils'
7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
8import HLSDownloader from 'hlsdownloader'
9import { logger } from '../helpers/logger' 8import { logger } from '../helpers/logger'
10import { parse } from 'url' 9import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
10import { generateRandomString } from '../helpers/utils'
11import { flatten, uniq } from 'lodash'
11 12
12async function updateMasterHLSPlaylist (video: VideoModel) { 13async function updateMasterHLSPlaylist (video: VideoModel) {
13 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) 14 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
@@ -37,66 +38,119 @@ async function updateMasterHLSPlaylist (video: VideoModel) {
37} 38}
38 39
39async function updateSha256Segments (video: VideoModel) { 40async function updateSha256Segments (video: VideoModel) {
40 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) 41 const json: { [filename: string]: { [range: string]: string } } = {}
41 const files = await readdir(directory) 42
42 const json: { [filename: string]: string} = {} 43 const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
44
45 // For all the resolutions available for this video
46 for (const file of video.VideoFiles) {
47 const rangeHashes: { [range: string]: string } = {}
48
49 const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution))
50 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
43 51
44 for (const file of files) { 52 // Maybe the playlist is not generated for this resolution yet
45 if (file.endsWith('.ts') === false) continue 53 if (!await pathExists(playlistPath)) continue
46 54
47 const buffer = await readFile(join(directory, file)) 55 const playlistContent = await readFile(playlistPath)
48 const filename = basename(file) 56 const ranges = getRangesFromPlaylist(playlistContent.toString())
49 57
50 json[filename] = sha256(buffer) 58 const fd = await open(videoPath, 'r')
59 for (const range of ranges) {
60 const buf = Buffer.alloc(range.length)
61 await read(fd, buf, 0, range.length, range.offset)
62
63 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
64 }
65 await close(fd)
66
67 const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)
68 json[videoFilename] = rangeHashes
51 } 69 }
52 70
53 const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) 71 const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
54 await outputJSON(outputPath, json) 72 await outputJSON(outputPath, json)
55} 73}
56 74
57function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { 75function getRangesFromPlaylist (playlistContent: string) {
58 let timer 76 const ranges: { offset: number, length: number }[] = []
77 const lines = playlistContent.split('\n')
78 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
59 79
60 logger.info('Importing HLS playlist %s', playlistUrl) 80 for (const line of lines) {
81 const captured = regex.exec(line)
61 82
62 const params = { 83 if (captured) {
63 playlistURL: playlistUrl, 84 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
64 destination: CONFIG.STORAGE.TMP_DIR 85 }
65 } 86 }
66 const downloader = new HLSDownloader(params)
67
68 const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname))
69 87
70 return new Promise<string>(async (res, rej) => { 88 return ranges
71 downloader.startDownload(err => { 89}
72 clearTimeout(timer)
73 90
74 if (err) { 91function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
75 deleteTmpDirectory(hlsDestinationDir) 92 let timer
76 93
77 return rej(err) 94 logger.info('Importing HLS playlist %s', playlistUrl)
78 }
79 95
80 move(hlsDestinationDir, destinationDir, { overwrite: true }) 96 return new Promise<string>(async (res, rej) => {
81 .then(() => res()) 97 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
82 .catch(err => {
83 deleteTmpDirectory(hlsDestinationDir)
84 98
85 return rej(err) 99 await ensureDir(tmpDirectory)
86 })
87 })
88 100
89 timer = setTimeout(() => { 101 timer = setTimeout(() => {
90 deleteTmpDirectory(hlsDestinationDir) 102 deleteTmpDirectory(tmpDirectory)
91 103
92 return rej(new Error('HLS download timeout.')) 104 return rej(new Error('HLS download timeout.'))
93 }, timeout) 105 }, timeout)
94 106
95 function deleteTmpDirectory (directory: string) { 107 try {
96 remove(directory) 108 // Fetch master playlist
97 .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) 109 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
110
111 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
112 const fileUrls = uniq(flatten(await Promise.all(subRequests)))
113
114 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
115
116 for (const fileUrl of fileUrls) {
117 const destPath = join(tmpDirectory, basename(fileUrl))
118
119 await doRequestAndSaveToFile({ uri: fileUrl }, destPath)
120 }
121
122 clearTimeout(timer)
123
124 await move(tmpDirectory, destinationDir, { overwrite: true })
125
126 return res()
127 } catch (err) {
128 deleteTmpDirectory(tmpDirectory)
129
130 return rej(err)
98 } 131 }
99 }) 132 })
133
134 function deleteTmpDirectory (directory: string) {
135 remove(directory)
136 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
137 }
138
139 async function fetchUniqUrls (playlistUrl: string) {
140 const { body } = await doRequest<string>({ uri: playlistUrl })
141
142 if (!body) return []
143
144 const urls = body.split('\n')
145 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
146 .map(url => {
147 if (url.startsWith('http://') || url.startsWith('https://')) return url
148
149 return `${dirname(playlistUrl)}/${url}`
150 })
151
152 return uniq(urls)
153 }
100} 154}
101 155
102// --------------------------------------------------------------------------- 156// ---------------------------------------------------------------------------
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 608badfef..086b860a2 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -100,7 +100,10 @@ async function generateHlsPlaylist (video: VideoModel, resolution: VideoResoluti
100 outputPath, 100 outputPath,
101 resolution, 101 resolution,
102 isPortraitMode, 102 isPortraitMode,
103 generateHlsPlaylist: true 103
104 hlsPlaylist: {
105 videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution)
106 }
104 } 107 }
105 108
106 await transcode(transcodeOptions) 109 await transcode(transcodeOptions)
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index bce537781..bf6f7b0c4 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -125,6 +125,10 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
125 return 'segments-sha256.json' 125 return 'segments-sha256.json'
126 } 126 }
127 127
128 static getHlsVideoName (uuid: string, resolution: number) {
129 return `${uuid}-${resolution}-fragmented.mp4`
130 }
131
128 static getHlsMasterPlaylistStaticPath (videoUUID: string) { 132 static getHlsMasterPlaylistStaticPath (videoUUID: string) {
129 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) 133 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
130 } 134 }
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 5b99309fb..778611fff 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -17,7 +17,7 @@ import {
17 viewVideo, 17 viewVideo,
18 wait, 18 wait,
19 waitUntilLog, 19 waitUntilLog,
20 checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer 20 checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer, checkSegmentHash
21} from '../../../../shared/utils' 21} from '../../../../shared/utils'
22import { waitJobs } from '../../../../shared/utils/server/jobs' 22import { waitJobs } from '../../../../shared/utils/server/jobs'
23 23
@@ -178,20 +178,24 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
178 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) 178 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
179 } 179 }
180 180
181 await makeGetRequest({ 181 const baseUrlPlaylist = servers[1].url + '/static/playlists/hls'
182 url: servers[0].url, 182 const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
183 statusCodeExpected: 200, 183
184 path: `/static/redundancy/hls/${videoUUID}/360_000.ts`, 184 const res = await getVideo(servers[0].url, videoUUID)
185 contentType: null 185 const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
186 }) 186
187 for (const resolution of [ 240, 360, 480, 720 ]) {
188 await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
189 }
187 190
188 for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) { 191 for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) {
189 const files = await readdir(join(root(), directory, videoUUID)) 192 const files = await readdir(join(root(), directory, videoUUID))
190 expect(files).to.have.length.at.least(4) 193 expect(files).to.have.length.at.least(4)
191 194
192 for (const resolution of [ 240, 360, 480, 720 ]) { 195 for (const resolution of [ 240, 360, 480, 720 ]) {
193 expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined 196 const filename = `${videoUUID}-${resolution}-fragmented.mp4`
194 expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined 197
198 expect(files.find(f => f === filename)).to.not.be.undefined
195 } 199 }
196 } 200 }
197} 201}
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
index 71d863b12..a1214bad1 100644
--- a/server/tests/api/videos/video-hls.ts
+++ b/server/tests/api/videos/video-hls.ts
@@ -4,13 +4,12 @@ import * as chai from 'chai'
4import 'mocha' 4import 'mocha'
5import { 5import {
6 checkDirectoryIsEmpty, 6 checkDirectoryIsEmpty,
7 checkSegmentHash,
7 checkTmpIsEmpty, 8 checkTmpIsEmpty,
8 doubleFollow, 9 doubleFollow,
9 flushAndRunMultipleServers, 10 flushAndRunMultipleServers,
10 flushTests, 11 flushTests,
11 getPlaylist, 12 getPlaylist,
12 getSegment,
13 getSegmentSha256,
14 getVideo, 13 getVideo,
15 killallServers, 14 killallServers,
16 removeVideo, 15 removeVideo,
@@ -22,7 +21,6 @@ import {
22} from '../../../../shared/utils' 21} from '../../../../shared/utils'
23import { VideoDetails } from '../../../../shared/models/videos' 22import { VideoDetails } from '../../../../shared/models/videos'
24import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type' 23import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type'
25import { sha256 } from '../../../helpers/core-utils'
26import { join } from 'path' 24import { join } from 'path'
27 25
28const expect = chai.expect 26const expect = chai.expect
@@ -56,19 +54,15 @@ async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) {
56 const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`) 54 const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`)
57 55
58 const subPlaylist = res2.text 56 const subPlaylist = res2.text
59 expect(subPlaylist).to.contain(resolution + '_000.ts') 57 expect(subPlaylist).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`)
60 } 58 }
61 } 59 }
62 60
63 { 61 {
64 for (const resolution of resolutions) { 62 const baseUrl = 'http://localhost:9001/static/playlists/hls'
65
66 const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`)
67 63
68 const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url) 64 for (const resolution of resolutions) {
69 65 await checkSegmentHash(baseUrl, baseUrl, videoUUID, resolution, hlsPlaylist)
70 const sha256Server = resSha.body[ resolution + '_000.ts' ]
71 expect(sha256(res2.body)).to.equal(sha256Server)
72 } 66 }
73 } 67 }
74 } 68 }
diff --git a/shared/models/activitypub/activitypub-ordered-collection.ts b/shared/models/activitypub/activitypub-ordered-collection.ts
index dfec0bb76..3de0890bb 100644
--- a/shared/models/activitypub/activitypub-ordered-collection.ts
+++ b/shared/models/activitypub/activitypub-ordered-collection.ts
@@ -2,6 +2,9 @@ export interface ActivityPubOrderedCollection<T> {
2 '@context': string[] 2 '@context': string[]
3 type: 'OrderedCollection' | 'OrderedCollectionPage' 3 type: 'OrderedCollection' | 'OrderedCollectionPage'
4 totalItems: number 4 totalItems: number
5 partOf?: string
6 orderedItems: T[] 5 orderedItems: T[]
6
7 partOf?: string
8 next?: string
9 first?: string
7} 10}
diff --git a/shared/utils/requests/requests.ts b/shared/utils/requests/requests.ts
index fc687c701..6b59e24fc 100644
--- a/shared/utils/requests/requests.ts
+++ b/shared/utils/requests/requests.ts
@@ -3,10 +3,10 @@ import { buildAbsoluteFixturePath, root } from '../miscs/miscs'
3import { isAbsolute, join } from 'path' 3import { isAbsolute, join } from 'path'
4import { parse } from 'url' 4import { parse } from 'url'
5 5
6function makeRawRequest (url: string, statusCodeExpected?: number) { 6function makeRawRequest (url: string, statusCodeExpected?: number, range?: string) {
7 const { host, protocol, pathname } = parse(url) 7 const { host, protocol, pathname } = parse(url)
8 8
9 return makeGetRequest({ url: `${protocol}//${host}`, path: pathname, statusCodeExpected }) 9 return makeGetRequest({ url: `${protocol}//${host}`, path: pathname, statusCodeExpected, range })
10} 10}
11 11
12function makeGetRequest (options: { 12function makeGetRequest (options: {
@@ -15,7 +15,8 @@ function makeGetRequest (options: {
15 query?: any, 15 query?: any,
16 token?: string, 16 token?: string,
17 statusCodeExpected?: number, 17 statusCodeExpected?: number,
18 contentType?: string 18 contentType?: string,
19 range?: string
19}) { 20}) {
20 if (!options.statusCodeExpected) options.statusCodeExpected = 400 21 if (!options.statusCodeExpected) options.statusCodeExpected = 400
21 if (options.contentType === undefined) options.contentType = 'application/json' 22 if (options.contentType === undefined) options.contentType = 'application/json'
@@ -25,6 +26,7 @@ function makeGetRequest (options: {
25 if (options.contentType) req.set('Accept', options.contentType) 26 if (options.contentType) req.set('Accept', options.contentType)
26 if (options.token) req.set('Authorization', 'Bearer ' + options.token) 27 if (options.token) req.set('Authorization', 'Bearer ' + options.token)
27 if (options.query) req.query(options.query) 28 if (options.query) req.query(options.query)
29 if (options.range) req.set('Range', options.range)
28 30
29 return req.expect(options.statusCodeExpected) 31 return req.expect(options.statusCodeExpected)
30} 32}
diff --git a/shared/utils/videos/video-playlists.ts b/shared/utils/videos/video-playlists.ts
index 9a0710ca6..eb25011cb 100644
--- a/shared/utils/videos/video-playlists.ts
+++ b/shared/utils/videos/video-playlists.ts
@@ -1,21 +1,51 @@
1import { makeRawRequest } from '../requests/requests' 1import { makeRawRequest } from '../requests/requests'
2import { sha256 } from '../../../server/helpers/core-utils'
3import { VideoStreamingPlaylist } from '../../models/videos/video-streaming-playlist.model'
4import { expect } from 'chai'
2 5
3function getPlaylist (url: string, statusCodeExpected = 200) { 6function getPlaylist (url: string, statusCodeExpected = 200) {
4 return makeRawRequest(url, statusCodeExpected) 7 return makeRawRequest(url, statusCodeExpected)
5} 8}
6 9
7function getSegment (url: string, statusCodeExpected = 200) { 10function getSegment (url: string, statusCodeExpected = 200, range?: string) {
8 return makeRawRequest(url, statusCodeExpected) 11 return makeRawRequest(url, statusCodeExpected, range)
9} 12}
10 13
11function getSegmentSha256 (url: string, statusCodeExpected = 200) { 14function getSegmentSha256 (url: string, statusCodeExpected = 200) {
12 return makeRawRequest(url, statusCodeExpected) 15 return makeRawRequest(url, statusCodeExpected)
13} 16}
14 17
18async function checkSegmentHash (
19 baseUrlPlaylist: string,
20 baseUrlSegment: string,
21 videoUUID: string,
22 resolution: number,
23 hlsPlaylist: VideoStreamingPlaylist
24) {
25 const res = await getPlaylist(`${baseUrlPlaylist}/${videoUUID}/${resolution}.m3u8`)
26 const playlist = res.text
27
28 const videoName = `${videoUUID}-${resolution}-fragmented.mp4`
29
30 const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist)
31
32 const length = parseInt(matches[1], 10)
33 const offset = parseInt(matches[2], 10)
34 const range = `${offset}-${offset + length - 1}`
35
36 const res2 = await getSegment(`${baseUrlSegment}/${videoUUID}/${videoName}`, 206, `bytes=${range}`)
37
38 const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url)
39
40 const sha256Server = resSha.body[ videoName ][range]
41 expect(sha256(res2.body)).to.equal(sha256Server)
42}
43
15// --------------------------------------------------------------------------- 44// ---------------------------------------------------------------------------
16 45
17export { 46export {
18 getPlaylist, 47 getPlaylist,
19 getSegment, 48 getSegment,
20 getSegmentSha256 49 getSegmentSha256,
50 checkSegmentHash
21} 51}
diff --git a/yarn.lock b/yarn.lock
index 47c0646e4..1e759af1b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,14 +2,6 @@
2# yarn lockfile v1 2# yarn lockfile v1
3 3
4 4
5"@babel/polyfill@^7.2.5":
6 version "7.2.5"
7 resolved "https://registry.yarnpkg.com/@babel/polyfill/-/polyfill-7.2.5.tgz#6c54b964f71ad27edddc567d065e57e87ed7fa7d"
8 integrity sha512-8Y/t3MWThtMLYr0YNC/Q76tqN1w30+b0uQMeFUYauG2UGTR19zyUtFrAzT23zNtBxPp+LbE5E/nwV/q/r3y6ug==
9 dependencies:
10 core-js "^2.5.7"
11 regenerator-runtime "^0.12.0"
12
13"@iamstarkov/listr-update-renderer@0.4.1": 5"@iamstarkov/listr-update-renderer@0.4.1":
14 version "0.4.1" 6 version "0.4.1"
15 resolved "https://registry.yarnpkg.com/@iamstarkov/listr-update-renderer/-/listr-update-renderer-0.4.1.tgz#d7c48092a2dcf90fd672b6c8b458649cb350c77e" 7 resolved "https://registry.yarnpkg.com/@iamstarkov/listr-update-renderer/-/listr-update-renderer-0.4.1.tgz#d7c48092a2dcf90fd672b6c8b458649cb350c77e"
@@ -3593,17 +3585,6 @@ hide-powered-by@1.0.0:
3593 resolved "https://registry.yarnpkg.com/hide-powered-by/-/hide-powered-by-1.0.0.tgz#4a85ad65881f62857fc70af7174a1184dccce32b" 3585 resolved "https://registry.yarnpkg.com/hide-powered-by/-/hide-powered-by-1.0.0.tgz#4a85ad65881f62857fc70af7174a1184dccce32b"
3594 integrity sha1-SoWtZYgfYoV/xwr3F0oRhNzM4ys= 3586 integrity sha1-SoWtZYgfYoV/xwr3F0oRhNzM4ys=
3595 3587
3596"hlsdownloader@https://github.com/Chocobozzz/hlsdownloader#build":
3597 version "0.0.0-semantic-release"
3598 resolved "https://github.com/Chocobozzz/hlsdownloader#e19f9d803dcfe7ec25fd734b4743184f19a9b0cc"
3599 dependencies:
3600 "@babel/polyfill" "^7.2.5"
3601 async "^2.6.1"
3602 minimist "^1.2.0"
3603 mkdirp "^0.5.1"
3604 request "^2.88.0"
3605 request-promise "^4.2.2"
3606
3607hosted-git-info@^2.1.4, hosted-git-info@^2.6.0, hosted-git-info@^2.7.1: 3588hosted-git-info@^2.1.4, hosted-git-info@^2.6.0, hosted-git-info@^2.7.1:
3608 version "2.7.1" 3589 version "2.7.1"
3609 resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047" 3590 resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047"
@@ -4870,7 +4851,7 @@ lodash@=3.10.1:
4870 resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6" 4851 resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
4871 integrity sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y= 4852 integrity sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=
4872 4853
4873lodash@^4.0.0, lodash@^4.13.1, lodash@^4.17.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.2, lodash@~4.17.10: 4854lodash@^4.0.0, lodash@^4.17.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.2, lodash@~4.17.10:
4874 version "4.17.11" 4855 version "4.17.11"
4875 resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" 4856 resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
4876 integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== 4857 integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
@@ -6651,11 +6632,6 @@ psl@^1.1.24:
6651 resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67" 6632 resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67"
6652 integrity sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ== 6633 integrity sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==
6653 6634
6654psl@^1.1.28:
6655 version "1.1.31"
6656 resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.31.tgz#e9aa86d0101b5b105cbe93ac6b784cd547276184"
6657 integrity sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==
6658
6659pstree.remy@^1.1.2: 6635pstree.remy@^1.1.2:
6660 version "1.1.2" 6636 version "1.1.2"
6661 resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.2.tgz#4448bbeb4b2af1fed242afc8dc7416a6f504951a" 6637 resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.2.tgz#4448bbeb4b2af1fed242afc8dc7416a6f504951a"
@@ -6699,7 +6675,7 @@ punycode@^1.4.1:
6699 resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" 6675 resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
6700 integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= 6676 integrity sha1-wNWmOycYgArY4esPpSachN1BhF4=
6701 6677
6702punycode@^2.1.0, punycode@^2.1.1: 6678punycode@^2.1.0:
6703 version "2.1.1" 6679 version "2.1.1"
6704 resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" 6680 resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
6705 integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== 6681 integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
@@ -6982,11 +6958,6 @@ reflect-metadata@^0.1.12:
6982 resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2" 6958 resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2"
6983 integrity sha512-n+IyV+nGz3+0q3/Yf1ra12KpCyi001bi4XFxSjbiWWjfqb52iTTtpGXmCCAOWWIAn9KEuFZKGqBERHmrtScZ3A== 6959 integrity sha512-n+IyV+nGz3+0q3/Yf1ra12KpCyi001bi4XFxSjbiWWjfqb52iTTtpGXmCCAOWWIAn9KEuFZKGqBERHmrtScZ3A==
6984 6960
6985regenerator-runtime@^0.12.0:
6986 version "0.12.1"
6987 resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.12.1.tgz#fa1a71544764c036f8c49b13a08b2594c9f8a0de"
6988 integrity sha512-odxIc1/vDlo4iZcfXqRYFj0vpXFNoGdKMAUieAlFYO6m/nl5e9KR/beGf41z4a1FI+aQgtjhuaSlDxQ0hmkrHg==
6989
6990regex-not@^1.0.0, regex-not@^1.0.2: 6961regex-not@^1.0.0, regex-not@^1.0.2:
6991 version "1.0.2" 6962 version "1.0.2"
6992 resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c" 6963 resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c"
@@ -7036,23 +7007,6 @@ repeat-string@^1.6.1:
7036 resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" 7007 resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637"
7037 integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= 7008 integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc=
7038 7009
7039request-promise-core@1.1.1:
7040 version "1.1.1"
7041 resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6"
7042 integrity sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=
7043 dependencies:
7044 lodash "^4.13.1"
7045
7046request-promise@^4.2.2:
7047 version "4.2.2"
7048 resolved "https://registry.yarnpkg.com/request-promise/-/request-promise-4.2.2.tgz#d1ea46d654a6ee4f8ee6a4fea1018c22911904b4"
7049 integrity sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=
7050 dependencies:
7051 bluebird "^3.5.0"
7052 request-promise-core "1.1.1"
7053 stealthy-require "^1.1.0"
7054 tough-cookie ">=2.3.3"
7055
7056request@^2.74.0, request@^2.81.0, request@^2.83.0, request@^2.87.0, request@^2.88.0: 7010request@^2.74.0, request@^2.81.0, request@^2.83.0, request@^2.87.0, request@^2.88.0:
7057 version "2.88.0" 7011 version "2.88.0"
7058 resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" 7012 resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
@@ -7970,11 +7924,6 @@ statuses@~1.4.0:
7970 resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087" 7924 resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087"
7971 integrity sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew== 7925 integrity sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==
7972 7926
7973stealthy-require@^1.1.0:
7974 version "1.1.1"
7975 resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
7976 integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
7977
7978stream-each@^1.1.0: 7927stream-each@^1.1.0:
7979 version "1.2.3" 7928 version "1.2.3"
7980 resolved "https://registry.yarnpkg.com/stream-each/-/stream-each-1.2.3.tgz#ebe27a0c389b04fbcc233642952e10731afa9bae" 7929 resolved "https://registry.yarnpkg.com/stream-each/-/stream-each-1.2.3.tgz#ebe27a0c389b04fbcc233642952e10731afa9bae"
@@ -8467,15 +8416,6 @@ touch@^3.1.0:
8467 dependencies: 8416 dependencies:
8468 nopt "~1.0.10" 8417 nopt "~1.0.10"
8469 8418
8470tough-cookie@>=2.3.3:
8471 version "3.0.1"
8472 resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-3.0.1.tgz#9df4f57e739c26930a018184887f4adb7dca73b2"
8473 integrity sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==
8474 dependencies:
8475 ip-regex "^2.1.0"
8476 psl "^1.1.28"
8477 punycode "^2.1.1"
8478
8479tough-cookie@~2.4.3: 8419tough-cookie@~2.4.3:
8480 version "2.4.3" 8420 version "2.4.3"
8481 resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" 8421 resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781"