From 092092969633bbcf6d4891a083ea497a7d5c3154 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 29 Jan 2019 08:37:25 +0100 Subject: Add hls support on server --- server/tests/api/check-params/config.ts | 3 + server/tests/api/redundancy/redundancy.ts | 212 +++++++++++++++++++++--------- server/tests/api/server/config.ts | 6 + server/tests/api/videos/index.ts | 1 + server/tests/api/videos/video-hls.ts | 145 ++++++++++++++++++++ 5 files changed, 302 insertions(+), 65 deletions(-) create mode 100644 server/tests/api/videos/video-hls.ts (limited to 'server/tests/api') diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 4038ecbf0..07de2b5a5 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -65,6 +65,9 @@ describe('Test config API validators', function () { '480p': true, '720p': false, '1080p': false + }, + hls: { + enabled: false } }, import: { diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 9d3ce8153..5b99309fb 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -17,7 +17,7 @@ import { viewVideo, wait, waitUntilLog, - checkVideoFilesWereRemoved, removeVideo, getVideoWithToken + checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer } from '../../../../shared/utils' import { waitJobs } from '../../../../shared/utils/server/jobs' @@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { const config = { + transcoding: { + hls: { + enabled: true + } + }, redundancy: { videos: { check_interval: '5 seconds', @@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: await waitJobs(servers) } -async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { +async function check1WebSeed (videoUUID?: string) { if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ @@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str ] for (const server of servers) { - { - // With token to avoid issues with video follow constraints - const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) + // With token to avoid issues with video follow constraints + const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) - const video: VideoDetails = res.body - for (const f of video.files) { - checkMagnetWebseeds(f, webseeds, server) - } + const video: VideoDetails = res.body + for (const f of video.files) { + checkMagnetWebseeds(f, webseeds, server) } } } -async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) - const stat = data.videosRedundancy[0] - - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(204800) - expect(stat.totalUsed).to.be.at.least(1).and.below(204801) - expect(stat.totalVideoFiles).to.equal(4) - expect(stat.totalVideos).to.equal(1) -} - -async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) - - const stat = data.videosRedundancy[0] - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(204800) - expect(stat.totalUsed).to.equal(0) - expect(stat.totalVideoFiles).to.equal(0) - expect(stat.totalVideos).to.equal(0) -} - -async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { +async function check2Webseeds (videoUUID?: string) { if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ @@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st await makeGetRequest({ url: servers[1].url, statusCodeExpected: 200, - path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, + path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`, contentType: null }) } @@ -174,6 +149,81 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st } } +async function check0PlaylistRedundancies (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + for (const server of servers) { + // With token to avoid issues with video follow constraints + const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) + const video: VideoDetails = res.body + + expect(video.streamingPlaylists).to.be.an('array') + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) + } +} + +async function check1PlaylistRedundancies (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video: VideoDetails = res.body + + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1) + + const redundancy = video.streamingPlaylists[0].redundancies[0] + + expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) + } + + await makeGetRequest({ + url: servers[0].url, + statusCodeExpected: 200, + path: `/static/redundancy/hls/${videoUUID}/360_000.ts`, + contentType: null + }) + + for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) { + const files = await readdir(join(root(), directory, videoUUID)) + expect(files).to.have.length.at.least(4) + + for (const resolution of [ 240, 360, 480, 720 ]) { + expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined + expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined + } + } +} + +async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + const stat = data.videosRedundancy[0] + + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(204800) + expect(stat.totalUsed).to.be.at.least(1).and.below(204801) + expect(stat.totalVideoFiles).to.equal(4) + expect(stat.totalVideos).to.equal(1) +} + +async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(204800) + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) +} + async function enableRedundancyOnServer1 () { await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) @@ -220,7 +270,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -229,27 +280,29 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) it('Should undo redundancy on server 1 and remove duplicated videos', async function () { - this.timeout(40000) + this.timeout(80000) await disableRedundancyOnServer1() await waitJobs(servers) await wait(5000) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ]) }) after(function () { @@ -267,7 +320,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -276,25 +330,27 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) it('Should unfollow on server 1 and remove duplicated videos', async function () { - this.timeout(40000) + this.timeout(80000) await unfollow(servers[0].url, servers[0].accessToken, servers[1]) await waitJobs(servers) await wait(5000) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) }) @@ -314,7 +370,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -323,18 +380,19 @@ describe('Test videos redundancy', function () { }) it('Should still have 1 webseed on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) await wait(15000) await waitJobs(servers) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) it('Should view 2 times the first video to have > min_views config', async function () { - this.timeout(40000) + this.timeout(80000) await viewVideo(servers[ 0 ].url, video1Server2UUID) await viewVideo(servers[ 2 ].url, video1Server2UUID) @@ -344,13 +402,14 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) @@ -405,7 +464,7 @@ describe('Test videos redundancy', function () { }) it('Should still have 2 webseeds after 10 seconds', async function () { - this.timeout(40000) + this.timeout(80000) await wait(10000) @@ -420,7 +479,7 @@ describe('Test videos redundancy', function () { }) it('Should stop server 1 and expire video redundancy', async function () { - this.timeout(40000) + this.timeout(80000) killallServers([ servers[0] ]) @@ -446,10 +505,11 @@ describe('Test videos redundancy', function () { await enableRedundancyOnServer1() await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) @@ -467,8 +527,10 @@ describe('Test videos redundancy', function () { await wait(1000) try { - await check1WebSeed(strategy, video1Server2UUID) - await check2Webseeds(strategy, video2Server2UUID) + await check1WebSeed(video1Server2UUID) + await check0PlaylistRedundancies(video1Server2UUID) + await check2Webseeds(video2Server2UUID) + await check1PlaylistRedundancies(video2Server2UUID) checked = true } catch { @@ -477,6 +539,26 @@ describe('Test videos redundancy', function () { } }) + it('Should disable strategy and remove redundancies', async function () { + this.timeout(80000) + + await waitJobs(servers) + + killallServers([ servers[ 0 ] ]) + await reRunServer(servers[ 0 ], { + redundancy: { + videos: { + check_interval: '1 second', + strategies: [] + } + } + }) + + await waitJobs(servers) + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ]) + }) + after(function () { return cleanServers() }) diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index bebfc7398..0dfe6e4fe 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) { expect(data.transcoding.resolutions['480p']).to.be.true expect(data.transcoding.resolutions['720p']).to.be.true expect(data.transcoding.resolutions['1080p']).to.be.true + expect(data.transcoding.hls.enabled).to.be.true + expect(data.import.videos.http.enabled).to.be.true expect(data.import.videos.torrent.enabled).to.be.true } @@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.transcoding.resolutions['480p']).to.be.true expect(data.transcoding.resolutions['720p']).to.be.false expect(data.transcoding.resolutions['1080p']).to.be.false + expect(data.transcoding.hls.enabled).to.be.false expect(data.import.videos.http.enabled).to.be.false expect(data.import.videos.torrent.enabled).to.be.false @@ -205,6 +208,9 @@ describe('Test config', function () { '480p': true, '720p': false, '1080p': false + }, + hls: { + enabled: false } }, import: { diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 97f467aae..a501a80b2 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -8,6 +8,7 @@ import './video-change-ownership' import './video-channels' import './video-comments' import './video-description' +import './video-hls' import './video-imports' import './video-nsfw' import './video-privacy' diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts new file mode 100644 index 000000000..71d863b12 --- /dev/null +++ b/server/tests/api/videos/video-hls.ts @@ -0,0 +1,145 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { + checkDirectoryIsEmpty, + checkTmpIsEmpty, + doubleFollow, + flushAndRunMultipleServers, + flushTests, + getPlaylist, + getSegment, + getSegmentSha256, + getVideo, + killallServers, + removeVideo, + ServerInfo, + setAccessTokensToServers, + updateVideo, + uploadVideo, + waitJobs +} from '../../../../shared/utils' +import { VideoDetails } from '../../../../shared/models/videos' +import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type' +import { sha256 } from '../../../helpers/core-utils' +import { join } from 'path' + +const expect = chai.expect + +async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) { + const resolutions = [ 240, 360, 480, 720 ] + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const videoDetails: VideoDetails = res.body + + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + + const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + expect(hlsPlaylist).to.not.be.undefined + + { + const res2 = await getPlaylist(hlsPlaylist.playlistUrl) + + const masterPlaylist = res2.text + + expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25') + + for (const resolution of resolutions) { + expect(masterPlaylist).to.contain(`${resolution}.m3u8`) + } + } + + { + for (const resolution of resolutions) { + const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`) + + const subPlaylist = res2.text + expect(subPlaylist).to.contain(resolution + '_000.ts') + } + } + + { + for (const resolution of resolutions) { + + const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`) + + const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url) + + const sha256Server = resSha.body[ resolution + '_000.ts' ] + expect(sha256(res2.body)).to.equal(sha256Server) + } + } + } +} + +describe('Test HLS videos', function () { + let servers: ServerInfo[] = [] + let videoUUID = '' + + before(async function () { + this.timeout(120000) + + servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } }) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + it('Should upload a video and transcode it to HLS', async function () { + this.timeout(120000) + + { + const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' }) + videoUUID = res.body.video.uuid + } + + await waitJobs(servers) + + await checkHlsPlaylist(servers, videoUUID) + }) + + it('Should update the video', async function () { + await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' }) + + await waitJobs(servers) + + await checkHlsPlaylist(servers, videoUUID) + }) + + it('Should delete the video', async function () { + await removeVideo(servers[0].url, servers[0].accessToken, videoUUID) + + await waitJobs(servers) + + for (const server of servers) { + await getVideo(server.url, videoUUID, 404) + } + }) + + it('Should have the playlists/segment deleted from the disk', async function () { + for (const server of servers) { + await checkDirectoryIsEmpty(server, 'videos') + await checkDirectoryIsEmpty(server, join('playlists', 'hls')) + } + }) + + it('Should have an empty tmp directory', async function () { + for (const server of servers) { + await checkTmpIsEmpty(server) + } + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) -- cgit v1.2.3