Add hls support on server

6 files changed, 311 insertions, 67 deletions

diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 4038ecbf0..07de2b5a5 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -65,6 +65,9 @@ describe('Test config API validators', function () {
         '480p': true,
         '720p': false,
         '1080p': false
+      },
+      hls: {
+        enabled: false
       }
     },
     import: {
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 9d3ce8153..5b99309fb 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -17,7 +17,7 @@ import {
   viewVideo,
   wait,
   waitUntilLog,
-  checkVideoFilesWereRemoved, removeVideo, getVideoWithToken
+  checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer
 } from '../../../../shared/utils'
 import { waitJobs } from '../../../../shared/utils/server/jobs'
 
@@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
 
 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
   const config = {
+    transcoding: {
+      hls: {
+        enabled: true
+      }
+    },
     redundancy: {
       videos: {
         check_interval: '5 seconds',
@@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
   await waitJobs(servers)
 }
 
-async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
+async function check1WebSeed (videoUUID?: string) {
   if (!videoUUID) videoUUID = video1Server2UUID
 
   const webseeds = [
@@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str
   ]
 
   for (const server of servers) {
-    {
-      // With token to avoid issues with video follow constraints
-      const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
+    // With token to avoid issues with video follow constraints
+    const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
 
-      const video: VideoDetails = res.body
-      for (const f of video.files) {
-        checkMagnetWebseeds(f, webseeds, server)
-      }
+    const video: VideoDetails = res.body
+    for (const f of video.files) {
+      checkMagnetWebseeds(f, webseeds, server)
     }
   }
 }
 
-async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
-  const res = await getStats(servers[0].url)
-  const data: ServerStats = res.body
-
-  expect(data.videosRedundancy).to.have.lengthOf(1)
-  const stat = data.videosRedundancy[0]
-
-  expect(stat.strategy).to.equal(strategy)
-  expect(stat.totalSize).to.equal(204800)
-  expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
-  expect(stat.totalVideoFiles).to.equal(4)
-  expect(stat.totalVideos).to.equal(1)
-}
-
-async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
-  const res = await getStats(servers[0].url)
-  const data: ServerStats = res.body
-
-  expect(data.videosRedundancy).to.have.lengthOf(1)
-
-  const stat = data.videosRedundancy[0]
-  expect(stat.strategy).to.equal(strategy)
-  expect(stat.totalSize).to.equal(204800)
-  expect(stat.totalUsed).to.equal(0)
-  expect(stat.totalVideoFiles).to.equal(0)
-  expect(stat.totalVideos).to.equal(0)
-}
-
-async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
+async function check2Webseeds (videoUUID?: string) {
   if (!videoUUID) videoUUID = video1Server2UUID
 
   const webseeds = [
@@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
       await makeGetRequest({
         url: servers[1].url,
         statusCodeExpected: 200,
-        path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
+        path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
         contentType: null
       })
     }
@@ -174,6 +149,81 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
   }
 }
 
+async function check0PlaylistRedundancies (videoUUID?: string) {
+  if (!videoUUID) videoUUID = video1Server2UUID
+
+  for (const server of servers) {
+    // With token to avoid issues with video follow constraints
+    const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
+    const video: VideoDetails = res.body
+
+    expect(video.streamingPlaylists).to.be.an('array')
+    expect(video.streamingPlaylists).to.have.lengthOf(1)
+    expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
+  }
+}
+
+async function check1PlaylistRedundancies (videoUUID?: string) {
+  if (!videoUUID) videoUUID = video1Server2UUID
+
+  for (const server of servers) {
+    const res = await getVideo(server.url, videoUUID)
+    const video: VideoDetails = res.body
+
+    expect(video.streamingPlaylists).to.have.lengthOf(1)
+    expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
+
+    const redundancy = video.streamingPlaylists[0].redundancies[0]
+
+    expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
+  }
+
+  await makeGetRequest({
+    url: servers[0].url,
+    statusCodeExpected: 200,
+    path: `/static/redundancy/hls/${videoUUID}/360_000.ts`,
+    contentType: null
+  })
+
+  for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) {
+    const files = await readdir(join(root(), directory, videoUUID))
+    expect(files).to.have.length.at.least(4)
+
+    for (const resolution of [ 240, 360, 480, 720 ]) {
+      expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined
+      expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined
+    }
+  }
+}
+
+async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
+  const res = await getStats(servers[0].url)
+  const data: ServerStats = res.body
+
+  expect(data.videosRedundancy).to.have.lengthOf(1)
+  const stat = data.videosRedundancy[0]
+
+  expect(stat.strategy).to.equal(strategy)
+  expect(stat.totalSize).to.equal(204800)
+  expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
+  expect(stat.totalVideoFiles).to.equal(4)
+  expect(stat.totalVideos).to.equal(1)
+}
+
+async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
+  const res = await getStats(servers[0].url)
+  const data: ServerStats = res.body
+
+  expect(data.videosRedundancy).to.have.lengthOf(1)
+
+  const stat = data.videosRedundancy[0]
+  expect(stat.strategy).to.equal(strategy)
+  expect(stat.totalSize).to.equal(204800)
+  expect(stat.totalUsed).to.equal(0)
+  expect(stat.totalVideoFiles).to.equal(0)
+  expect(stat.totalVideos).to.equal(0)
+}
+
 async function enableRedundancyOnServer1 () {
   await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 
@@ -220,7 +270,8 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should have 1 webseed on the first video', async function () {
-      await check1WebSeed(strategy)
+      await check1WebSeed()
+      await check0PlaylistRedundancies()
       await checkStatsWith1Webseed(strategy)
     })
 
@@ -229,27 +280,29 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should have 2 webseeds on the first video', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       await waitJobs(servers)
-      await waitUntilLog(servers[0], 'Duplicated ', 4)
+      await waitUntilLog(servers[0], 'Duplicated ', 5)
       await waitJobs(servers)
 
-      await check2Webseeds(strategy)
+      await check2Webseeds()
+      await check1PlaylistRedundancies()
       await checkStatsWith2Webseed(strategy)
     })
 
     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       await disableRedundancyOnServer1()
 
       await waitJobs(servers)
       await wait(5000)
 
-      await check1WebSeed(strategy)
+      await check1WebSeed()
+      await check0PlaylistRedundancies()
 
-      await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
+      await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
     })
 
     after(function () {
@@ -267,7 +320,8 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should have 1 webseed on the first video', async function () {
-      await check1WebSeed(strategy)
+      await check1WebSeed()
+      await check0PlaylistRedundancies()
       await checkStatsWith1Webseed(strategy)
     })
 
@@ -276,25 +330,27 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should have 2 webseeds on the first video', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       await waitJobs(servers)
-      await waitUntilLog(servers[0], 'Duplicated ', 4)
+      await waitUntilLog(servers[0], 'Duplicated ', 5)
       await waitJobs(servers)
 
-      await check2Webseeds(strategy)
+      await check2Webseeds()
+      await check1PlaylistRedundancies()
       await checkStatsWith2Webseed(strategy)
     })
 
     it('Should unfollow on server 1 and remove duplicated videos', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       await unfollow(servers[0].url, servers[0].accessToken, servers[1])
 
       await waitJobs(servers)
       await wait(5000)
 
-      await check1WebSeed(strategy)
+      await check1WebSeed()
+      await check0PlaylistRedundancies()
 
       await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
     })
@@ -314,7 +370,8 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should have 1 webseed on the first video', async function () {
-      await check1WebSeed(strategy)
+      await check1WebSeed()
+      await check0PlaylistRedundancies()
       await checkStatsWith1Webseed(strategy)
     })
 
@@ -323,18 +380,19 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should still have 1 webseed on the first video', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       await waitJobs(servers)
       await wait(15000)
       await waitJobs(servers)
 
-      await check1WebSeed(strategy)
+      await check1WebSeed()
+      await check0PlaylistRedundancies()
       await checkStatsWith1Webseed(strategy)
     })
 
     it('Should view 2 times the first video to have > min_views config', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       await viewVideo(servers[ 0 ].url, video1Server2UUID)
       await viewVideo(servers[ 2 ].url, video1Server2UUID)
@@ -344,13 +402,14 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should have 2 webseeds on the first video', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       await waitJobs(servers)
-      await waitUntilLog(servers[0], 'Duplicated ', 4)
+      await waitUntilLog(servers[0], 'Duplicated ', 5)
       await waitJobs(servers)
 
-      await check2Webseeds(strategy)
+      await check2Webseeds()
+      await check1PlaylistRedundancies()
       await checkStatsWith2Webseed(strategy)
     })
 
@@ -405,7 +464,7 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should still have 2 webseeds after 10 seconds', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       await wait(10000)
 
@@ -420,7 +479,7 @@ describe('Test videos redundancy', function () {
     })
 
     it('Should stop server 1 and expire video redundancy', async function () {
-      this.timeout(40000)
+      this.timeout(80000)
 
       killallServers([ servers[0] ])
 
@@ -446,10 +505,11 @@ describe('Test videos redundancy', function () {
       await enableRedundancyOnServer1()
 
       await waitJobs(servers)
-      await waitUntilLog(servers[0], 'Duplicated ', 4)
+      await waitUntilLog(servers[0], 'Duplicated ', 5)
       await waitJobs(servers)
 
-      await check2Webseeds(strategy)
+      await check2Webseeds()
+      await check1PlaylistRedundancies()
       await checkStatsWith2Webseed(strategy)
 
       const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
@@ -467,8 +527,10 @@ describe('Test videos redundancy', function () {
         await wait(1000)
 
         try {
-          await check1WebSeed(strategy, video1Server2UUID)
-          await check2Webseeds(strategy, video2Server2UUID)
+          await check1WebSeed(video1Server2UUID)
+          await check0PlaylistRedundancies(video1Server2UUID)
+          await check2Webseeds(video2Server2UUID)
+          await check1PlaylistRedundancies(video2Server2UUID)
 
           checked = true
         } catch {
@@ -477,6 +539,26 @@ describe('Test videos redundancy', function () {
       }
     })
 
+    it('Should disable strategy and remove redundancies', async function () {
+      this.timeout(80000)
+
+      await waitJobs(servers)
+
+      killallServers([ servers[ 0 ] ])
+      await reRunServer(servers[ 0 ], {
+        redundancy: {
+          videos: {
+            check_interval: '1 second',
+            strategies: []
+          }
+        }
+      })
+
+      await waitJobs(servers)
+
+      await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
+    })
+
     after(function () {
       return cleanServers()
     })
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index bebfc7398..0dfe6e4fe 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) {
   expect(data.transcoding.resolutions['480p']).to.be.true
   expect(data.transcoding.resolutions['720p']).to.be.true
   expect(data.transcoding.resolutions['1080p']).to.be.true
+  expect(data.transcoding.hls.enabled).to.be.true
+
   expect(data.import.videos.http.enabled).to.be.true
   expect(data.import.videos.torrent.enabled).to.be.true
 }
@@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) {
   expect(data.transcoding.resolutions['480p']).to.be.true
   expect(data.transcoding.resolutions['720p']).to.be.false
   expect(data.transcoding.resolutions['1080p']).to.be.false
+  expect(data.transcoding.hls.enabled).to.be.false
 
   expect(data.import.videos.http.enabled).to.be.false
   expect(data.import.videos.torrent.enabled).to.be.false
@@ -205,6 +208,9 @@ describe('Test config', function () {
           '480p': true,
           '720p': false,
           '1080p': false
+        },
+        hls: {
+          enabled: false
         }
       },
       import: {
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 97f467aae..a501a80b2 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -8,6 +8,7 @@ import './video-change-ownership'
 import './video-channels'
 import './video-comments'
 import './video-description'
+import './video-hls'
 import './video-imports'
 import './video-nsfw'
 import './video-privacy'
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
new file mode 100644
index 000000000..71d863b12
--- /dev/null
+++ b/server/tests/api/videos/video-hls.ts
@@ -0,0 +1,145 @@
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import {
+  checkDirectoryIsEmpty,
+  checkTmpIsEmpty,
+  doubleFollow,
+  flushAndRunMultipleServers,
+  flushTests,
+  getPlaylist,
+  getSegment,
+  getSegmentSha256,
+  getVideo,
+  killallServers,
+  removeVideo,
+  ServerInfo,
+  setAccessTokensToServers,
+  updateVideo,
+  uploadVideo,
+  waitJobs
+} from '../../../../shared/utils'
+import { VideoDetails } from '../../../../shared/models/videos'
+import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type'
+import { sha256 } from '../../../helpers/core-utils'
+import { join } from 'path'
+
+const expect = chai.expect
+
+async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) {
+  const resolutions = [ 240, 360, 480, 720 ]
+
+  for (const server of servers) {
+    const res = await getVideo(server.url, videoUUID)
+    const videoDetails: VideoDetails = res.body
+
+    expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
+
+    const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
+    expect(hlsPlaylist).to.not.be.undefined
+
+    {
+      const res2 = await getPlaylist(hlsPlaylist.playlistUrl)
+
+      const masterPlaylist = res2.text
+
+      expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25')
+
+      for (const resolution of resolutions) {
+        expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
+      }
+    }
+
+    {
+      for (const resolution of resolutions) {
+        const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`)
+
+        const subPlaylist = res2.text
+        expect(subPlaylist).to.contain(resolution + '_000.ts')
+      }
+    }
+
+    {
+      for (const resolution of resolutions) {
+
+        const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`)
+
+        const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url)
+
+        const sha256Server = resSha.body[ resolution + '_000.ts' ]
+        expect(sha256(res2.body)).to.equal(sha256Server)
+      }
+    }
+  }
+}
+
+describe('Test HLS videos', function () {
+  let servers: ServerInfo[] = []
+  let videoUUID = ''
+
+  before(async function () {
+    this.timeout(120000)
+
+    servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } })
+
+    // Get the access tokens
+    await setAccessTokensToServers(servers)
+
+    // Server 1 and server 2 follow each other
+    await doubleFollow(servers[0], servers[1])
+  })
+
+  it('Should upload a video and transcode it to HLS', async function () {
+    this.timeout(120000)
+
+    {
+      const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' })
+      videoUUID = res.body.video.uuid
+    }
+
+    await waitJobs(servers)
+
+    await checkHlsPlaylist(servers, videoUUID)
+  })
+
+  it('Should update the video', async function () {
+    await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' })
+
+    await waitJobs(servers)
+
+    await checkHlsPlaylist(servers, videoUUID)
+  })
+
+  it('Should delete the video', async function () {
+    await removeVideo(servers[0].url, servers[0].accessToken, videoUUID)
+
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      await getVideo(server.url, videoUUID, 404)
+    }
+  })
+
+  it('Should have the playlists/segment deleted from the disk', async function () {
+    for (const server of servers) {
+      await checkDirectoryIsEmpty(server, 'videos')
+      await checkDirectoryIsEmpty(server, join('playlists', 'hls'))
+    }
+  })
+
+  it('Should have an empty tmp directory', async function () {
+    for (const server of servers) {
+      await checkTmpIsEmpty(server)
+    }
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})
diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts
index 811ea6a9f..d38bb4331 100644
--- a/server/tests/cli/update-host.ts
+++ b/server/tests/cli/update-host.ts
@@ -86,6 +86,13 @@ describe('Test update host scripts', function () {
       const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid)
 
       expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid)
+
+      const res = await getVideo(server.url, video.uuid)
+      const videoDetails: VideoDetails = res.body
+
+      expect(videoDetails.trackerUrls[0]).to.include(server.host)
+      expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host)
+      expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host)
     }
   })
 
@@ -100,7 +107,7 @@ describe('Test update host scripts', function () {
     }
   })
 
-  it('Should have update accounts url', async function () {
+  it('Should have updated accounts url', async function () {
     const res = await getAccountsList(server.url)
     expect(res.body.total).to.equal(3)
 
@@ -112,7 +119,7 @@ describe('Test update host scripts', function () {
     }
   })
 
-  it('Should update torrent hosts', async function () {
+  it('Should have updated torrent hosts', async function () {
     this.timeout(30000)
 
     const res = await getVideosList(server.url)