Add ability to save replay of permanent lives

7 files changed, 208 insertions, 67 deletions

diff --git a/server/tests/api/check-params/live.ts b/server/tests/api/check-params/live.ts
index b253f5e20..2f1c1257e 100644
--- a/server/tests/api/check-params/live.ts
+++ b/server/tests/api/check-params/live.ts
@@ -212,12 +212,6 @@ describe('Test video lives API validator', function () {
       await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
     })
 
-    it('Should fail with save replay and permanent live set to true', async function () {
-      const fields = { ...baseCorrectParams, saveReplay: true, permanentLive: true }
-
-      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
-    })
-
     it('Should fail with bad latency setting', async function () {
       const fields = { ...baseCorrectParams, latencyMode: 42 }
 
diff --git a/server/tests/api/live/live-constraints.ts b/server/tests/api/live/live-constraints.ts
index 909399836..b92dc7b89 100644
--- a/server/tests/api/live/live-constraints.ts
+++ b/server/tests/api/live/live-constraints.ts
@@ -14,7 +14,7 @@ import {
   setDefaultVideoChannel,
   waitJobs
 } from '@shared/server-commands'
-import { checkLiveCleanupAfterSave } from '../../shared'
+import { checkLiveCleanup } from '../../shared'
 
 const expect = chai.expect
 
@@ -43,7 +43,7 @@ describe('Test live constraints', function () {
       expect(video.duration).to.be.greaterThan(0)
     }
 
-    await checkLiveCleanupAfterSave(servers[0], videoId, resolutions)
+    await checkLiveCleanup(servers[0], videoId, resolutions)
   }
 
   async function waitUntilLivePublishedOnAllServers (videoId: string) {
diff --git a/server/tests/api/live/live-permanent.ts b/server/tests/api/live/live-permanent.ts
index 3e6fec453..a88d71dd9 100644
--- a/server/tests/api/live/live-permanent.ts
+++ b/server/tests/api/live/live-permanent.ts
@@ -121,7 +121,7 @@ describe('Permanent live', function () {
     await waitJobs(servers)
   })
 
-  it('Should not have cleaned up this live', async function () {
+  it('Should have cleaned up this live', async function () {
     this.timeout(40000)
 
     await wait(5000)
@@ -129,7 +129,8 @@ describe('Permanent live', function () {
 
     for (const server of servers) {
       const videoDetails = await server.videos.get({ id: videoUUID })
-      expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
+
+      expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
     }
   })
 
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index 95a342b01..ba68a4287 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -3,7 +3,7 @@
 import 'mocha'
 import * as chai from 'chai'
 import { FfmpegCommand } from 'fluent-ffmpeg'
-import { checkLiveCleanupAfterSave } from '@server/tests/shared'
+import { checkLiveCleanup } from '@server/tests/shared'
 import { wait } from '@shared/core-utils'
 import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models'
 import {
@@ -11,6 +11,7 @@ import {
   ConfigCommand,
   createMultipleServers,
   doubleFollow,
+  findExternalSavedVideo,
   PeerTubeServer,
   setAccessTokensToServers,
   setDefaultVideoChannel,
@@ -18,7 +19,8 @@ import {
   testFfmpegStreamError,
   waitJobs,
   waitUntilLivePublishedOnAllServers,
-  waitUntilLiveSavedOnAllServers
+  waitUntilLiveReplacedByReplayOnAllServers,
+  waitUntilLiveWaitingOnAllServers
 } from '@shared/server-commands'
 
 const expect = chai.expect
@@ -28,7 +30,7 @@ describe('Save replay setting', function () {
   let liveVideoUUID: string
   let ffmpegCommand: FfmpegCommand
 
-  async function createLiveWrapper (saveReplay: boolean) {
+  async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) {
     if (liveVideoUUID) {
       try {
         await servers[0].videos.remove({ id: liveVideoUUID })
@@ -40,7 +42,8 @@ describe('Save replay setting', function () {
       channelId: servers[0].store.channel.id,
       privacy: VideoPrivacy.PUBLIC,
       name: 'my super live',
-      saveReplay
+      saveReplay: options.replay,
+      permanentLive: options.permanent
     }
 
     const { uuid } = await servers[0].live.create({ fields: attributes })
@@ -104,7 +107,7 @@ describe('Save replay setting', function () {
     it('Should correctly create and federate the "waiting for stream" live', async function () {
       this.timeout(20000)
 
-      liveVideoUUID = await createLiveWrapper(false)
+      liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 
       await waitJobs(servers)
 
@@ -140,13 +143,13 @@ describe('Save replay setting', function () {
       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 
       // No resolutions saved since we did not save replay
-      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
+      await checkLiveCleanup(servers[0], liveVideoUUID, [])
     })
 
     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
       this.timeout(40000)
 
-      liveVideoUUID = await createLiveWrapper(false)
+      liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 
       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 
@@ -169,13 +172,13 @@ describe('Save replay setting', function () {
 
       await wait(5000)
       await waitJobs(servers)
-      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
+      await checkLiveCleanup(servers[0], liveVideoUUID, [])
     })
 
     it('Should correctly terminate the stream on delete and delete the video', async function () {
       this.timeout(40000)
 
-      liveVideoUUID = await createLiveWrapper(false)
+      liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 
       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 
@@ -193,16 +196,16 @@ describe('Save replay setting', function () {
       await waitJobs(servers)
 
       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
-      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
+      await checkLiveCleanup(servers[0], liveVideoUUID, [])
     })
   })
 
-  describe('With save replay enabled', function () {
+  describe('With save replay enabled on non permanent live', function () {
 
     it('Should correctly create and federate the "waiting for stream" live', async function () {
       this.timeout(20000)
 
-      liveVideoUUID = await createLiveWrapper(true)
+      liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 
       await waitJobs(servers)
 
@@ -227,7 +230,7 @@ describe('Save replay setting', function () {
 
       await stopFfmpeg(ffmpegCommand)
 
-      await waitUntilLiveSavedOnAllServers(servers, liveVideoUUID)
+      await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
       await waitJobs(servers)
 
       // Live has been transcoded
@@ -249,13 +252,13 @@ describe('Save replay setting', function () {
     })
 
     it('Should have cleaned up the live files', async function () {
-      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
+      await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
     })
 
     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
       this.timeout(40000)
 
-      liveVideoUUID = await createLiveWrapper(true)
+      liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 
       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
@@ -277,13 +280,13 @@ describe('Save replay setting', function () {
 
       await wait(5000)
       await waitJobs(servers)
-      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
+      await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
     })
 
     it('Should correctly terminate the stream on delete and delete the video', async function () {
       this.timeout(40000)
 
-      liveVideoUUID = await createLiveWrapper(true)
+      liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 
       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
@@ -300,7 +303,123 @@ describe('Save replay setting', function () {
       await waitJobs(servers)
 
       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
-      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
+      await checkLiveCleanup(servers[0], liveVideoUUID, [])
+    })
+  })
+
+  describe('With save replay enabled on permanent live', function () {
+    let lastReplayUUID: string
+
+    it('Should correctly create and federate the "waiting for stream" live', async function () {
+      this.timeout(20000)
+
+      liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
+
+      await waitJobs(servers)
+
+      await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
+      await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
+    })
+
+    it('Should correctly have updated the live and federated it when streaming in the live', async function () {
+      this.timeout(20000)
+
+      ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
+      await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
+
+      await waitJobs(servers)
+
+      await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
+      await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
+    })
+
+    it('Should correctly have saved the live and federated it after the streaming', async function () {
+      this.timeout(30000)
+
+      const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
+
+      await stopFfmpeg(ffmpegCommand)
+
+      await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
+      await waitJobs(servers)
+
+      const video = await findExternalSavedVideo(servers[0], liveDetails)
+      expect(video).to.exist
+
+      for (const server of servers) {
+        await server.videos.get({ id: video.uuid })
+      }
+
+      lastReplayUUID = video.uuid
+    })
+
+    it('Should have cleaned up the live files', async function () {
+      await checkLiveCleanup(servers[0], liveVideoUUID, [])
+    })
+
+    it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
+      this.timeout(60000)
+
+      await servers[0].videos.remove({ id: lastReplayUUID })
+
+      liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
+
+      ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
+      await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
+
+      const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
+
+      await waitJobs(servers)
+      await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
+
+      await Promise.all([
+        servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
+        testFfmpegStreamError(ffmpegCommand, true)
+      ])
+
+      await waitJobs(servers)
+      await wait(5000)
+      await waitJobs(servers)
+
+      const replay = await findExternalSavedVideo(servers[0], liveDetails)
+      expect(replay).to.exist
+
+      for (const videoId of [ liveVideoUUID, replay.uuid ]) {
+        await checkVideosExist(videoId, false)
+
+        await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
+        await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+      }
+
+      await checkLiveCleanup(servers[0], liveVideoUUID, [])
+    })
+
+    it('Should correctly terminate the stream on delete and not save the video', async function () {
+      this.timeout(40000)
+
+      liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
+
+      ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
+      await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
+
+      const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
+
+      await waitJobs(servers)
+      await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
+
+      await Promise.all([
+        servers[0].videos.remove({ id: liveVideoUUID }),
+        testFfmpegStreamError(ffmpegCommand, true)
+      ])
+
+      await wait(5000)
+      await waitJobs(servers)
+
+      const replay = await findExternalSavedVideo(servers[0], liveDetails)
+      expect(replay).to.not.exist
+
+      await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
+      await checkLiveCleanup(servers[0], liveVideoUUID, [])
     })
   })
 
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index aeb039696..6e7b77bce 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -4,7 +4,7 @@ import 'mocha'
 import * as chai from 'chai'
 import { basename, join } from 'path'
 import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg'
-import { checkLiveCleanupAfterSave, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared'
+import { checkLiveCleanup, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared'
 import { wait } from '@shared/core-utils'
 import {
   HttpStatusCode,
@@ -583,7 +583,7 @@ describe('Test live', function () {
     it('Should correctly have cleaned up the live files', async function () {
       this.timeout(30000)
 
-      await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ])
+      await checkLiveCleanup(servers[0], liveVideoId, [ 240, 360, 720 ])
     })
   })
 
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts
index 0cb0a6e34..5d6281dec 100644
--- a/server/tests/api/object-storage/live.ts
+++ b/server/tests/api/object-storage/live.ts
@@ -2,13 +2,13 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { FfmpegCommand } from 'fluent-ffmpeg'
 import { expectStartWith } from '@server/tests/shared'
 import { areObjectStorageTestsDisabled } from '@shared/core-utils'
 import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models'
 import {
   createMultipleServers,
   doubleFollow,
+  findExternalSavedVideo,
   killallServers,
   makeRawRequest,
   ObjectStorageCommand,
@@ -18,17 +18,19 @@ import {
   stopFfmpeg,
   waitJobs,
   waitUntilLivePublishedOnAllServers,
-  waitUntilLiveSavedOnAllServers
+  waitUntilLiveReplacedByReplayOnAllServers,
+  waitUntilLiveWaitingOnAllServers
 } from '@shared/server-commands'
 
 const expect = chai.expect
 
-async function createLive (server: PeerTubeServer) {
+async function createLive (server: PeerTubeServer, permanent: boolean) {
   const attributes: LiveVideoCreate = {
     channelId: server.store.channel.id,
     privacy: VideoPrivacy.PUBLIC,
     name: 'my super live',
-    saveReplay: true
+    saveReplay: true,
+    permanentLive: permanent
   }
 
   const { uuid } = await server.live.create({ fields: attributes })
@@ -44,12 +46,39 @@ async function checkFiles (files: VideoFile[]) {
   }
 }
 
+async function getFiles (server: PeerTubeServer, videoUUID: string) {
+  const video = await server.videos.get({ id: videoUUID })
+
+  expect(video.files).to.have.lengthOf(0)
+  expect(video.streamingPlaylists).to.have.lengthOf(1)
+
+  return video.streamingPlaylists[0].files
+}
+
+async function streamAndEnd (servers: PeerTubeServer[], liveUUID: string) {
+  const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveUUID })
+  await waitUntilLivePublishedOnAllServers(servers, liveUUID)
+
+  const videoLiveDetails = await servers[0].videos.get({ id: liveUUID })
+  const liveDetails = await servers[0].live.get({ videoId: liveUUID })
+
+  await stopFfmpeg(ffmpegCommand)
+
+  if (liveDetails.permanentLive) {
+    await waitUntilLiveWaitingOnAllServers(servers, liveUUID)
+  } else {
+    await waitUntilLiveReplacedByReplayOnAllServers(servers, liveUUID)
+  }
+
+  await waitJobs(servers)
+
+  return { videoLiveDetails, liveDetails }
+}
+
 describe('Object storage for lives', function () {
   if (areObjectStorageTestsDisabled()) return
 
-  let ffmpegCommand: FfmpegCommand
   let servers: PeerTubeServer[]
-  let videoUUID: string
 
   before(async function () {
     this.timeout(120000)
@@ -66,31 +95,22 @@ describe('Object storage for lives', function () {
   })
 
   describe('Without live transcoding', async function () {
+    let videoUUID: string
 
     before(async function () {
       await servers[0].config.enableLive({ transcoding: false })
 
-      videoUUID = await createLive(servers[0])
+      videoUUID = await createLive(servers[0], false)
     })
 
     it('Should create a live and save the replay on object storage', async function () {
       this.timeout(220000)
 
-      ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
-      await waitUntilLivePublishedOnAllServers(servers, videoUUID)
-
-      await stopFfmpeg(ffmpegCommand)
-
-      await waitUntilLiveSavedOnAllServers(servers, videoUUID)
-      await waitJobs(servers)
+      await streamAndEnd(servers, videoUUID)
 
       for (const server of servers) {
-        const video = await server.videos.get({ id: videoUUID })
-
-        expect(video.files).to.have.lengthOf(0)
-        expect(video.streamingPlaylists).to.have.lengthOf(1)
-
-        const files = video.streamingPlaylists[0].files
+        const files = await getFiles(server, videoUUID)
+        expect(files).to.have.lengthOf(1)
 
         await checkFiles(files)
       }
@@ -98,31 +118,38 @@ describe('Object storage for lives', function () {
   })
 
   describe('With live transcoding', async function () {
+    let videoUUIDPermanent: string
+    let videoUUIDNonPermanent: string
 
     before(async function () {
       await servers[0].config.enableLive({ transcoding: true })
 
-      videoUUID = await createLive(servers[0])
+      videoUUIDPermanent = await createLive(servers[0], true)
+      videoUUIDNonPermanent = await createLive(servers[0], false)
     })
 
-    it('Should import a video and have sent it to object storage', async function () {
+    it('Should create a live and save the replay on object storage', async function () {
       this.timeout(240000)
 
-      ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
-      await waitUntilLivePublishedOnAllServers(servers, videoUUID)
+      await streamAndEnd(servers, videoUUIDNonPermanent)
 
-      await stopFfmpeg(ffmpegCommand)
+      for (const server of servers) {
+        const files = await getFiles(server, videoUUIDNonPermanent)
+        expect(files).to.have.lengthOf(5)
 
-      await waitUntilLiveSavedOnAllServers(servers, videoUUID)
-      await waitJobs(servers)
+        await checkFiles(files)
+      }
+    })
 
-      for (const server of servers) {
-        const video = await server.videos.get({ id: videoUUID })
+    it('Should create a live and save the replay of permanent live on object storage', async function () {
+      this.timeout(240000)
+
+      const { videoLiveDetails } = await streamAndEnd(servers, videoUUIDPermanent)
 
-        expect(video.files).to.have.lengthOf(0)
-        expect(video.streamingPlaylists).to.have.lengthOf(1)
+      const replay = await findExternalSavedVideo(servers[0], videoLiveDetails)
 
-        const files = video.streamingPlaylists[0].files
+      for (const server of servers) {
+        const files = await getFiles(server, replay.uuid)
         expect(files).to.have.lengthOf(5)
 
         await checkFiles(files)
diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts
index 72e3e27f6..6ee4899b0 100644
--- a/server/tests/shared/live.ts
+++ b/server/tests/shared/live.ts
@@ -5,11 +5,11 @@ import { pathExists, readdir } from 'fs-extra'
 import { join } from 'path'
 import { PeerTubeServer } from '@shared/server-commands'
 
-async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
+async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) {
   const basePath = server.servers.buildDirectory('streaming-playlists')
   const hlsPath = join(basePath, 'hls', videoUUID)
 
-  if (resolutions.length === 0) {
+  if (savedResolutions.length === 0) {
     const result = await pathExists(hlsPath)
     expect(result).to.be.false
 
@@ -19,9 +19,9 @@ async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: str
   const files = await readdir(hlsPath)
 
   // fragmented file and playlist per resolution + master playlist + segments sha256 json file
-  expect(files).to.have.lengthOf(resolutions.length * 2 + 2)
+  expect(files).to.have.lengthOf(savedResolutions.length * 2 + 2)
 
-  for (const resolution of resolutions) {
+  for (const resolution of savedResolutions) {
     const fragmentedFile = files.find(f => f.endsWith(`-${resolution}-fragmented.mp4`))
     expect(fragmentedFile).to.exist
 
@@ -37,5 +37,5 @@ async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: str
 }
 
 export {
-  checkLiveCleanupAfterSave
+  checkLiveCleanup
 }