Cleanup lives on server restart

3 files changed, 105 insertions, 2 deletions

diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts
index 2702437c4..fe5b33322 100644
--- a/server/lib/live-manager.ts
+++ b/server/lib/live-manager.ts
@@ -99,6 +99,10 @@ class LiveManager {
       }
     })
 
+    // Cleanup broken lives, that were terminated by a server restart for example
+    this.handleBrokenLives()
+      .catch(err => logger.error('Cannot handle broken lives.', { err }))
+
     setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE)
   }
 
@@ -468,6 +472,14 @@ class LiveManager {
     }
   }
 
+  private async handleBrokenLives () {
+    const videoIds = await VideoModel.listPublishedLiveIds()
+
+    for (const id of videoIds) {
+      await this.onEndTransmuxing(id, true)
+    }
+  }
+
   static get Instance () {
     return this.instance || (this.instance = new this())
   }
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 70839aa89..f3055a494 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -988,6 +988,19 @@ export class VideoModel extends Model<VideoModel> {
     })
   }
 
+  static listPublishedLiveIds () {
+    const options = {
+      attributes: [ 'id' ],
+      where: {
+        isLive: true,
+        state: VideoState.PUBLISHED
+      }
+    }
+
+    return VideoModel.findAll(options)
+      .map(v => v.id)
+  }
+
   static listUserVideosForApi (
     accountId: number,
     start: number,
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index 29081c6cc..aa2e1318a 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -2,22 +2,28 @@
 
 import 'mocha'
 import * as chai from 'chai'
+import { FfmpegCommand } from 'fluent-ffmpeg'
 import { getLiveNotificationSocket } from '@shared/extra-utils/socket/socket-io'
 import { LiveVideo, LiveVideoCreate, Video, VideoDetails, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models'
 import {
   addVideoToBlacklist,
   checkLiveCleanup,
+  checkLiveSegmentHash,
   checkResolutionsInMasterPlaylist,
+  checkSegmentHash,
   cleanupTests,
   createLive,
   doubleFollow,
   flushAndRunMultipleServers,
   getLive,
+  getPlaylist,
   getVideo,
   getVideoIdFromUUID,
   getVideosList,
+  killallServers,
   makeRawRequest,
   removeVideo,
+  reRunServer,
   sendRTMPStream,
   sendRTMPStreamInVideo,
   ServerInfo,
@@ -31,9 +37,9 @@ import {
   viewVideo,
   wait,
   waitJobs,
-  waitUntilLiveStarts
+  waitUntilLiveStarts,
+  waitUntilLog
 } from '../../../../shared/extra-utils'
-import { FfmpegCommand } from 'fluent-ffmpeg'
 
 const expect = chai.expect
 
@@ -316,6 +322,19 @@ describe('Test live', function () {
         expect(hlsPlaylist.files).to.have.lengthOf(0)
 
         await checkResolutionsInMasterPlaylist(hlsPlaylist.playlistUrl, resolutions)
+
+        for (let i = 0; i < resolutions.length; i++) {
+          const segmentName = `${i}-000001.ts`
+          await waitUntilLog(servers[0], `${video.uuid}/${segmentName}`, 1, false)
+
+          const res = await getPlaylist(`${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`)
+          const subPlaylist = res.text
+
+          expect(subPlaylist).to.contain(segmentName)
+
+          const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
+          await checkLiveSegmentHash(baseUrlAndPath, video.uuid, segmentName, hlsPlaylist)
+        }
       }
     }
 
@@ -580,6 +599,65 @@ describe('Test live', function () {
     })
   })
 
+  describe('After a server restart', function () {
+    let liveVideoId: string
+    let liveVideoReplayId: string
+
+    async function createLiveWrapper (saveReplay: boolean) {
+      const liveAttributes = {
+        name: 'live video',
+        channelId: servers[0].videoChannel.id,
+        privacy: VideoPrivacy.PUBLIC,
+        saveReplay
+      }
+
+      const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
+      return res.body.video.uuid
+    }
+
+    before(async function () {
+      this.timeout(60000)
+
+      liveVideoId = await createLiveWrapper(false)
+      liveVideoReplayId = await createLiveWrapper(true)
+
+      await Promise.all([
+        sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId),
+        sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoReplayId)
+      ])
+
+      await Promise.all([
+        waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoId),
+        waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoReplayId)
+      ])
+
+      await killallServers([ servers[0] ])
+      await reRunServer(servers[0])
+
+      await wait(5000)
+    })
+
+    it('Should cleanup lives', async function () {
+      this.timeout(60000)
+
+      const res = await getVideo(servers[0].url, liveVideoId)
+      const video: VideoDetails = res.body
+
+      expect(video.state.id).to.equal(VideoState.LIVE_ENDED)
+    })
+
+    it('Should save a live replay', async function () {
+      this.timeout(60000)
+
+      await waitJobs(servers)
+
+      const res = await getVideo(servers[0].url, liveVideoReplayId)
+      const video: VideoDetails = res.body
+
+      expect(video.state.id).to.equal(VideoState.PUBLISHED)
+    })
+  })
+
   after(async function () {
     await cleanupTests(servers)
   })