Cleanup lives on server restart

8 files changed, 151 insertions, 10 deletions

diff --git a/client/src/app/+videos/+video-watch/video-watch.component.html b/client/src/app/+videos/+video-watch/video-watch.component.html
index daee4be2a..9e469f0b0 100644
--- a/client/src/app/+videos/+video-watch/video-watch.component.html
+++ b/client/src/app/+videos/+video-watch/video-watch.component.html
@@ -34,7 +34,7 @@
     </div>
 
     <div i18n class="col-md-12 alert alert-info" *ngIf="isLiveEnded()">
-      This live is finished.
+      This live has ended.
     </div>
 
     <div class="col-md-12 alert alert-danger" *ngIf="video?.blacklisted">
@@ -51,8 +51,14 @@
           <div class="d-block d-md-none"> <!-- only shown on medium devices, has its counterpart for larger viewports below -->
             <h1 class="video-info-name">{{ video.name }}</h1>
 
-            <div i18n class="video-info-date-views">
-              Published <my-date-toggle [date]="video.publishedAt"></my-date-toggle> <span class="views"> • {{ video.views | myNumberFormatter }} views</span>
+            <div class="video-info-date-views">
+              <ng-container i18n>Published <my-date-toggle [date]="video.publishedAt"></my-date-toggle></ng-container>
+
+              <span i18n class="views">
+                • {{ video.views | myNumberFormatter }}
+                <ng-container *ngIf="!video.isLive">views</ng-container>
+                <ng-container *ngIf="video.isLive">viewers</ng-container>
+              </span>
             </div>
           </div>
 
@@ -62,8 +68,14 @@
             </div>
 
             <div class="video-info-first-row-bottom">
-              <div i18n class="d-none d-md-block video-info-date-views">
-                Published <my-date-toggle [date]="video.publishedAt"></my-date-toggle> <span class="views"> • {{ video.views | myNumberFormatter }} views</span>
+              <div class="d-none d-md-block video-info-date-views">
+                <ng-container i18n>Published <my-date-toggle [date]="video.publishedAt"></my-date-toggle></ng-container>
+
+                <span i18n class="views">
+                  • {{ video.views | myNumberFormatter }}
+                  <ng-container *ngIf="!video.isLive">views</ng-container>
+                  <ng-container *ngIf="video.isLive">viewers</ng-container>
+                </span>
               </div>
 
               <div class="video-actions-rates">
diff --git a/client/src/app/shared/shared-thumbnail/video-thumbnail.component.html b/client/src/app/shared/shared-thumbnail/video-thumbnail.component.html
index be07844ab..4fea0cc1c 100644
--- a/client/src/app/shared/shared-thumbnail/video-thumbnail.component.html
+++ b/client/src/app/shared/shared-thumbnail/video-thumbnail.component.html
@@ -27,7 +27,10 @@
   <div class="video-thumbnail-label-overlay danger"><ng-content select="label-danger"></ng-content></div>
 
   <div class="video-thumbnail-duration-overlay" *ngIf="!video.isLive">{{ video.durationLabel }}</div>
-  <div i18n class="video-thumbnail-live-overlay" *ngIf="video.isLive">LIVE</div>
+  <div  class="video-thumbnail-live-overlay" [ngClass]="{ 'live-ended': isLiveEnded() }" *ngIf="video.isLive">
+    <ng-container i18n *ngIf="!isLiveEnded()">LIVE</ng-container>
+    <ng-container i18n *ngIf="isLiveEnded()">LIVE ENDED</ng-container>
+  </div>
 
   <div class="play-overlay">
     <div class="icon"></div>
diff --git a/client/src/app/shared/shared-thumbnail/video-thumbnail.component.scss b/client/src/app/shared/shared-thumbnail/video-thumbnail.component.scss
index 1b6151c89..4f53ffaf6 100644
--- a/client/src/app/shared/shared-thumbnail/video-thumbnail.component.scss
+++ b/client/src/app/shared/shared-thumbnail/video-thumbnail.component.scss
@@ -51,9 +51,12 @@
   }
 
   .video-thumbnail-live-overlay {
-    background-color: rgba(224, 8, 8, 0.7);
-    color: #fff;
     font-weight: $font-semibold;
+    color: #fff;
+
+    &:not(.live-ended) {
+      background-color: rgba(224, 8, 8, 0.7);
+    }
   }
 
   .video-thumbnail-actions-overlay {
diff --git a/client/src/app/shared/shared-thumbnail/video-thumbnail.component.ts b/client/src/app/shared/shared-thumbnail/video-thumbnail.component.ts
index b2a2cf240..67a9b0028 100644
--- a/client/src/app/shared/shared-thumbnail/video-thumbnail.component.ts
+++ b/client/src/app/shared/shared-thumbnail/video-thumbnail.component.ts
@@ -1,5 +1,6 @@
 import { Component, EventEmitter, Input, Output } from '@angular/core'
 import { ScreenService } from '@app/core'
+import { VideoState } from '@shared/models'
 import { Video } from '../shared-main'
 
 @Component({
@@ -29,6 +30,10 @@ export class VideoThumbnailComponent {
     this.addedToWatchLaterText = $localize`Remove from watch later`
   }
 
+  isLiveEnded () {
+    return this.video.state.id === VideoState.LIVE_ENDED
+  }
+
   getImageUrl () {
     if (!this.video) return ''
 
diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts
index 2702437c4..fe5b33322 100644
--- a/server/lib/live-manager.ts
+++ b/server/lib/live-manager.ts
@@ -99,6 +99,10 @@ class LiveManager {
       }
     })
 
+    // Cleanup broken lives, that were terminated by a server restart for example
+    this.handleBrokenLives()
+      .catch(err => logger.error('Cannot handle broken lives.', { err }))
+
     setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE)
   }
 
@@ -468,6 +472,14 @@ class LiveManager {
     }
   }
 
+  private async handleBrokenLives () {
+    const videoIds = await VideoModel.listPublishedLiveIds()
+
+    for (const id of videoIds) {
+      await this.onEndTransmuxing(id, true)
+    }
+  }
+
   static get Instance () {
     return this.instance || (this.instance = new this())
   }
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 70839aa89..f3055a494 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -988,6 +988,19 @@ export class VideoModel extends Model<VideoModel> {
     })
   }
 
+  static listPublishedLiveIds () {
+    const options = {
+      attributes: [ 'id' ],
+      where: {
+        isLive: true,
+        state: VideoState.PUBLISHED
+      }
+    }
+
+    return VideoModel.findAll(options)
+      .map(v => v.id)
+  }
+
   static listUserVideosForApi (
     accountId: number,
     start: number,
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index 29081c6cc..aa2e1318a 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -2,22 +2,28 @@
 
 import 'mocha'
 import * as chai from 'chai'
+import { FfmpegCommand } from 'fluent-ffmpeg'
 import { getLiveNotificationSocket } from '@shared/extra-utils/socket/socket-io'
 import { LiveVideo, LiveVideoCreate, Video, VideoDetails, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models'
 import {
   addVideoToBlacklist,
   checkLiveCleanup,
+  checkLiveSegmentHash,
   checkResolutionsInMasterPlaylist,
+  checkSegmentHash,
   cleanupTests,
   createLive,
   doubleFollow,
   flushAndRunMultipleServers,
   getLive,
+  getPlaylist,
   getVideo,
   getVideoIdFromUUID,
   getVideosList,
+  killallServers,
   makeRawRequest,
   removeVideo,
+  reRunServer,
   sendRTMPStream,
   sendRTMPStreamInVideo,
   ServerInfo,
@@ -31,9 +37,9 @@ import {
   viewVideo,
   wait,
   waitJobs,
-  waitUntilLiveStarts
+  waitUntilLiveStarts,
+  waitUntilLog
 } from '../../../../shared/extra-utils'
-import { FfmpegCommand } from 'fluent-ffmpeg'
 
 const expect = chai.expect
 
@@ -316,6 +322,19 @@ describe('Test live', function () {
         expect(hlsPlaylist.files).to.have.lengthOf(0)
 
         await checkResolutionsInMasterPlaylist(hlsPlaylist.playlistUrl, resolutions)
+
+        for (let i = 0; i < resolutions.length; i++) {
+          const segmentName = `${i}-000001.ts`
+          await waitUntilLog(servers[0], `${video.uuid}/${segmentName}`, 1, false)
+
+          const res = await getPlaylist(`${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`)
+          const subPlaylist = res.text
+
+          expect(subPlaylist).to.contain(segmentName)
+
+          const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
+          await checkLiveSegmentHash(baseUrlAndPath, video.uuid, segmentName, hlsPlaylist)
+        }
       }
     }
 
@@ -580,6 +599,65 @@ describe('Test live', function () {
     })
   })
 
+  describe('After a server restart', function () {
+    let liveVideoId: string
+    let liveVideoReplayId: string
+
+    async function createLiveWrapper (saveReplay: boolean) {
+      const liveAttributes = {
+        name: 'live video',
+        channelId: servers[0].videoChannel.id,
+        privacy: VideoPrivacy.PUBLIC,
+        saveReplay
+      }
+
+      const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
+      return res.body.video.uuid
+    }
+
+    before(async function () {
+      this.timeout(60000)
+
+      liveVideoId = await createLiveWrapper(false)
+      liveVideoReplayId = await createLiveWrapper(true)
+
+      await Promise.all([
+        sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId),
+        sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoReplayId)
+      ])
+
+      await Promise.all([
+        waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoId),
+        waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoReplayId)
+      ])
+
+      await killallServers([ servers[0] ])
+      await reRunServer(servers[0])
+
+      await wait(5000)
+    })
+
+    it('Should cleanup lives', async function () {
+      this.timeout(60000)
+
+      const res = await getVideo(servers[0].url, liveVideoId)
+      const video: VideoDetails = res.body
+
+      expect(video.state.id).to.equal(VideoState.LIVE_ENDED)
+    })
+
+    it('Should save a live replay', async function () {
+      this.timeout(60000)
+
+      await waitJobs(servers)
+
+      const res = await getVideo(servers[0].url, liveVideoReplayId)
+      const video: VideoDetails = res.body
+
+      expect(video.state.id).to.equal(VideoState.PUBLISHED)
+    })
+  })
+
   after(async function () {
     await cleanupTests(servers)
   })
diff --git a/shared/extra-utils/videos/video-streaming-playlists.ts b/shared/extra-utils/videos/video-streaming-playlists.ts
index 8cf0e4930..b386e77c3 100644
--- a/shared/extra-utils/videos/video-streaming-playlists.ts
+++ b/shared/extra-utils/videos/video-streaming-playlists.ts
@@ -41,6 +41,20 @@ async function checkSegmentHash (
   expect(sha256(res2.body)).to.equal(sha256Server)
 }
 
+async function checkLiveSegmentHash (
+  baseUrlSegment: string,
+  videoUUID: string,
+  segmentName: string,
+  hlsPlaylist: VideoStreamingPlaylist
+) {
+  const res2 = await getSegment(`${baseUrlSegment}/${videoUUID}/${segmentName}`)
+
+  const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url)
+
+  const sha256Server = resSha.body[segmentName]
+  expect(sha256(res2.body)).to.equal(sha256Server)
+}
+
 async function checkResolutionsInMasterPlaylist (playlistUrl: string, resolutions: number[]) {
   const res = await getPlaylist(playlistUrl)
 
@@ -62,5 +76,6 @@ export {
   getSegment,
   checkResolutionsInMasterPlaylist,
   getSegmentSha256,
+  checkLiveSegmentHash,
   checkSegmentHash
 }