From b6e2b5df73d3b67e275000f612907859c39d90d1 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 24 Jun 2022 09:34:56 +0200 Subject: [PATCH] Fix removed sha segments on fast restream --- server/initializers/constants.ts | 2 +- .../lib/job-queue/handlers/video-live-ending.ts | 13 +++---------- server/lib/live/live-manager.ts | 5 +---- server/lib/live/live-utils.ts | 15 +++++++++++---- server/tests/api/live/live-fast-restream.ts | 5 +++-- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6c6628d28..88f370a9a 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -865,7 +865,7 @@ if (isTestInstance() === true) { PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000 - VIDEO_LIVE.CLEANUP_DELAY = getIntEnv('PEERTUBE_TEST_CONSTANTS.VIDEO_LIVE.CLEANUP_DELAY') ?? 5000 + VIDEO_LIVE.CLEANUP_DELAY = getIntEnv('PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY') ?? 5000 VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY = 2 VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY = 1 VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION = 1 diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 450bda2fd..b2c42664a 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -4,13 +4,8 @@ import { join } from 'path' import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamDuration } from '@server/helpers/ffmpeg' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { cleanupUnsavedNormalLive, cleanupPermanentLive, cleanupTMPLiveFiles, LiveSegmentShaStore } from '@server/lib/live' -import { - generateHLSMasterPlaylistFilename, - generateHlsSha256SegmentsFilename, - getLiveDirectory, - getLiveReplayBaseDirectory -} from '@server/lib/paths' +import { cleanupPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' import { generateVideoMiniature } from '@server/lib/thumbnail' import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' import { moveToNextState } from '@server/lib/video-state' @@ -44,8 +39,6 @@ async function processVideoLiveEnding (job: Job) { return } - LiveSegmentShaStore.Instance.cleanupShaSegments(liveVideo.uuid) - if (live.saveReplay !== true) { return cleanupLiveAndFederate({ live, video: liveVideo, streamingPlaylistId: payload.streamingPlaylistId }) } @@ -137,7 +130,7 @@ async function replaceLiveByReplay (options: { }) { const { liveVideo, liveSession, live, replayDirectory } = options - await cleanupTMPLiveFiles(getLiveDirectory(liveVideo)) + await cleanupTMPLiveFiles(liveVideo) await live.destroy() diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index f2f064b49..d499b4b1a 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts @@ -27,7 +27,6 @@ import { JobQueue } from '../job-queue' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' import { PeerTubeSocket } from '../peertube-socket' import { LiveQuotaStore } from './live-quota-store' -import { LiveSegmentShaStore } from './live-segment-sha-store' import { cleanupPermanentLive } from './live-utils' import { MuxingSession } from './shared' @@ -219,9 +218,7 @@ class LiveManager { return this.abortSession(sessionId) } - // Cleanup old potential live files (could happen with a permanent live) - LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) - + // Cleanup old potential live (could happen with a permanent live) const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) if (oldStreamingPlaylist) { if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts index 6305a97a8..bba876642 100644 --- a/server/lib/live/live-utils.ts +++ b/server/lib/live/live-utils.ts @@ -3,6 +3,7 @@ import { basename, join } from 'path' import { logger } from '@server/helpers/logger' import { MStreamingPlaylist, MVideo } from '@server/types/models' import { getLiveDirectory } from '../paths' +import { LiveSegmentShaStore } from './live-segment-sha-store' function buildConcatenatedName (segmentOrPlaylistPath: string) { const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) @@ -11,9 +12,7 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) { } async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { - const hlsDirectory = getLiveDirectory(video) - - await cleanupTMPLiveFiles(hlsDirectory) + await cleanupTMPLiveFiles(video) await streamingPlaylist.destroy() } @@ -24,11 +23,19 @@ async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStre await remove(hlsDirectory) await streamingPlaylist.destroy() + + LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) } -async function cleanupTMPLiveFiles (hlsDirectory: string) { +async function cleanupTMPLiveFiles (video: MVideo) { + const hlsDirectory = getLiveDirectory(video) + + LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) + if (!await pathExists(hlsDirectory)) return + logger.info('Cleanup TMP live files of %s.', hlsDirectory) + const files = await readdir(hlsDirectory) for (const filename of files) { diff --git a/server/tests/api/live/live-fast-restream.ts b/server/tests/api/live/live-fast-restream.ts index 4b5d041ec..b524cbe7f 100644 --- a/server/tests/api/live/live-fast-restream.ts +++ b/server/tests/api/live/live-fast-restream.ts @@ -57,13 +57,14 @@ describe('Fast restream in live', function () { } async function ensureLastLiveWorks (liveId: string) { - // Equivalent to PEERTUBE_TEST_CONSTANTS.VIDEO_LIVE.CLEANUP_DELAY + // Equivalent to PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY for (let i = 0; i < 100; i++) { const video = await server.videos.get({ id: liveId }) expect(video.streamingPlaylists).to.have.lengthOf(1) await server.live.getSegment({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200) + await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200) await wait(100) } @@ -101,7 +102,7 @@ describe('Fast restream in live', function () { before(async function () { this.timeout(120000) - const env = { 'PEERTUBE_TEST_CONSTANTS.VIDEO_LIVE.CLEANUP_DELAY': '10000' } + const env = { 'PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY': '10000' } server = await createSingleServer(1, {}, { env }) // Get the access tokens -- 2.41.0