aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-10-25 14:18:59 +0200
committerChocobozzz <me@florianbigard.com>2022-10-25 14:18:59 +0200
commitaa887096f9a305568dc3a846d0c9cdf7e45f1c67 (patch)
tree5dc455d680ebdd24644ebe29b2f129afb903b421
parent508c1b1e9f3b26752a961e945b7fa59b72b30827 (diff)
downloadPeerTube-aa887096f9a305568dc3a846d0c9cdf7e45f1c67.tar.gz
PeerTube-aa887096f9a305568dc3a846d0c9cdf7e45f1c67.tar.zst
PeerTube-aa887096f9a305568dc3a846d0c9cdf7e45f1c67.zip
Correctly delete live files from object storage
-rw-r--r--server/lib/live/live-utils.ts6
-rw-r--r--server/lib/live/shared/muxing-session.ts4
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts9
-rw-r--r--server/lib/object-storage/videos.ts16
-rw-r--r--server/models/video/video.ts8
-rw-r--r--server/tests/api/live/live-constraints.ts2
-rw-r--r--server/tests/api/live/live-permanent.ts12
-rw-r--r--server/tests/api/live/live-save-replay.ts18
-rw-r--r--server/tests/shared/live.ts17
9 files changed, 69 insertions, 23 deletions
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
index d2b8e3a55..c0dec9829 100644
--- a/server/lib/live/live-utils.ts
+++ b/server/lib/live/live-utils.ts
@@ -3,7 +3,7 @@ import { basename, join } from 'path'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' 4import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
5import { VideoStorage } from '@shared/models' 5import { VideoStorage } from '@shared/models'
6import { listHLSFileKeysOf, removeHLSFileObjectStorage, removeHLSObjectStorage } from '../object-storage' 6import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage'
7import { getLiveDirectory } from '../paths' 7import { getLiveDirectory } from '../paths'
8 8
9function buildConcatenatedName (segmentOrPlaylistPath: string) { 9function buildConcatenatedName (segmentOrPlaylistPath: string) {
@@ -77,11 +77,13 @@ async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) {
77async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) { 77async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) {
78 if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return 78 if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
79 79
80 logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid)
81
80 const keys = await listHLSFileKeysOf(streamingPlaylist) 82 const keys = await listHLSFileKeysOf(streamingPlaylist)
81 83
82 for (const key of keys) { 84 for (const key of keys) {
83 if (isTMPLiveFile(key)) { 85 if (isTMPLiveFile(key)) {
84 await removeHLSFileObjectStorage(streamingPlaylist, key) 86 await removeHLSFileObjectStorageByFullKey(key)
85 } 87 }
86 } 88 }
87} 89}
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index 64add2611..6ec126955 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -10,7 +10,7 @@ import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers
10import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' 10import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
11import { CONFIG } from '@server/initializers/config' 11import { CONFIG } from '@server/initializers/config'
12import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' 12import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
13import { removeHLSFileObjectStorage, storeHLSFileFromFilename, storeHLSFileFromPath } from '@server/lib/object-storage' 13import { removeHLSFileObjectStorageByPath, storeHLSFileFromFilename, storeHLSFileFromPath } from '@server/lib/object-storage'
14import { VideoFileModel } from '@server/models/video/video-file' 14import { VideoFileModel } from '@server/models/video/video-file'
15import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' 15import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
16import { VideoStorage } from '@shared/models' 16import { VideoStorage } from '@shared/models'
@@ -341,7 +341,7 @@ class MuxingSession extends EventEmitter {
341 341
342 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { 342 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
343 try { 343 try {
344 await removeHLSFileObjectStorage(this.streamingPlaylist, segmentPath) 344 await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath)
345 } catch (err) { 345 } catch (err) {
346 logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() }) 346 logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() })
347 } 347 }
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
index d13c25798..3046d76bc 100644
--- a/server/lib/object-storage/shared/object-storage-helpers.ts
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -110,11 +110,15 @@ function updatePrefixACL (options: {
110function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { 110function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
111 const key = buildKey(objectStorageKey, bucketInfo) 111 const key = buildKey(objectStorageKey, bucketInfo)
112 112
113 logger.debug('Removing file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) 113 return removeObjectByFullKey(key, bucketInfo)
114}
115
116function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
117 logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
114 118
115 const command = new DeleteObjectCommand({ 119 const command = new DeleteObjectCommand({
116 Bucket: bucketInfo.BUCKET_NAME, 120 Bucket: bucketInfo.BUCKET_NAME,
117 Key: key 121 Key: fullKey
118 }) 122 })
119 123
120 return getClient().send(command) 124 return getClient().send(command)
@@ -195,6 +199,7 @@ export {
195 storeObject, 199 storeObject,
196 200
197 removeObject, 201 removeObject,
202 removeObjectByFullKey,
198 removePrefix, 203 removePrefix,
199 204
200 makeAvailable, 205 makeAvailable,
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 003807826..b764e4b22 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -11,6 +11,7 @@ import {
11 lTags, 11 lTags,
12 makeAvailable, 12 makeAvailable,
13 removeObject, 13 removeObject,
14 removeObjectByFullKey,
14 removePrefix, 15 removePrefix,
15 storeObject, 16 storeObject,
16 updateObjectACL, 17 updateObjectACL,
@@ -76,10 +77,18 @@ function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
76 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 77 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
77} 78}
78 79
79function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename: string) { 80function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
80 return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 81 return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
81} 82}
82 83
84function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
85 return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
86}
87
88function removeHLSFileObjectStorageByFullKey (key: string) {
89 return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
90}
91
83// --------------------------------------------------------------------------- 92// ---------------------------------------------------------------------------
84 93
85function removeWebTorrentObjectStorage (videoFile: MVideoFile) { 94function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
@@ -162,7 +171,10 @@ export {
162 updateHLSFilesACL, 171 updateHLSFilesACL,
163 172
164 removeHLSObjectStorage, 173 removeHLSObjectStorage,
165 removeHLSFileObjectStorage, 174 removeHLSFileObjectStorageByFilename,
175 removeHLSFileObjectStorageByPath,
176 removeHLSFileObjectStorageByFullKey,
177
166 removeWebTorrentObjectStorage, 178 removeWebTorrentObjectStorage,
167 179
168 makeWebTorrentFileAvailable, 180 makeWebTorrentFileAvailable,
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 9399712b8..2ff92cbf1 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -26,7 +26,7 @@ import {
26} from 'sequelize-typescript' 26} from 'sequelize-typescript'
27import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' 27import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
28import { LiveManager } from '@server/lib/live/live-manager' 28import { LiveManager } from '@server/lib/live/live-manager'
29import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' 29import { removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
30import { tracer } from '@server/lib/opentelemetry/tracing' 30import { tracer } from '@server/lib/opentelemetry/tracing'
31import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 31import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
32import { VideoPathManager } from '@server/lib/video-path-manager' 32import { VideoPathManager } from '@server/lib/video-path-manager'
@@ -1830,8 +1830,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1830 await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename)) 1830 await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename))
1831 1831
1832 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { 1832 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
1833 await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename) 1833 await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), videoFile.filename)
1834 await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), resolutionFilename) 1834 await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), resolutionFilename)
1835 } 1835 }
1836 } 1836 }
1837 1837
@@ -1840,7 +1840,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1840 await remove(filePath) 1840 await remove(filePath)
1841 1841
1842 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { 1842 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
1843 await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), filename) 1843 await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), filename)
1844 } 1844 }
1845 } 1845 }
1846 1846
diff --git a/server/tests/api/live/live-constraints.ts b/server/tests/api/live/live-constraints.ts
index 64ef73a2a..c82585a9e 100644
--- a/server/tests/api/live/live-constraints.ts
+++ b/server/tests/api/live/live-constraints.ts
@@ -49,7 +49,7 @@ describe('Test live constraints', function () {
49 expect(video.duration).to.be.greaterThan(0) 49 expect(video.duration).to.be.greaterThan(0)
50 } 50 }
51 51
52 await checkLiveCleanup(servers[0], videoId, resolutions) 52 await checkLiveCleanup({ server: servers[0], permanent: false, videoUUID: videoId, savedResolutions: resolutions })
53 } 53 }
54 54
55 function updateQuota (options: { total: number, daily: number }) { 55 function updateQuota (options: { total: number, daily: number }) {
diff --git a/server/tests/api/live/live-permanent.ts b/server/tests/api/live/live-permanent.ts
index 5d227200e..4203b1bfc 100644
--- a/server/tests/api/live/live-permanent.ts
+++ b/server/tests/api/live/live-permanent.ts
@@ -1,6 +1,7 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { checkLiveCleanup } from '@server/tests/shared'
4import { wait } from '@shared/core-utils' 5import { wait } from '@shared/core-utils'
5import { LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' 6import { LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models'
6import { 7import {
@@ -129,6 +130,8 @@ describe('Permanent live', function () {
129 130
130 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) 131 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
131 } 132 }
133
134 await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID })
132 }) 135 })
133 136
134 it('Should have set this live to waiting for live state', async function () { 137 it('Should have set this live to waiting for live state', async function () {
@@ -186,6 +189,15 @@ describe('Permanent live', function () {
186 } 189 }
187 }) 190 })
188 191
192 it('Should remove the live and have cleaned up the directory', async function () {
193 this.timeout(60000)
194
195 await servers[0].videos.remove({ id: videoUUID })
196 await waitJobs(servers)
197
198 await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID })
199 })
200
189 after(async function () { 201 after(async function () {
190 await cleanupTests(servers) 202 await cleanupTests(servers)
191 }) 203 })
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index 7014292d0..8f17b4566 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -186,7 +186,7 @@ describe('Save replay setting', function () {
186 await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) 186 await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
187 187
188 // No resolutions saved since we did not save replay 188 // No resolutions saved since we did not save replay
189 await checkLiveCleanup(servers[0], liveVideoUUID, []) 189 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
190 }) 190 })
191 191
192 it('Should have appropriate ended session', async function () { 192 it('Should have appropriate ended session', async function () {
@@ -220,7 +220,7 @@ describe('Save replay setting', function () {
220 220
221 await wait(5000) 221 await wait(5000)
222 await waitJobs(servers) 222 await waitJobs(servers)
223 await checkLiveCleanup(servers[0], liveVideoUUID, []) 223 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
224 }) 224 })
225 225
226 it('Should have blacklisted session error', async function () { 226 it('Should have blacklisted session error', async function () {
@@ -238,7 +238,7 @@ describe('Save replay setting', function () {
238 await publishLiveAndDelete({ permanent: false, replay: false }) 238 await publishLiveAndDelete({ permanent: false, replay: false })
239 239
240 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) 240 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
241 await checkLiveCleanup(servers[0], liveVideoUUID, []) 241 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
242 }) 242 })
243 }) 243 })
244 244
@@ -317,7 +317,7 @@ describe('Save replay setting', function () {
317 }) 317 })
318 318
319 it('Should have cleaned up the live files', async function () { 319 it('Should have cleaned up the live files', async function () {
320 await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) 320 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] })
321 }) 321 })
322 322
323 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { 323 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
@@ -332,7 +332,7 @@ describe('Save replay setting', function () {
332 332
333 await wait(5000) 333 await wait(5000)
334 await waitJobs(servers) 334 await waitJobs(servers)
335 await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) 335 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] })
336 }) 336 })
337 337
338 it('Should correctly terminate the stream on delete and delete the video', async function () { 338 it('Should correctly terminate the stream on delete and delete the video', async function () {
@@ -341,7 +341,7 @@ describe('Save replay setting', function () {
341 await publishLiveAndDelete({ permanent: false, replay: true }) 341 await publishLiveAndDelete({ permanent: false, replay: true })
342 342
343 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) 343 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
344 await checkLiveCleanup(servers[0], liveVideoUUID, []) 344 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
345 }) 345 })
346 }) 346 })
347 347
@@ -413,7 +413,7 @@ describe('Save replay setting', function () {
413 }) 413 })
414 414
415 it('Should have cleaned up the live files', async function () { 415 it('Should have cleaned up the live files', async function () {
416 await checkLiveCleanup(servers[0], liveVideoUUID, []) 416 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
417 }) 417 })
418 418
419 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { 419 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
@@ -432,7 +432,7 @@ describe('Save replay setting', function () {
432 await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 432 await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
433 } 433 }
434 434
435 await checkLiveCleanup(servers[0], liveVideoUUID, []) 435 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
436 }) 436 })
437 437
438 it('Should correctly terminate the stream on delete and not save the video', async function () { 438 it('Should correctly terminate the stream on delete and not save the video', async function () {
@@ -444,7 +444,7 @@ describe('Save replay setting', function () {
444 expect(replay).to.not.exist 444 expect(replay).to.not.exist
445 445
446 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) 446 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
447 await checkLiveCleanup(servers[0], liveVideoUUID, []) 447 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
448 }) 448 })
449 }) 449 })
450 450
diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts
index 78e29f575..47e0dc481 100644
--- a/server/tests/shared/live.ts
+++ b/server/tests/shared/live.ts
@@ -7,10 +7,25 @@ import { LiveVideo, VideoStreamingPlaylistType } from '@shared/models'
7import { ObjectStorageCommand, PeerTubeServer } from '@shared/server-commands' 7import { ObjectStorageCommand, PeerTubeServer } from '@shared/server-commands'
8import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists' 8import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists'
9 9
10async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) { 10async function checkLiveCleanup (options: {
11 server: PeerTubeServer
12 videoUUID: string
13 permanent: boolean
14 savedResolutions?: number[]
15}) {
16 const { server, videoUUID, permanent, savedResolutions = [] } = options
17
11 const basePath = server.servers.buildDirectory('streaming-playlists') 18 const basePath = server.servers.buildDirectory('streaming-playlists')
12 const hlsPath = join(basePath, 'hls', videoUUID) 19 const hlsPath = join(basePath, 'hls', videoUUID)
13 20
21 if (permanent) {
22 if (!await pathExists(hlsPath)) return
23
24 const files = await readdir(hlsPath)
25 expect(files).to.have.lengthOf(0)
26 return
27 }
28
14 if (savedResolutions.length === 0) { 29 if (savedResolutions.length === 0) {
15 return checkUnsavedLiveCleanup(server, videoUUID, hlsPath) 30 return checkUnsavedLiveCleanup(server, videoUUID, hlsPath)
16 } 31 }