diff options
author | Chocobozzz <me@florianbigard.com> | 2022-10-04 10:03:17 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2022-10-04 10:03:17 +0200 |
commit | cfd57d2ca0bb058087f7dc90fcc3e8442b0288e1 (patch) | |
tree | dc899a1504ecac588e5580553e02571e0f5d7e4b /server | |
parent | 9c0cdc5047918b959ebd5e075ddad81eb7fb93f0 (diff) | |
download | PeerTube-cfd57d2ca0bb058087f7dc90fcc3e8442b0288e1.tar.gz PeerTube-cfd57d2ca0bb058087f7dc90fcc3e8442b0288e1.tar.zst PeerTube-cfd57d2ca0bb058087f7dc90fcc3e8442b0288e1.zip |
Live supports object storage
* Sync live files (segments, master playlist, resolution playlist,
segment sha file) into object storage
* Automatically delete them when the live ends
* Segment sha file is now a file on disk, and not stored in memory
anymore
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/index.ts | 1 | ||||
-rw-r--r-- | server/controllers/live.ts | 32 | ||||
-rw-r--r-- | server/lib/hls.ts | 6 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/move-to-object-storage.ts | 10 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-live-ending.ts | 23 | ||||
-rw-r--r-- | server/lib/live/live-manager.ts | 12 | ||||
-rw-r--r-- | server/lib/live/live-segment-sha-store.ts | 75 | ||||
-rw-r--r-- | server/lib/live/live-utils.ts | 67 | ||||
-rw-r--r-- | server/lib/live/shared/muxing-session.ts | 106 | ||||
-rw-r--r-- | server/lib/object-storage/shared/object-storage-helpers.ts | 25 | ||||
-rw-r--r-- | server/lib/object-storage/videos.ts | 37 | ||||
-rw-r--r-- | server/models/video/video-streaming-playlist.ts | 24 | ||||
-rw-r--r-- | server/tests/api/live/live-fast-restream.ts | 2 | ||||
-rw-r--r-- | server/tests/api/live/live.ts | 87 | ||||
-rw-r--r-- | server/tests/api/object-storage/live.ts | 183 | ||||
-rw-r--r-- | server/tests/shared/live.ts | 116 | ||||
-rw-r--r-- | server/tests/shared/streaming-playlists.ts | 13 |
17 files changed, 546 insertions, 273 deletions
diff --git a/server/controllers/index.ts b/server/controllers/index.ts index e8833d58c..8574a9e7b 100644 --- a/server/controllers/index.ts +++ b/server/controllers/index.ts | |||
@@ -6,7 +6,6 @@ export * from './feeds' | |||
6 | export * from './services' | 6 | export * from './services' |
7 | export * from './static' | 7 | export * from './static' |
8 | export * from './lazy-static' | 8 | export * from './lazy-static' |
9 | export * from './live' | ||
10 | export * from './misc' | 9 | export * from './misc' |
11 | export * from './webfinger' | 10 | export * from './webfinger' |
12 | export * from './tracker' | 11 | export * from './tracker' |
diff --git a/server/controllers/live.ts b/server/controllers/live.ts deleted file mode 100644 index 81008f120..000000000 --- a/server/controllers/live.ts +++ /dev/null | |||
@@ -1,32 +0,0 @@ | |||
1 | import cors from 'cors' | ||
2 | import express from 'express' | ||
3 | import { mapToJSON } from '@server/helpers/core-utils' | ||
4 | import { LiveSegmentShaStore } from '@server/lib/live' | ||
5 | import { HttpStatusCode } from '@shared/models' | ||
6 | |||
7 | const liveRouter = express.Router() | ||
8 | |||
9 | liveRouter.use('/segments-sha256/:videoUUID', | ||
10 | cors(), | ||
11 | getSegmentsSha256 | ||
12 | ) | ||
13 | |||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
16 | export { | ||
17 | liveRouter | ||
18 | } | ||
19 | |||
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | function getSegmentsSha256 (req: express.Request, res: express.Response) { | ||
23 | const videoUUID = req.params.videoUUID | ||
24 | |||
25 | const result = LiveSegmentShaStore.Instance.getSegmentsSha256(videoUUID) | ||
26 | |||
27 | if (!result) { | ||
28 | return res.status(HttpStatusCode.NOT_FOUND_404).end() | ||
29 | } | ||
30 | |||
31 | return res.json(mapToJSON(result)) | ||
32 | } | ||
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index a0a5afc0f..a41f1ae48 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -15,7 +15,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers | |||
15 | import { sequelizeTypescript } from '../initializers/database' | 15 | import { sequelizeTypescript } from '../initializers/database' |
16 | import { VideoFileModel } from '../models/video/video-file' | 16 | import { VideoFileModel } from '../models/video/video-file' |
17 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 17 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' |
18 | import { storeHLSFile } from './object-storage' | 18 | import { storeHLSFileFromFilename } from './object-storage' |
19 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' | 19 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' |
20 | import { VideoPathManager } from './video-path-manager' | 20 | import { VideoPathManager } from './video-path-manager' |
21 | 21 | ||
@@ -95,7 +95,7 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist | |||
95 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | 95 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') |
96 | 96 | ||
97 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | 97 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { |
98 | playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename) | 98 | playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) |
99 | await remove(masterPlaylistPath) | 99 | await remove(masterPlaylistPath) |
100 | } | 100 | } |
101 | 101 | ||
@@ -146,7 +146,7 @@ function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist | |||
146 | await outputJSON(outputPath, json) | 146 | await outputJSON(outputPath, json) |
147 | 147 | ||
148 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | 148 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { |
149 | playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename) | 149 | playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) |
150 | await remove(outputPath) | 150 | await remove(outputPath) |
151 | } | 151 | } |
152 | 152 | ||
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index 25bdebeea..28c3d325d 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts | |||
@@ -5,7 +5,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger' | |||
5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' | 5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' |
6 | import { CONFIG } from '@server/initializers/config' | 6 | import { CONFIG } from '@server/initializers/config' |
7 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' | 7 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' |
8 | import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' | 8 | import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage' |
9 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | 9 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' |
10 | import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' | 10 | import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' |
11 | import { VideoModel } from '@server/models/video/video' | 11 | import { VideoModel } from '@server/models/video/video' |
@@ -88,10 +88,10 @@ async function moveHLSFiles (video: MVideoWithAllFiles) { | |||
88 | 88 | ||
89 | // Resolution playlist | 89 | // Resolution playlist |
90 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | 90 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) |
91 | await storeHLSFile(playlistWithVideo, playlistFilename) | 91 | await storeHLSFileFromFilename(playlistWithVideo, playlistFilename) |
92 | 92 | ||
93 | // Resolution fragmented file | 93 | // Resolution fragmented file |
94 | const fileUrl = await storeHLSFile(playlistWithVideo, file.filename) | 94 | const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename) |
95 | 95 | ||
96 | const oldPath = join(getHLSDirectory(video), file.filename) | 96 | const oldPath = join(getHLSDirectory(video), file.filename) |
97 | 97 | ||
@@ -113,9 +113,9 @@ async function doAfterLastJob (options: { | |||
113 | const playlistWithVideo = playlist.withVideo(video) | 113 | const playlistWithVideo = playlist.withVideo(video) |
114 | 114 | ||
115 | // Master playlist | 115 | // Master playlist |
116 | playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename) | 116 | playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename) |
117 | // Sha256 segments file | 117 | // Sha256 segments file |
118 | playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename) | 118 | playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename) |
119 | 119 | ||
120 | playlist.storage = VideoStorage.OBJECT_STORAGE | 120 | playlist.storage = VideoStorage.OBJECT_STORAGE |
121 | 121 | ||
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 8a3ee09a2..abfaf1cd7 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts | |||
@@ -4,7 +4,7 @@ import { join } from 'path' | |||
4 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' | 4 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' |
5 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 5 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
6 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | 6 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' |
7 | import { cleanupPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' | 7 | import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' |
8 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' | 8 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' |
9 | import { generateVideoMiniature } from '@server/lib/thumbnail' | 9 | import { generateVideoMiniature } from '@server/lib/thumbnail' |
10 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' | 10 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' |
@@ -141,23 +141,22 @@ async function replaceLiveByReplay (options: { | |||
141 | }) { | 141 | }) { |
142 | const { video, liveSession, live, permanentLive, replayDirectory } = options | 142 | const { video, liveSession, live, permanentLive, replayDirectory } = options |
143 | 143 | ||
144 | await cleanupTMPLiveFiles(video) | 144 | const videoWithFiles = await VideoModel.loadFull(video.id) |
145 | const hlsPlaylist = videoWithFiles.getHLSPlaylist() | ||
146 | |||
147 | await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist) | ||
145 | 148 | ||
146 | await live.destroy() | 149 | await live.destroy() |
147 | 150 | ||
148 | video.isLive = false | 151 | videoWithFiles.isLive = false |
149 | video.waitTranscoding = true | 152 | videoWithFiles.waitTranscoding = true |
150 | video.state = VideoState.TO_TRANSCODE | 153 | videoWithFiles.state = VideoState.TO_TRANSCODE |
151 | 154 | ||
152 | await video.save() | 155 | await videoWithFiles.save() |
153 | 156 | ||
154 | liveSession.replayVideoId = video.id | 157 | liveSession.replayVideoId = videoWithFiles.id |
155 | await liveSession.save() | 158 | await liveSession.save() |
156 | 159 | ||
157 | // Remove old HLS playlist video files | ||
158 | const videoWithFiles = await VideoModel.loadFull(video.id) | ||
159 | |||
160 | const hlsPlaylist = videoWithFiles.getHLSPlaylist() | ||
161 | await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) | 160 | await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) |
162 | 161 | ||
163 | // Reset playlist | 162 | // Reset playlist |
@@ -234,7 +233,7 @@ async function cleanupLiveAndFederate (options: { | |||
234 | 233 | ||
235 | if (streamingPlaylist) { | 234 | if (streamingPlaylist) { |
236 | if (permanentLive) { | 235 | if (permanentLive) { |
237 | await cleanupPermanentLive(video, streamingPlaylist) | 236 | await cleanupAndDestroyPermanentLive(video, streamingPlaylist) |
238 | } else { | 237 | } else { |
239 | await cleanupUnsavedNormalLive(video, streamingPlaylist) | 238 | await cleanupUnsavedNormalLive(video, streamingPlaylist) |
240 | } | 239 | } |
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 16715862b..9470b530b 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts | |||
@@ -21,14 +21,14 @@ import { VideoLiveSessionModel } from '@server/models/video/video-live-session' | |||
21 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 21 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
22 | import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models' | 22 | import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models' |
23 | import { pick, wait } from '@shared/core-utils' | 23 | import { pick, wait } from '@shared/core-utils' |
24 | import { LiveVideoError, VideoState, VideoStreamingPlaylistType } from '@shared/models' | 24 | import { LiveVideoError, VideoState, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' |
25 | import { federateVideoIfNeeded } from '../activitypub/videos' | 25 | import { federateVideoIfNeeded } from '../activitypub/videos' |
26 | import { JobQueue } from '../job-queue' | 26 | import { JobQueue } from '../job-queue' |
27 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' | 27 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' |
28 | import { PeerTubeSocket } from '../peertube-socket' | 28 | import { PeerTubeSocket } from '../peertube-socket' |
29 | import { Hooks } from '../plugins/hooks' | 29 | import { Hooks } from '../plugins/hooks' |
30 | import { LiveQuotaStore } from './live-quota-store' | 30 | import { LiveQuotaStore } from './live-quota-store' |
31 | import { cleanupPermanentLive } from './live-utils' | 31 | import { cleanupAndDestroyPermanentLive } from './live-utils' |
32 | import { MuxingSession } from './shared' | 32 | import { MuxingSession } from './shared' |
33 | 33 | ||
34 | const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') | 34 | const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') |
@@ -224,7 +224,7 @@ class LiveManager { | |||
224 | if (oldStreamingPlaylist) { | 224 | if (oldStreamingPlaylist) { |
225 | if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) | 225 | if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) |
226 | 226 | ||
227 | await cleanupPermanentLive(video, oldStreamingPlaylist) | 227 | await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) |
228 | } | 228 | } |
229 | 229 | ||
230 | this.videoSessions.set(video.id, sessionId) | 230 | this.videoSessions.set(video.id, sessionId) |
@@ -301,7 +301,7 @@ class LiveManager { | |||
301 | ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) | 301 | ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) |
302 | }) | 302 | }) |
303 | 303 | ||
304 | muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) | 304 | muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) |
305 | 305 | ||
306 | muxingSession.on('bad-socket-health', ({ videoId }) => { | 306 | muxingSession.on('bad-socket-health', ({ videoId }) => { |
307 | logger.error( | 307 | logger.error( |
@@ -485,6 +485,10 @@ class LiveManager { | |||
485 | 485 | ||
486 | playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) | 486 | playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) |
487 | 487 | ||
488 | playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED | ||
489 | ? VideoStorage.OBJECT_STORAGE | ||
490 | : VideoStorage.FILE_SYSTEM | ||
491 | |||
488 | return playlist.save() | 492 | return playlist.save() |
489 | } | 493 | } |
490 | 494 | ||
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts index 4af6f3ebf..faf03dccf 100644 --- a/server/lib/live/live-segment-sha-store.ts +++ b/server/lib/live/live-segment-sha-store.ts | |||
@@ -1,62 +1,73 @@ | |||
1 | import { writeJson } from 'fs-extra' | ||
1 | import { basename } from 'path' | 2 | import { basename } from 'path' |
3 | import { mapToJSON } from '@server/helpers/core-utils' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
5 | import { MStreamingPlaylistVideo } from '@server/types/models' | ||
3 | import { buildSha256Segment } from '../hls' | 6 | import { buildSha256Segment } from '../hls' |
7 | import { storeHLSFileFromPath } from '../object-storage' | ||
4 | 8 | ||
5 | const lTags = loggerTagsFactory('live') | 9 | const lTags = loggerTagsFactory('live') |
6 | 10 | ||
7 | class LiveSegmentShaStore { | 11 | class LiveSegmentShaStore { |
8 | 12 | ||
9 | private static instance: LiveSegmentShaStore | 13 | private readonly segmentsSha256 = new Map<string, string>() |
10 | 14 | ||
11 | private readonly segmentsSha256 = new Map<string, Map<string, string>>() | 15 | private readonly videoUUID: string |
12 | 16 | private readonly sha256Path: string | |
13 | private constructor () { | 17 | private readonly streamingPlaylist: MStreamingPlaylistVideo |
18 | private readonly sendToObjectStorage: boolean | ||
19 | |||
20 | constructor (options: { | ||
21 | videoUUID: string | ||
22 | sha256Path: string | ||
23 | streamingPlaylist: MStreamingPlaylistVideo | ||
24 | sendToObjectStorage: boolean | ||
25 | }) { | ||
26 | this.videoUUID = options.videoUUID | ||
27 | this.sha256Path = options.sha256Path | ||
28 | this.streamingPlaylist = options.streamingPlaylist | ||
29 | this.sendToObjectStorage = options.sendToObjectStorage | ||
14 | } | 30 | } |
15 | 31 | ||
16 | getSegmentsSha256 (videoUUID: string) { | 32 | async addSegmentSha (segmentPath: string) { |
17 | return this.segmentsSha256.get(videoUUID) | 33 | logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID)) |
18 | } | ||
19 | |||
20 | async addSegmentSha (videoUUID: string, segmentPath: string) { | ||
21 | const segmentName = basename(segmentPath) | ||
22 | logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID)) | ||
23 | 34 | ||
24 | const shaResult = await buildSha256Segment(segmentPath) | 35 | const shaResult = await buildSha256Segment(segmentPath) |
25 | 36 | ||
26 | if (!this.segmentsSha256.has(videoUUID)) { | 37 | const segmentName = basename(segmentPath) |
27 | this.segmentsSha256.set(videoUUID, new Map()) | 38 | this.segmentsSha256.set(segmentName, shaResult) |
28 | } | ||
29 | 39 | ||
30 | const filesMap = this.segmentsSha256.get(videoUUID) | 40 | await this.writeToDisk() |
31 | filesMap.set(segmentName, shaResult) | ||
32 | } | 41 | } |
33 | 42 | ||
34 | removeSegmentSha (videoUUID: string, segmentPath: string) { | 43 | async removeSegmentSha (segmentPath: string) { |
35 | const segmentName = basename(segmentPath) | 44 | const segmentName = basename(segmentPath) |
36 | 45 | ||
37 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID)) | 46 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) |
38 | 47 | ||
39 | const filesMap = this.segmentsSha256.get(videoUUID) | 48 | if (!this.segmentsSha256.has(segmentName)) { |
40 | if (!filesMap) { | 49 | logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID)) |
41 | logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID)) | ||
42 | return | 50 | return |
43 | } | 51 | } |
44 | 52 | ||
45 | if (!filesMap.has(segmentName)) { | 53 | this.segmentsSha256.delete(segmentName) |
46 | logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID)) | ||
47 | return | ||
48 | } | ||
49 | 54 | ||
50 | filesMap.delete(segmentName) | 55 | await this.writeToDisk() |
51 | } | 56 | } |
52 | 57 | ||
53 | cleanupShaSegments (videoUUID: string) { | 58 | private async writeToDisk () { |
54 | this.segmentsSha256.delete(videoUUID) | 59 | await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256)) |
55 | } | ||
56 | 60 | ||
57 | static get Instance () { | 61 | if (this.sendToObjectStorage) { |
58 | return this.instance || (this.instance = new this()) | 62 | const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) |
63 | |||
64 | if (this.streamingPlaylist.segmentsSha256Url !== url) { | ||
65 | this.streamingPlaylist.segmentsSha256Url = url | ||
66 | await this.streamingPlaylist.save() | ||
67 | } | ||
68 | } | ||
59 | } | 69 | } |
70 | |||
60 | } | 71 | } |
61 | 72 | ||
62 | export { | 73 | export { |
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts index bba876642..d2b8e3a55 100644 --- a/server/lib/live/live-utils.ts +++ b/server/lib/live/live-utils.ts | |||
@@ -1,9 +1,10 @@ | |||
1 | import { pathExists, readdir, remove } from 'fs-extra' | 1 | import { pathExists, readdir, remove } from 'fs-extra' |
2 | import { basename, join } from 'path' | 2 | import { basename, join } from 'path' |
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
4 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 4 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' |
5 | import { VideoStorage } from '@shared/models' | ||
6 | import { listHLSFileKeysOf, removeHLSFileObjectStorage, removeHLSObjectStorage } from '../object-storage' | ||
5 | import { getLiveDirectory } from '../paths' | 7 | import { getLiveDirectory } from '../paths' |
6 | import { LiveSegmentShaStore } from './live-segment-sha-store' | ||
7 | 8 | ||
8 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | 9 | function buildConcatenatedName (segmentOrPlaylistPath: string) { |
9 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | 10 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) |
@@ -11,8 +12,8 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) { | |||
11 | return 'concat-' + num[1] + '.ts' | 12 | return 'concat-' + num[1] + '.ts' |
12 | } | 13 | } |
13 | 14 | ||
14 | async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | 15 | async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
15 | await cleanupTMPLiveFiles(video) | 16 | await cleanupTMPLiveFiles(video, streamingPlaylist) |
16 | 17 | ||
17 | await streamingPlaylist.destroy() | 18 | await streamingPlaylist.destroy() |
18 | } | 19 | } |
@@ -20,32 +21,51 @@ async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamin | |||
20 | async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | 21 | async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
21 | const hlsDirectory = getLiveDirectory(video) | 22 | const hlsDirectory = getLiveDirectory(video) |
22 | 23 | ||
24 | // We uploaded files to object storage too, remove them | ||
25 | if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
26 | await removeHLSObjectStorage(streamingPlaylist.withVideo(video)) | ||
27 | } | ||
28 | |||
23 | await remove(hlsDirectory) | 29 | await remove(hlsDirectory) |
24 | 30 | ||
25 | await streamingPlaylist.destroy() | 31 | await streamingPlaylist.destroy() |
32 | } | ||
26 | 33 | ||
27 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) | 34 | async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
35 | await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video)) | ||
36 | |||
37 | await cleanupTMPLiveFilesFromFilesystem(video) | ||
28 | } | 38 | } |
29 | 39 | ||
30 | async function cleanupTMPLiveFiles (video: MVideo) { | 40 | export { |
31 | const hlsDirectory = getLiveDirectory(video) | 41 | cleanupAndDestroyPermanentLive, |
42 | cleanupUnsavedNormalLive, | ||
43 | cleanupTMPLiveFiles, | ||
44 | buildConcatenatedName | ||
45 | } | ||
46 | |||
47 | // --------------------------------------------------------------------------- | ||
32 | 48 | ||
33 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) | 49 | function isTMPLiveFile (name: string) { |
50 | return name.endsWith('.ts') || | ||
51 | name.endsWith('.m3u8') || | ||
52 | name.endsWith('.json') || | ||
53 | name.endsWith('.mpd') || | ||
54 | name.endsWith('.m4s') || | ||
55 | name.endsWith('.tmp') | ||
56 | } | ||
57 | |||
58 | async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) { | ||
59 | const hlsDirectory = getLiveDirectory(video) | ||
34 | 60 | ||
35 | if (!await pathExists(hlsDirectory)) return | 61 | if (!await pathExists(hlsDirectory)) return |
36 | 62 | ||
37 | logger.info('Cleanup TMP live files of %s.', hlsDirectory) | 63 | logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory) |
38 | 64 | ||
39 | const files = await readdir(hlsDirectory) | 65 | const files = await readdir(hlsDirectory) |
40 | 66 | ||
41 | for (const filename of files) { | 67 | for (const filename of files) { |
42 | if ( | 68 | if (isTMPLiveFile(filename)) { |
43 | filename.endsWith('.ts') || | ||
44 | filename.endsWith('.m3u8') || | ||
45 | filename.endsWith('.mpd') || | ||
46 | filename.endsWith('.m4s') || | ||
47 | filename.endsWith('.tmp') | ||
48 | ) { | ||
49 | const p = join(hlsDirectory, filename) | 69 | const p = join(hlsDirectory, filename) |
50 | 70 | ||
51 | remove(p) | 71 | remove(p) |
@@ -54,9 +74,14 @@ async function cleanupTMPLiveFiles (video: MVideo) { | |||
54 | } | 74 | } |
55 | } | 75 | } |
56 | 76 | ||
57 | export { | 77 | async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) { |
58 | cleanupPermanentLive, | 78 | if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return |
59 | cleanupUnsavedNormalLive, | 79 | |
60 | cleanupTMPLiveFiles, | 80 | const keys = await listHLSFileKeysOf(streamingPlaylist) |
61 | buildConcatenatedName | 81 | |
82 | for (const key of keys) { | ||
83 | if (isTMPLiveFile(key)) { | ||
84 | await removeHLSFileObjectStorage(streamingPlaylist, key) | ||
85 | } | ||
86 | } | ||
62 | } | 87 | } |
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index 505717dce..4c27d5dd8 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -9,8 +9,10 @@ import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers | |||
9 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | 9 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' |
10 | import { CONFIG } from '@server/initializers/config' | 10 | import { CONFIG } from '@server/initializers/config' |
11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | 11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' |
12 | import { removeHLSFileObjectStorage, storeHLSFileFromFilename, storeHLSFileFromPath } from '@server/lib/object-storage' | ||
12 | import { VideoFileModel } from '@server/models/video/video-file' | 13 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | 14 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' |
15 | import { VideoStorage } from '@shared/models' | ||
14 | import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' | 16 | import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' |
15 | import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' | 17 | import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' |
16 | import { isAbleToUploadVideo } from '../../user' | 18 | import { isAbleToUploadVideo } from '../../user' |
@@ -21,7 +23,7 @@ import { buildConcatenatedName } from '../live-utils' | |||
21 | import memoizee = require('memoizee') | 23 | import memoizee = require('memoizee') |
22 | 24 | ||
23 | interface MuxingSessionEvents { | 25 | interface MuxingSessionEvents { |
24 | 'master-playlist-created': (options: { videoId: number }) => void | 26 | 'live-ready': (options: { videoId: number }) => void |
25 | 27 | ||
26 | 'bad-socket-health': (options: { videoId: number }) => void | 28 | 'bad-socket-health': (options: { videoId: number }) => void |
27 | 'duration-exceeded': (options: { videoId: number }) => void | 29 | 'duration-exceeded': (options: { videoId: number }) => void |
@@ -68,12 +70,18 @@ class MuxingSession extends EventEmitter { | |||
68 | private readonly outDirectory: string | 70 | private readonly outDirectory: string |
69 | private readonly replayDirectory: string | 71 | private readonly replayDirectory: string |
70 | 72 | ||
73 | private readonly liveSegmentShaStore: LiveSegmentShaStore | ||
74 | |||
71 | private readonly lTags: LoggerTagsFn | 75 | private readonly lTags: LoggerTagsFn |
72 | 76 | ||
73 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} | 77 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} |
74 | 78 | ||
75 | private tsWatcher: FSWatcher | 79 | private tsWatcher: FSWatcher |
76 | private masterWatcher: FSWatcher | 80 | private masterWatcher: FSWatcher |
81 | private m3u8Watcher: FSWatcher | ||
82 | |||
83 | private masterPlaylistCreated = false | ||
84 | private liveReady = false | ||
77 | 85 | ||
78 | private aborted = false | 86 | private aborted = false |
79 | 87 | ||
@@ -123,6 +131,13 @@ class MuxingSession extends EventEmitter { | |||
123 | this.outDirectory = getLiveDirectory(this.videoLive.Video) | 131 | this.outDirectory = getLiveDirectory(this.videoLive.Video) |
124 | this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) | 132 | this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) |
125 | 133 | ||
134 | this.liveSegmentShaStore = new LiveSegmentShaStore({ | ||
135 | videoUUID: this.videoLive.Video.uuid, | ||
136 | sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename), | ||
137 | streamingPlaylist: this.streamingPlaylist, | ||
138 | sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED | ||
139 | }) | ||
140 | |||
126 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) | 141 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) |
127 | } | 142 | } |
128 | 143 | ||
@@ -159,8 +174,9 @@ class MuxingSession extends EventEmitter { | |||
159 | 174 | ||
160 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) | 175 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) |
161 | 176 | ||
162 | this.watchTSFiles() | ||
163 | this.watchMasterFile() | 177 | this.watchMasterFile() |
178 | this.watchTSFiles() | ||
179 | this.watchM3U8File() | ||
164 | 180 | ||
165 | let ffmpegShellCommand: string | 181 | let ffmpegShellCommand: string |
166 | this.ffmpegCommand.on('start', cmdline => { | 182 | this.ffmpegCommand.on('start', cmdline => { |
@@ -219,7 +235,7 @@ class MuxingSession extends EventEmitter { | |||
219 | setTimeout(() => { | 235 | setTimeout(() => { |
220 | // Wait latest segments generation, and close watchers | 236 | // Wait latest segments generation, and close watchers |
221 | 237 | ||
222 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ]) | 238 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ]) |
223 | .then(() => { | 239 | .then(() => { |
224 | // Process remaining segments hash | 240 | // Process remaining segments hash |
225 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | 241 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { |
@@ -240,14 +256,41 @@ class MuxingSession extends EventEmitter { | |||
240 | private watchMasterFile () { | 256 | private watchMasterFile () { |
241 | this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) | 257 | this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) |
242 | 258 | ||
243 | this.masterWatcher.on('add', () => { | 259 | this.masterWatcher.on('add', async () => { |
244 | this.emit('master-playlist-created', { videoId: this.videoId }) | 260 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { |
261 | try { | ||
262 | const url = await storeHLSFileFromFilename(this.streamingPlaylist, this.streamingPlaylist.playlistFilename) | ||
263 | |||
264 | this.streamingPlaylist.playlistUrl = url | ||
265 | await this.streamingPlaylist.save() | ||
266 | } catch (err) { | ||
267 | logger.error('Cannot upload live master file to object storage.', { err, ...this.lTags() }) | ||
268 | } | ||
269 | } | ||
270 | |||
271 | this.masterPlaylistCreated = true | ||
245 | 272 | ||
246 | this.masterWatcher.close() | 273 | this.masterWatcher.close() |
247 | .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) | 274 | .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) |
248 | }) | 275 | }) |
249 | } | 276 | } |
250 | 277 | ||
278 | private watchM3U8File () { | ||
279 | this.m3u8Watcher = watch(this.outDirectory + '/*.m3u8') | ||
280 | |||
281 | const onChangeOrAdd = async (m3u8Path: string) => { | ||
282 | if (this.streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return | ||
283 | |||
284 | try { | ||
285 | await storeHLSFileFromPath(this.streamingPlaylist, m3u8Path) | ||
286 | } catch (err) { | ||
287 | logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() }) | ||
288 | } | ||
289 | } | ||
290 | |||
291 | this.m3u8Watcher.on('change', onChangeOrAdd) | ||
292 | } | ||
293 | |||
251 | private watchTSFiles () { | 294 | private watchTSFiles () { |
252 | const startStreamDateTime = new Date().getTime() | 295 | const startStreamDateTime = new Date().getTime() |
253 | 296 | ||
@@ -282,7 +325,21 @@ class MuxingSession extends EventEmitter { | |||
282 | } | 325 | } |
283 | } | 326 | } |
284 | 327 | ||
285 | const deleteHandler = (segmentPath: string) => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath) | 328 | const deleteHandler = async (segmentPath: string) => { |
329 | try { | ||
330 | await this.liveSegmentShaStore.removeSegmentSha(segmentPath) | ||
331 | } catch (err) { | ||
332 | logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() }) | ||
333 | } | ||
334 | |||
335 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
336 | try { | ||
337 | await removeHLSFileObjectStorage(this.streamingPlaylist, segmentPath) | ||
338 | } catch (err) { | ||
339 | logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() }) | ||
340 | } | ||
341 | } | ||
342 | } | ||
286 | 343 | ||
287 | this.tsWatcher.on('add', p => addHandler(p)) | 344 | this.tsWatcher.on('add', p => addHandler(p)) |
288 | this.tsWatcher.on('unlink', p => deleteHandler(p)) | 345 | this.tsWatcher.on('unlink', p => deleteHandler(p)) |
@@ -315,6 +372,7 @@ class MuxingSession extends EventEmitter { | |||
315 | extname: '.ts', | 372 | extname: '.ts', |
316 | infoHash: null, | 373 | infoHash: null, |
317 | fps: this.fps, | 374 | fps: this.fps, |
375 | storage: this.streamingPlaylist.storage, | ||
318 | videoStreamingPlaylistId: this.streamingPlaylist.id | 376 | videoStreamingPlaylistId: this.streamingPlaylist.id |
319 | }) | 377 | }) |
320 | 378 | ||
@@ -343,18 +401,36 @@ class MuxingSession extends EventEmitter { | |||
343 | } | 401 | } |
344 | 402 | ||
345 | private processSegments (segmentPaths: string[]) { | 403 | private processSegments (segmentPaths: string[]) { |
346 | mapSeries(segmentPaths, async previousSegment => { | 404 | mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment)) |
347 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | 405 | .catch(err => { |
348 | await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment) | 406 | if (this.aborted) return |
407 | |||
408 | logger.error('Cannot process segments', { err, ...this.lTags() }) | ||
409 | }) | ||
410 | } | ||
349 | 411 | ||
350 | if (this.saveReplay) { | 412 | private async processSegment (segmentPath: string) { |
351 | await this.addSegmentToReplay(previousSegment) | 413 | // Add sha hash of previous segments, because ffmpeg should have finished generating them |
414 | await this.liveSegmentShaStore.addSegmentSha(segmentPath) | ||
415 | |||
416 | if (this.saveReplay) { | ||
417 | await this.addSegmentToReplay(segmentPath) | ||
418 | } | ||
419 | |||
420 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
421 | try { | ||
422 | await storeHLSFileFromPath(this.streamingPlaylist, segmentPath) | ||
423 | } catch (err) { | ||
424 | logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() }) | ||
352 | } | 425 | } |
353 | }).catch(err => { | 426 | } |
354 | if (this.aborted) return | ||
355 | 427 | ||
356 | logger.error('Cannot process segments', { err, ...this.lTags() }) | 428 | // Master playlist and segment JSON file are created, live is ready |
357 | }) | 429 | if (this.masterPlaylistCreated && !this.liveReady) { |
430 | this.liveReady = true | ||
431 | |||
432 | this.emit('live-ready', { videoId: this.videoId }) | ||
433 | } | ||
358 | } | 434 | } |
359 | 435 | ||
360 | private hasClientSocketInBadHealth (sessionId: string) { | 436 | private hasClientSocketInBadHealth (sessionId: string) { |
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts index 16161362c..c131977e8 100644 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ b/server/lib/object-storage/shared/object-storage-helpers.ts | |||
@@ -22,6 +22,24 @@ type BucketInfo = { | |||
22 | PREFIX?: string | 22 | PREFIX?: string |
23 | } | 23 | } |
24 | 24 | ||
25 | async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { | ||
26 | const s3Client = getClient() | ||
27 | |||
28 | const commandPrefix = bucketInfo.PREFIX + prefix | ||
29 | const listCommand = new ListObjectsV2Command({ | ||
30 | Bucket: bucketInfo.BUCKET_NAME, | ||
31 | Prefix: commandPrefix | ||
32 | }) | ||
33 | |||
34 | const listedObjects = await s3Client.send(listCommand) | ||
35 | |||
36 | if (isArray(listedObjects.Contents) !== true) return [] | ||
37 | |||
38 | return listedObjects.Contents.map(c => c.Key) | ||
39 | } | ||
40 | |||
41 | // --------------------------------------------------------------------------- | ||
42 | |||
25 | async function storeObject (options: { | 43 | async function storeObject (options: { |
26 | inputPath: string | 44 | inputPath: string |
27 | objectStorageKey: string | 45 | objectStorageKey: string |
@@ -36,6 +54,8 @@ async function storeObject (options: { | |||
36 | return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) | 54 | return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) |
37 | } | 55 | } |
38 | 56 | ||
57 | // --------------------------------------------------------------------------- | ||
58 | |||
39 | async function removeObject (filename: string, bucketInfo: BucketInfo) { | 59 | async function removeObject (filename: string, bucketInfo: BucketInfo) { |
40 | const command = new DeleteObjectCommand({ | 60 | const command = new DeleteObjectCommand({ |
41 | Bucket: bucketInfo.BUCKET_NAME, | 61 | Bucket: bucketInfo.BUCKET_NAME, |
@@ -89,6 +109,8 @@ async function removePrefix (prefix: string, bucketInfo: BucketInfo) { | |||
89 | if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) | 109 | if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) |
90 | } | 110 | } |
91 | 111 | ||
112 | // --------------------------------------------------------------------------- | ||
113 | |||
92 | async function makeAvailable (options: { | 114 | async function makeAvailable (options: { |
93 | key: string | 115 | key: string |
94 | destination: string | 116 | destination: string |
@@ -122,7 +144,8 @@ export { | |||
122 | storeObject, | 144 | storeObject, |
123 | removeObject, | 145 | removeObject, |
124 | removePrefix, | 146 | removePrefix, |
125 | makeAvailable | 147 | makeAvailable, |
148 | listKeysOfPrefix | ||
126 | } | 149 | } |
127 | 150 | ||
128 | // --------------------------------------------------------------------------- | 151 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts index 66e738200..62aae248b 100644 --- a/server/lib/object-storage/videos.ts +++ b/server/lib/object-storage/videos.ts | |||
@@ -1,19 +1,35 @@ | |||
1 | import { join } from 'path' | 1 | import { basename, join } from 'path' |
2 | import { logger } from '@server/helpers/logger' | 2 | import { logger } from '@server/helpers/logger' |
3 | import { CONFIG } from '@server/initializers/config' | 3 | import { CONFIG } from '@server/initializers/config' |
4 | import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' | 4 | import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' |
5 | import { getHLSDirectory } from '../paths' | 5 | import { getHLSDirectory } from '../paths' |
6 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' | 6 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' |
7 | import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' | 7 | import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' |
8 | 8 | ||
9 | function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) { | 9 | function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { |
10 | return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
11 | } | ||
12 | |||
13 | // --------------------------------------------------------------------------- | ||
14 | |||
15 | function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { | ||
10 | return storeObject({ | 16 | return storeObject({ |
11 | inputPath: path ?? join(getHLSDirectory(playlist.Video), filename), | 17 | inputPath: join(getHLSDirectory(playlist.Video), filename), |
12 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), | 18 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), |
13 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | 19 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS |
14 | }) | 20 | }) |
15 | } | 21 | } |
16 | 22 | ||
23 | function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { | ||
24 | return storeObject({ | ||
25 | inputPath: path, | ||
26 | objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), | ||
27 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | ||
28 | }) | ||
29 | } | ||
30 | |||
31 | // --------------------------------------------------------------------------- | ||
32 | |||
17 | function storeWebTorrentFile (filename: string) { | 33 | function storeWebTorrentFile (filename: string) { |
18 | return storeObject({ | 34 | return storeObject({ |
19 | inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), | 35 | inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), |
@@ -22,6 +38,8 @@ function storeWebTorrentFile (filename: string) { | |||
22 | }) | 38 | }) |
23 | } | 39 | } |
24 | 40 | ||
41 | // --------------------------------------------------------------------------- | ||
42 | |||
25 | function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { | 43 | function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { |
26 | return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | 44 | return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) |
27 | } | 45 | } |
@@ -30,10 +48,14 @@ function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename | |||
30 | return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | 48 | return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) |
31 | } | 49 | } |
32 | 50 | ||
51 | // --------------------------------------------------------------------------- | ||
52 | |||
33 | function removeWebTorrentObjectStorage (videoFile: MVideoFile) { | 53 | function removeWebTorrentObjectStorage (videoFile: MVideoFile) { |
34 | return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) | 54 | return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) |
35 | } | 55 | } |
36 | 56 | ||
57 | // --------------------------------------------------------------------------- | ||
58 | |||
37 | async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { | 59 | async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { |
38 | const key = generateHLSObjectStorageKey(playlist, filename) | 60 | const key = generateHLSObjectStorageKey(playlist, filename) |
39 | 61 | ||
@@ -62,9 +84,14 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin | |||
62 | return destination | 84 | return destination |
63 | } | 85 | } |
64 | 86 | ||
87 | // --------------------------------------------------------------------------- | ||
88 | |||
65 | export { | 89 | export { |
90 | listHLSFileKeysOf, | ||
91 | |||
66 | storeWebTorrentFile, | 92 | storeWebTorrentFile, |
67 | storeHLSFile, | 93 | storeHLSFileFromFilename, |
94 | storeHLSFileFromPath, | ||
68 | 95 | ||
69 | removeHLSObjectStorage, | 96 | removeHLSObjectStorage, |
70 | removeHLSFileObjectStorage, | 97 | removeHLSFileObjectStorage, |
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index f587989dc..2b6771f27 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts | |||
@@ -245,21 +245,25 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
245 | } | 245 | } |
246 | 246 | ||
247 | getMasterPlaylistUrl (video: MVideo) { | 247 | getMasterPlaylistUrl (video: MVideo) { |
248 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | 248 | if (video.isOwned()) { |
249 | return getHLSPublicFileUrl(this.playlistUrl) | 249 | if (this.storage === VideoStorage.OBJECT_STORAGE) { |
250 | } | 250 | return getHLSPublicFileUrl(this.playlistUrl) |
251 | } | ||
251 | 252 | ||
252 | if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) | 253 | return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) |
254 | } | ||
253 | 255 | ||
254 | return this.playlistUrl | 256 | return this.playlistUrl |
255 | } | 257 | } |
256 | 258 | ||
257 | getSha256SegmentsUrl (video: MVideo) { | 259 | getSha256SegmentsUrl (video: MVideo) { |
258 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | 260 | if (video.isOwned()) { |
259 | return getHLSPublicFileUrl(this.segmentsSha256Url) | 261 | if (this.storage === VideoStorage.OBJECT_STORAGE) { |
260 | } | 262 | return getHLSPublicFileUrl(this.segmentsSha256Url) |
263 | } | ||
261 | 264 | ||
262 | if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) | 265 | return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid) |
266 | } | ||
263 | 267 | ||
264 | return this.segmentsSha256Url | 268 | return this.segmentsSha256Url |
265 | } | 269 | } |
@@ -287,9 +291,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
287 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename) | 291 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename) |
288 | } | 292 | } |
289 | 293 | ||
290 | private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) { | 294 | private getSha256SegmentsStaticPath (videoUUID: string) { |
291 | if (isLive) return join('/live', 'segments-sha256', videoUUID) | ||
292 | |||
293 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename) | 295 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename) |
294 | } | 296 | } |
295 | } | 297 | } |
diff --git a/server/tests/api/live/live-fast-restream.ts b/server/tests/api/live/live-fast-restream.ts index 502959258..3ea6be9ff 100644 --- a/server/tests/api/live/live-fast-restream.ts +++ b/server/tests/api/live/live-fast-restream.ts | |||
@@ -59,7 +59,7 @@ describe('Fast restream in live', function () { | |||
59 | const video = await server.videos.get({ id: liveId }) | 59 | const video = await server.videos.get({ id: liveId }) |
60 | expect(video.streamingPlaylists).to.have.lengthOf(1) | 60 | expect(video.streamingPlaylists).to.have.lengthOf(1) |
61 | 61 | ||
62 | await server.live.getSegment({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) | 62 | await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) |
63 | await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200) | 63 | await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200) |
64 | await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200) | 64 | await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200) |
65 | 65 | ||
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts index 4e070832d..5dd2bd9ab 100644 --- a/server/tests/api/live/live.ts +++ b/server/tests/api/live/live.ts | |||
@@ -3,7 +3,7 @@ | |||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { basename, join } from 'path' | 4 | import { basename, join } from 'path' |
5 | import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg' | 5 | import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg' |
6 | import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared' | 6 | import { testImage, testVideoResolutions } from '@server/tests/shared' |
7 | import { getAllFiles, wait } from '@shared/core-utils' | 7 | import { getAllFiles, wait } from '@shared/core-utils' |
8 | import { | 8 | import { |
9 | HttpStatusCode, | 9 | HttpStatusCode, |
@@ -372,46 +372,6 @@ describe('Test live', function () { | |||
372 | return uuid | 372 | return uuid |
373 | } | 373 | } |
374 | 374 | ||
375 | async function testVideoResolutions (liveVideoId: string, resolutions: number[]) { | ||
376 | for (const server of servers) { | ||
377 | const { data } = await server.videos.list() | ||
378 | expect(data.find(v => v.uuid === liveVideoId)).to.exist | ||
379 | |||
380 | const video = await server.videos.get({ id: liveVideoId }) | ||
381 | |||
382 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
383 | |||
384 | const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) | ||
385 | expect(hlsPlaylist).to.exist | ||
386 | |||
387 | // Only finite files are displayed | ||
388 | expect(hlsPlaylist.files).to.have.lengthOf(0) | ||
389 | |||
390 | await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) | ||
391 | |||
392 | for (let i = 0; i < resolutions.length; i++) { | ||
393 | const segmentNum = 3 | ||
394 | const segmentName = `${i}-00000${segmentNum}.ts` | ||
395 | await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, playlistNumber: i, segment: segmentNum }) | ||
396 | |||
397 | const subPlaylist = await servers[0].streamingPlaylists.get({ | ||
398 | url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8` | ||
399 | }) | ||
400 | |||
401 | expect(subPlaylist).to.contain(segmentName) | ||
402 | |||
403 | const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls' | ||
404 | await checkLiveSegmentHash({ | ||
405 | server, | ||
406 | baseUrlSegment: baseUrlAndPath, | ||
407 | videoUUID: video.uuid, | ||
408 | segmentName, | ||
409 | hlsPlaylist | ||
410 | }) | ||
411 | } | ||
412 | } | ||
413 | } | ||
414 | |||
415 | function updateConf (resolutions: number[]) { | 375 | function updateConf (resolutions: number[]) { |
416 | return servers[0].config.updateCustomSubConfig({ | 376 | return servers[0].config.updateCustomSubConfig({ |
417 | newConfig: { | 377 | newConfig: { |
@@ -449,7 +409,14 @@ describe('Test live', function () { | |||
449 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 409 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
450 | await waitJobs(servers) | 410 | await waitJobs(servers) |
451 | 411 | ||
452 | await testVideoResolutions(liveVideoId, [ 720 ]) | 412 | await testVideoResolutions({ |
413 | originServer: servers[0], | ||
414 | servers, | ||
415 | liveVideoId, | ||
416 | resolutions: [ 720 ], | ||
417 | objectStorage: false, | ||
418 | transcoded: true | ||
419 | }) | ||
453 | 420 | ||
454 | await stopFfmpeg(ffmpegCommand) | 421 | await stopFfmpeg(ffmpegCommand) |
455 | }) | 422 | }) |
@@ -477,7 +444,14 @@ describe('Test live', function () { | |||
477 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 444 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
478 | await waitJobs(servers) | 445 | await waitJobs(servers) |
479 | 446 | ||
480 | await testVideoResolutions(liveVideoId, resolutions.concat([ 720 ])) | 447 | await testVideoResolutions({ |
448 | originServer: servers[0], | ||
449 | servers, | ||
450 | liveVideoId, | ||
451 | resolutions: resolutions.concat([ 720 ]), | ||
452 | objectStorage: false, | ||
453 | transcoded: true | ||
454 | }) | ||
481 | 455 | ||
482 | await stopFfmpeg(ffmpegCommand) | 456 | await stopFfmpeg(ffmpegCommand) |
483 | }) | 457 | }) |
@@ -522,7 +496,14 @@ describe('Test live', function () { | |||
522 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 496 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
523 | await waitJobs(servers) | 497 | await waitJobs(servers) |
524 | 498 | ||
525 | await testVideoResolutions(liveVideoId, resolutions) | 499 | await testVideoResolutions({ |
500 | originServer: servers[0], | ||
501 | servers, | ||
502 | liveVideoId, | ||
503 | resolutions, | ||
504 | objectStorage: false, | ||
505 | transcoded: true | ||
506 | }) | ||
526 | 507 | ||
527 | await stopFfmpeg(ffmpegCommand) | 508 | await stopFfmpeg(ffmpegCommand) |
528 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) | 509 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) |
@@ -611,7 +592,14 @@ describe('Test live', function () { | |||
611 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 592 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
612 | await waitJobs(servers) | 593 | await waitJobs(servers) |
613 | 594 | ||
614 | await testVideoResolutions(liveVideoId, resolutions) | 595 | await testVideoResolutions({ |
596 | originServer: servers[0], | ||
597 | servers, | ||
598 | liveVideoId, | ||
599 | resolutions, | ||
600 | objectStorage: false, | ||
601 | transcoded: true | ||
602 | }) | ||
615 | 603 | ||
616 | await stopFfmpeg(ffmpegCommand) | 604 | await stopFfmpeg(ffmpegCommand) |
617 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) | 605 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) |
@@ -640,7 +628,14 @@ describe('Test live', function () { | |||
640 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 628 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
641 | await waitJobs(servers) | 629 | await waitJobs(servers) |
642 | 630 | ||
643 | await testVideoResolutions(liveVideoId, [ 720 ]) | 631 | await testVideoResolutions({ |
632 | originServer: servers[0], | ||
633 | servers, | ||
634 | liveVideoId, | ||
635 | resolutions: [ 720 ], | ||
636 | objectStorage: false, | ||
637 | transcoded: true | ||
638 | }) | ||
644 | 639 | ||
645 | await stopFfmpeg(ffmpegCommand) | 640 | await stopFfmpeg(ffmpegCommand) |
646 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) | 641 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) |
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts index 0958ffe0f..7e16b4c89 100644 --- a/server/tests/api/object-storage/live.ts +++ b/server/tests/api/object-storage/live.ts | |||
@@ -1,9 +1,9 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { expectStartWith } from '@server/tests/shared' | 4 | import { expectStartWith, testVideoResolutions } from '@server/tests/shared' |
5 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 5 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' |
6 | import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models' | 6 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models' |
7 | import { | 7 | import { |
8 | createMultipleServers, | 8 | createMultipleServers, |
9 | doubleFollow, | 9 | doubleFollow, |
@@ -35,41 +35,43 @@ async function createLive (server: PeerTubeServer, permanent: boolean) { | |||
35 | return uuid | 35 | return uuid |
36 | } | 36 | } |
37 | 37 | ||
38 | async function checkFiles (files: VideoFile[]) { | 38 | async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, numberOfFiles: number) { |
39 | for (const file of files) { | 39 | for (const server of servers) { |
40 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | 40 | const video = await server.videos.get({ id: videoUUID }) |
41 | 41 | ||
42 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 42 | expect(video.files).to.have.lengthOf(0) |
43 | } | 43 | expect(video.streamingPlaylists).to.have.lengthOf(1) |
44 | } | ||
45 | 44 | ||
46 | async function getFiles (server: PeerTubeServer, videoUUID: string) { | 45 | const files = video.streamingPlaylists[0].files |
47 | const video = await server.videos.get({ id: videoUUID }) | 46 | expect(files).to.have.lengthOf(numberOfFiles) |
48 | 47 | ||
49 | expect(video.files).to.have.lengthOf(0) | 48 | for (const file of files) { |
50 | expect(video.streamingPlaylists).to.have.lengthOf(1) | 49 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) |
51 | 50 | ||
52 | return video.streamingPlaylists[0].files | 51 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) |
52 | } | ||
53 | } | ||
53 | } | 54 | } |
54 | 55 | ||
55 | async function streamAndEnd (servers: PeerTubeServer[], liveUUID: string) { | 56 | async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[]) { |
56 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveUUID }) | 57 | const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`) |
57 | await waitUntilLivePublishedOnAllServers(servers, liveUUID) | ||
58 | |||
59 | const videoLiveDetails = await servers[0].videos.get({ id: liveUUID }) | ||
60 | const liveDetails = await servers[0].live.get({ videoId: liveUUID }) | ||
61 | 58 | ||
62 | await stopFfmpeg(ffmpegCommand) | 59 | for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) { |
63 | 60 | await server.live.getPlaylistFile({ | |
64 | if (liveDetails.permanentLive) { | 61 | videoUUID, |
65 | await waitUntilLiveWaitingOnAllServers(servers, liveUUID) | 62 | playlistName, |
66 | } else { | 63 | expectedStatus: HttpStatusCode.NOT_FOUND_404, |
67 | await waitUntilLiveReplacedByReplayOnAllServers(servers, liveUUID) | 64 | objectStorage: true |
65 | }) | ||
68 | } | 66 | } |
69 | 67 | ||
70 | await waitJobs(servers) | 68 | await server.live.getSegmentFile({ |
71 | 69 | videoUUID, | |
72 | return { videoLiveDetails, liveDetails } | 70 | playlistNumber: 0, |
71 | segment: 0, | ||
72 | objectStorage: true, | ||
73 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
74 | }) | ||
73 | } | 75 | } |
74 | 76 | ||
75 | describe('Object storage for lives', function () { | 77 | describe('Object storage for lives', function () { |
@@ -100,57 +102,124 @@ describe('Object storage for lives', function () { | |||
100 | videoUUID = await createLive(servers[0], false) | 102 | videoUUID = await createLive(servers[0], false) |
101 | }) | 103 | }) |
102 | 104 | ||
103 | it('Should create a live and save the replay on object storage', async function () { | 105 | it('Should create a live and publish it on object storage', async function () { |
106 | this.timeout(220000) | ||
107 | |||
108 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
109 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
110 | |||
111 | await testVideoResolutions({ | ||
112 | originServer: servers[0], | ||
113 | servers, | ||
114 | liveVideoId: videoUUID, | ||
115 | resolutions: [ 720 ], | ||
116 | transcoded: false, | ||
117 | objectStorage: true | ||
118 | }) | ||
119 | |||
120 | await stopFfmpeg(ffmpegCommand) | ||
121 | }) | ||
122 | |||
123 | it('Should have saved the replay on object storage', async function () { | ||
104 | this.timeout(220000) | 124 | this.timeout(220000) |
105 | 125 | ||
106 | await streamAndEnd(servers, videoUUID) | 126 | await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID) |
127 | await waitJobs(servers) | ||
107 | 128 | ||
108 | for (const server of servers) { | 129 | await checkFilesExist(servers, videoUUID, 1) |
109 | const files = await getFiles(server, videoUUID) | 130 | }) |
110 | expect(files).to.have.lengthOf(1) | ||
111 | 131 | ||
112 | await checkFiles(files) | 132 | it('Should have cleaned up live files from object storage', async function () { |
113 | } | 133 | await checkFilesCleanup(servers[0], videoUUID, [ 720 ]) |
114 | }) | 134 | }) |
115 | }) | 135 | }) |
116 | 136 | ||
117 | describe('With live transcoding', async function () { | 137 | describe('With live transcoding', async function () { |
118 | let videoUUIDPermanent: string | 138 | const resolutions = [ 720, 480, 360, 240, 144 ] |
119 | let videoUUIDNonPermanent: string | ||
120 | 139 | ||
121 | before(async function () { | 140 | before(async function () { |
122 | await servers[0].config.enableLive({ transcoding: true }) | 141 | await servers[0].config.enableLive({ transcoding: true }) |
123 | |||
124 | videoUUIDPermanent = await createLive(servers[0], true) | ||
125 | videoUUIDNonPermanent = await createLive(servers[0], false) | ||
126 | }) | 142 | }) |
127 | 143 | ||
128 | it('Should create a live and save the replay on object storage', async function () { | 144 | describe('Normal replay', function () { |
129 | this.timeout(240000) | 145 | let videoUUIDNonPermanent: string |
146 | |||
147 | before(async function () { | ||
148 | videoUUIDNonPermanent = await createLive(servers[0], false) | ||
149 | }) | ||
150 | |||
151 | it('Should create a live and publish it on object storage', async function () { | ||
152 | this.timeout(240000) | ||
153 | |||
154 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDNonPermanent }) | ||
155 | await waitUntilLivePublishedOnAllServers(servers, videoUUIDNonPermanent) | ||
156 | |||
157 | await testVideoResolutions({ | ||
158 | originServer: servers[0], | ||
159 | servers, | ||
160 | liveVideoId: videoUUIDNonPermanent, | ||
161 | resolutions, | ||
162 | transcoded: true, | ||
163 | objectStorage: true | ||
164 | }) | ||
165 | |||
166 | await stopFfmpeg(ffmpegCommand) | ||
167 | }) | ||
130 | 168 | ||
131 | await streamAndEnd(servers, videoUUIDNonPermanent) | 169 | it('Should have saved the replay on object storage', async function () { |
170 | this.timeout(220000) | ||
132 | 171 | ||
133 | for (const server of servers) { | 172 | await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent) |
134 | const files = await getFiles(server, videoUUIDNonPermanent) | 173 | await waitJobs(servers) |
135 | expect(files).to.have.lengthOf(5) | ||
136 | 174 | ||
137 | await checkFiles(files) | 175 | await checkFilesExist(servers, videoUUIDNonPermanent, 5) |
138 | } | 176 | }) |
177 | |||
178 | it('Should have cleaned up live files from object storage', async function () { | ||
179 | await checkFilesCleanup(servers[0], videoUUIDNonPermanent, resolutions) | ||
180 | }) | ||
139 | }) | 181 | }) |
140 | 182 | ||
141 | it('Should create a live and save the replay of permanent live on object storage', async function () { | 183 | describe('Permanent replay', function () { |
142 | this.timeout(240000) | 184 | let videoUUIDPermanent: string |
185 | |||
186 | before(async function () { | ||
187 | videoUUIDPermanent = await createLive(servers[0], true) | ||
188 | }) | ||
189 | |||
190 | it('Should create a live and publish it on object storage', async function () { | ||
191 | this.timeout(240000) | ||
192 | |||
193 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) | ||
194 | await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) | ||
195 | |||
196 | await testVideoResolutions({ | ||
197 | originServer: servers[0], | ||
198 | servers, | ||
199 | liveVideoId: videoUUIDPermanent, | ||
200 | resolutions, | ||
201 | transcoded: true, | ||
202 | objectStorage: true | ||
203 | }) | ||
204 | |||
205 | await stopFfmpeg(ffmpegCommand) | ||
206 | }) | ||
207 | |||
208 | it('Should have saved the replay on object storage', async function () { | ||
209 | this.timeout(220000) | ||
143 | 210 | ||
144 | const { videoLiveDetails } = await streamAndEnd(servers, videoUUIDPermanent) | 211 | await waitUntilLiveWaitingOnAllServers(servers, videoUUIDPermanent) |
212 | await waitJobs(servers) | ||
145 | 213 | ||
146 | const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) | 214 | const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent }) |
215 | const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) | ||
147 | 216 | ||
148 | for (const server of servers) { | 217 | await checkFilesExist(servers, replay.uuid, 5) |
149 | const files = await getFiles(server, replay.uuid) | 218 | }) |
150 | expect(files).to.have.lengthOf(5) | ||
151 | 219 | ||
152 | await checkFiles(files) | 220 | it('Should have cleaned up live files from object storage', async function () { |
153 | } | 221 | await checkFilesCleanup(servers[0], videoUUIDPermanent, resolutions) |
222 | }) | ||
154 | }) | 223 | }) |
155 | }) | 224 | }) |
156 | 225 | ||
diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts index 4bd4786fc..aa79622cb 100644 --- a/server/tests/shared/live.ts +++ b/server/tests/shared/live.ts | |||
@@ -3,39 +3,92 @@ | |||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { pathExists, readdir } from 'fs-extra' | 4 | import { pathExists, readdir } from 'fs-extra' |
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { LiveVideo } from '@shared/models' | 6 | import { wait } from '@shared/core-utils' |
7 | import { PeerTubeServer } from '@shared/server-commands' | 7 | import { LiveVideo, VideoStreamingPlaylistType } from '@shared/models' |
8 | import { ObjectStorageCommand, PeerTubeServer } from '@shared/server-commands' | ||
9 | import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists' | ||
8 | 10 | ||
9 | async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) { | 11 | async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) { |
10 | let live: LiveVideo | ||
11 | |||
12 | try { | ||
13 | live = await server.live.get({ videoId: videoUUID }) | ||
14 | } catch {} | ||
15 | |||
16 | const basePath = server.servers.buildDirectory('streaming-playlists') | 12 | const basePath = server.servers.buildDirectory('streaming-playlists') |
17 | const hlsPath = join(basePath, 'hls', videoUUID) | 13 | const hlsPath = join(basePath, 'hls', videoUUID) |
18 | 14 | ||
19 | if (savedResolutions.length === 0) { | 15 | if (savedResolutions.length === 0) { |
16 | return checkUnsavedLiveCleanup(server, videoUUID, hlsPath) | ||
17 | } | ||
18 | |||
19 | return checkSavedLiveCleanup(hlsPath, savedResolutions) | ||
20 | } | ||
21 | |||
22 | // --------------------------------------------------------------------------- | ||
20 | 23 | ||
21 | if (live?.permanentLive) { | 24 | async function testVideoResolutions (options: { |
22 | expect(await pathExists(hlsPath)).to.be.true | 25 | originServer: PeerTubeServer |
26 | servers: PeerTubeServer[] | ||
27 | liveVideoId: string | ||
28 | resolutions: number[] | ||
29 | transcoded: boolean | ||
30 | objectStorage: boolean | ||
31 | }) { | ||
32 | const { originServer, servers, liveVideoId, resolutions, transcoded, objectStorage } = options | ||
23 | 33 | ||
24 | const hlsFiles = await readdir(hlsPath) | 34 | for (const server of servers) { |
25 | expect(hlsFiles).to.have.lengthOf(1) // Only replays directory | 35 | const { data } = await server.videos.list() |
36 | expect(data.find(v => v.uuid === liveVideoId)).to.exist | ||
26 | 37 | ||
27 | const replayDir = join(hlsPath, 'replay') | 38 | const video = await server.videos.get({ id: liveVideoId }) |
28 | expect(await pathExists(replayDir)).to.be.true | 39 | expect(video.streamingPlaylists).to.have.lengthOf(1) |
29 | 40 | ||
30 | const replayFiles = await readdir(join(hlsPath, 'replay')) | 41 | const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) |
31 | expect(replayFiles).to.have.lengthOf(0) | 42 | expect(hlsPlaylist).to.exist |
32 | } else { | 43 | expect(hlsPlaylist.files).to.have.lengthOf(0) // Only fragmented mp4 files are displayed |
33 | expect(await pathExists(hlsPath)).to.be.false | 44 | |
45 | await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions, transcoded }) | ||
46 | |||
47 | if (objectStorage) { | ||
48 | expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getPlaylistBaseUrl()) | ||
34 | } | 49 | } |
35 | 50 | ||
36 | return | 51 | for (let i = 0; i < resolutions.length; i++) { |
52 | const segmentNum = 3 | ||
53 | const segmentName = `${i}-00000${segmentNum}.ts` | ||
54 | await originServer.live.waitUntilSegmentGeneration({ videoUUID: video.uuid, playlistNumber: i, segment: segmentNum }) | ||
55 | |||
56 | const baseUrl = objectStorage | ||
57 | ? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls' | ||
58 | : originServer.url + '/static/streaming-playlists/hls' | ||
59 | |||
60 | if (objectStorage) { | ||
61 | // Playlist file upload | ||
62 | await wait(500) | ||
63 | |||
64 | expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getPlaylistBaseUrl()) | ||
65 | } | ||
66 | |||
67 | const subPlaylist = await originServer.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/${i}.m3u8` }) | ||
68 | |||
69 | expect(subPlaylist).to.contain(segmentName) | ||
70 | |||
71 | await checkLiveSegmentHash({ | ||
72 | server, | ||
73 | baseUrlSegment: baseUrl, | ||
74 | videoUUID: video.uuid, | ||
75 | segmentName, | ||
76 | hlsPlaylist | ||
77 | }) | ||
78 | } | ||
37 | } | 79 | } |
80 | } | ||
81 | |||
82 | // --------------------------------------------------------------------------- | ||
83 | |||
84 | export { | ||
85 | checkLiveCleanup, | ||
86 | testVideoResolutions | ||
87 | } | ||
38 | 88 | ||
89 | // --------------------------------------------------------------------------- | ||
90 | |||
91 | async function checkSavedLiveCleanup (hlsPath: string, savedResolutions: number[] = []) { | ||
39 | const files = await readdir(hlsPath) | 92 | const files = await readdir(hlsPath) |
40 | 93 | ||
41 | // fragmented file and playlist per resolution + master playlist + segments sha256 json file | 94 | // fragmented file and playlist per resolution + master playlist + segments sha256 json file |
@@ -56,6 +109,27 @@ async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, save | |||
56 | expect(shaFile).to.exist | 109 | expect(shaFile).to.exist |
57 | } | 110 | } |
58 | 111 | ||
59 | export { | 112 | async function checkUnsavedLiveCleanup (server: PeerTubeServer, videoUUID: string, hlsPath: string) { |
60 | checkLiveCleanup | 113 | let live: LiveVideo |
114 | |||
115 | try { | ||
116 | live = await server.live.get({ videoId: videoUUID }) | ||
117 | } catch {} | ||
118 | |||
119 | if (live?.permanentLive) { | ||
120 | expect(await pathExists(hlsPath)).to.be.true | ||
121 | |||
122 | const hlsFiles = await readdir(hlsPath) | ||
123 | expect(hlsFiles).to.have.lengthOf(1) // Only replays directory | ||
124 | |||
125 | const replayDir = join(hlsPath, 'replay') | ||
126 | expect(await pathExists(replayDir)).to.be.true | ||
127 | |||
128 | const replayFiles = await readdir(join(hlsPath, 'replay')) | ||
129 | expect(replayFiles).to.have.lengthOf(0) | ||
130 | |||
131 | return | ||
132 | } | ||
133 | |||
134 | expect(await pathExists(hlsPath)).to.be.false | ||
61 | } | 135 | } |
diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts index 4d82b3654..eff34944b 100644 --- a/server/tests/shared/streaming-playlists.ts +++ b/server/tests/shared/streaming-playlists.ts | |||
@@ -26,7 +26,7 @@ async function checkSegmentHash (options: { | |||
26 | const offset = parseInt(matches[2], 10) | 26 | const offset = parseInt(matches[2], 10) |
27 | const range = `${offset}-${offset + length - 1}` | 27 | const range = `${offset}-${offset + length - 1}` |
28 | 28 | ||
29 | const segmentBody = await command.getSegment({ | 29 | const segmentBody = await command.getFragmentedSegment({ |
30 | url: `${baseUrlSegment}/${videoName}`, | 30 | url: `${baseUrlSegment}/${videoName}`, |
31 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, | 31 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, |
32 | range: `bytes=${range}` | 32 | range: `bytes=${range}` |
@@ -46,7 +46,7 @@ async function checkLiveSegmentHash (options: { | |||
46 | const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options | 46 | const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options |
47 | const command = server.streamingPlaylists | 47 | const command = server.streamingPlaylists |
48 | 48 | ||
49 | const segmentBody = await command.getSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` }) | 49 | const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` }) |
50 | const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) | 50 | const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) |
51 | 51 | ||
52 | expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) | 52 | expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) |
@@ -56,15 +56,16 @@ async function checkResolutionsInMasterPlaylist (options: { | |||
56 | server: PeerTubeServer | 56 | server: PeerTubeServer |
57 | playlistUrl: string | 57 | playlistUrl: string |
58 | resolutions: number[] | 58 | resolutions: number[] |
59 | transcoded?: boolean // default true | ||
59 | }) { | 60 | }) { |
60 | const { server, playlistUrl, resolutions } = options | 61 | const { server, playlistUrl, resolutions, transcoded = true } = options |
61 | 62 | ||
62 | const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl }) | 63 | const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl }) |
63 | 64 | ||
64 | for (const resolution of resolutions) { | 65 | for (const resolution of resolutions) { |
65 | const reg = new RegExp( | 66 | const reg = transcoded |
66 | '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"' | 67 | ? new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"') |
67 | ) | 68 | : new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + '') |
68 | 69 | ||
69 | expect(masterPlaylist).to.match(reg) | 70 | expect(masterPlaylist).to.match(reg) |
70 | } | 71 | } |