aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-02-01 14:19:44 +0100
committerChocobozzz <me@florianbigard.com>2022-02-01 14:19:44 +0100
commita2caee9f5162232234de2e8aae6957cc7f38c853 (patch)
treef6ca87b03ed80ca4b6625a7b2b31706cd4f831ad /server/lib
parent0f11ec8dd32b50897c18588db948e96cf0fc2c70 (diff)
downloadPeerTube-a2caee9f5162232234de2e8aae6957cc7f38c853.tar.gz
PeerTube-a2caee9f5162232234de2e8aae6957cc7f38c853.tar.zst
PeerTube-a2caee9f5162232234de2e8aae6957cc7f38c853.zip
Fix HLS re transcoding with object storage enabled
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/hls.ts15
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts17
-rw-r--r--server/lib/object-storage/videos.ts6
3 files changed, 28 insertions, 10 deletions
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 1574ff27b..985f50587 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -3,6 +3,7 @@ import { flatten, uniq } from 'lodash'
3import { basename, dirname, join } from 'path' 3import { basename, dirname, join } from 'path'
4import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' 4import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
5import { sha256 } from '@shared/extra-utils' 5import { sha256 } from '@shared/extra-utils'
6import { VideoStorage } from '@shared/models'
6import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' 7import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
7import { logger } from '../helpers/logger' 8import { logger } from '../helpers/logger'
8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' 9import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
@@ -12,6 +13,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
12import { sequelizeTypescript } from '../initializers/database' 13import { sequelizeTypescript } from '../initializers/database'
13import { VideoFileModel } from '../models/video/video-file' 14import { VideoFileModel } from '../models/video/video-file'
14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 15import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
16import { storeHLSFile } from './object-storage'
15import { getHlsResolutionPlaylistFilename } from './paths' 17import { getHlsResolutionPlaylistFilename } from './paths'
16import { VideoPathManager } from './video-path-manager' 18import { VideoPathManager } from './video-path-manager'
17 19
@@ -58,8 +60,12 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl
58 }) 60 })
59 } 61 }
60 62
61 await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => { 63 await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => {
62 return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') 64 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
65
66 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
67 await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath)
68 }
63 }) 69 })
64} 70}
65 71
@@ -94,6 +100,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP
94 100
95 const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) 101 const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
96 await outputJSON(outputPath, json) 102 await outputJSON(outputPath, json)
103
104 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
105 await storeHLSFile(playlist, playlist.segmentsSha256Filename)
106 await remove(outputPath)
107 }
97} 108}
98 109
99async function buildSha256Segment (segmentPath: string) { 110async function buildSha256Segment (segmentPath: string) {
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 9e39322a8..69b441176 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -1,7 +1,7 @@
1import { Job } from 'bull' 1import { Job } from 'bull'
2import { remove } from 'fs-extra' 2import { remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { logger } from '@server/helpers/logger' 4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { updateTorrentMetadata } from '@server/helpers/webtorrent' 5import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' 7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
@@ -13,6 +13,8 @@ import { VideoJobInfoModel } from '@server/models/video/video-job-info'
13import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' 13import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
14import { MoveObjectStoragePayload, VideoStorage } from '@shared/models' 14import { MoveObjectStoragePayload, VideoStorage } from '@shared/models'
15 15
16const lTagsBase = loggerTagsFactory('move-object-storage')
17
16export async function processMoveToObjectStorage (job: Job) { 18export async function processMoveToObjectStorage (job: Job) {
17 const payload = job.data as MoveObjectStoragePayload 19 const payload = job.data as MoveObjectStoragePayload
18 logger.info('Moving video %s in job %d.', payload.videoUUID, job.id) 20 logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
@@ -20,26 +22,33 @@ export async function processMoveToObjectStorage (job: Job) {
20 const video = await VideoModel.loadWithFiles(payload.videoUUID) 22 const video = await VideoModel.loadWithFiles(payload.videoUUID)
21 // No video, maybe deleted? 23 // No video, maybe deleted?
22 if (!video) { 24 if (!video) {
23 logger.info('Can\'t process job %d, video does not exist.', job.id) 25 logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
24 return undefined 26 return undefined
25 } 27 }
26 28
29 const lTags = lTagsBase(video.uuid, video.url)
30
27 try { 31 try {
28 if (video.VideoFiles) { 32 if (video.VideoFiles) {
33 logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags)
34
29 await moveWebTorrentFiles(video) 35 await moveWebTorrentFiles(video)
30 } 36 }
31 37
32 if (video.VideoStreamingPlaylists) { 38 if (video.VideoStreamingPlaylists) {
39 logger.debug('Moving HLS playlist of %s.', video.uuid)
40
33 await moveHLSFiles(video) 41 await moveHLSFiles(video)
34 } 42 }
35 43
36 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') 44 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
37 if (pendingMove === 0) { 45 if (pendingMove === 0) {
38 logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id) 46 logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id, lTags)
47
39 await doAfterLastJob(video, payload.isNewVideo) 48 await doAfterLastJob(video, payload.isNewVideo)
40 } 49 }
41 } catch (err) { 50 } catch (err) {
42 logger.error('Cannot move video %s to object storage.', video.url, { err }) 51 logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTags })
43 52
44 await moveToFailedMoveToObjectStorageState(video) 53 await moveToFailedMoveToObjectStorageState(video)
45 await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') 54 await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 8988f3e2a..066b48ab0 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -6,11 +6,9 @@ import { getHLSDirectory } from '../paths'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' 6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' 7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8 8
9function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) { 9function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) {
10 const baseHlsDirectory = getHLSDirectory(playlist.Video)
11
12 return storeObject({ 10 return storeObject({
13 inputPath: join(baseHlsDirectory, filename), 11 inputPath: path ?? join(getHLSDirectory(playlist.Video), filename),
14 objectStorageKey: generateHLSObjectStorageKey(playlist, filename), 12 objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
15 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS 13 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
16 }) 14 })