aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/transcoding
diff options
context:
space:
mode:
authorJelle Besseling <jelle@pingiun.com>2021-08-17 08:26:20 +0200
committerGitHub <noreply@github.com>2021-08-17 08:26:20 +0200
commit0305db28c98fd6cf43a3c50ba92c76215e99d512 (patch)
tree33b753a19728d9f453c1aa4f19b36ac797e5fe80 /server/lib/transcoding
parentf88ae8f5bc223579313b28582de9101944a4a814 (diff)
downloadPeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.gz
PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.zst
PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.zip
Add support for saving video files to object storage (#4290)
* Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/lib/transcoding')
-rw-r--r--server/lib/transcoding/video-transcoding.ts228
1 files changed, 115 insertions, 113 deletions
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index d2a556360..ee228c011 100644
--- a/server/lib/transcoding/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -4,13 +4,13 @@ import { basename, extname as extnameUtil, join } from 'path'
4import { toEven } from '@server/helpers/core-utils' 4import { toEven } from '@server/helpers/core-utils'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
7import { VideoResolution } from '../../../shared/models/videos' 7import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' 9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils'
10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' 10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
11import { logger } from '../../helpers/logger' 11import { logger } from '../../helpers/logger'
12import { CONFIG } from '../../initializers/config' 12import { CONFIG } from '../../initializers/config'
13import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' 13import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
14import { VideoFileModel } from '../../models/video/video-file' 14import { VideoFileModel } from '../../models/video/video-file'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' 15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' 16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
@@ -19,9 +19,9 @@ import {
19 generateHlsSha256SegmentsFilename, 19 generateHlsSha256SegmentsFilename,
20 generateHLSVideoFilename, 20 generateHLSVideoFilename,
21 generateWebTorrentVideoFilename, 21 generateWebTorrentVideoFilename,
22 getHlsResolutionPlaylistFilename, 22 getHlsResolutionPlaylistFilename
23 getVideoFilePath 23} from '../paths'
24} from '../video-paths' 24import { VideoPathManager } from '../video-path-manager'
25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' 25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
26 26
27/** 27/**
@@ -32,159 +32,162 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
32 */ 32 */
33 33
34// Optimize the original video file and replace it. The resolution is not changed. 34// Optimize the original video file and replace it. The resolution is not changed.
35async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { 35function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) {
36 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 36 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
37 const newExtname = '.mp4' 37 const newExtname = '.mp4'
38 38
39 const videoInputPath = getVideoFilePath(video, inputVideoFile) 39 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => {
40 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 40 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
41 41
42 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) 42 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath)
43 ? 'quick-transcode' 43 ? 'quick-transcode'
44 : 'video' 44 : 'video'
45 45
46 const resolution = toEven(inputVideoFile.resolution) 46 const resolution = toEven(inputVideoFile.resolution)
47 47
48 const transcodeOptions: TranscodeOptions = { 48 const transcodeOptions: TranscodeOptions = {
49 type: transcodeType, 49 type: transcodeType,
50 50
51 inputPath: videoInputPath, 51 inputPath: videoInputPath,
52 outputPath: videoTranscodedPath, 52 outputPath: videoTranscodedPath,
53 53
54 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 54 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
55 profile: CONFIG.TRANSCODING.PROFILE, 55 profile: CONFIG.TRANSCODING.PROFILE,
56 56
57 resolution, 57 resolution,
58 58
59 job 59 job
60 } 60 }
61 61
62 // Could be very long! 62 // Could be very long!
63 await transcode(transcodeOptions) 63 await transcode(transcodeOptions)
64 64
65 try { 65 try {
66 await remove(videoInputPath) 66 await remove(videoInputPath)
67 67
68 // Important to do this before getVideoFilename() to take in account the new filename 68 // Important to do this before getVideoFilename() to take in account the new filename
69 inputVideoFile.extname = newExtname 69 inputVideoFile.extname = newExtname
70 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) 70 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
71 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
71 72
72 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 73 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
73 74
74 await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 75 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
75 76
76 return transcodeType 77 return { transcodeType, videoFile }
77 } catch (err) { 78 } catch (err) {
78 // Auto destruction... 79 // Auto destruction...
79 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) 80 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err }))
80 81
81 throw err 82 throw err
82 } 83 }
84 })
83} 85}
84 86
85// Transcode the original video file to a lower resolution. 87// Transcode the original video file to a lower resolution
86async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { 88// We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
89function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) {
87 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 90 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
88 const extname = '.mp4' 91 const extname = '.mp4'
89 92
90 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed 93 return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => {
91 const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile()) 94 const newVideoFile = new VideoFileModel({
95 resolution,
96 extname,
97 filename: generateWebTorrentVideoFilename(resolution, extname),
98 size: 0,
99 videoId: video.id
100 })
92 101
93 const newVideoFile = new VideoFileModel({ 102 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
94 resolution, 103 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
95 extname,
96 filename: generateWebTorrentVideoFilename(resolution, extname),
97 size: 0,
98 videoId: video.id
99 })
100 104
101 const videoOutputPath = getVideoFilePath(video, newVideoFile) 105 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
102 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) 106 ? {
107 type: 'only-audio' as 'only-audio',
103 108
104 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO 109 inputPath: videoInputPath,
105 ? { 110 outputPath: videoTranscodedPath,
106 type: 'only-audio' as 'only-audio',
107 111
108 inputPath: videoInputPath, 112 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
109 outputPath: videoTranscodedPath, 113 profile: CONFIG.TRANSCODING.PROFILE,
110 114
111 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 115 resolution,
112 profile: CONFIG.TRANSCODING.PROFILE,
113 116
114 resolution, 117 job
118 }
119 : {
120 type: 'video' as 'video',
121 inputPath: videoInputPath,
122 outputPath: videoTranscodedPath,
115 123
116 job 124 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
117 } 125 profile: CONFIG.TRANSCODING.PROFILE,
118 : {
119 type: 'video' as 'video',
120 inputPath: videoInputPath,
121 outputPath: videoTranscodedPath,
122 126
123 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 127 resolution,
124 profile: CONFIG.TRANSCODING.PROFILE, 128 isPortraitMode: isPortrait,
125 129
126 resolution, 130 job
127 isPortraitMode: isPortrait, 131 }
128 132
129 job 133 await transcode(transcodeOptions)
130 }
131
132 await transcode(transcodeOptions)
133 134
134 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) 135 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
136 })
135} 137}
136 138
137// Merge an image with an audio file to create a video 139// Merge an image with an audio file to create a video
138async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { 140function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) {
139 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 141 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
140 const newExtname = '.mp4' 142 const newExtname = '.mp4'
141 143
142 const inputVideoFile = video.getMinQualityFile() 144 const inputVideoFile = video.getMinQualityFile()
143 145
144 const audioInputPath = getVideoFilePath(video, inputVideoFile) 146 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => {
145 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 147 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
146 148
147 // If the user updates the video preview during transcoding 149 // If the user updates the video preview during transcoding
148 const previewPath = video.getPreview().getPath() 150 const previewPath = video.getPreview().getPath()
149 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) 151 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
150 await copyFile(previewPath, tmpPreviewPath) 152 await copyFile(previewPath, tmpPreviewPath)
151 153
152 const transcodeOptions = { 154 const transcodeOptions = {
153 type: 'merge-audio' as 'merge-audio', 155 type: 'merge-audio' as 'merge-audio',
154 156
155 inputPath: tmpPreviewPath, 157 inputPath: tmpPreviewPath,
156 outputPath: videoTranscodedPath, 158 outputPath: videoTranscodedPath,
157 159
158 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 160 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
159 profile: CONFIG.TRANSCODING.PROFILE, 161 profile: CONFIG.TRANSCODING.PROFILE,
160 162
161 audioPath: audioInputPath, 163 audioPath: audioInputPath,
162 resolution, 164 resolution,
163 165
164 job 166 job
165 } 167 }
166 168
167 try { 169 try {
168 await transcode(transcodeOptions) 170 await transcode(transcodeOptions)
169 171
170 await remove(audioInputPath) 172 await remove(audioInputPath)
171 await remove(tmpPreviewPath) 173 await remove(tmpPreviewPath)
172 } catch (err) { 174 } catch (err) {
173 await remove(tmpPreviewPath) 175 await remove(tmpPreviewPath)
174 throw err 176 throw err
175 } 177 }
176 178
177 // Important to do this before getVideoFilename() to take in account the new file extension 179 // Important to do this before getVideoFilename() to take in account the new file extension
178 inputVideoFile.extname = newExtname 180 inputVideoFile.extname = newExtname
179 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) 181 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
180 182
181 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 183 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
182 // ffmpeg generated a new video file, so update the video duration 184 // ffmpeg generated a new video file, so update the video duration
183 // See https://trac.ffmpeg.org/ticket/5456 185 // See https://trac.ffmpeg.org/ticket/5456
184 video.duration = await getDurationFromVideoFile(videoTranscodedPath) 186 video.duration = await getDurationFromVideoFile(videoTranscodedPath)
185 await video.save() 187 await video.save()
186 188
187 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 189 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
190 })
188} 191}
189 192
190// Concat TS segments from a live video to a fragmented mp4 HLS playlist 193// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@@ -258,7 +261,7 @@ async function onWebTorrentVideoFileTranscoding (
258 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 261 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
259 video.VideoFiles = await video.$get('VideoFiles') 262 video.VideoFiles = await video.$get('VideoFiles')
260 263
261 return video 264 return { video, videoFile }
262} 265}
263 266
264async function generateHlsPlaylistCommon (options: { 267async function generateHlsPlaylistCommon (options: {
@@ -335,14 +338,13 @@ async function generateHlsPlaylistCommon (options: {
335 videoStreamingPlaylistId: playlist.id 338 videoStreamingPlaylistId: playlist.id
336 }) 339 })
337 340
338 const videoFilePath = getVideoFilePath(playlist, newVideoFile) 341 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
339 342
340 // Move files from tmp transcoded directory to the appropriate place 343 // Move files from tmp transcoded directory to the appropriate place
341 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 344 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
342 await ensureDir(baseHlsDirectory)
343 345
344 // Move playlist file 346 // Move playlist file
345 const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) 347 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
346 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) 348 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
347 // Move video file 349 // Move video file
348 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) 350 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
@@ -355,7 +357,7 @@ async function generateHlsPlaylistCommon (options: {
355 357
356 await createTorrentAndSetInfoHash(playlist, newVideoFile) 358 await createTorrentAndSetInfoHash(playlist, newVideoFile)
357 359
358 await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) 360 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
359 361
360 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo 362 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
361 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') 363 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
@@ -368,5 +370,5 @@ async function generateHlsPlaylistCommon (options: {
368 await updateMasterHLSPlaylist(video, playlistWithFiles) 370 await updateMasterHLSPlaylist(video, playlistWithFiles)
369 await updateSha256VODSegments(video, playlistWithFiles) 371 await updateSha256VODSegments(video, playlistWithFiles)
370 372
371 return resolutionPlaylistPath 373 return { resolutionPlaylistPath, videoFile: savedVideoFile }
372} 374}