aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorJelle Besseling <jelle@pingiun.com>2021-08-17 08:26:20 +0200
committerGitHub <noreply@github.com>2021-08-17 08:26:20 +0200
commit0305db28c98fd6cf43a3c50ba92c76215e99d512 (patch)
tree33b753a19728d9f453c1aa4f19b36ac797e5fe80 /server/lib
parentf88ae8f5bc223579313b28582de9101944a4a814 (diff)
downloadPeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.gz
PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.zst
PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.zip
Add support for saving video files to object storage (#4290)
* Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts2
-rw-r--r--server/lib/hls.ts82
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts114
-rw-r--r--server/lib/job-queue/handlers/video-file-import.ts27
-rw-r--r--server/lib/job-queue/handlers/video-import.ts16
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts18
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts128
-rw-r--r--server/lib/job-queue/job-queue.ts13
-rw-r--r--server/lib/live/live-manager.ts2
-rw-r--r--server/lib/live/live-utils.ts4
-rw-r--r--server/lib/live/shared/muxing-session.ts4
-rw-r--r--server/lib/object-storage/index.ts3
-rw-r--r--server/lib/object-storage/keys.ts20
-rw-r--r--server/lib/object-storage/shared/client.ts56
-rw-r--r--server/lib/object-storage/shared/index.ts3
-rw-r--r--server/lib/object-storage/shared/logger.ts7
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts229
-rw-r--r--server/lib/object-storage/urls.ts40
-rw-r--r--server/lib/object-storage/videos.ts72
-rw-r--r--server/lib/paths.ts (renamed from server/lib/video-paths.ts)66
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts2
-rw-r--r--server/lib/thumbnail.ts30
-rw-r--r--server/lib/transcoding/video-transcoding.ts228
-rw-r--r--server/lib/video-path-manager.ts139
-rw-r--r--server/lib/video-state.ts99
-rw-r--r--server/lib/video-urls.ts31
-rw-r--r--server/lib/video.ts47
27 files changed, 1141 insertions, 341 deletions
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index 1fa16295d..bd9ed45a9 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -6,7 +6,7 @@ import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/vide
6import { logger } from '@server/helpers/logger' 6import { logger } from '@server/helpers/logger'
7import { getExtFromMimetype } from '@server/helpers/video' 7import { getExtFromMimetype } from '@server/helpers/video'
8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' 8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { generateTorrentFileName } from '@server/lib/video-paths' 9import { generateTorrentFileName } from '@server/lib/paths'
10import { VideoCaptionModel } from '@server/models/video/video-caption' 10import { VideoCaptionModel } from '@server/models/video/video-caption'
11import { VideoFileModel } from '@server/models/video/video-file' 11import { VideoFileModel } from '@server/models/video/video-file'
12import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 12import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 0e77ab9fa..0828a2d0f 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,4 +1,4 @@
1import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra' 1import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
2import { flatten, uniq } from 'lodash' 2import { flatten, uniq } from 'lodash'
3import { basename, dirname, join } from 'path' 3import { basename, dirname, join } from 'path'
4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' 4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
@@ -8,11 +8,12 @@ import { logger } from '../helpers/logger'
8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' 8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
9import { generateRandomString } from '../helpers/utils' 9import { generateRandomString } from '../helpers/utils'
10import { CONFIG } from '../initializers/config' 10import { CONFIG } from '../initializers/config'
11import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' 11import { P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants'
12import { sequelizeTypescript } from '../initializers/database' 12import { sequelizeTypescript } from '../initializers/database'
13import { VideoFileModel } from '../models/video/video-file' 13import { VideoFileModel } from '../models/video/video-file'
14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
15import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' 15import { getHlsResolutionPlaylistFilename } from './paths'
16import { VideoPathManager } from './video-path-manager'
16 17
17async function updateStreamingPlaylistsInfohashesIfNeeded () { 18async function updateStreamingPlaylistsInfohashesIfNeeded () {
18 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() 19 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
@@ -31,75 +32,66 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () {
31} 32}
32 33
33async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { 34async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
34 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
35
36 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] 35 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
37 36
38 const masterPlaylistPath = join(directory, playlist.playlistFilename)
39
40 for (const file of playlist.VideoFiles) { 37 for (const file of playlist.VideoFiles) {
41 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) 38 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
42 39
43 // If we did not generated a playlist for this resolution, skip 40 await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => {
44 const filePlaylistPath = join(directory, playlistFilename) 41 const size = await getVideoStreamSize(videoFilePath)
45 if (await pathExists(filePlaylistPath) === false) continue
46
47 const videoFilePath = getVideoFilePath(playlist, file)
48 42
49 const size = await getVideoStreamSize(videoFilePath) 43 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
44 const resolution = `RESOLUTION=${size.width}x${size.height}`
50 45
51 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) 46 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
52 const resolution = `RESOLUTION=${size.width}x${size.height}` 47 if (file.fps) line += ',FRAME-RATE=' + file.fps
53 48
54 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` 49 const codecs = await Promise.all([
55 if (file.fps) line += ',FRAME-RATE=' + file.fps 50 getVideoStreamCodec(videoFilePath),
51 getAudioStreamCodec(videoFilePath)
52 ])
56 53
57 const codecs = await Promise.all([ 54 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
58 getVideoStreamCodec(videoFilePath),
59 getAudioStreamCodec(videoFilePath)
60 ])
61 55
62 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` 56 masterPlaylists.push(line)
63 57 masterPlaylists.push(playlistFilename)
64 masterPlaylists.push(line) 58 })
65 masterPlaylists.push(playlistFilename)
66 } 59 }
67 60
68 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') 61 await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
62 return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
63 })
69} 64}
70 65
71async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { 66async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
72 const json: { [filename: string]: { [range: string]: string } } = {} 67 const json: { [filename: string]: { [range: string]: string } } = {}
73 68
74 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
75
76 // For all the resolutions available for this video 69 // For all the resolutions available for this video
77 for (const file of playlist.VideoFiles) { 70 for (const file of playlist.VideoFiles) {
78 const rangeHashes: { [range: string]: string } = {} 71 const rangeHashes: { [range: string]: string } = {}
79 72
80 const videoPath = getVideoFilePath(playlist, file) 73 await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => {
81 const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename))
82
83 // Maybe the playlist is not generated for this resolution yet
84 if (!await pathExists(resolutionPlaylistPath)) continue
85 74
86 const playlistContent = await readFile(resolutionPlaylistPath) 75 return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => {
87 const ranges = getRangesFromPlaylist(playlistContent.toString()) 76 const playlistContent = await readFile(resolutionPlaylistPath)
77 const ranges = getRangesFromPlaylist(playlistContent.toString())
88 78
89 const fd = await open(videoPath, 'r') 79 const fd = await open(videoPath, 'r')
90 for (const range of ranges) { 80 for (const range of ranges) {
91 const buf = Buffer.alloc(range.length) 81 const buf = Buffer.alloc(range.length)
92 await read(fd, buf, 0, range.length, range.offset) 82 await read(fd, buf, 0, range.length, range.offset)
93 83
94 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) 84 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
95 } 85 }
96 await close(fd) 86 await close(fd)
97 87
98 const videoFilename = file.filename 88 const videoFilename = file.filename
99 json[videoFilename] = rangeHashes 89 json[videoFilename] = rangeHashes
90 })
91 })
100 } 92 }
101 93
102 const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) 94 const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
103 await outputJSON(outputPath, json) 95 await outputJSON(outputPath, json)
104} 96}
105 97
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
new file mode 100644
index 000000000..a0c58d211
--- /dev/null
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -0,0 +1,114 @@
1import * as Bull from 'bull'
2import { remove } from 'fs-extra'
3import { join } from 'path'
4import { logger } from '@server/helpers/logger'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config'
7import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
9import { moveToNextState } from '@server/lib/video-state'
10import { VideoModel } from '@server/models/video/video'
11import { VideoJobInfoModel } from '@server/models/video/video-job-info'
12import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
13import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared'
14
15export async function processMoveToObjectStorage (job: Bull.Job) {
16 const payload = job.data as MoveObjectStoragePayload
17 logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
18
19 const video = await VideoModel.loadWithFiles(payload.videoUUID)
20 // No video, maybe deleted?
21 if (!video) {
22 logger.info('Can\'t process job %d, video does not exist.', job.id)
23 return undefined
24 }
25
26 if (video.VideoFiles) {
27 await moveWebTorrentFiles(video)
28 }
29
30 if (video.VideoStreamingPlaylists) {
31 await moveHLSFiles(video)
32 }
33
34 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
35 if (pendingMove === 0) {
36 logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
37 await doAfterLastJob(video, payload.isNewVideo)
38 }
39
40 return payload.videoUUID
41}
42
43// ---------------------------------------------------------------------------
44
45async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
46 for (const file of video.VideoFiles) {
47 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
48
49 const fileUrl = await storeWebTorrentFile(file.filename)
50
51 const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename)
52 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
53 }
54}
55
56async function moveHLSFiles (video: MVideoWithAllFiles) {
57 for (const playlist of video.VideoStreamingPlaylists) {
58
59 for (const file of playlist.VideoFiles) {
60 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
61
62 // Resolution playlist
63 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
64 await storeHLSFile(playlist, video, playlistFilename)
65
66 // Resolution fragmented file
67 const fileUrl = await storeHLSFile(playlist, video, file.filename)
68
69 const oldPath = join(getHLSDirectory(video), file.filename)
70
71 await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
72 }
73 }
74}
75
76async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) {
77 for (const playlist of video.VideoStreamingPlaylists) {
78 if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
79
80 // Master playlist
81 playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename)
82 // Sha256 segments file
83 playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename)
84
85 playlist.storage = VideoStorage.OBJECT_STORAGE
86
87 await playlist.save()
88 }
89
90 // Remove empty hls video directory
91 if (video.VideoStreamingPlaylists) {
92 await remove(getHLSDirectory(video))
93 }
94
95 await moveToNextState(video, isNewVideo)
96}
97
98async function onFileMoved (options: {
99 videoOrPlaylist: MVideo | MStreamingPlaylistVideo
100 file: MVideoFile
101 fileUrl: string
102 oldPath: string
103}) {
104 const { videoOrPlaylist, file, fileUrl, oldPath } = options
105
106 file.fileUrl = fileUrl
107 file.storage = VideoStorage.OBJECT_STORAGE
108
109 await createTorrentAndSetInfoHash(videoOrPlaylist, file)
110 await file.save()
111
112 logger.debug('Removing %s because it\'s now on object storage', oldPath)
113 await remove(oldPath)
114}
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 2f4abf730..e8ee1f759 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -2,15 +2,19 @@ import * as Bull from 'bull'
2import { copy, stat } from 'fs-extra' 2import { copy, stat } from 'fs-extra'
3import { getLowercaseExtension } from '@server/helpers/core-utils' 3import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
5import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 5import { CONFIG } from '@server/initializers/config'
6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
7import { generateWebTorrentVideoFilename } from '@server/lib/paths'
8import { addMoveToObjectStorageJob } from '@server/lib/video'
9import { VideoPathManager } from '@server/lib/video-path-manager'
6import { UserModel } from '@server/models/user/user' 10import { UserModel } from '@server/models/user/user'
7import { MVideoFullLight } from '@server/types/models' 11import { MVideoFullLight } from '@server/types/models'
8import { VideoFileImportPayload } from '@shared/models' 12import { VideoFileImportPayload, VideoStorage } from '@shared/models'
9import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' 13import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
10import { logger } from '../../../helpers/logger' 14import { logger } from '../../../helpers/logger'
11import { VideoModel } from '../../../models/video/video' 15import { VideoModel } from '../../../models/video/video'
12import { VideoFileModel } from '../../../models/video/video-file' 16import { VideoFileModel } from '../../../models/video/video-file'
13import { onNewWebTorrentFileResolution } from './video-transcoding' 17import { createHlsJobIfEnabled } from './video-transcoding'
14 18
15async function processVideoFileImport (job: Bull.Job) { 19async function processVideoFileImport (job: Bull.Job) {
16 const payload = job.data as VideoFileImportPayload 20 const payload = job.data as VideoFileImportPayload
@@ -29,15 +33,19 @@ async function processVideoFileImport (job: Bull.Job) {
29 33
30 const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) 34 const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId)
31 35
32 const newResolutionPayload = { 36 await createHlsJobIfEnabled(user, {
33 type: 'new-resolution-to-webtorrent' as 'new-resolution-to-webtorrent',
34 videoUUID: video.uuid, 37 videoUUID: video.uuid,
35 resolution: data.resolution, 38 resolution: data.resolution,
36 isPortraitMode: data.isPortraitMode, 39 isPortraitMode: data.isPortraitMode,
37 copyCodecs: false, 40 copyCodecs: true,
38 isNewVideo: false 41 isMaxQuality: false
42 })
43
44 if (CONFIG.OBJECT_STORAGE.ENABLED) {
45 await addMoveToObjectStorageJob(video)
46 } else {
47 await federateVideoIfNeeded(video, false)
39 } 48 }
40 await onNewWebTorrentFileResolution(video, user, newResolutionPayload)
41 49
42 return video 50 return video
43} 51}
@@ -72,12 +80,13 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
72 resolution, 80 resolution,
73 extname: fileExt, 81 extname: fileExt,
74 filename: generateWebTorrentVideoFilename(resolution, fileExt), 82 filename: generateWebTorrentVideoFilename(resolution, fileExt),
83 storage: VideoStorage.FILE_SYSTEM,
75 size, 84 size,
76 fps, 85 fps,
77 videoId: video.id 86 videoId: video.id
78 }) 87 })
79 88
80 const outputPath = getVideoFilePath(video, newVideoFile) 89 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
81 await copy(inputFilePath, outputPath) 90 await copy(inputFilePath, outputPath)
82 91
83 video.VideoFiles.push(newVideoFile) 92 video.VideoFiles.push(newVideoFile)
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index fec553f2b..a5fa204f5 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -4,11 +4,13 @@ import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { retryTransactionWrapper } from '@server/helpers/database-utils' 4import { retryTransactionWrapper } from '@server/helpers/database-utils'
5import { YoutubeDL } from '@server/helpers/youtube-dl' 5import { YoutubeDL } from '@server/helpers/youtube-dl'
6import { isPostImportVideoAccepted } from '@server/lib/moderation' 6import { isPostImportVideoAccepted } from '@server/lib/moderation'
7import { generateWebTorrentVideoFilename } from '@server/lib/paths'
7import { Hooks } from '@server/lib/plugins/hooks' 8import { Hooks } from '@server/lib/plugins/hooks'
8import { ServerConfigManager } from '@server/lib/server-config-manager' 9import { ServerConfigManager } from '@server/lib/server-config-manager'
9import { isAbleToUploadVideo } from '@server/lib/user' 10import { isAbleToUploadVideo } from '@server/lib/user'
10import { addOptimizeOrMergeAudioJob } from '@server/lib/video' 11import { addMoveToObjectStorageJob, addOptimizeOrMergeAudioJob } from '@server/lib/video'
11import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { buildNextVideoState } from '@server/lib/video-state'
12import { ThumbnailModel } from '@server/models/video/thumbnail' 14import { ThumbnailModel } from '@server/models/video/thumbnail'
13import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' 15import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
14import { 16import {
@@ -25,7 +27,6 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
25import { logger } from '../../../helpers/logger' 27import { logger } from '../../../helpers/logger'
26import { getSecureTorrentName } from '../../../helpers/utils' 28import { getSecureTorrentName } from '../../../helpers/utils'
27import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' 29import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
28import { CONFIG } from '../../../initializers/config'
29import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' 30import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
30import { sequelizeTypescript } from '../../../initializers/database' 31import { sequelizeTypescript } from '../../../initializers/database'
31import { VideoModel } from '../../../models/video/video' 32import { VideoModel } from '../../../models/video/video'
@@ -100,7 +101,6 @@ type ProcessFileOptions = {
100} 101}
101async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { 102async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
102 let tempVideoPath: string 103 let tempVideoPath: string
103 let videoDestFile: string
104 let videoFile: VideoFileModel 104 let videoFile: VideoFileModel
105 105
106 try { 106 try {
@@ -159,7 +159,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
159 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) 159 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
160 160
161 // Move file 161 // Move file
162 videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile) 162 const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
163 await move(tempVideoPath, videoDestFile) 163 await move(tempVideoPath, videoDestFile)
164 tempVideoPath = null // This path is not used anymore 164 tempVideoPath = null // This path is not used anymore
165 165
@@ -204,7 +204,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
204 204
205 // Update video DB object 205 // Update video DB object
206 video.duration = duration 206 video.duration = duration
207 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED 207 video.state = buildNextVideoState(video.state)
208 await video.save({ transaction: t }) 208 await video.save({ transaction: t })
209 209
210 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) 210 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
@@ -245,6 +245,10 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
245 Notifier.Instance.notifyOnNewVideoIfNeeded(video) 245 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
246 } 246 }
247 247
248 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
249 return addMoveToObjectStorageJob(videoImportUpdated.Video)
250 }
251
248 // Create transcoding jobs? 252 // Create transcoding jobs?
249 if (video.state === VideoState.TO_TRANSCODE) { 253 if (video.state === VideoState.TO_TRANSCODE) {
250 await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) 254 await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User)
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index aa5bd573a..9ccf724c2 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -4,10 +4,11 @@ import { join } from 'path'
4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { VIDEO_LIVE } from '@server/initializers/constants' 5import { VIDEO_LIVE } from '@server/initializers/constants'
6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' 6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live'
7import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveDirectory } from '@server/lib/paths'
7import { generateVideoMiniature } from '@server/lib/thumbnail' 8import { generateVideoMiniature } from '@server/lib/thumbnail'
8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' 9import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
9import { publishAndFederateIfNeeded } from '@server/lib/video' 10import { VideoPathManager } from '@server/lib/video-path-manager'
10import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths' 11import { moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 12import { VideoModel } from '@server/models/video/video'
12import { VideoFileModel } from '@server/models/video/video-file' 13import { VideoFileModel } from '@server/models/video/video-file'
13import { VideoLiveModel } from '@server/models/video/video-live' 14import { VideoLiveModel } from '@server/models/video/video-live'
@@ -55,16 +56,15 @@ export {
55// --------------------------------------------------------------------------- 56// ---------------------------------------------------------------------------
56 57
57async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { 58async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) {
58 const hlsDirectory = getHLSDirectory(video, false) 59 const replayDirectory = VideoPathManager.Instance.getFSHLSOutputPath(video, VIDEO_LIVE.REPLAY_DIRECTORY)
59 const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY)
60 60
61 const rootFiles = await readdir(hlsDirectory) 61 const rootFiles = await readdir(getLiveDirectory(video))
62 62
63 const playlistFiles = rootFiles.filter(file => { 63 const playlistFiles = rootFiles.filter(file => {
64 return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename 64 return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename
65 }) 65 })
66 66
67 await cleanupLiveFiles(hlsDirectory) 67 await cleanupTMPLiveFiles(getLiveDirectory(video))
68 68
69 await live.destroy() 69 await live.destroy()
70 70
@@ -98,7 +98,7 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt
98 98
99 const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) 99 const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe)
100 100
101 const outputPath = await generateHlsPlaylistResolutionFromTS({ 101 const { resolutionPlaylistPath: outputPath } = await generateHlsPlaylistResolutionFromTS({
102 video: videoWithFiles, 102 video: videoWithFiles,
103 concatenatedTsFilePath, 103 concatenatedTsFilePath,
104 resolution, 104 resolution,
@@ -133,10 +133,10 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt
133 }) 133 })
134 } 134 }
135 135
136 await publishAndFederateIfNeeded(videoWithFiles, true) 136 await moveToNextState(videoWithFiles, false)
137} 137}
138 138
139async function cleanupLiveFiles (hlsDirectory: string) { 139async function cleanupTMPLiveFiles (hlsDirectory: string) {
140 if (!await pathExists(hlsDirectory)) return 140 if (!await pathExists(hlsDirectory)) return
141 141
142 const files = await readdir(hlsDirectory) 142 const files = await readdir(hlsDirectory)
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 876d1460c..b3149dde8 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -1,9 +1,11 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' 2import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils'
3import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' 3import { addTranscodingJob, getTranscodingJobPriority } from '@server/lib/video'
4import { getVideoFilePath } from '@server/lib/video-paths' 4import { VideoPathManager } from '@server/lib/video-path-manager'
5import { moveToNextState } from '@server/lib/video-state'
5import { UserModel } from '@server/models/user/user' 6import { UserModel } from '@server/models/user/user'
6import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' 7import { VideoJobInfoModel } from '@server/models/video/video-job-info'
8import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models'
7import { 9import {
8 HLSTranscodingPayload, 10 HLSTranscodingPayload,
9 MergeAudioTranscodingPayload, 11 MergeAudioTranscodingPayload,
@@ -16,17 +18,14 @@ import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
16import { logger } from '../../../helpers/logger' 18import { logger } from '../../../helpers/logger'
17import { CONFIG } from '../../../initializers/config' 19import { CONFIG } from '../../../initializers/config'
18import { VideoModel } from '../../../models/video/video' 20import { VideoModel } from '../../../models/video/video'
19import { federateVideoIfNeeded } from '../../activitypub/videos'
20import { Notifier } from '../../notifier'
21import { 21import {
22 generateHlsPlaylistResolution, 22 generateHlsPlaylistResolution,
23 mergeAudioVideofile, 23 mergeAudioVideofile,
24 optimizeOriginalVideofile, 24 optimizeOriginalVideofile,
25 transcodeNewWebTorrentResolution 25 transcodeNewWebTorrentResolution
26} from '../../transcoding/video-transcoding' 26} from '../../transcoding/video-transcoding'
27import { JobQueue } from '../job-queue'
28 27
29type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> 28type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
30 29
31const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { 30const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = {
32 'new-resolution-to-hls': handleHLSJob, 31 'new-resolution-to-hls': handleHLSJob,
@@ -69,15 +68,16 @@ async function handleHLSJob (job: Bull.Job, payload: HLSTranscodingPayload, vide
69 : video.getMaxQualityFile() 68 : video.getMaxQualityFile()
70 69
71 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() 70 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
72 const videoInputPath = getVideoFilePath(videoOrStreamingPlaylist, videoFileInput)
73 71
74 await generateHlsPlaylistResolution({ 72 await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => {
75 video, 73 return generateHlsPlaylistResolution({
76 videoInputPath, 74 video,
77 resolution: payload.resolution, 75 videoInputPath,
78 copyCodecs: payload.copyCodecs, 76 resolution: payload.resolution,
79 isPortraitMode: payload.isPortraitMode || false, 77 copyCodecs: payload.copyCodecs,
80 job 78 isPortraitMode: payload.isPortraitMode || false,
79 job
80 })
81 }) 81 })
82 82
83 await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) 83 await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload)
@@ -101,7 +101,7 @@ async function handleWebTorrentMergeAudioJob (job: Bull.Job, payload: MergeAudio
101} 101}
102 102
103async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { 103async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
104 const transcodeType = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) 104 const { transcodeType } = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job)
105 105
106 await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) 106 await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user)
107} 107}
@@ -121,10 +121,18 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay
121 video.VideoFiles = [] 121 video.VideoFiles = []
122 122
123 // Create HLS new resolution jobs 123 // Create HLS new resolution jobs
124 await createLowerResolutionsJobs(video, user, payload.resolution, payload.isPortraitMode, 'hls') 124 await createLowerResolutionsJobs({
125 video,
126 user,
127 videoFileResolution: payload.resolution,
128 isPortraitMode: payload.isPortraitMode,
129 isNewVideo: payload.isNewVideo ?? true,
130 type: 'hls'
131 })
125 } 132 }
126 133
127 return publishAndFederateIfNeeded(video) 134 await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
135 await moveToNextState(video, payload.isNewVideo)
128} 136}
129 137
130async function onVideoFileOptimizer ( 138async function onVideoFileOptimizer (
@@ -143,58 +151,54 @@ async function onVideoFileOptimizer (
143 // Video does not exist anymore 151 // Video does not exist anymore
144 if (!videoDatabase) return undefined 152 if (!videoDatabase) return undefined
145 153
146 let videoPublished = false
147
148 // Generate HLS version of the original file 154 // Generate HLS version of the original file
149 const originalFileHLSPayload = Object.assign({}, payload, { 155 const originalFileHLSPayload = {
156 ...payload,
157
150 isPortraitMode, 158 isPortraitMode,
151 resolution: videoDatabase.getMaxQualityFile().resolution, 159 resolution: videoDatabase.getMaxQualityFile().resolution,
152 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues 160 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
153 copyCodecs: transcodeType !== 'quick-transcode', 161 copyCodecs: transcodeType !== 'quick-transcode',
154 isMaxQuality: true 162 isMaxQuality: true
155 }) 163 }
156 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) 164 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
165 const hasNewResolutions = await createLowerResolutionsJobs({
166 video: videoDatabase,
167 user,
168 videoFileResolution: resolution,
169 isPortraitMode,
170 type: 'webtorrent',
171 isNewVideo: payload.isNewVideo ?? true
172 })
157 173
158 const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, resolution, isPortraitMode, 'webtorrent') 174 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
159 175
176 // Move to next state if there are no other resolutions to generate
160 if (!hasHls && !hasNewResolutions) { 177 if (!hasHls && !hasNewResolutions) {
161 // No transcoding to do, it's now published 178 await moveToNextState(videoDatabase, payload.isNewVideo)
162 videoPublished = await videoDatabase.publishIfNeededAndSave(undefined)
163 } 179 }
164
165 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo)
166
167 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
168 if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
169} 180}
170 181
171async function onNewWebTorrentFileResolution ( 182async function onNewWebTorrentFileResolution (
172 video: MVideoUUID, 183 video: MVideo,
173 user: MUserId, 184 user: MUserId,
174 payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload 185 payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload
175) { 186) {
176 await publishAndFederateIfNeeded(video) 187 await createHlsJobIfEnabled(user, { ...payload, copyCodecs: true, isMaxQuality: false })
188 await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
177 189
178 await createHlsJobIfEnabled(user, Object.assign({}, payload, { copyCodecs: true, isMaxQuality: false })) 190 await moveToNextState(video, payload.isNewVideo)
179} 191}
180 192
181// ---------------------------------------------------------------------------
182
183export {
184 processVideoTranscoding,
185 onNewWebTorrentFileResolution
186}
187
188// ---------------------------------------------------------------------------
189
190async function createHlsJobIfEnabled (user: MUserId, payload: { 193async function createHlsJobIfEnabled (user: MUserId, payload: {
191 videoUUID: string 194 videoUUID: string
192 resolution: number 195 resolution: number
193 isPortraitMode?: boolean 196 isPortraitMode?: boolean
194 copyCodecs: boolean 197 copyCodecs: boolean
195 isMaxQuality: boolean 198 isMaxQuality: boolean
199 isNewVideo?: boolean
196}) { 200}) {
197 if (!payload || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false 201 if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false
198 202
199 const jobOptions = { 203 const jobOptions = {
200 priority: await getTranscodingJobPriority(user) 204 priority: await getTranscodingJobPriority(user)
@@ -206,21 +210,35 @@ async function createHlsJobIfEnabled (user: MUserId, payload: {
206 resolution: payload.resolution, 210 resolution: payload.resolution,
207 isPortraitMode: payload.isPortraitMode, 211 isPortraitMode: payload.isPortraitMode,
208 copyCodecs: payload.copyCodecs, 212 copyCodecs: payload.copyCodecs,
209 isMaxQuality: payload.isMaxQuality 213 isMaxQuality: payload.isMaxQuality,
214 isNewVideo: payload.isNewVideo
210 } 215 }
211 216
212 JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }, jobOptions) 217 await addTranscodingJob(hlsTranscodingPayload, jobOptions)
213 218
214 return true 219 return true
215} 220}
216 221
217async function createLowerResolutionsJobs ( 222// ---------------------------------------------------------------------------
218 video: MVideoFullLight, 223
219 user: MUserId, 224export {
220 videoFileResolution: number, 225 processVideoTranscoding,
221 isPortraitMode: boolean, 226 createHlsJobIfEnabled,
227 onNewWebTorrentFileResolution
228}
229
230// ---------------------------------------------------------------------------
231
232async function createLowerResolutionsJobs (options: {
233 video: MVideoFullLight
234 user: MUserId
235 videoFileResolution: number
236 isPortraitMode: boolean
237 isNewVideo: boolean
222 type: 'hls' | 'webtorrent' 238 type: 'hls' | 'webtorrent'
223) { 239}) {
240 const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options
241
224 // Create transcoding jobs if there are enabled resolutions 242 // Create transcoding jobs if there are enabled resolutions
225 const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') 243 const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod')
226 const resolutionCreated: number[] = [] 244 const resolutionCreated: number[] = []
@@ -234,7 +252,8 @@ async function createLowerResolutionsJobs (
234 type: 'new-resolution-to-webtorrent', 252 type: 'new-resolution-to-webtorrent',
235 videoUUID: video.uuid, 253 videoUUID: video.uuid,
236 resolution, 254 resolution,
237 isPortraitMode 255 isPortraitMode,
256 isNewVideo
238 } 257 }
239 } 258 }
240 259
@@ -245,7 +264,8 @@ async function createLowerResolutionsJobs (
245 resolution, 264 resolution,
246 isPortraitMode, 265 isPortraitMode,
247 copyCodecs: false, 266 copyCodecs: false,
248 isMaxQuality: false 267 isMaxQuality: false,
268 isNewVideo
249 } 269 }
250 } 270 }
251 271
@@ -257,7 +277,7 @@ async function createLowerResolutionsJobs (
257 priority: await getTranscodingJobPriority(user) 277 priority: await getTranscodingJobPriority(user)
258 } 278 }
259 279
260 JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }, jobOptions) 280 await addTranscodingJob(dataInput, jobOptions)
261 } 281 }
262 282
263 if (resolutionCreated.length === 0) { 283 if (resolutionCreated.length === 0) {
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 42e8347b1..7a3a1bf82 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -11,6 +11,7 @@ import {
11 EmailPayload, 11 EmailPayload,
12 JobState, 12 JobState,
13 JobType, 13 JobType,
14 MoveObjectStoragePayload,
14 RefreshPayload, 15 RefreshPayload,
15 VideoFileImportPayload, 16 VideoFileImportPayload,
16 VideoImportPayload, 17 VideoImportPayload,
@@ -34,6 +35,7 @@ import { processVideoImport } from './handlers/video-import'
34import { processVideoLiveEnding } from './handlers/video-live-ending' 35import { processVideoLiveEnding } from './handlers/video-live-ending'
35import { processVideoTranscoding } from './handlers/video-transcoding' 36import { processVideoTranscoding } from './handlers/video-transcoding'
36import { processVideosViews } from './handlers/video-views' 37import { processVideosViews } from './handlers/video-views'
38import { processMoveToObjectStorage } from './handlers/move-to-object-storage'
37 39
38type CreateJobArgument = 40type CreateJobArgument =
39 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 41 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -49,9 +51,10 @@ type CreateJobArgument =
49 { type: 'videos-views', payload: {} } | 51 { type: 'videos-views', payload: {} } |
50 { type: 'video-live-ending', payload: VideoLiveEndingPayload } | 52 { type: 'video-live-ending', payload: VideoLiveEndingPayload } |
51 { type: 'actor-keys', payload: ActorKeysPayload } | 53 { type: 'actor-keys', payload: ActorKeysPayload } |
52 { type: 'video-redundancy', payload: VideoRedundancyPayload } 54 { type: 'video-redundancy', payload: VideoRedundancyPayload } |
55 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload }
53 56
54type CreateJobOptions = { 57export type CreateJobOptions = {
55 delay?: number 58 delay?: number
56 priority?: number 59 priority?: number
57} 60}
@@ -70,7 +73,8 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = {
70 'activitypub-refresher': refreshAPObject, 73 'activitypub-refresher': refreshAPObject,
71 'video-live-ending': processVideoLiveEnding, 74 'video-live-ending': processVideoLiveEnding,
72 'actor-keys': processActorKeys, 75 'actor-keys': processActorKeys,
73 'video-redundancy': processVideoRedundancy 76 'video-redundancy': processVideoRedundancy,
77 'move-to-object-storage': processMoveToObjectStorage
74} 78}
75 79
76const jobTypes: JobType[] = [ 80const jobTypes: JobType[] = [
@@ -87,7 +91,8 @@ const jobTypes: JobType[] = [
87 'activitypub-refresher', 91 'activitypub-refresher',
88 'video-redundancy', 92 'video-redundancy',
89 'actor-keys', 93 'actor-keys',
90 'video-live-ending' 94 'video-live-ending',
95 'move-to-object-storage'
91] 96]
92 97
93class JobQueue { 98class JobQueue {
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 2a429fb33..d7dc841d9 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -20,7 +20,7 @@ import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
20import { federateVideoIfNeeded } from '../activitypub/videos' 20import { federateVideoIfNeeded } from '../activitypub/videos'
21import { JobQueue } from '../job-queue' 21import { JobQueue } from '../job-queue'
22import { PeerTubeSocket } from '../peertube-socket' 22import { PeerTubeSocket } from '../peertube-socket'
23import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths' 23import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../paths'
24import { LiveQuotaStore } from './live-quota-store' 24import { LiveQuotaStore } from './live-quota-store'
25import { LiveSegmentShaStore } from './live-segment-sha-store' 25import { LiveSegmentShaStore } from './live-segment-sha-store'
26import { cleanupLive } from './live-utils' 26import { cleanupLive } from './live-utils'
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
index e4526c7a5..3bf723b98 100644
--- a/server/lib/live/live-utils.ts
+++ b/server/lib/live/live-utils.ts
@@ -1,7 +1,7 @@
1import { remove } from 'fs-extra' 1import { remove } from 'fs-extra'
2import { basename } from 'path' 2import { basename } from 'path'
3import { MStreamingPlaylist, MVideo } from '@server/types/models' 3import { MStreamingPlaylist, MVideo } from '@server/types/models'
4import { getHLSDirectory } from '../video-paths' 4import { getLiveDirectory } from '../paths'
5 5
6function buildConcatenatedName (segmentOrPlaylistPath: string) { 6function buildConcatenatedName (segmentOrPlaylistPath: string) {
7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) 7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
@@ -10,7 +10,7 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) {
10} 10}
11 11
12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
13 const hlsDirectory = getHLSDirectory(video) 13 const hlsDirectory = getLiveDirectory(video)
14 14
15 await remove(hlsDirectory) 15 await remove(hlsDirectory)
16 16
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index a80abc843..9b5b6c4fc 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -11,9 +11,9 @@ import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' 11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
12import { VideoFileModel } from '@server/models/video/video-file' 12import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' 13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
14import { getLiveDirectory } from '../../paths'
14import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' 15import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles'
15import { isAbleToUploadVideo } from '../../user' 16import { isAbleToUploadVideo } from '../../user'
16import { getHLSDirectory } from '../../video-paths'
17import { LiveQuotaStore } from '../live-quota-store' 17import { LiveQuotaStore } from '../live-quota-store'
18import { LiveSegmentShaStore } from '../live-segment-sha-store' 18import { LiveSegmentShaStore } from '../live-segment-sha-store'
19import { buildConcatenatedName } from '../live-utils' 19import { buildConcatenatedName } from '../live-utils'
@@ -282,7 +282,7 @@ class MuxingSession extends EventEmitter {
282 } 282 }
283 283
284 private async prepareDirectories () { 284 private async prepareDirectories () {
285 const outPath = getHLSDirectory(this.videoLive.Video) 285 const outPath = getLiveDirectory(this.videoLive.Video)
286 await ensureDir(outPath) 286 await ensureDir(outPath)
287 287
288 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) 288 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts
new file mode 100644
index 000000000..8b413a40e
--- /dev/null
+++ b/server/lib/object-storage/index.ts
@@ -0,0 +1,3 @@
1export * from './keys'
2export * from './urls'
3export * from './videos'
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts
new file mode 100644
index 000000000..519474775
--- /dev/null
+++ b/server/lib/object-storage/keys.ts
@@ -0,0 +1,20 @@
1import { join } from 'path'
2import { MStreamingPlaylist, MVideoUUID } from '@server/types/models'
3
4function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
5 return join(generateHLSObjectBaseStorageKey(playlist, video), filename)
6}
7
8function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) {
9 return playlist.getStringType() + '_' + video.uuid
10}
11
12function generateWebTorrentObjectStorageKey (filename: string) {
13 return filename
14}
15
16export {
17 generateHLSObjectStorageKey,
18 generateHLSObjectBaseStorageKey,
19 generateWebTorrentObjectStorageKey
20}
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts
new file mode 100644
index 000000000..c9a614593
--- /dev/null
+++ b/server/lib/object-storage/shared/client.ts
@@ -0,0 +1,56 @@
1import { S3Client } from '@aws-sdk/client-s3'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { lTags } from './logger'
5
6let endpointParsed: URL
7function getEndpointParsed () {
8 if (endpointParsed) return endpointParsed
9
10 endpointParsed = new URL(getEndpoint())
11
12 return endpointParsed
13}
14
15let s3Client: S3Client
16function getClient () {
17 if (s3Client) return s3Client
18
19 const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE
20
21 s3Client = new S3Client({
22 endpoint: getEndpoint(),
23 region: OBJECT_STORAGE.REGION,
24 credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID
25 ? {
26 accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
27 secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
28 }
29 : undefined
30 })
31
32 logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
33
34 return s3Client
35}
36
37// ---------------------------------------------------------------------------
38
39export {
40 getEndpointParsed,
41 getClient
42}
43
44// ---------------------------------------------------------------------------
45
46let endpoint: string
47function getEndpoint () {
48 if (endpoint) return endpoint
49
50 const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT
51 endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://')
52 ? CONFIG.OBJECT_STORAGE.ENDPOINT
53 : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT
54
55 return endpoint
56}
diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts
new file mode 100644
index 000000000..11e10aa9f
--- /dev/null
+++ b/server/lib/object-storage/shared/index.ts
@@ -0,0 +1,3 @@
1export * from './client'
2export * from './logger'
3export * from './object-storage-helpers'
diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts
new file mode 100644
index 000000000..8ab7cbd71
--- /dev/null
+++ b/server/lib/object-storage/shared/logger.ts
@@ -0,0 +1,7 @@
1import { loggerTagsFactory } from '@server/helpers/logger'
2
3const lTags = loggerTagsFactory('object-storage')
4
5export {
6 lTags
7}
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
new file mode 100644
index 000000000..e23216907
--- /dev/null
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -0,0 +1,229 @@
1import { close, createReadStream, createWriteStream, ensureDir, open, ReadStream, stat } from 'fs-extra'
2import { min } from 'lodash'
3import { dirname } from 'path'
4import { Readable } from 'stream'
5import {
6 CompletedPart,
7 CompleteMultipartUploadCommand,
8 CreateMultipartUploadCommand,
9 DeleteObjectCommand,
10 GetObjectCommand,
11 ListObjectsV2Command,
12 PutObjectCommand,
13 UploadPartCommand
14} from '@aws-sdk/client-s3'
15import { pipelinePromise } from '@server/helpers/core-utils'
16import { isArray } from '@server/helpers/custom-validators/misc'
17import { logger } from '@server/helpers/logger'
18import { CONFIG } from '@server/initializers/config'
19import { getPrivateUrl } from '../urls'
20import { getClient } from './client'
21import { lTags } from './logger'
22
23type BucketInfo = {
24 BUCKET_NAME: string
25 PREFIX?: string
26}
27
28async function storeObject (options: {
29 inputPath: string
30 objectStorageKey: string
31 bucketInfo: BucketInfo
32}): Promise<string> {
33 const { inputPath, objectStorageKey, bucketInfo } = options
34
35 logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
36
37 const stats = await stat(inputPath)
38
39 // If bigger than max allowed size we do a multipart upload
40 if (stats.size > CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART) {
41 return multiPartUpload({ inputPath, objectStorageKey, bucketInfo })
42 }
43
44 const fileStream = createReadStream(inputPath)
45 return objectStoragePut({ objectStorageKey, content: fileStream, bucketInfo })
46}
47
48async function removeObject (filename: string, bucketInfo: BucketInfo) {
49 const command = new DeleteObjectCommand({
50 Bucket: bucketInfo.BUCKET_NAME,
51 Key: buildKey(filename, bucketInfo)
52 })
53
54 return getClient().send(command)
55}
56
57async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
58 const s3Client = getClient()
59
60 const commandPrefix = bucketInfo.PREFIX + prefix
61 const listCommand = new ListObjectsV2Command({
62 Bucket: bucketInfo.BUCKET_NAME,
63 Prefix: commandPrefix
64 })
65
66 const listedObjects = await s3Client.send(listCommand)
67
68 // FIXME: use bulk delete when s3ninja will support this operation
69 // const deleteParams = {
70 // Bucket: bucketInfo.BUCKET_NAME,
71 // Delete: { Objects: [] }
72 // }
73
74 if (isArray(listedObjects.Contents) !== true) {
75 const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
76
77 logger.error(message, { response: listedObjects, ...lTags() })
78 throw new Error(message)
79 }
80
81 for (const object of listedObjects.Contents) {
82 const command = new DeleteObjectCommand({
83 Bucket: bucketInfo.BUCKET_NAME,
84 Key: object.Key
85 })
86
87 await s3Client.send(command)
88
89 // FIXME: use bulk delete when s3ninja will support this operation
90 // deleteParams.Delete.Objects.push({ Key: object.Key })
91 }
92
93 // FIXME: use bulk delete when s3ninja will support this operation
94 // const deleteCommand = new DeleteObjectsCommand(deleteParams)
95 // await s3Client.send(deleteCommand)
96
97 // Repeat if not all objects could be listed at once (limit of 1000?)
98 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
99}
100
101async function makeAvailable (options: {
102 key: string
103 destination: string
104 bucketInfo: BucketInfo
105}) {
106 const { key, destination, bucketInfo } = options
107
108 await ensureDir(dirname(options.destination))
109
110 const command = new GetObjectCommand({
111 Bucket: bucketInfo.BUCKET_NAME,
112 Key: buildKey(key, bucketInfo)
113 })
114 const response = await getClient().send(command)
115
116 const file = createWriteStream(destination)
117 await pipelinePromise(response.Body as Readable, file)
118
119 file.close()
120}
121
122function buildKey (key: string, bucketInfo: BucketInfo) {
123 return bucketInfo.PREFIX + key
124}
125
126// ---------------------------------------------------------------------------
127
128export {
129 BucketInfo,
130 buildKey,
131 storeObject,
132 removeObject,
133 removePrefix,
134 makeAvailable
135}
136
137// ---------------------------------------------------------------------------
138
139async function objectStoragePut (options: {
140 objectStorageKey: string
141 content: ReadStream
142 bucketInfo: BucketInfo
143}) {
144 const { objectStorageKey, content, bucketInfo } = options
145
146 const command = new PutObjectCommand({
147 Bucket: bucketInfo.BUCKET_NAME,
148 Key: buildKey(objectStorageKey, bucketInfo),
149 Body: content
150 })
151
152 await getClient().send(command)
153
154 return getPrivateUrl(bucketInfo, objectStorageKey)
155}
156
157async function multiPartUpload (options: {
158 inputPath: string
159 objectStorageKey: string
160 bucketInfo: BucketInfo
161}) {
162 const { objectStorageKey, inputPath, bucketInfo } = options
163
164 const key = buildKey(objectStorageKey, bucketInfo)
165 const s3Client = getClient()
166
167 const statResult = await stat(inputPath)
168
169 const createMultipartCommand = new CreateMultipartUploadCommand({
170 Bucket: bucketInfo.BUCKET_NAME,
171 Key: key
172 })
173 const createResponse = await s3Client.send(createMultipartCommand)
174
175 const fd = await open(inputPath, 'r')
176 let partNumber = 1
177 const parts: CompletedPart[] = []
178 const partSize = CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART
179
180 for (let start = 0; start < statResult.size; start += partSize) {
181 logger.debug(
182 'Uploading part %d of file to %s%s in bucket %s',
183 partNumber, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()
184 )
185
186 // FIXME: Remove when https://github.com/aws/aws-sdk-js-v3/pull/2637 is released
187 // The s3 sdk needs to know the length of the http body beforehand, but doesn't support
188 // streams with start and end set, so it just tries to stat the file in stream.path.
189 // This fails for us because we only want to send part of the file. The stream type
190 // is modified so we can set the byteLength here, which s3 detects because array buffers
191 // have this field set
192 const stream: ReadStream & { byteLength: number } =
193 createReadStream(
194 inputPath,
195 { fd, autoClose: false, start, end: (start + partSize) - 1 }
196 ) as ReadStream & { byteLength: number }
197
198 // Calculate if the part size is more than what's left over, and in that case use left over bytes for byteLength
199 stream.byteLength = min([ statResult.size - start, partSize ])
200
201 const uploadPartCommand = new UploadPartCommand({
202 Bucket: bucketInfo.BUCKET_NAME,
203 Key: key,
204 UploadId: createResponse.UploadId,
205 PartNumber: partNumber,
206 Body: stream
207 })
208 const uploadResponse = await s3Client.send(uploadPartCommand)
209
210 parts.push({ ETag: uploadResponse.ETag, PartNumber: partNumber })
211 partNumber += 1
212 }
213 await close(fd)
214
215 const completeUploadCommand = new CompleteMultipartUploadCommand({
216 Bucket: bucketInfo.BUCKET_NAME,
217 Key: objectStorageKey,
218 UploadId: createResponse.UploadId,
219 MultipartUpload: { Parts: parts }
220 })
221 await s3Client.send(completeUploadCommand)
222
223 logger.debug(
224 'Completed %s%s in bucket %s in %d parts',
225 bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, partNumber - 1, lTags()
226 )
227
228 return getPrivateUrl(bucketInfo, objectStorageKey)
229}
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts
new file mode 100644
index 000000000..2a889190b
--- /dev/null
+++ b/server/lib/object-storage/urls.ts
@@ -0,0 +1,40 @@
1import { CONFIG } from '@server/initializers/config'
2import { BucketInfo, buildKey, getEndpointParsed } from './shared'
3
4function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) {
5 return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
6}
7
8function getWebTorrentPublicFileUrl (fileUrl: string) {
9 const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL
10 if (!baseUrl) return fileUrl
11
12 return replaceByBaseUrl(fileUrl, baseUrl)
13}
14
15function getHLSPublicFileUrl (fileUrl: string) {
16 const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL
17 if (!baseUrl) return fileUrl
18
19 return replaceByBaseUrl(fileUrl, baseUrl)
20}
21
22export {
23 getPrivateUrl,
24 getWebTorrentPublicFileUrl,
25 replaceByBaseUrl,
26 getHLSPublicFileUrl
27}
28
29// ---------------------------------------------------------------------------
30
31function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {
32 if (baseUrl) return baseUrl
33
34 return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/`
35}
36
37const regex = new RegExp('https?://[^/]+')
38function replaceByBaseUrl (fileUrl: string, baseUrl: string) {
39 return fileUrl.replace(regex, baseUrl)
40}
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
new file mode 100644
index 000000000..15b8f58d5
--- /dev/null
+++ b/server/lib/object-storage/videos.ts
@@ -0,0 +1,72 @@
1import { join } from 'path'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models'
5import { getHLSDirectory } from '../paths'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8
9function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
10 const baseHlsDirectory = getHLSDirectory(video)
11
12 return storeObject({
13 inputPath: join(baseHlsDirectory, filename),
14 objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename),
15 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
16 })
17}
18
19function storeWebTorrentFile (filename: string) {
20 return storeObject({
21 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
22 objectStorageKey: generateWebTorrentObjectStorageKey(filename),
23 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
24 })
25}
26
27function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) {
28 return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
29}
30
31function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
32 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
33}
34
35async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) {
36 const key = generateHLSObjectStorageKey(playlist, video, filename)
37
38 logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
39
40 await makeAvailable({
41 key,
42 destination,
43 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
44 })
45
46 return destination
47}
48
49async function makeWebTorrentFileAvailable (filename: string, destination: string) {
50 const key = generateWebTorrentObjectStorageKey(filename)
51
52 logger.info('Fetching WebTorrent file %s from object storage to %s.', key, destination, lTags())
53
54 await makeAvailable({
55 key,
56 destination,
57 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
58 })
59
60 return destination
61}
62
63export {
64 storeWebTorrentFile,
65 storeHLSFile,
66
67 removeHLSObjectStorage,
68 removeWebTorrentObjectStorage,
69
70 makeWebTorrentFileAvailable,
71 makeHLSFileAvailable
72}
diff --git a/server/lib/video-paths.ts b/server/lib/paths.ts
index 1e4382108..434e637c6 100644
--- a/server/lib/video-paths.ts
+++ b/server/lib/paths.ts
@@ -1,9 +1,8 @@
1import { join } from 'path' 1import { join } from 'path'
2import { extractVideo } from '@server/helpers/video'
3import { CONFIG } from '@server/initializers/config'
4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
5import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
6import { buildUUID } from '@server/helpers/uuid' 2import { buildUUID } from '@server/helpers/uuid'
3import { CONFIG } from '@server/initializers/config'
4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
5import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
7import { removeFragmentedMP4Ext } from '@shared/core-utils' 6import { removeFragmentedMP4Ext } from '@shared/core-utils'
8 7
9// ################## Video file name ################## 8// ################## Video file name ##################
@@ -16,39 +15,18 @@ function generateHLSVideoFilename (resolution: number) {
16 return `${buildUUID()}-${resolution}-fragmented.mp4` 15 return `${buildUUID()}-${resolution}-fragmented.mp4`
17} 16}
18 17
19function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { 18// ################## Streaming playlist ##################
20 if (videoFile.isHLS()) {
21 const video = extractVideo(videoOrPlaylist)
22
23 return join(getHLSDirectory(video), videoFile.filename)
24 }
25
26 const baseDir = isRedundancy
27 ? CONFIG.STORAGE.REDUNDANCY_DIR
28 : CONFIG.STORAGE.VIDEOS_DIR
29
30 return join(baseDir, videoFile.filename)
31}
32
33// ################## Redundancy ##################
34 19
35function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { 20function getLiveDirectory (video: MVideoUUID) {
36 // Base URL used by our HLS player 21 return getHLSDirectory(video)
37 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid
38} 22}
39 23
40function generateWebTorrentRedundancyUrl (file: MVideoFile) { 24function getHLSDirectory (video: MVideoUUID) {
41 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename 25 return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
42} 26}
43 27
44// ################## Streaming playlist ################## 28function getHLSRedundancyDirectory (video: MVideoUUID) {
45 29 return join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
46function getHLSDirectory (video: MVideoUUID, isRedundancy = false) {
47 const baseDir = isRedundancy
48 ? HLS_REDUNDANCY_DIRECTORY
49 : HLS_STREAMING_PLAYLIST_DIRECTORY
50
51 return join(baseDir, video.uuid)
52} 30}
53 31
54function getHlsResolutionPlaylistFilename (videoFilename: string) { 32function getHlsResolutionPlaylistFilename (videoFilename: string) {
@@ -81,36 +59,24 @@ function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVi
81 return uuid + '-' + resolution + extension 59 return uuid + '-' + resolution + extension
82} 60}
83 61
84function getTorrentFilePath (videoFile: MVideoFile) { 62function getFSTorrentFilePath (videoFile: MVideoFile) {
85 return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) 63 return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
86} 64}
87 65
88// ################## Meta data ##################
89
90function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) {
91 const path = '/api/v1/videos/'
92
93 return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id
94}
95
96// --------------------------------------------------------------------------- 66// ---------------------------------------------------------------------------
97 67
98export { 68export {
99 generateHLSVideoFilename, 69 generateHLSVideoFilename,
100 generateWebTorrentVideoFilename, 70 generateWebTorrentVideoFilename,
101 71
102 getVideoFilePath,
103
104 generateTorrentFileName, 72 generateTorrentFileName,
105 getTorrentFilePath, 73 getFSTorrentFilePath,
106 74
107 getHLSDirectory, 75 getHLSDirectory,
76 getLiveDirectory,
77 getHLSRedundancyDirectory,
78
108 generateHLSMasterPlaylistFilename, 79 generateHLSMasterPlaylistFilename,
109 generateHlsSha256SegmentsFilename, 80 generateHlsSha256SegmentsFilename,
110 getHlsResolutionPlaylistFilename, 81 getHlsResolutionPlaylistFilename
111
112 getLocalVideoFileMetadataUrl,
113
114 generateWebTorrentRedundancyUrl,
115 generateHLSRedundancyUrl
116} 82}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 137ae53a0..ebfd015b5 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -24,7 +24,7 @@ import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlayli
24import { getOrCreateAPVideo } from '../activitypub/videos' 24import { getOrCreateAPVideo } from '../activitypub/videos'
25import { downloadPlaylistSegments } from '../hls' 25import { downloadPlaylistSegments } from '../hls'
26import { removeVideoRedundancy } from '../redundancy' 26import { removeVideoRedundancy } from '../redundancy'
27import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' 27import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls'
28import { AbstractScheduler } from './abstract-scheduler' 28import { AbstractScheduler } from './abstract-scheduler'
29 29
30type CandidateToDuplicate = { 30type CandidateToDuplicate = {
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index c08523988..d2384f53c 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -1,5 +1,4 @@
1import { join } from 'path' 1import { join } from 'path'
2
3import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' 2import { ThumbnailType } from '../../shared/models/videos/thumbnail.type'
4import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' 3import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils'
5import { generateImageFilename, processImage } from '../helpers/image-utils' 4import { generateImageFilename, processImage } from '../helpers/image-utils'
@@ -10,7 +9,7 @@ import { ThumbnailModel } from '../models/video/thumbnail'
10import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' 9import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models'
11import { MThumbnail } from '../types/models/video/thumbnail' 10import { MThumbnail } from '../types/models/video/thumbnail'
12import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' 11import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
13import { getVideoFilePath } from './video-paths' 12import { VideoPathManager } from './video-path-manager'
14 13
15type ImageSize = { height?: number, width?: number } 14type ImageSize = { height?: number, width?: number }
16 15
@@ -116,21 +115,22 @@ function generateVideoMiniature (options: {
116}) { 115}) {
117 const { video, videoFile, type } = options 116 const { video, videoFile, type } = options
118 117
119 const input = getVideoFilePath(video, videoFile) 118 return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => {
119 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
120 120
121 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) 121 const thumbnailCreator = videoFile.isAudio()
122 const thumbnailCreator = videoFile.isAudio() 122 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true)
123 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) 123 : () => generateImageFromVideoFile(input, basePath, filename, { height, width })
124 : () => generateImageFromVideoFile(input, basePath, filename, { height, width })
125 124
126 return updateThumbnailFromFunction({ 125 return updateThumbnailFromFunction({
127 thumbnailCreator, 126 thumbnailCreator,
128 filename, 127 filename,
129 height, 128 height,
130 width, 129 width,
131 type, 130 type,
132 automaticallyGenerated: true, 131 automaticallyGenerated: true,
133 existingThumbnail 132 existingThumbnail
133 })
134 }) 134 })
135} 135}
136 136
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index d2a556360..ee228c011 100644
--- a/server/lib/transcoding/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -4,13 +4,13 @@ import { basename, extname as extnameUtil, join } from 'path'
4import { toEven } from '@server/helpers/core-utils' 4import { toEven } from '@server/helpers/core-utils'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
7import { VideoResolution } from '../../../shared/models/videos' 7import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' 9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils'
10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' 10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
11import { logger } from '../../helpers/logger' 11import { logger } from '../../helpers/logger'
12import { CONFIG } from '../../initializers/config' 12import { CONFIG } from '../../initializers/config'
13import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' 13import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
14import { VideoFileModel } from '../../models/video/video-file' 14import { VideoFileModel } from '../../models/video/video-file'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' 15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' 16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
@@ -19,9 +19,9 @@ import {
19 generateHlsSha256SegmentsFilename, 19 generateHlsSha256SegmentsFilename,
20 generateHLSVideoFilename, 20 generateHLSVideoFilename,
21 generateWebTorrentVideoFilename, 21 generateWebTorrentVideoFilename,
22 getHlsResolutionPlaylistFilename, 22 getHlsResolutionPlaylistFilename
23 getVideoFilePath 23} from '../paths'
24} from '../video-paths' 24import { VideoPathManager } from '../video-path-manager'
25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' 25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
26 26
27/** 27/**
@@ -32,159 +32,162 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
32 */ 32 */
33 33
34// Optimize the original video file and replace it. The resolution is not changed. 34// Optimize the original video file and replace it. The resolution is not changed.
35async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { 35function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) {
36 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 36 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
37 const newExtname = '.mp4' 37 const newExtname = '.mp4'
38 38
39 const videoInputPath = getVideoFilePath(video, inputVideoFile) 39 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => {
40 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 40 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
41 41
42 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) 42 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath)
43 ? 'quick-transcode' 43 ? 'quick-transcode'
44 : 'video' 44 : 'video'
45 45
46 const resolution = toEven(inputVideoFile.resolution) 46 const resolution = toEven(inputVideoFile.resolution)
47 47
48 const transcodeOptions: TranscodeOptions = { 48 const transcodeOptions: TranscodeOptions = {
49 type: transcodeType, 49 type: transcodeType,
50 50
51 inputPath: videoInputPath, 51 inputPath: videoInputPath,
52 outputPath: videoTranscodedPath, 52 outputPath: videoTranscodedPath,
53 53
54 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 54 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
55 profile: CONFIG.TRANSCODING.PROFILE, 55 profile: CONFIG.TRANSCODING.PROFILE,
56 56
57 resolution, 57 resolution,
58 58
59 job 59 job
60 } 60 }
61 61
62 // Could be very long! 62 // Could be very long!
63 await transcode(transcodeOptions) 63 await transcode(transcodeOptions)
64 64
65 try { 65 try {
66 await remove(videoInputPath) 66 await remove(videoInputPath)
67 67
68 // Important to do this before getVideoFilename() to take in account the new filename 68 // Important to do this before getVideoFilename() to take in account the new filename
69 inputVideoFile.extname = newExtname 69 inputVideoFile.extname = newExtname
70 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) 70 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
71 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
71 72
72 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 73 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
73 74
74 await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 75 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
75 76
76 return transcodeType 77 return { transcodeType, videoFile }
77 } catch (err) { 78 } catch (err) {
78 // Auto destruction... 79 // Auto destruction...
79 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) 80 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err }))
80 81
81 throw err 82 throw err
82 } 83 }
84 })
83} 85}
84 86
85// Transcode the original video file to a lower resolution. 87// Transcode the original video file to a lower resolution
86async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { 88// We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
89function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) {
87 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 90 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
88 const extname = '.mp4' 91 const extname = '.mp4'
89 92
90 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed 93 return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => {
91 const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile()) 94 const newVideoFile = new VideoFileModel({
95 resolution,
96 extname,
97 filename: generateWebTorrentVideoFilename(resolution, extname),
98 size: 0,
99 videoId: video.id
100 })
92 101
93 const newVideoFile = new VideoFileModel({ 102 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
94 resolution, 103 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
95 extname,
96 filename: generateWebTorrentVideoFilename(resolution, extname),
97 size: 0,
98 videoId: video.id
99 })
100 104
101 const videoOutputPath = getVideoFilePath(video, newVideoFile) 105 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
102 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) 106 ? {
107 type: 'only-audio' as 'only-audio',
103 108
104 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO 109 inputPath: videoInputPath,
105 ? { 110 outputPath: videoTranscodedPath,
106 type: 'only-audio' as 'only-audio',
107 111
108 inputPath: videoInputPath, 112 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
109 outputPath: videoTranscodedPath, 113 profile: CONFIG.TRANSCODING.PROFILE,
110 114
111 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 115 resolution,
112 profile: CONFIG.TRANSCODING.PROFILE,
113 116
114 resolution, 117 job
118 }
119 : {
120 type: 'video' as 'video',
121 inputPath: videoInputPath,
122 outputPath: videoTranscodedPath,
115 123
116 job 124 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
117 } 125 profile: CONFIG.TRANSCODING.PROFILE,
118 : {
119 type: 'video' as 'video',
120 inputPath: videoInputPath,
121 outputPath: videoTranscodedPath,
122 126
123 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 127 resolution,
124 profile: CONFIG.TRANSCODING.PROFILE, 128 isPortraitMode: isPortrait,
125 129
126 resolution, 130 job
127 isPortraitMode: isPortrait, 131 }
128 132
129 job 133 await transcode(transcodeOptions)
130 }
131
132 await transcode(transcodeOptions)
133 134
134 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) 135 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
136 })
135} 137}
136 138
137// Merge an image with an audio file to create a video 139// Merge an image with an audio file to create a video
138async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { 140function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) {
139 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 141 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
140 const newExtname = '.mp4' 142 const newExtname = '.mp4'
141 143
142 const inputVideoFile = video.getMinQualityFile() 144 const inputVideoFile = video.getMinQualityFile()
143 145
144 const audioInputPath = getVideoFilePath(video, inputVideoFile) 146 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => {
145 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 147 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
146 148
147 // If the user updates the video preview during transcoding 149 // If the user updates the video preview during transcoding
148 const previewPath = video.getPreview().getPath() 150 const previewPath = video.getPreview().getPath()
149 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) 151 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
150 await copyFile(previewPath, tmpPreviewPath) 152 await copyFile(previewPath, tmpPreviewPath)
151 153
152 const transcodeOptions = { 154 const transcodeOptions = {
153 type: 'merge-audio' as 'merge-audio', 155 type: 'merge-audio' as 'merge-audio',
154 156
155 inputPath: tmpPreviewPath, 157 inputPath: tmpPreviewPath,
156 outputPath: videoTranscodedPath, 158 outputPath: videoTranscodedPath,
157 159
158 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 160 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
159 profile: CONFIG.TRANSCODING.PROFILE, 161 profile: CONFIG.TRANSCODING.PROFILE,
160 162
161 audioPath: audioInputPath, 163 audioPath: audioInputPath,
162 resolution, 164 resolution,
163 165
164 job 166 job
165 } 167 }
166 168
167 try { 169 try {
168 await transcode(transcodeOptions) 170 await transcode(transcodeOptions)
169 171
170 await remove(audioInputPath) 172 await remove(audioInputPath)
171 await remove(tmpPreviewPath) 173 await remove(tmpPreviewPath)
172 } catch (err) { 174 } catch (err) {
173 await remove(tmpPreviewPath) 175 await remove(tmpPreviewPath)
174 throw err 176 throw err
175 } 177 }
176 178
177 // Important to do this before getVideoFilename() to take in account the new file extension 179 // Important to do this before getVideoFilename() to take in account the new file extension
178 inputVideoFile.extname = newExtname 180 inputVideoFile.extname = newExtname
179 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) 181 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
180 182
181 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 183 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
182 // ffmpeg generated a new video file, so update the video duration 184 // ffmpeg generated a new video file, so update the video duration
183 // See https://trac.ffmpeg.org/ticket/5456 185 // See https://trac.ffmpeg.org/ticket/5456
184 video.duration = await getDurationFromVideoFile(videoTranscodedPath) 186 video.duration = await getDurationFromVideoFile(videoTranscodedPath)
185 await video.save() 187 await video.save()
186 188
187 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 189 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
190 })
188} 191}
189 192
190// Concat TS segments from a live video to a fragmented mp4 HLS playlist 193// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@@ -258,7 +261,7 @@ async function onWebTorrentVideoFileTranscoding (
258 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 261 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
259 video.VideoFiles = await video.$get('VideoFiles') 262 video.VideoFiles = await video.$get('VideoFiles')
260 263
261 return video 264 return { video, videoFile }
262} 265}
263 266
264async function generateHlsPlaylistCommon (options: { 267async function generateHlsPlaylistCommon (options: {
@@ -335,14 +338,13 @@ async function generateHlsPlaylistCommon (options: {
335 videoStreamingPlaylistId: playlist.id 338 videoStreamingPlaylistId: playlist.id
336 }) 339 })
337 340
338 const videoFilePath = getVideoFilePath(playlist, newVideoFile) 341 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
339 342
340 // Move files from tmp transcoded directory to the appropriate place 343 // Move files from tmp transcoded directory to the appropriate place
341 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 344 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
342 await ensureDir(baseHlsDirectory)
343 345
344 // Move playlist file 346 // Move playlist file
345 const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) 347 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
346 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) 348 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
347 // Move video file 349 // Move video file
348 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) 350 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
@@ -355,7 +357,7 @@ async function generateHlsPlaylistCommon (options: {
355 357
356 await createTorrentAndSetInfoHash(playlist, newVideoFile) 358 await createTorrentAndSetInfoHash(playlist, newVideoFile)
357 359
358 await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) 360 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
359 361
360 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo 362 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
361 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') 363 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
@@ -368,5 +370,5 @@ async function generateHlsPlaylistCommon (options: {
368 await updateMasterHLSPlaylist(video, playlistWithFiles) 370 await updateMasterHLSPlaylist(video, playlistWithFiles)
369 await updateSha256VODSegments(video, playlistWithFiles) 371 await updateSha256VODSegments(video, playlistWithFiles)
370 372
371 return resolutionPlaylistPath 373 return { resolutionPlaylistPath, videoFile: savedVideoFile }
372} 374}
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
new file mode 100644
index 000000000..4c5d0c89d
--- /dev/null
+++ b/server/lib/video-path-manager.ts
@@ -0,0 +1,139 @@
1import { remove } from 'fs-extra'
2import { extname, join } from 'path'
3import { buildUUID } from '@server/helpers/uuid'
4import { extractVideo } from '@server/helpers/video'
5import { CONFIG } from '@server/initializers/config'
6import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
7import { VideoStorage } from '@shared/models'
8import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
9import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
10
11type MakeAvailableCB <T> = (path: string) => Promise<T> | T
12
13class VideoPathManager {
14
15 private static instance: VideoPathManager
16
17 private constructor () {}
18
19 getFSHLSOutputPath (video: MVideoUUID, filename?: string) {
20 const base = getHLSDirectory(video)
21 if (!filename) return base
22
23 return join(base, filename)
24 }
25
26 getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
27 if (videoFile.isHLS()) {
28 const video = extractVideo(videoOrPlaylist)
29
30 return join(getHLSRedundancyDirectory(video), videoFile.filename)
31 }
32
33 return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename)
34 }
35
36 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
37 if (videoFile.isHLS()) {
38 const video = extractVideo(videoOrPlaylist)
39
40 return join(getHLSDirectory(video), videoFile.filename)
41 }
42
43 return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename)
44 }
45
46 async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
47 if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
48 return this.makeAvailableFactory(
49 () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile),
50 false,
51 cb
52 )
53 }
54
55 const destination = this.buildTMPDestination(videoFile.filename)
56
57 if (videoFile.isHLS()) {
58 const video = extractVideo(videoOrPlaylist)
59
60 return this.makeAvailableFactory(
61 () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination),
62 true,
63 cb
64 )
65 }
66
67 return this.makeAvailableFactory(
68 () => makeWebTorrentFileAvailable(videoFile.filename, destination),
69 true,
70 cb
71 )
72 }
73
74 async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
75 const filename = getHlsResolutionPlaylistFilename(videoFile.filename)
76
77 if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
78 return this.makeAvailableFactory(
79 () => join(getHLSDirectory(playlist.Video), filename),
80 false,
81 cb
82 )
83 }
84
85 return this.makeAvailableFactory(
86 () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
87 true,
88 cb
89 )
90 }
91
92 async makeAvailablePlaylistFile <T> (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB<T>) {
93 if (playlist.storage === VideoStorage.FILE_SYSTEM) {
94 return this.makeAvailableFactory(
95 () => join(getHLSDirectory(playlist.Video), filename),
96 false,
97 cb
98 )
99 }
100
101 return this.makeAvailableFactory(
102 () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
103 true,
104 cb
105 )
106 }
107
108 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) {
109 let result: T
110
111 const destination = await method()
112
113 try {
114 result = await cb(destination)
115 } catch (err) {
116 if (destination && clean) await remove(destination)
117 throw err
118 }
119
120 if (clean) await remove(destination)
121
122 return result
123 }
124
125 private buildTMPDestination (filename: string) {
126 return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename))
127
128 }
129
130 static get Instance () {
131 return this.instance || (this.instance = new this())
132 }
133}
134
135// ---------------------------------------------------------------------------
136
137export {
138 VideoPathManager
139}
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
new file mode 100644
index 000000000..0613d94bf
--- /dev/null
+++ b/server/lib/video-state.ts
@@ -0,0 +1,99 @@
1import { Transaction } from 'sequelize'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { VideoModel } from '@server/models/video/video'
6import { VideoJobInfoModel } from '@server/models/video/video-job-info'
7import { MVideoFullLight, MVideoUUID } from '@server/types/models'
8import { VideoState } from '@shared/models'
9import { federateVideoIfNeeded } from './activitypub/videos'
10import { Notifier } from './notifier'
11import { addMoveToObjectStorageJob } from './video'
12
13function buildNextVideoState (currentState?: VideoState) {
14 if (currentState === VideoState.PUBLISHED) {
15 throw new Error('Video is already in its final state')
16 }
17
18 if (
19 currentState !== VideoState.TO_TRANSCODE &&
20 currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
21 CONFIG.TRANSCODING.ENABLED
22 ) {
23 return VideoState.TO_TRANSCODE
24 }
25
26 if (
27 currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
28 CONFIG.OBJECT_STORAGE.ENABLED
29 ) {
30 return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
31 }
32
33 return VideoState.PUBLISHED
34}
35
36function moveToNextState (video: MVideoUUID, isNewVideo = true) {
37 return sequelizeTypescript.transaction(async t => {
38 // Maybe the video changed in database, refresh it
39 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
40 // Video does not exist anymore
41 if (!videoDatabase) return undefined
42
43 // Already in its final state
44 if (videoDatabase.state === VideoState.PUBLISHED) {
45 return federateVideoIfNeeded(videoDatabase, false, t)
46 }
47
48 const newState = buildNextVideoState(videoDatabase.state)
49
50 if (newState === VideoState.PUBLISHED) {
51 return moveToPublishedState(videoDatabase, isNewVideo, t)
52 }
53
54 if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
55 return moveToExternalStorageState(videoDatabase, isNewVideo, t)
56 }
57 })
58}
59
60// ---------------------------------------------------------------------------
61
62export {
63 buildNextVideoState,
64 moveToNextState
65}
66
67// ---------------------------------------------------------------------------
68
69async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) {
70 logger.info('Publishing video %s.', video.uuid, { tags: [ video.uuid ] })
71
72 const previousState = video.state
73 await video.setNewState(VideoState.PUBLISHED, transaction)
74
75 // If the video was not published, we consider it is a new one for other instances
76 // Live videos are always federated, so it's not a new video
77 await federateVideoIfNeeded(video, isNewVideo, transaction)
78
79 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
80
81 if (previousState === VideoState.TO_TRANSCODE) {
82 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video)
83 }
84}
85
86async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) {
87 const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction)
88 const pendingTranscode = videoJobInfo?.pendingTranscode || 0
89
90 // We want to wait all transcoding jobs before moving the video on an external storage
91 if (pendingTranscode !== 0) return
92
93 await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, transaction)
94
95 logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
96
97 addMoveToObjectStorageJob(video, isNewVideo)
98 .catch(err => logger.error('Cannot add move to object storage job', { err }))
99}
diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts
new file mode 100644
index 000000000..64c2c9bf9
--- /dev/null
+++ b/server/lib/video-urls.ts
@@ -0,0 +1,31 @@
1
2import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
3import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
4
5// ################## Redundancy ##################
6
7function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) {
8 // Base URL used by our HLS player
9 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid
10}
11
12function generateWebTorrentRedundancyUrl (file: MVideoFile) {
13 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename
14}
15
16// ################## Meta data ##################
17
18function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) {
19 const path = '/api/v1/videos/'
20
21 return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id
22}
23
24// ---------------------------------------------------------------------------
25
26export {
27 getLocalVideoFileMetadataUrl,
28
29 generateWebTorrentRedundancyUrl,
30 generateHLSRedundancyUrl
31}
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 61fee4949..0a2b93cc0 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -1,15 +1,13 @@
1import { UploadFiles } from 'express' 1import { UploadFiles } from 'express'
2import { Transaction } from 'sequelize/types' 2import { Transaction } from 'sequelize/types'
3import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' 3import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { TagModel } from '@server/models/video/tag' 4import { TagModel } from '@server/models/video/tag'
6import { VideoModel } from '@server/models/video/video' 5import { VideoModel } from '@server/models/video/video'
6import { VideoJobInfoModel } from '@server/models/video/video-job-info'
7import { FilteredModelAttributes } from '@server/types' 7import { FilteredModelAttributes } from '@server/types'
8import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' 8import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' 9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
10import { federateVideoIfNeeded } from './activitypub/videos' 10import { CreateJobOptions, JobQueue } from './job-queue/job-queue'
11import { JobQueue } from './job-queue/job-queue'
12import { Notifier } from './notifier'
13import { updateVideoMiniatureFromExisting } from './thumbnail' 11import { updateVideoMiniatureFromExisting } from './thumbnail'
14 12
15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 13function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
@@ -82,29 +80,6 @@ async function setVideoTags (options: {
82 video.Tags = tagInstances 80 video.Tags = tagInstances
83} 81}
84 82
85async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) {
86 const result = await sequelizeTypescript.transaction(async t => {
87 // Maybe the video changed in database, refresh it
88 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
89 // Video does not exist anymore
90 if (!videoDatabase) return undefined
91
92 // We transcoded the video file in another format, now we can publish it
93 const videoPublished = await videoDatabase.publishIfNeededAndSave(t)
94
95 // If the video was not published, we consider it is a new one for other instances
96 // Live videos are always federated, so it's not a new video
97 await federateVideoIfNeeded(videoDatabase, !wasLive && videoPublished, t)
98
99 return { videoDatabase, videoPublished }
100 })
101
102 if (result?.videoPublished) {
103 Notifier.Instance.notifyOnNewVideoIfNeeded(result.videoDatabase)
104 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(result.videoDatabase)
105 }
106}
107
108async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { 83async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) {
109 let dataInput: VideoTranscodingPayload 84 let dataInput: VideoTranscodingPayload
110 85
@@ -127,7 +102,20 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF
127 priority: await getTranscodingJobPriority(user) 102 priority: await getTranscodingJobPriority(user)
128 } 103 }
129 104
130 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput }, jobOptions) 105 return addTranscodingJob(dataInput, jobOptions)
106}
107
108async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) {
109 await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
110
111 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options)
112}
113
114async function addMoveToObjectStorageJob (video: MVideoUUID, isNewVideo = true) {
115 await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove')
116
117 const dataInput = { videoUUID: video.uuid, isNewVideo }
118 return JobQueue.Instance.createJobWithPromise({ type: 'move-to-object-storage', payload: dataInput })
131} 119}
132 120
133async function getTranscodingJobPriority (user: MUserId) { 121async function getTranscodingJobPriority (user: MUserId) {
@@ -143,9 +131,10 @@ async function getTranscodingJobPriority (user: MUserId) {
143 131
144export { 132export {
145 buildLocalVideoFromReq, 133 buildLocalVideoFromReq,
146 publishAndFederateIfNeeded,
147 buildVideoThumbnailsFromReq, 134 buildVideoThumbnailsFromReq,
148 setVideoTags, 135 setVideoTags,
149 addOptimizeOrMergeAudioJob, 136 addOptimizeOrMergeAudioJob,
137 addTranscodingJob,
138 addMoveToObjectStorageJob,
150 getTranscodingJobPriority 139 getTranscodingJobPriority
151} 140}
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-12 08:11:22 +0000