aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/auth/oauth.ts9
-rw-r--r--server/lib/job-queue/handlers/manage-video-torrent.ts2
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts6
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts22
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts95
-rw-r--r--server/lib/object-storage/videos.ts9
-rw-r--r--server/lib/paths.ts17
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts66
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts4
-rw-r--r--server/lib/transcoding/transcoding.ts367
-rw-r--r--server/lib/video-path-manager.ts51
-rw-r--r--server/lib/video-privacy.ts96
-rw-r--r--server/lib/video-tokens-manager.ts49
-rw-r--r--server/lib/video.ts61
14 files changed, 601 insertions, 253 deletions
diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts
index 35b05ec5a..bc0d4301f 100644
--- a/server/lib/auth/oauth.ts
+++ b/server/lib/auth/oauth.ts
@@ -95,14 +95,9 @@ async function handleOAuthToken (req: express.Request, options: { refreshTokenAu
95 95
96function handleOAuthAuthenticate ( 96function handleOAuthAuthenticate (
97 req: express.Request, 97 req: express.Request,
98 res: express.Response, 98 res: express.Response
99 authenticateInQuery = false
100) { 99) {
101 const options = authenticateInQuery 100 return oAuthServer.authenticate(new Request(req), new Response(res))
102 ? { allowBearerTokensInQueryString: true }
103 : {}
104
105 return oAuthServer.authenticate(new Request(req), new Response(res), options)
106} 101}
107 102
108export { 103export {
diff --git a/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/lib/job-queue/handlers/manage-video-torrent.ts
index 03aa414c9..425915c96 100644
--- a/server/lib/job-queue/handlers/manage-video-torrent.ts
+++ b/server/lib/job-queue/handlers/manage-video-torrent.ts
@@ -82,7 +82,7 @@ async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) {
82async function loadFileOrLog (videoFileId: number) { 82async function loadFileOrLog (videoFileId: number) {
83 if (!videoFileId) return undefined 83 if (!videoFileId) return undefined
84 84
85 const file = await VideoFileModel.loadWithVideo(videoFileId) 85 const file = await VideoFileModel.load(videoFileId)
86 86
87 if (!file) { 87 if (!file) {
88 logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) 88 logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId)
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 28c3d325d..0b68555d1 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -3,10 +3,10 @@ import { remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { logger, loggerTagsFactory } from '@server/helpers/logger' 4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { updateTorrentMetadata } from '@server/helpers/webtorrent' 5import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' 6import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
8import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage' 7import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage'
9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
9import { VideoPathManager } from '@server/lib/video-path-manager'
10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' 10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
12import { VideoJobInfoModel } from '@server/models/video/video-job-info' 12import { VideoJobInfoModel } from '@server/models/video/video-job-info'
@@ -72,9 +72,9 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
72 for (const file of video.VideoFiles) { 72 for (const file of video.VideoFiles) {
73 if (file.storage !== VideoStorage.FILE_SYSTEM) continue 73 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
74 74
75 const fileUrl = await storeWebTorrentFile(file.filename) 75 const fileUrl = await storeWebTorrentFile(video, file)
76 76
77 const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) 77 const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
78 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) 78 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
79 } 79 }
80} 80}
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 7dbffc955..c6263f55a 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -18,6 +18,7 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin
18import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' 18import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models'
19import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 19import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
20import { logger, loggerTagsFactory } from '../../../helpers/logger' 20import { logger, loggerTagsFactory } from '../../../helpers/logger'
21import { VideoPathManager } from '@server/lib/video-path-manager'
21 22
22const lTags = loggerTagsFactory('live', 'job') 23const lTags = loggerTagsFactory('live', 'job')
23 24
@@ -205,18 +206,27 @@ async function assignReplayFilesToVideo (options: {
205 const concatenatedTsFiles = await readdir(replayDirectory) 206 const concatenatedTsFiles = await readdir(replayDirectory)
206 207
207 for (const concatenatedTsFile of concatenatedTsFiles) { 208 for (const concatenatedTsFile of concatenatedTsFiles) {
209 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
210
208 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) 211 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
209 212
210 const probe = await ffprobePromise(concatenatedTsFilePath) 213 const probe = await ffprobePromise(concatenatedTsFilePath)
211 const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) 214 const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
212 const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) 215 const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
213 216
214 await generateHlsPlaylistResolutionFromTS({ 217 try {
215 video, 218 await generateHlsPlaylistResolutionFromTS({
216 concatenatedTsFilePath, 219 video,
217 resolution, 220 inputFileMutexReleaser,
218 isAAC: audioStream?.codec_name === 'aac' 221 concatenatedTsFilePath,
219 }) 222 resolution,
223 isAAC: audioStream?.codec_name === 'aac'
224 })
225 } catch (err) {
226 logger.error('Cannot generate HLS playlist resolution from TS files.', { err })
227 }
228
229 inputFileMutexReleaser()
220 } 230 }
221 231
222 return video 232 return video
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index b0e92acf7..48c675678 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -94,15 +94,24 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
94 94
95 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() 95 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
96 96
97 await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { 97 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
98 return generateHlsPlaylistResolution({ 98
99 video, 99 try {
100 videoInputPath, 100 await videoFileInput.getVideo().reload()
101 resolution: payload.resolution, 101
102 copyCodecs: payload.copyCodecs, 102 await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
103 job 103 return generateHlsPlaylistResolution({
104 video,
105 videoInputPath,
106 inputFileMutexReleaser,
107 resolution: payload.resolution,
108 copyCodecs: payload.copyCodecs,
109 job
110 })
104 }) 111 })
105 }) 112 } finally {
113 inputFileMutexReleaser()
114 }
106 115
107 logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) 116 logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid))
108 117
@@ -177,38 +186,44 @@ async function onVideoFirstWebTorrentTranscoding (
177 transcodeType: TranscodeVODOptionsType, 186 transcodeType: TranscodeVODOptionsType,
178 user: MUserId 187 user: MUserId
179) { 188) {
180 const { resolution, audioStream } = await videoArg.probeMaxQualityFile() 189 const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
181 190
182 // Maybe the video changed in database, refresh it 191 try {
183 const videoDatabase = await VideoModel.loadFull(videoArg.uuid) 192 // Maybe the video changed in database, refresh it
184 // Video does not exist anymore 193 const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
185 if (!videoDatabase) return undefined 194 // Video does not exist anymore
186 195 if (!videoDatabase) return undefined
187 // Generate HLS version of the original file 196
188 const originalFileHLSPayload = { 197 const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile()
189 ...payload, 198
190 199 // Generate HLS version of the original file
191 hasAudio: !!audioStream, 200 const originalFileHLSPayload = {
192 resolution: videoDatabase.getMaxQualityFile().resolution, 201 ...payload,
193 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues 202
194 copyCodecs: transcodeType !== 'quick-transcode', 203 hasAudio: !!audioStream,
195 isMaxQuality: true 204 resolution: videoDatabase.getMaxQualityFile().resolution,
196 } 205 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
197 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) 206 copyCodecs: transcodeType !== 'quick-transcode',
198 const hasNewResolutions = await createLowerResolutionsJobs({ 207 isMaxQuality: true
199 video: videoDatabase, 208 }
200 user, 209 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
201 videoFileResolution: resolution, 210 const hasNewResolutions = await createLowerResolutionsJobs({
202 hasAudio: !!audioStream, 211 video: videoDatabase,
203 type: 'webtorrent', 212 user,
204 isNewVideo: payload.isNewVideo ?? true 213 videoFileResolution: resolution,
205 }) 214 hasAudio: !!audioStream,
206 215 type: 'webtorrent',
207 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') 216 isNewVideo: payload.isNewVideo ?? true
208 217 })
209 // Move to next state if there are no other resolutions to generate 218
210 if (!hasHls && !hasNewResolutions) { 219 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
211 await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) 220
221 // Move to next state if there are no other resolutions to generate
222 if (!hasHls && !hasNewResolutions) {
223 await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
224 }
225 } finally {
226 mutexReleaser()
212 } 227 }
213} 228}
214 229
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 62aae248b..e323baaa2 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -1,8 +1,9 @@
1import { basename, join } from 'path' 1import { basename, join } from 'path'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' 4import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models'
5import { getHLSDirectory } from '../paths' 5import { getHLSDirectory } from '../paths'
6import { VideoPathManager } from '../video-path-manager'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' 7import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' 8import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8 9
@@ -30,10 +31,10 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string)
30 31
31// --------------------------------------------------------------------------- 32// ---------------------------------------------------------------------------
32 33
33function storeWebTorrentFile (filename: string) { 34function storeWebTorrentFile (video: MVideo, file: MVideoFile) {
34 return storeObject({ 35 return storeObject({
35 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), 36 inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
36 objectStorageKey: generateWebTorrentObjectStorageKey(filename), 37 objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
37 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS 38 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
38 }) 39 })
39} 40}
diff --git a/server/lib/paths.ts b/server/lib/paths.ts
index b29854700..470970f55 100644
--- a/server/lib/paths.ts
+++ b/server/lib/paths.ts
@@ -1,9 +1,10 @@
1import { join } from 'path' 1import { join } from 'path'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, VIDEO_LIVE } from '@server/initializers/constants' 3import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants'
4import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' 4import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
5import { removeFragmentedMP4Ext } from '@shared/core-utils' 5import { removeFragmentedMP4Ext } from '@shared/core-utils'
6import { buildUUID } from '@shared/extra-utils' 6import { buildUUID } from '@shared/extra-utils'
7import { isVideoInPrivateDirectory } from './video-privacy'
7 8
8// ################## Video file name ################## 9// ################## Video file name ##################
9 10
@@ -17,20 +18,24 @@ function generateHLSVideoFilename (resolution: number) {
17 18
18// ################## Streaming playlist ################## 19// ################## Streaming playlist ##################
19 20
20function getLiveDirectory (video: MVideoUUID) { 21function getLiveDirectory (video: MVideo) {
21 return getHLSDirectory(video) 22 return getHLSDirectory(video)
22} 23}
23 24
24function getLiveReplayBaseDirectory (video: MVideoUUID) { 25function getLiveReplayBaseDirectory (video: MVideo) {
25 return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) 26 return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY)
26} 27}
27 28
28function getHLSDirectory (video: MVideoUUID) { 29function getHLSDirectory (video: MVideo) {
29 return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 30 if (isVideoInPrivateDirectory(video.privacy)) {
31 return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid)
32 }
33
34 return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid)
30} 35}
31 36
32function getHLSRedundancyDirectory (video: MVideoUUID) { 37function getHLSRedundancyDirectory (video: MVideoUUID) {
33 return join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 38 return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
34} 39}
35 40
36function getHlsResolutionPlaylistFilename (videoFilename: string) { 41function getHlsResolutionPlaylistFilename (videoFilename: string) {
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 5bfbc3cd2..30bf189db 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -1,11 +1,14 @@
1import { VideoModel } from '@server/models/video/video' 1import { VideoModel } from '@server/models/video/video'
2import { MVideoFullLight } from '@server/types/models' 2import { MScheduleVideoUpdate } from '@server/types/models'
3import { VideoPrivacy, VideoState } from '@shared/models'
3import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
4import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
5import { sequelizeTypescript } from '../../initializers/database' 6import { sequelizeTypescript } from '../../initializers/database'
6import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' 7import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update'
7import { federateVideoIfNeeded } from '../activitypub/videos'
8import { Notifier } from '../notifier' 8import { Notifier } from '../notifier'
9import { addVideoJobsAfterUpdate } from '../video'
10import { VideoPathManager } from '../video-path-manager'
11import { setVideoPrivacy } from '../video-privacy'
9import { AbstractScheduler } from './abstract-scheduler' 12import { AbstractScheduler } from './abstract-scheduler'
10 13
11export class UpdateVideosScheduler extends AbstractScheduler { 14export class UpdateVideosScheduler extends AbstractScheduler {
@@ -26,35 +29,54 @@ export class UpdateVideosScheduler extends AbstractScheduler {
26 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined 29 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
27 30
28 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() 31 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate()
29 const publishedVideos: MVideoFullLight[] = []
30 32
31 for (const schedule of schedules) { 33 for (const schedule of schedules) {
32 await sequelizeTypescript.transaction(async t => { 34 const videoOnly = await VideoModel.load(schedule.videoId)
33 const video = await VideoModel.loadFull(schedule.videoId, t) 35 const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)
34 36
35 logger.info('Executing scheduled video update on %s.', video.uuid) 37 try {
38 const { video, published } = await this.updateAVideo(schedule)
36 39
37 if (schedule.privacy) { 40 if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)
38 const wasConfidentialVideo = video.isConfidential() 41 } catch (err) {
39 const isNewVideo = video.isNewVideo(schedule.privacy) 42 logger.error('Cannot update video', { err })
43 }
40 44
41 video.setPrivacy(schedule.privacy) 45 mutexReleaser()
42 await video.save({ transaction: t }) 46 }
43 await federateVideoIfNeeded(video, isNewVideo, t) 47 }
48
49 private async updateAVideo (schedule: MScheduleVideoUpdate) {
50 let oldPrivacy: VideoPrivacy
51 let isNewVideo: boolean
52 let published = false
53
54 const video = await sequelizeTypescript.transaction(async t => {
55 const video = await VideoModel.loadFull(schedule.videoId, t)
56 if (video.state === VideoState.TO_TRANSCODE) return
57
58 logger.info('Executing scheduled video update on %s.', video.uuid)
59
60 if (schedule.privacy) {
61 isNewVideo = video.isNewVideo(schedule.privacy)
62 oldPrivacy = video.privacy
44 63
45 if (wasConfidentialVideo) { 64 setVideoPrivacy(video, schedule.privacy)
46 publishedVideos.push(video) 65 await video.save({ transaction: t })
47 } 66
67 if (oldPrivacy === VideoPrivacy.PRIVATE) {
68 published = true
48 } 69 }
70 }
49 71
50 await schedule.destroy({ transaction: t }) 72 await schedule.destroy({ transaction: t })
51 })
52 }
53 73
54 for (const v of publishedVideos) { 74 return video
55 Notifier.Instance.notifyOnNewVideoIfNeeded(v) 75 })
56 Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v) 76
57 } 77 await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false })
78
79 return { video, published }
58 } 80 }
59 81
60 static get Instance () { 82 static get Instance () {
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 91c217615..78245fa6a 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -16,7 +16,7 @@ import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
16import { logger, loggerTagsFactory } from '../../helpers/logger' 16import { logger, loggerTagsFactory } from '../../helpers/logger'
17import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 17import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
18import { CONFIG } from '../../initializers/config' 18import { CONFIG } from '../../initializers/config'
19import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' 19import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
20import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 20import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
21import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 21import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
22import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' 22import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
@@ -262,7 +262,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
262 262
263 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) 263 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid))
264 264
265 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 265 const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
266 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) 266 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
267 267
268 const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 268 const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000
diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts
index 44e26754d..736e96e65 100644
--- a/server/lib/transcoding/transcoding.ts
+++ b/server/lib/transcoding/transcoding.ts
@@ -1,3 +1,4 @@
1import { MutexInterface } from 'async-mutex'
1import { Job } from 'bullmq' 2import { Job } from 'bullmq'
2import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' 3import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
3import { basename, extname as extnameUtil, join } from 'path' 4import { basename, extname as extnameUtil, join } from 'path'
@@ -6,11 +7,13 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils'
6import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 7import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
7import { sequelizeTypescript } from '@server/initializers/database' 8import { sequelizeTypescript } from '@server/initializers/database'
8import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 9import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
10import { pick } from '@shared/core-utils'
9import { VideoResolution, VideoStorage } from '../../../shared/models/videos' 11import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
10import { 12import {
11 buildFileMetadata, 13 buildFileMetadata,
12 canDoQuickTranscode, 14 canDoQuickTranscode,
13 computeResolutionsToTranscode, 15 computeResolutionsToTranscode,
16 ffprobePromise,
14 getVideoStreamDuration, 17 getVideoStreamDuration,
15 getVideoStreamFPS, 18 getVideoStreamFPS,
16 transcodeVOD, 19 transcodeVOD,
@@ -33,7 +36,7 @@ import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
33 */ 36 */
34 37
35// Optimize the original video file and replace it. The resolution is not changed. 38// Optimize the original video file and replace it. The resolution is not changed.
36function optimizeOriginalVideofile (options: { 39async function optimizeOriginalVideofile (options: {
37 video: MVideoFullLight 40 video: MVideoFullLight
38 inputVideoFile: MVideoFile 41 inputVideoFile: MVideoFile
39 job: Job 42 job: Job
@@ -43,49 +46,61 @@ function optimizeOriginalVideofile (options: {
43 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 46 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
44 const newExtname = '.mp4' 47 const newExtname = '.mp4'
45 48
46 return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => { 49 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
47 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
48 50
49 const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) 51 try {
50 ? 'quick-transcode' 52 await video.reload()
51 : 'video'
52 53
53 const resolution = buildOriginalFileResolution(inputVideoFile.resolution) 54 const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
54 55
55 const transcodeOptions: TranscodeVODOptions = { 56 const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
56 type: transcodeType, 57 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
57 58
58 inputPath: videoInputPath, 59 const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
59 outputPath: videoTranscodedPath, 60 ? 'quick-transcode'
61 : 'video'
60 62
61 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 63 const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
62 profile: CONFIG.TRANSCODING.PROFILE,
63 64
64 resolution, 65 const transcodeOptions: TranscodeVODOptions = {
66 type: transcodeType,
65 67
66 job 68 inputPath: videoInputPath,
67 } 69 outputPath: videoTranscodedPath,
68 70
69 // Could be very long! 71 inputFileMutexReleaser,
70 await transcodeVOD(transcodeOptions)
71 72
72 // Important to do this before getVideoFilename() to take in account the new filename 73 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
73 inputVideoFile.resolution = resolution 74 profile: CONFIG.TRANSCODING.PROFILE,
74 inputVideoFile.extname = newExtname
75 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
76 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
77 75
78 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) 76 resolution,
79 77
80 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 78 job
81 await remove(videoInputPath) 79 }
82 80
83 return { transcodeType, videoFile } 81 // Could be very long!
84 }) 82 await transcodeVOD(transcodeOptions)
83
84 // Important to do this before getVideoFilename() to take in account the new filename
85 inputVideoFile.resolution = resolution
86 inputVideoFile.extname = newExtname
87 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
88 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
89
90 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
91 await remove(videoInputPath)
92
93 return { transcodeType, videoFile }
94 })
95
96 return result
97 } finally {
98 inputFileMutexReleaser()
99 }
85} 100}
86 101
87// Transcode the original video file to a lower resolution compatible with WebTorrent 102// Transcode the original video file to a lower resolution compatible with WebTorrent
88function transcodeNewWebTorrentResolution (options: { 103async function transcodeNewWebTorrentResolution (options: {
89 video: MVideoFullLight 104 video: MVideoFullLight
90 resolution: VideoResolution 105 resolution: VideoResolution
91 job: Job 106 job: Job
@@ -95,53 +110,68 @@ function transcodeNewWebTorrentResolution (options: {
95 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 110 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
96 const newExtname = '.mp4' 111 const newExtname = '.mp4'
97 112
98 return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => { 113 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
99 const newVideoFile = new VideoFileModel({
100 resolution,
101 extname: newExtname,
102 filename: generateWebTorrentVideoFilename(resolution, newExtname),
103 size: 0,
104 videoId: video.id
105 })
106 114
107 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) 115 try {
108 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) 116 await video.reload()
109 117
110 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO 118 const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
111 ? {
112 type: 'only-audio' as 'only-audio',
113 119
114 inputPath: videoInputPath, 120 const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
115 outputPath: videoTranscodedPath, 121 const newVideoFile = new VideoFileModel({
122 resolution,
123 extname: newExtname,
124 filename: generateWebTorrentVideoFilename(resolution, newExtname),
125 size: 0,
126 videoId: video.id
127 })
116 128
117 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 129 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
118 profile: CONFIG.TRANSCODING.PROFILE,
119 130
120 resolution, 131 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
132 ? {
133 type: 'only-audio' as 'only-audio',
121 134
122 job 135 inputPath: videoInputPath,
123 } 136 outputPath: videoTranscodedPath,
124 : {
125 type: 'video' as 'video',
126 inputPath: videoInputPath,
127 outputPath: videoTranscodedPath,
128 137
129 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 138 inputFileMutexReleaser,
130 profile: CONFIG.TRANSCODING.PROFILE,
131 139
132 resolution, 140 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
141 profile: CONFIG.TRANSCODING.PROFILE,
133 142
134 job 143 resolution,
135 }
136 144
137 await transcodeVOD(transcodeOptions) 145 job
146 }
147 : {
148 type: 'video' as 'video',
149 inputPath: videoInputPath,
150 outputPath: videoTranscodedPath,
138 151
139 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) 152 inputFileMutexReleaser,
140 }) 153
154 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
155 profile: CONFIG.TRANSCODING.PROFILE,
156
157 resolution,
158
159 job
160 }
161
162 await transcodeVOD(transcodeOptions)
163
164 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
165 })
166
167 return result
168 } finally {
169 inputFileMutexReleaser()
170 }
141} 171}
142 172
143// Merge an image with an audio file to create a video 173// Merge an image with an audio file to create a video
144function mergeAudioVideofile (options: { 174async function mergeAudioVideofile (options: {
145 video: MVideoFullLight 175 video: MVideoFullLight
146 resolution: VideoResolution 176 resolution: VideoResolution
147 job: Job 177 job: Job
@@ -151,54 +181,67 @@ function mergeAudioVideofile (options: {
151 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 181 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
152 const newExtname = '.mp4' 182 const newExtname = '.mp4'
153 183
154 const inputVideoFile = video.getMinQualityFile() 184 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
155 185
156 return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => { 186 try {
157 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 187 await video.reload()
158 188
159 // If the user updates the video preview during transcoding 189 const inputVideoFile = video.getMinQualityFile()
160 const previewPath = video.getPreview().getPath()
161 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
162 await copyFile(previewPath, tmpPreviewPath)
163 190
164 const transcodeOptions = { 191 const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
165 type: 'merge-audio' as 'merge-audio',
166 192
167 inputPath: tmpPreviewPath, 193 const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
168 outputPath: videoTranscodedPath, 194 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
169 195
170 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 196 // If the user updates the video preview during transcoding
171 profile: CONFIG.TRANSCODING.PROFILE, 197 const previewPath = video.getPreview().getPath()
198 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
199 await copyFile(previewPath, tmpPreviewPath)
172 200
173 audioPath: audioInputPath, 201 const transcodeOptions = {
174 resolution, 202 type: 'merge-audio' as 'merge-audio',
175 203
176 job 204 inputPath: tmpPreviewPath,
177 } 205 outputPath: videoTranscodedPath,
178 206
179 try { 207 inputFileMutexReleaser,
180 await transcodeVOD(transcodeOptions)
181 208
182 await remove(audioInputPath) 209 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
183 await remove(tmpPreviewPath) 210 profile: CONFIG.TRANSCODING.PROFILE,
184 } catch (err) {
185 await remove(tmpPreviewPath)
186 throw err
187 }
188 211
189 // Important to do this before getVideoFilename() to take in account the new file extension 212 audioPath: audioInputPath,
190 inputVideoFile.extname = newExtname 213 resolution,
191 inputVideoFile.resolution = resolution
192 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
193 214
194 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) 215 job
195 // ffmpeg generated a new video file, so update the video duration 216 }
196 // See https://trac.ffmpeg.org/ticket/5456
197 video.duration = await getVideoStreamDuration(videoTranscodedPath)
198 await video.save()
199 217
200 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 218 try {
201 }) 219 await transcodeVOD(transcodeOptions)
220
221 await remove(audioInputPath)
222 await remove(tmpPreviewPath)
223 } catch (err) {
224 await remove(tmpPreviewPath)
225 throw err
226 }
227
228 // Important to do this before getVideoFilename() to take in account the new file extension
229 inputVideoFile.extname = newExtname
230 inputVideoFile.resolution = resolution
231 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
232
233 // ffmpeg generated a new video file, so update the video duration
234 // See https://trac.ffmpeg.org/ticket/5456
235 video.duration = await getVideoStreamDuration(videoTranscodedPath)
236 await video.save()
237
238 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
239 })
240
241 return result
242 } finally {
243 inputFileMutexReleaser()
244 }
202} 245}
203 246
204// Concat TS segments from a live video to a fragmented mp4 HLS playlist 247// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@@ -207,13 +250,13 @@ async function generateHlsPlaylistResolutionFromTS (options: {
207 concatenatedTsFilePath: string 250 concatenatedTsFilePath: string
208 resolution: VideoResolution 251 resolution: VideoResolution
209 isAAC: boolean 252 isAAC: boolean
253 inputFileMutexReleaser: MutexInterface.Releaser
210}) { 254}) {
211 return generateHlsPlaylistCommon({ 255 return generateHlsPlaylistCommon({
212 video: options.video,
213 resolution: options.resolution,
214 inputPath: options.concatenatedTsFilePath,
215 type: 'hls-from-ts' as 'hls-from-ts', 256 type: 'hls-from-ts' as 'hls-from-ts',
216 isAAC: options.isAAC 257 inputPath: options.concatenatedTsFilePath,
258
259 ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
217 }) 260 })
218} 261}
219 262
@@ -223,15 +266,14 @@ function generateHlsPlaylistResolution (options: {
223 videoInputPath: string 266 videoInputPath: string
224 resolution: VideoResolution 267 resolution: VideoResolution
225 copyCodecs: boolean 268 copyCodecs: boolean
269 inputFileMutexReleaser: MutexInterface.Releaser
226 job?: Job 270 job?: Job
227}) { 271}) {
228 return generateHlsPlaylistCommon({ 272 return generateHlsPlaylistCommon({
229 video: options.video,
230 resolution: options.resolution,
231 copyCodecs: options.copyCodecs,
232 inputPath: options.videoInputPath,
233 type: 'hls' as 'hls', 273 type: 'hls' as 'hls',
234 job: options.job 274 inputPath: options.videoInputPath,
275
276 ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
235 }) 277 })
236} 278}
237 279
@@ -251,27 +293,39 @@ async function onWebTorrentVideoFileTranscoding (
251 video: MVideoFullLight, 293 video: MVideoFullLight,
252 videoFile: MVideoFile, 294 videoFile: MVideoFile,
253 transcodingPath: string, 295 transcodingPath: string,
254 outputPath: string 296 newVideoFile: MVideoFile
255) { 297) {
256 const stats = await stat(transcodingPath) 298 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
257 const fps = await getVideoStreamFPS(transcodingPath) 299
258 const metadata = await buildFileMetadata(transcodingPath) 300 try {
301 await video.reload()
302
303 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
259 304
260 await move(transcodingPath, outputPath, { overwrite: true }) 305 const stats = await stat(transcodingPath)
261 306
262 videoFile.size = stats.size 307 const probe = await ffprobePromise(transcodingPath)
263 videoFile.fps = fps 308 const fps = await getVideoStreamFPS(transcodingPath, probe)
264 videoFile.metadata = metadata 309 const metadata = await buildFileMetadata(transcodingPath, probe)
265 310
266 await createTorrentAndSetInfoHash(video, videoFile) 311 await move(transcodingPath, outputPath, { overwrite: true })
267 312
268 const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) 313 videoFile.size = stats.size
269 if (oldFile) await video.removeWebTorrentFile(oldFile) 314 videoFile.fps = fps
315 videoFile.metadata = metadata
270 316
271 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 317 await createTorrentAndSetInfoHash(video, videoFile)
272 video.VideoFiles = await video.$get('VideoFiles')
273 318
274 return { video, videoFile } 319 const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
320 if (oldFile) await video.removeWebTorrentFile(oldFile)
321
322 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
323 video.VideoFiles = await video.$get('VideoFiles')
324
325 return { video, videoFile }
326 } finally {
327 mutexReleaser()
328 }
275} 329}
276 330
277async function generateHlsPlaylistCommon (options: { 331async function generateHlsPlaylistCommon (options: {
@@ -279,12 +333,15 @@ async function generateHlsPlaylistCommon (options: {
279 video: MVideo 333 video: MVideo
280 inputPath: string 334 inputPath: string
281 resolution: VideoResolution 335 resolution: VideoResolution
336
337 inputFileMutexReleaser: MutexInterface.Releaser
338
282 copyCodecs?: boolean 339 copyCodecs?: boolean
283 isAAC?: boolean 340 isAAC?: boolean
284 341
285 job?: Job 342 job?: Job
286}) { 343}) {
287 const { type, video, inputPath, resolution, copyCodecs, isAAC, job } = options 344 const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
288 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 345 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
289 346
290 const videoTranscodedBasePath = join(transcodeDirectory, type) 347 const videoTranscodedBasePath = join(transcodeDirectory, type)
@@ -308,6 +365,8 @@ async function generateHlsPlaylistCommon (options: {
308 365
309 isAAC, 366 isAAC,
310 367
368 inputFileMutexReleaser,
369
311 hlsPlaylist: { 370 hlsPlaylist: {
312 videoFilename 371 videoFilename
313 }, 372 },
@@ -333,40 +392,54 @@ async function generateHlsPlaylistCommon (options: {
333 videoStreamingPlaylistId: playlist.id 392 videoStreamingPlaylistId: playlist.id
334 }) 393 })
335 394
336 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) 395 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
337 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
338 396
339 // Move playlist file 397 try {
340 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) 398 // VOD transcoding is a long task, refresh video attributes
341 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) 399 await video.reload()
342 // Move video file
343 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
344 400
345 // Update video duration if it was not set (in case of a live for example) 401 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
346 if (!video.duration) { 402 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
347 video.duration = await getVideoStreamDuration(videoFilePath)
348 await video.save()
349 }
350 403
351 const stats = await stat(videoFilePath) 404 // Move playlist file
405 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
406 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
407 // Move video file
408 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
352 409
353 newVideoFile.size = stats.size 410 // Update video duration if it was not set (in case of a live for example)
354 newVideoFile.fps = await getVideoStreamFPS(videoFilePath) 411 if (!video.duration) {
355 newVideoFile.metadata = await buildFileMetadata(videoFilePath) 412 video.duration = await getVideoStreamDuration(videoFilePath)
413 await video.save()
414 }
356 415
357 await createTorrentAndSetInfoHash(playlist, newVideoFile) 416 const stats = await stat(videoFilePath)
358 417
359 const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) 418 newVideoFile.size = stats.size
360 if (oldFile) { 419 newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
361 await video.removeStreamingPlaylistVideoFile(playlist, oldFile) 420 newVideoFile.metadata = await buildFileMetadata(videoFilePath)
362 await oldFile.destroy() 421
363 } 422 await createTorrentAndSetInfoHash(playlist, newVideoFile)
364 423
365 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) 424 const oldFile = await VideoFileModel.loadHLSFile({
425 playlistId: playlist.id,
426 fps: newVideoFile.fps,
427 resolution: newVideoFile.resolution
428 })
429
430 if (oldFile) {
431 await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
432 await oldFile.destroy()
433 }
366 434
367 await updatePlaylistAfterFileChange(video, playlist) 435 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
368 436
369 return { resolutionPlaylistPath, videoFile: savedVideoFile } 437 await updatePlaylistAfterFileChange(video, playlist)
438
439 return { resolutionPlaylistPath, videoFile: savedVideoFile }
440 } finally {
441 mutexReleaser()
442 }
370} 443}
371 444
372function buildOriginalFileResolution (inputResolution: number) { 445function buildOriginalFileResolution (inputResolution: number) {
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
index c3f55fd95..9953cae5d 100644
--- a/server/lib/video-path-manager.ts
+++ b/server/lib/video-path-manager.ts
@@ -1,29 +1,31 @@
1import { Mutex } from 'async-mutex'
1import { remove } from 'fs-extra' 2import { remove } from 'fs-extra'
2import { extname, join } from 'path' 3import { extname, join } from 'path'
4import { logger, loggerTagsFactory } from '@server/helpers/logger'
3import { extractVideo } from '@server/helpers/video' 5import { extractVideo } from '@server/helpers/video'
4import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
5import { 7import { DIRECTORIES } from '@server/initializers/constants'
6 MStreamingPlaylistVideo, 8import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models'
7 MVideo,
8 MVideoFile,
9 MVideoFileStreamingPlaylistVideo,
10 MVideoFileVideo,
11 MVideoUUID
12} from '@server/types/models'
13import { buildUUID } from '@shared/extra-utils' 9import { buildUUID } from '@shared/extra-utils'
14import { VideoStorage } from '@shared/models' 10import { VideoStorage } from '@shared/models'
15import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' 11import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
16import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' 12import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
13import { isVideoInPrivateDirectory } from './video-privacy'
17 14
18type MakeAvailableCB <T> = (path: string) => Promise<T> | T 15type MakeAvailableCB <T> = (path: string) => Promise<T> | T
19 16
17const lTags = loggerTagsFactory('video-path-manager')
18
20class VideoPathManager { 19class VideoPathManager {
21 20
22 private static instance: VideoPathManager 21 private static instance: VideoPathManager
23 22
23 // Key is a video UUID
24 private readonly videoFileMutexStore = new Map<string, Mutex>()
25
24 private constructor () {} 26 private constructor () {}
25 27
26 getFSHLSOutputPath (video: MVideoUUID, filename?: string) { 28 getFSHLSOutputPath (video: MVideo, filename?: string) {
27 const base = getHLSDirectory(video) 29 const base = getHLSDirectory(video)
28 if (!filename) return base 30 if (!filename) return base
29 31
@@ -41,13 +43,17 @@ class VideoPathManager {
41 } 43 }
42 44
43 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { 45 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
44 if (videoFile.isHLS()) { 46 const video = extractVideo(videoOrPlaylist)
45 const video = extractVideo(videoOrPlaylist)
46 47
48 if (videoFile.isHLS()) {
47 return join(getHLSDirectory(video), videoFile.filename) 49 return join(getHLSDirectory(video), videoFile.filename)
48 } 50 }
49 51
50 return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) 52 if (isVideoInPrivateDirectory(video.privacy)) {
53 return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename)
54 }
55
56 return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename)
51 } 57 }
52 58
53 async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) { 59 async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
@@ -113,6 +119,27 @@ class VideoPathManager {
113 ) 119 )
114 } 120 }
115 121
122 async lockFiles (videoUUID: string) {
123 if (!this.videoFileMutexStore.has(videoUUID)) {
124 this.videoFileMutexStore.set(videoUUID, new Mutex())
125 }
126
127 const mutex = this.videoFileMutexStore.get(videoUUID)
128 const releaser = await mutex.acquire()
129
130 logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID))
131
132 return releaser
133 }
134
135 unlockFiles (videoUUID: string) {
136 const mutex = this.videoFileMutexStore.get(videoUUID)
137
138 mutex.release()
139
140 logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID))
141 }
142
116 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { 143 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) {
117 let result: T 144 let result: T
118 145
diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts
new file mode 100644
index 000000000..1a4a5a22d
--- /dev/null
+++ b/server/lib/video-privacy.ts
@@ -0,0 +1,96 @@
1import { move } from 'fs-extra'
2import { join } from 'path'
3import { logger } from '@server/helpers/logger'
4import { DIRECTORIES } from '@server/initializers/constants'
5import { MVideo, MVideoFullLight } from '@server/types/models'
6import { VideoPrivacy } from '@shared/models'
7
8function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) {
9 if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
10 video.publishedAt = new Date()
11 }
12
13 video.privacy = newPrivacy
14}
15
16function isVideoInPrivateDirectory (privacy: VideoPrivacy) {
17 return privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.INTERNAL
18}
19
20function isVideoInPublicDirectory (privacy: VideoPrivacy) {
21 return !isVideoInPrivateDirectory(privacy)
22}
23
24async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) {
25 // Now public, previously private
26 if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) {
27 await moveFiles({ type: 'private-to-public', video })
28
29 return true
30 }
31
32 // Now private, previously public
33 if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) {
34 await moveFiles({ type: 'public-to-private', video })
35
36 return true
37 }
38
39 return false
40}
41
42export {
43 setVideoPrivacy,
44
45 isVideoInPrivateDirectory,
46 isVideoInPublicDirectory,
47
48 moveFilesIfPrivacyChanged
49}
50
51// ---------------------------------------------------------------------------
52
53async function moveFiles (options: {
54 type: 'private-to-public' | 'public-to-private'
55 video: MVideoFullLight
56}) {
57 const { type, video } = options
58
59 const directories = type === 'private-to-public'
60 ? {
61 webtorrent: { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC },
62 hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC }
63 }
64 : {
65 webtorrent: { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE },
66 hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE }
67 }
68
69 for (const file of video.VideoFiles) {
70 const source = join(directories.webtorrent.old, file.filename)
71 const destination = join(directories.webtorrent.new, file.filename)
72
73 try {
74 logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
75
76 await move(source, destination)
77 } catch (err) {
78 logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err })
79 }
80 }
81
82 const hls = video.getHLSPlaylist()
83
84 if (hls) {
85 const source = join(directories.hls.old, video.uuid)
86 const destination = join(directories.hls.new, video.uuid)
87
88 try {
89 logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
90
91 await move(source, destination)
92 } catch (err) {
93 logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err })
94 }
95 }
96}
diff --git a/server/lib/video-tokens-manager.ts b/server/lib/video-tokens-manager.ts
new file mode 100644
index 000000000..c43085d16
--- /dev/null
+++ b/server/lib/video-tokens-manager.ts
@@ -0,0 +1,49 @@
1import LRUCache from 'lru-cache'
2import { LRU_CACHE } from '@server/initializers/constants'
3import { buildUUID } from '@shared/extra-utils'
4
5// ---------------------------------------------------------------------------
6// Create temporary tokens that can be used as URL query parameters to access video static files
7// ---------------------------------------------------------------------------
8
9class VideoTokensManager {
10
11 private static instance: VideoTokensManager
12
13 private readonly lruCache = new LRUCache<string, string>({
14 max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE,
15 ttl: LRU_CACHE.VIDEO_TOKENS.TTL
16 })
17
18 private constructor () {}
19
20 create (videoUUID: string) {
21 const token = buildUUID()
22
23 const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL)
24
25 this.lruCache.set(token, videoUUID)
26
27 return { token, expires }
28 }
29
30 hasToken (options: {
31 token: string
32 videoUUID: string
33 }) {
34 const value = this.lruCache.get(options.token)
35 if (!value) return false
36
37 return value === options.videoUUID
38 }
39
40 static get Instance () {
41 return this.instance || (this.instance = new this())
42 }
43}
44
45// ---------------------------------------------------------------------------
46
47export {
48 VideoTokensManager
49}
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 6c4f3ce7b..aacc41a7a 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -7,10 +7,11 @@ import { TagModel } from '@server/models/video/tag'
7import { VideoModel } from '@server/models/video/video' 7import { VideoModel } from '@server/models/video/video'
8import { VideoJobInfoModel } from '@server/models/video/video-job-info' 8import { VideoJobInfoModel } from '@server/models/video/video-job-info'
9import { FilteredModelAttributes } from '@server/types' 9import { FilteredModelAttributes } from '@server/types'
10import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' 10import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
11import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models' 11import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
12import { CreateJobOptions } from './job-queue/job-queue' 12import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue'
13import { updateVideoMiniatureFromExisting } from './thumbnail' 13import { updateVideoMiniatureFromExisting } from './thumbnail'
14import { moveFilesIfPrivacyChanged } from './video-privacy'
14 15
15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 16function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
16 return { 17 return {
@@ -177,6 +178,59 @@ const getCachedVideoDuration = memoizee(getVideoDuration, {
177 178
178// --------------------------------------------------------------------------- 179// ---------------------------------------------------------------------------
179 180
181async function addVideoJobsAfterUpdate (options: {
182 video: MVideoFullLight
183 isNewVideo: boolean
184
185 nameChanged: boolean
186 oldPrivacy: VideoPrivacy
187}) {
188 const { video, nameChanged, oldPrivacy, isNewVideo } = options
189 const jobs: CreateJobArgument[] = []
190
191 const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy)
192
193 if (!video.isLive && (nameChanged || filePathChanged)) {
194 for (const file of (video.VideoFiles || [])) {
195 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
196
197 jobs.push({ type: 'manage-video-torrent', payload })
198 }
199
200 const hls = video.getHLSPlaylist()
201
202 for (const file of (hls?.VideoFiles || [])) {
203 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
204
205 jobs.push({ type: 'manage-video-torrent', payload })
206 }
207 }
208
209 jobs.push({
210 type: 'federate-video',
211 payload: {
212 videoUUID: video.uuid,
213 isNewVideo
214 }
215 })
216
217 const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy)
218
219 if (wasConfidentialVideo) {
220 jobs.push({
221 type: 'notify',
222 payload: {
223 action: 'new-video',
224 videoUUID: video.uuid
225 }
226 })
227 }
228
229 return JobQueue.Instance.createSequentialJobFlow(...jobs)
230}
231
232// ---------------------------------------------------------------------------
233
180export { 234export {
181 buildLocalVideoFromReq, 235 buildLocalVideoFromReq,
182 buildVideoThumbnailsFromReq, 236 buildVideoThumbnailsFromReq,
@@ -185,5 +239,6 @@ export {
185 buildTranscodingJob, 239 buildTranscodingJob,
186 buildMoveToObjectStorageJob, 240 buildMoveToObjectStorageJob,
187 getTranscodingJobPriority, 241 getTranscodingJobPriority,
242 addVideoJobsAfterUpdate,
188 getCachedVideoDuration 243 getCachedVideoDuration
189} 244}