aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/studio.ts40
-rw-r--r--server/initializers/config.ts1
-rw-r--r--server/lib/job-queue/handlers/video-studio-edition.ts90
-rw-r--r--server/lib/video-studio.ts23
-rw-r--r--server/middlewares/validators/videos/video-studio.ts6
-rw-r--r--server/tests/api/transcoding/video-studio.ts25
-rw-r--r--server/tests/shared/directories.ts5
7 files changed, 133 insertions, 57 deletions
diff --git a/server/controllers/api/videos/studio.ts b/server/controllers/api/videos/studio.ts
index 6667532bf..2ccb2fb89 100644
--- a/server/controllers/api/videos/studio.ts
+++ b/server/controllers/api/videos/studio.ts
@@ -1,8 +1,12 @@
1import Bluebird from 'bluebird'
1import express from 'express' 2import express from 'express'
3import { move } from 'fs-extra'
4import { basename, join } from 'path'
2import { createAnyReqFiles } from '@server/helpers/express-utils' 5import { createAnyReqFiles } from '@server/helpers/express-utils'
6import { CONFIG } from '@server/initializers/config'
3import { MIMETYPES } from '@server/initializers/constants' 7import { MIMETYPES } from '@server/initializers/constants'
4import { JobQueue } from '@server/lib/job-queue' 8import { JobQueue } from '@server/lib/job-queue'
5import { buildTaskFileFieldname, getTaskFile } from '@server/lib/video-studio' 9import { buildTaskFileFieldname, getTaskFileFromReq } from '@server/lib/video-studio'
6import { 10import {
7 HttpStatusCode, 11 HttpStatusCode,
8 VideoState, 12 VideoState,
@@ -68,7 +72,7 @@ async function createEditionTasks (req: express.Request, res: express.Response)
68 72
69 const payload = { 73 const payload = {
70 videoUUID: video.uuid, 74 videoUUID: video.uuid,
71 tasks: body.tasks.map((t, i) => buildTaskPayload(t, i, files)) 75 tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files))
72 } 76 }
73 77
74 JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload }) 78 JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload })
@@ -77,7 +81,11 @@ async function createEditionTasks (req: express.Request, res: express.Response)
77} 81}
78 82
79const taskPayloadBuilders: { 83const taskPayloadBuilders: {
80 [id in VideoStudioTask['name']]: (task: VideoStudioTask, indice?: number, files?: Express.Multer.File[]) => VideoStudioTaskPayload 84 [id in VideoStudioTask['name']]: (
85 task: VideoStudioTask,
86 indice?: number,
87 files?: Express.Multer.File[]
88 ) => Promise<VideoStudioTaskPayload>
81} = { 89} = {
82 'add-intro': buildIntroOutroTask, 90 'add-intro': buildIntroOutroTask,
83 'add-outro': buildIntroOutroTask, 91 'add-outro': buildIntroOutroTask,
@@ -85,34 +93,46 @@ const taskPayloadBuilders: {
85 'add-watermark': buildWatermarkTask 93 'add-watermark': buildWatermarkTask
86} 94}
87 95
88function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): VideoStudioTaskPayload { 96function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): Promise<VideoStudioTaskPayload> {
89 return taskPayloadBuilders[task.name](task, indice, files) 97 return taskPayloadBuilders[task.name](task, indice, files)
90} 98}
91 99
92function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) { 100async function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) {
101 const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path)
102
93 return { 103 return {
94 name: task.name, 104 name: task.name,
95 options: { 105 options: {
96 file: getTaskFile(files, indice).path 106 file: destination
97 } 107 }
98 } 108 }
99} 109}
100 110
101function buildCutTask (task: VideoStudioTaskCut) { 111function buildCutTask (task: VideoStudioTaskCut) {
102 return { 112 return Promise.resolve({
103 name: task.name, 113 name: task.name,
104 options: { 114 options: {
105 start: task.options.start, 115 start: task.options.start,
106 end: task.options.end 116 end: task.options.end
107 } 117 }
108 } 118 })
109} 119}
110 120
111function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) { 121async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) {
122 const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path)
123
112 return { 124 return {
113 name: task.name, 125 name: task.name,
114 options: { 126 options: {
115 file: getTaskFile(files, indice).path 127 file: destination
116 } 128 }
117 } 129 }
118} 130}
131
132async function moveStudioFileToPersistentTMP (file: string) {
133 const destination = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, basename(file))
134
135 await move(file, destination)
136
137 return destination
138}
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 699dd4704..f2d8f99b5 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -98,6 +98,7 @@ const CONFIG = {
98 98
99 STORAGE: { 99 STORAGE: {
100 TMP_DIR: buildPath(config.get<string>('storage.tmp')), 100 TMP_DIR: buildPath(config.get<string>('storage.tmp')),
101 TMP_PERSISTENT_DIR: buildPath(config.get<string>('storage.tmp_persistent')),
101 BIN_DIR: buildPath(config.get<string>('storage.bin')), 102 BIN_DIR: buildPath(config.get<string>('storage.bin')),
102 ACTOR_IMAGES: buildPath(config.get<string>('storage.avatars')), 103 ACTOR_IMAGES: buildPath(config.get<string>('storage.avatars')),
103 LOG_DIR: buildPath(config.get<string>('storage.logs')), 104 LOG_DIR: buildPath(config.get<string>('storage.logs')),
diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts
index fbb55a388..5e8dd4f51 100644
--- a/server/lib/job-queue/handlers/video-studio-edition.ts
+++ b/server/lib/job-queue/handlers/video-studio-edition.ts
@@ -12,7 +12,7 @@ import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default
12import { isAbleToUploadVideo } from '@server/lib/user' 12import { isAbleToUploadVideo } from '@server/lib/user'
13import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' 13import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
14import { VideoPathManager } from '@server/lib/video-path-manager' 14import { VideoPathManager } from '@server/lib/video-path-manager'
15import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' 15import { approximateIntroOutroAdditionalSize, safeCleanupStudioTMPFiles } from '@server/lib/video-studio'
16import { UserModel } from '@server/models/user/user' 16import { UserModel } from '@server/models/user/user'
17import { VideoModel } from '@server/models/video/video' 17import { VideoModel } from '@server/models/video/video'
18import { VideoFileModel } from '@server/models/video/video-file' 18import { VideoFileModel } from '@server/models/video/video-file'
@@ -39,63 +39,73 @@ async function processVideoStudioEdition (job: Job) {
39 39
40 logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) 40 logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
41 41
42 const video = await VideoModel.loadFull(payload.videoUUID) 42 try {
43 const video = await VideoModel.loadFull(payload.videoUUID)
43 44
44 // No video, maybe deleted? 45 // No video, maybe deleted?
45 if (!video) { 46 if (!video) {
46 logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) 47 logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
47 return undefined
48 }
49 48
50 await checkUserQuotaOrThrow(video, payload) 49 await safeCleanupStudioTMPFiles(payload)
50 return undefined
51 }
51 52
52 const inputFile = video.getMaxQualityFile() 53 await checkUserQuotaOrThrow(video, payload)
53 54
54 const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { 55 const inputFile = video.getMaxQualityFile()
55 let tmpInputFilePath: string
56 let outputPath: string
57 56
58 for (const task of payload.tasks) { 57 const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => {
59 const outputFilename = buildUUID() + inputFile.extname 58 let tmpInputFilePath: string
60 outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) 59 let outputPath: string
61 60
62 await processTask({ 61 for (const task of payload.tasks) {
63 inputPath: tmpInputFilePath ?? originalFilePath, 62 const outputFilename = buildUUID() + inputFile.extname
64 video, 63 outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
65 outputPath,
66 task,
67 lTags
68 })
69 64
70 if (tmpInputFilePath) await remove(tmpInputFilePath) 65 await processTask({
66 inputPath: tmpInputFilePath ?? originalFilePath,
67 video,
68 outputPath,
69 task,
70 lTags
71 })
71 72
72 // For the next iteration 73 if (tmpInputFilePath) await remove(tmpInputFilePath)
73 tmpInputFilePath = outputPath
74 }
75 74
76 return outputPath 75 // For the next iteration
77 }) 76 tmpInputFilePath = outputPath
77 }
78 78
79 logger.info('Video edition ended for video %s.', video.uuid, lTags) 79 return outputPath
80 })
80 81
81 const newFile = await buildNewFile(video, editionResultPath) 82 logger.info('Video edition ended for video %s.', video.uuid, lTags)
82 83
83 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) 84 const newFile = await buildNewFile(video, editionResultPath)
84 await move(editionResultPath, outputPath)
85 85
86 await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) 86 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile)
87 await removeAllFiles(video, newFile) 87 await move(editionResultPath, outputPath)
88 88
89 await newFile.save() 89 await safeCleanupStudioTMPFiles(payload)
90 90
91 video.duration = await getVideoStreamDuration(outputPath) 91 await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)
92 await video.save() 92 await removeAllFiles(video, newFile)
93 93
94 await federateVideoIfNeeded(video, false, undefined) 94 await newFile.save()
95 95
96 const user = await UserModel.loadByVideoId(video.id) 96 video.duration = await getVideoStreamDuration(outputPath)
97 await video.save()
97 98
98 await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) 99 await federateVideoIfNeeded(video, false, undefined)
100
101 const user = await UserModel.loadByVideoId(video.id)
102
103 await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false })
104 } catch (err) {
105 await safeCleanupStudioTMPFiles(payload)
106
107 throw err
108 }
99} 109}
100 110
101// --------------------------------------------------------------------------- 111// ---------------------------------------------------------------------------
diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts
index b392bdb00..beda326a0 100644
--- a/server/lib/video-studio.ts
+++ b/server/lib/video-studio.ts
@@ -1,15 +1,31 @@
1import { logger } from '@server/helpers/logger'
1import { MVideoFullLight } from '@server/types/models' 2import { MVideoFullLight } from '@server/types/models'
2import { getVideoStreamDuration } from '@shared/ffmpeg' 3import { getVideoStreamDuration } from '@shared/ffmpeg'
3import { VideoStudioTask } from '@shared/models' 4import { VideoStudioEditionPayload, VideoStudioTask } from '@shared/models'
5import { remove } from 'fs-extra'
4 6
5function buildTaskFileFieldname (indice: number, fieldName = 'file') { 7function buildTaskFileFieldname (indice: number, fieldName = 'file') {
6 return `tasks[${indice}][options][${fieldName}]` 8 return `tasks[${indice}][options][${fieldName}]`
7} 9}
8 10
9function getTaskFile (files: Express.Multer.File[], indice: number, fieldName = 'file') { 11function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') {
10 return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) 12 return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName))
11} 13}
12 14
15async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) {
16 for (const task of payload.tasks) {
17 try {
18 if (task.name === 'add-intro' || task.name === 'add-outro') {
19 await remove(task.options.file)
20 } else if (task.name === 'add-watermark') {
21 await remove(task.options.file)
22 }
23 } catch (err) {
24 logger.error('Cannot remove studio file', { err })
25 }
26 }
27}
28
13async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) { 29async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) {
14 let additionalDuration = 0 30 let additionalDuration = 0
15 31
@@ -28,5 +44,6 @@ async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, task
28export { 44export {
29 approximateIntroOutroAdditionalSize, 45 approximateIntroOutroAdditionalSize,
30 buildTaskFileFieldname, 46 buildTaskFileFieldname,
31 getTaskFile 47 getTaskFileFromReq,
48 safeCleanupStudioTMPFiles
32} 49}
diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts
index 4397e887e..7a68f88e5 100644
--- a/server/middlewares/validators/videos/video-studio.ts
+++ b/server/middlewares/validators/videos/video-studio.ts
@@ -9,7 +9,7 @@ import {
9} from '@server/helpers/custom-validators/video-studio' 9} from '@server/helpers/custom-validators/video-studio'
10import { cleanUpReqFiles } from '@server/helpers/express-utils' 10import { cleanUpReqFiles } from '@server/helpers/express-utils'
11import { CONFIG } from '@server/initializers/config' 11import { CONFIG } from '@server/initializers/config'
12import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio' 12import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio'
13import { isAudioFile } from '@shared/ffmpeg' 13import { isAudioFile } from '@shared/ffmpeg'
14import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' 14import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models'
15import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' 15import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared'
@@ -49,7 +49,7 @@ const videoStudioAddEditionValidator = [
49 } 49 }
50 50
51 if (task.name === 'add-intro' || task.name === 'add-outro') { 51 if (task.name === 'add-intro' || task.name === 'add-outro') {
52 const filePath = getTaskFile(files, i).path 52 const filePath = getTaskFileFromReq(files, i).path
53 53
54 // Our concat filter needs a video stream 54 // Our concat filter needs a video stream
55 if (await isAudioFile(filePath)) { 55 if (await isAudioFile(filePath)) {
@@ -79,7 +79,7 @@ const videoStudioAddEditionValidator = [
79 if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) 79 if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
80 80
81 // Try to make an approximation of bytes added by the intro/outro 81 // Try to make an approximation of bytes added by the intro/outro
82 const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFile(files, i).path) 82 const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFileFromReq(files, i).path)
83 if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req) 83 if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req)
84 84
85 return next() 85 return next()
diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts
index ab08e8fb6..30f72e6e9 100644
--- a/server/tests/api/transcoding/video-studio.ts
+++ b/server/tests/api/transcoding/video-studio.ts
@@ -1,5 +1,5 @@
1import { expect } from 'chai' 1import { expect } from 'chai'
2import { expectStartWith } from '@server/tests/shared' 2import { checkPersistentTmpIsEmpty, expectStartWith } from '@server/tests/shared'
3import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' 3import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils'
4import { VideoStudioTask } from '@shared/models' 4import { VideoStudioTask } from '@shared/models'
5import { 5import {
@@ -356,6 +356,29 @@ describe('Test video studio', function () {
356 }) 356 })
357 }) 357 })
358 358
359 describe('Server restart', function () {
360
361 it('Should still be able to run video edition after a server restart', async function () {
362 this.timeout(240_000)
363
364 await renewVideo()
365 await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks: VideoStudioCommand.getComplexTask() })
366
367 await servers[0].kill()
368 await servers[0].run()
369
370 await waitJobs(servers)
371
372 for (const server of servers) {
373 await checkDuration(server, 9)
374 }
375 })
376
377 it('Should have an empty persistent tmp directory', async function () {
378 await checkPersistentTmpIsEmpty(servers[0])
379 })
380 })
381
359 after(async function () { 382 after(async function () {
360 await cleanupTests(servers) 383 await cleanupTests(servers)
361 }) 384 })
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts
index 90d534a06..a614cef7c 100644
--- a/server/tests/shared/directories.ts
+++ b/server/tests/shared/directories.ts
@@ -12,6 +12,10 @@ async function checkTmpIsEmpty (server: PeerTubeServer) {
12 } 12 }
13} 13}
14 14
15async function checkPersistentTmpIsEmpty (server: PeerTubeServer) {
16 await checkDirectoryIsEmpty(server, 'tmp-persistent')
17}
18
15async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { 19async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) {
16 const directoryPath = server.getDirectoryPath(directory) 20 const directoryPath = server.getDirectoryPath(directory)
17 21
@@ -26,5 +30,6 @@ async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string,
26 30
27export { 31export {
28 checkTmpIsEmpty, 32 checkTmpIsEmpty,
33 checkPersistentTmpIsEmpty,
29 checkDirectoryIsEmpty 34 checkDirectoryIsEmpty
30} 35}