diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/videos/studio.ts | 40 | ||||
-rw-r--r-- | server/initializers/config.ts | 1 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-studio-edition.ts | 90 | ||||
-rw-r--r-- | server/lib/video-studio.ts | 23 | ||||
-rw-r--r-- | server/middlewares/validators/videos/video-studio.ts | 6 | ||||
-rw-r--r-- | server/tests/api/transcoding/video-studio.ts | 25 | ||||
-rw-r--r-- | server/tests/shared/directories.ts | 5 |
7 files changed, 133 insertions, 57 deletions
diff --git a/server/controllers/api/videos/studio.ts b/server/controllers/api/videos/studio.ts index 6667532bf..2ccb2fb89 100644 --- a/server/controllers/api/videos/studio.ts +++ b/server/controllers/api/videos/studio.ts | |||
@@ -1,8 +1,12 @@ | |||
1 | import Bluebird from 'bluebird' | ||
1 | import express from 'express' | 2 | import express from 'express' |
3 | import { move } from 'fs-extra' | ||
4 | import { basename, join } from 'path' | ||
2 | import { createAnyReqFiles } from '@server/helpers/express-utils' | 5 | import { createAnyReqFiles } from '@server/helpers/express-utils' |
6 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { MIMETYPES } from '@server/initializers/constants' | 7 | import { MIMETYPES } from '@server/initializers/constants' |
4 | import { JobQueue } from '@server/lib/job-queue' | 8 | import { JobQueue } from '@server/lib/job-queue' |
5 | import { buildTaskFileFieldname, getTaskFile } from '@server/lib/video-studio' | 9 | import { buildTaskFileFieldname, getTaskFileFromReq } from '@server/lib/video-studio' |
6 | import { | 10 | import { |
7 | HttpStatusCode, | 11 | HttpStatusCode, |
8 | VideoState, | 12 | VideoState, |
@@ -68,7 +72,7 @@ async function createEditionTasks (req: express.Request, res: express.Response) | |||
68 | 72 | ||
69 | const payload = { | 73 | const payload = { |
70 | videoUUID: video.uuid, | 74 | videoUUID: video.uuid, |
71 | tasks: body.tasks.map((t, i) => buildTaskPayload(t, i, files)) | 75 | tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files)) |
72 | } | 76 | } |
73 | 77 | ||
74 | JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload }) | 78 | JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload }) |
@@ -77,7 +81,11 @@ async function createEditionTasks (req: express.Request, res: express.Response) | |||
77 | } | 81 | } |
78 | 82 | ||
79 | const taskPayloadBuilders: { | 83 | const taskPayloadBuilders: { |
80 | [id in VideoStudioTask['name']]: (task: VideoStudioTask, indice?: number, files?: Express.Multer.File[]) => VideoStudioTaskPayload | 84 | [id in VideoStudioTask['name']]: ( |
85 | task: VideoStudioTask, | ||
86 | indice?: number, | ||
87 | files?: Express.Multer.File[] | ||
88 | ) => Promise<VideoStudioTaskPayload> | ||
81 | } = { | 89 | } = { |
82 | 'add-intro': buildIntroOutroTask, | 90 | 'add-intro': buildIntroOutroTask, |
83 | 'add-outro': buildIntroOutroTask, | 91 | 'add-outro': buildIntroOutroTask, |
@@ -85,34 +93,46 @@ const taskPayloadBuilders: { | |||
85 | 'add-watermark': buildWatermarkTask | 93 | 'add-watermark': buildWatermarkTask |
86 | } | 94 | } |
87 | 95 | ||
88 | function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): VideoStudioTaskPayload { | 96 | function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): Promise<VideoStudioTaskPayload> { |
89 | return taskPayloadBuilders[task.name](task, indice, files) | 97 | return taskPayloadBuilders[task.name](task, indice, files) |
90 | } | 98 | } |
91 | 99 | ||
92 | function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) { | 100 | async function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) { |
101 | const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path) | ||
102 | |||
93 | return { | 103 | return { |
94 | name: task.name, | 104 | name: task.name, |
95 | options: { | 105 | options: { |
96 | file: getTaskFile(files, indice).path | 106 | file: destination |
97 | } | 107 | } |
98 | } | 108 | } |
99 | } | 109 | } |
100 | 110 | ||
101 | function buildCutTask (task: VideoStudioTaskCut) { | 111 | function buildCutTask (task: VideoStudioTaskCut) { |
102 | return { | 112 | return Promise.resolve({ |
103 | name: task.name, | 113 | name: task.name, |
104 | options: { | 114 | options: { |
105 | start: task.options.start, | 115 | start: task.options.start, |
106 | end: task.options.end | 116 | end: task.options.end |
107 | } | 117 | } |
108 | } | 118 | }) |
109 | } | 119 | } |
110 | 120 | ||
111 | function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) { | 121 | async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) { |
122 | const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path) | ||
123 | |||
112 | return { | 124 | return { |
113 | name: task.name, | 125 | name: task.name, |
114 | options: { | 126 | options: { |
115 | file: getTaskFile(files, indice).path | 127 | file: destination |
116 | } | 128 | } |
117 | } | 129 | } |
118 | } | 130 | } |
131 | |||
132 | async function moveStudioFileToPersistentTMP (file: string) { | ||
133 | const destination = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, basename(file)) | ||
134 | |||
135 | await move(file, destination) | ||
136 | |||
137 | return destination | ||
138 | } | ||
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 699dd4704..f2d8f99b5 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -98,6 +98,7 @@ const CONFIG = { | |||
98 | 98 | ||
99 | STORAGE: { | 99 | STORAGE: { |
100 | TMP_DIR: buildPath(config.get<string>('storage.tmp')), | 100 | TMP_DIR: buildPath(config.get<string>('storage.tmp')), |
101 | TMP_PERSISTENT_DIR: buildPath(config.get<string>('storage.tmp_persistent')), | ||
101 | BIN_DIR: buildPath(config.get<string>('storage.bin')), | 102 | BIN_DIR: buildPath(config.get<string>('storage.bin')), |
102 | ACTOR_IMAGES: buildPath(config.get<string>('storage.avatars')), | 103 | ACTOR_IMAGES: buildPath(config.get<string>('storage.avatars')), |
103 | LOG_DIR: buildPath(config.get<string>('storage.logs')), | 104 | LOG_DIR: buildPath(config.get<string>('storage.logs')), |
diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts index fbb55a388..5e8dd4f51 100644 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/lib/job-queue/handlers/video-studio-edition.ts | |||
@@ -12,7 +12,7 @@ import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default | |||
12 | import { isAbleToUploadVideo } from '@server/lib/user' | 12 | import { isAbleToUploadVideo } from '@server/lib/user' |
13 | import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' | 13 | import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' |
14 | import { VideoPathManager } from '@server/lib/video-path-manager' | 14 | import { VideoPathManager } from '@server/lib/video-path-manager' |
15 | import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' | 15 | import { approximateIntroOutroAdditionalSize, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' |
16 | import { UserModel } from '@server/models/user/user' | 16 | import { UserModel } from '@server/models/user/user' |
17 | import { VideoModel } from '@server/models/video/video' | 17 | import { VideoModel } from '@server/models/video/video' |
18 | import { VideoFileModel } from '@server/models/video/video-file' | 18 | import { VideoFileModel } from '@server/models/video/video-file' |
@@ -39,63 +39,73 @@ async function processVideoStudioEdition (job: Job) { | |||
39 | 39 | ||
40 | logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) | 40 | logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) |
41 | 41 | ||
42 | const video = await VideoModel.loadFull(payload.videoUUID) | 42 | try { |
43 | const video = await VideoModel.loadFull(payload.videoUUID) | ||
43 | 44 | ||
44 | // No video, maybe deleted? | 45 | // No video, maybe deleted? |
45 | if (!video) { | 46 | if (!video) { |
46 | logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) | 47 | logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) |
47 | return undefined | ||
48 | } | ||
49 | 48 | ||
50 | await checkUserQuotaOrThrow(video, payload) | 49 | await safeCleanupStudioTMPFiles(payload) |
50 | return undefined | ||
51 | } | ||
51 | 52 | ||
52 | const inputFile = video.getMaxQualityFile() | 53 | await checkUserQuotaOrThrow(video, payload) |
53 | 54 | ||
54 | const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { | 55 | const inputFile = video.getMaxQualityFile() |
55 | let tmpInputFilePath: string | ||
56 | let outputPath: string | ||
57 | 56 | ||
58 | for (const task of payload.tasks) { | 57 | const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { |
59 | const outputFilename = buildUUID() + inputFile.extname | 58 | let tmpInputFilePath: string |
60 | outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) | 59 | let outputPath: string |
61 | 60 | ||
62 | await processTask({ | 61 | for (const task of payload.tasks) { |
63 | inputPath: tmpInputFilePath ?? originalFilePath, | 62 | const outputFilename = buildUUID() + inputFile.extname |
64 | video, | 63 | outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) |
65 | outputPath, | ||
66 | task, | ||
67 | lTags | ||
68 | }) | ||
69 | 64 | ||
70 | if (tmpInputFilePath) await remove(tmpInputFilePath) | 65 | await processTask({ |
66 | inputPath: tmpInputFilePath ?? originalFilePath, | ||
67 | video, | ||
68 | outputPath, | ||
69 | task, | ||
70 | lTags | ||
71 | }) | ||
71 | 72 | ||
72 | // For the next iteration | 73 | if (tmpInputFilePath) await remove(tmpInputFilePath) |
73 | tmpInputFilePath = outputPath | ||
74 | } | ||
75 | 74 | ||
76 | return outputPath | 75 | // For the next iteration |
77 | }) | 76 | tmpInputFilePath = outputPath |
77 | } | ||
78 | 78 | ||
79 | logger.info('Video edition ended for video %s.', video.uuid, lTags) | 79 | return outputPath |
80 | }) | ||
80 | 81 | ||
81 | const newFile = await buildNewFile(video, editionResultPath) | 82 | logger.info('Video edition ended for video %s.', video.uuid, lTags) |
82 | 83 | ||
83 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) | 84 | const newFile = await buildNewFile(video, editionResultPath) |
84 | await move(editionResultPath, outputPath) | ||
85 | 85 | ||
86 | await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) | 86 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) |
87 | await removeAllFiles(video, newFile) | 87 | await move(editionResultPath, outputPath) |
88 | 88 | ||
89 | await newFile.save() | 89 | await safeCleanupStudioTMPFiles(payload) |
90 | 90 | ||
91 | video.duration = await getVideoStreamDuration(outputPath) | 91 | await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) |
92 | await video.save() | 92 | await removeAllFiles(video, newFile) |
93 | 93 | ||
94 | await federateVideoIfNeeded(video, false, undefined) | 94 | await newFile.save() |
95 | 95 | ||
96 | const user = await UserModel.loadByVideoId(video.id) | 96 | video.duration = await getVideoStreamDuration(outputPath) |
97 | await video.save() | ||
97 | 98 | ||
98 | await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) | 99 | await federateVideoIfNeeded(video, false, undefined) |
100 | |||
101 | const user = await UserModel.loadByVideoId(video.id) | ||
102 | |||
103 | await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) | ||
104 | } catch (err) { | ||
105 | await safeCleanupStudioTMPFiles(payload) | ||
106 | |||
107 | throw err | ||
108 | } | ||
99 | } | 109 | } |
100 | 110 | ||
101 | // --------------------------------------------------------------------------- | 111 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts index b392bdb00..beda326a0 100644 --- a/server/lib/video-studio.ts +++ b/server/lib/video-studio.ts | |||
@@ -1,15 +1,31 @@ | |||
1 | import { logger } from '@server/helpers/logger' | ||
1 | import { MVideoFullLight } from '@server/types/models' | 2 | import { MVideoFullLight } from '@server/types/models' |
2 | import { getVideoStreamDuration } from '@shared/ffmpeg' | 3 | import { getVideoStreamDuration } from '@shared/ffmpeg' |
3 | import { VideoStudioTask } from '@shared/models' | 4 | import { VideoStudioEditionPayload, VideoStudioTask } from '@shared/models' |
5 | import { remove } from 'fs-extra' | ||
4 | 6 | ||
5 | function buildTaskFileFieldname (indice: number, fieldName = 'file') { | 7 | function buildTaskFileFieldname (indice: number, fieldName = 'file') { |
6 | return `tasks[${indice}][options][${fieldName}]` | 8 | return `tasks[${indice}][options][${fieldName}]` |
7 | } | 9 | } |
8 | 10 | ||
9 | function getTaskFile (files: Express.Multer.File[], indice: number, fieldName = 'file') { | 11 | function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { |
10 | return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) | 12 | return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) |
11 | } | 13 | } |
12 | 14 | ||
15 | async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) { | ||
16 | for (const task of payload.tasks) { | ||
17 | try { | ||
18 | if (task.name === 'add-intro' || task.name === 'add-outro') { | ||
19 | await remove(task.options.file) | ||
20 | } else if (task.name === 'add-watermark') { | ||
21 | await remove(task.options.file) | ||
22 | } | ||
23 | } catch (err) { | ||
24 | logger.error('Cannot remove studio file', { err }) | ||
25 | } | ||
26 | } | ||
27 | } | ||
28 | |||
13 | async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) { | 29 | async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) { |
14 | let additionalDuration = 0 | 30 | let additionalDuration = 0 |
15 | 31 | ||
@@ -28,5 +44,6 @@ async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, task | |||
28 | export { | 44 | export { |
29 | approximateIntroOutroAdditionalSize, | 45 | approximateIntroOutroAdditionalSize, |
30 | buildTaskFileFieldname, | 46 | buildTaskFileFieldname, |
31 | getTaskFile | 47 | getTaskFileFromReq, |
48 | safeCleanupStudioTMPFiles | ||
32 | } | 49 | } |
diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts index 4397e887e..7a68f88e5 100644 --- a/server/middlewares/validators/videos/video-studio.ts +++ b/server/middlewares/validators/videos/video-studio.ts | |||
@@ -9,7 +9,7 @@ import { | |||
9 | } from '@server/helpers/custom-validators/video-studio' | 9 | } from '@server/helpers/custom-validators/video-studio' |
10 | import { cleanUpReqFiles } from '@server/helpers/express-utils' | 10 | import { cleanUpReqFiles } from '@server/helpers/express-utils' |
11 | import { CONFIG } from '@server/initializers/config' | 11 | import { CONFIG } from '@server/initializers/config' |
12 | import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio' | 12 | import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio' |
13 | import { isAudioFile } from '@shared/ffmpeg' | 13 | import { isAudioFile } from '@shared/ffmpeg' |
14 | import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' | 14 | import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' |
15 | import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' | 15 | import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' |
@@ -49,7 +49,7 @@ const videoStudioAddEditionValidator = [ | |||
49 | } | 49 | } |
50 | 50 | ||
51 | if (task.name === 'add-intro' || task.name === 'add-outro') { | 51 | if (task.name === 'add-intro' || task.name === 'add-outro') { |
52 | const filePath = getTaskFile(files, i).path | 52 | const filePath = getTaskFileFromReq(files, i).path |
53 | 53 | ||
54 | // Our concat filter needs a video stream | 54 | // Our concat filter needs a video stream |
55 | if (await isAudioFile(filePath)) { | 55 | if (await isAudioFile(filePath)) { |
@@ -79,7 +79,7 @@ const videoStudioAddEditionValidator = [ | |||
79 | if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) | 79 | if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) |
80 | 80 | ||
81 | // Try to make an approximation of bytes added by the intro/outro | 81 | // Try to make an approximation of bytes added by the intro/outro |
82 | const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFile(files, i).path) | 82 | const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFileFromReq(files, i).path) |
83 | if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req) | 83 | if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req) |
84 | 84 | ||
85 | return next() | 85 | return next() |
diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts index ab08e8fb6..30f72e6e9 100644 --- a/server/tests/api/transcoding/video-studio.ts +++ b/server/tests/api/transcoding/video-studio.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { expect } from 'chai' | 1 | import { expect } from 'chai' |
2 | import { expectStartWith } from '@server/tests/shared' | 2 | import { checkPersistentTmpIsEmpty, expectStartWith } from '@server/tests/shared' |
3 | import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' | 3 | import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' |
4 | import { VideoStudioTask } from '@shared/models' | 4 | import { VideoStudioTask } from '@shared/models' |
5 | import { | 5 | import { |
@@ -356,6 +356,29 @@ describe('Test video studio', function () { | |||
356 | }) | 356 | }) |
357 | }) | 357 | }) |
358 | 358 | ||
359 | describe('Server restart', function () { | ||
360 | |||
361 | it('Should still be able to run video edition after a server restart', async function () { | ||
362 | this.timeout(240_000) | ||
363 | |||
364 | await renewVideo() | ||
365 | await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks: VideoStudioCommand.getComplexTask() }) | ||
366 | |||
367 | await servers[0].kill() | ||
368 | await servers[0].run() | ||
369 | |||
370 | await waitJobs(servers) | ||
371 | |||
372 | for (const server of servers) { | ||
373 | await checkDuration(server, 9) | ||
374 | } | ||
375 | }) | ||
376 | |||
377 | it('Should have an empty persistent tmp directory', async function () { | ||
378 | await checkPersistentTmpIsEmpty(servers[0]) | ||
379 | }) | ||
380 | }) | ||
381 | |||
359 | after(async function () { | 382 | after(async function () { |
360 | await cleanupTests(servers) | 383 | await cleanupTests(servers) |
361 | }) | 384 | }) |
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts index 90d534a06..a614cef7c 100644 --- a/server/tests/shared/directories.ts +++ b/server/tests/shared/directories.ts | |||
@@ -12,6 +12,10 @@ async function checkTmpIsEmpty (server: PeerTubeServer) { | |||
12 | } | 12 | } |
13 | } | 13 | } |
14 | 14 | ||
15 | async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { | ||
16 | await checkDirectoryIsEmpty(server, 'tmp-persistent') | ||
17 | } | ||
18 | |||
15 | async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { | 19 | async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { |
16 | const directoryPath = server.getDirectoryPath(directory) | 20 | const directoryPath = server.getDirectoryPath(directory) |
17 | 21 | ||
@@ -26,5 +30,6 @@ async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, | |||
26 | 30 | ||
27 | export { | 31 | export { |
28 | checkTmpIsEmpty, | 32 | checkTmpIsEmpty, |
33 | checkPersistentTmpIsEmpty, | ||
29 | checkDirectoryIsEmpty | 34 | checkDirectoryIsEmpty |
30 | } | 35 | } |