Add TMP persistent directory

To store files that must be preserved between peertube restarts

17 files changed, 143 insertions, 57 deletions

diff --git a/config/default.yaml b/config/default.yaml
index 986b2e999..f3f29ecb9 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -120,6 +120,7 @@ defaults:
 # From the project root directory
 storage:
   tmp: 'storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing...
+  tmp_persistent: 'storage/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts
   bin: 'storage/bin/'
   avatars: 'storage/avatars/'
   videos: 'storage/videos/'
diff --git a/config/production.yaml.example b/config/production.yaml.example
index bd01375cd..ea6d77306 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -118,6 +118,7 @@ defaults:
 # From the project root directory
 storage:
   tmp: '/var/www/peertube/storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing...
+  tmp_persistent: '/var/www/peertube/storage/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts
   bin: '/var/www/peertube/storage/bin/'
   avatars: '/var/www/peertube/storage/avatars/'
   videos: '/var/www/peertube/storage/videos/'
diff --git a/config/test-1.yaml b/config/test-1.yaml
index 1d1020214..7b62e3d0c 100644
--- a/config/test-1.yaml
+++ b/config/test-1.yaml
@@ -10,6 +10,7 @@ database:
 # From the project root directory
 storage:
   tmp: 'test1/tmp/'
+  tmp_persistent: 'test1/tmp-persistent/'
   bin: 'test1/bin/'
   avatars: 'test1/avatars/'
   videos: 'test1/videos/'
diff --git a/config/test-2.yaml b/config/test-2.yaml
index d155b017d..ba36369a6 100644
--- a/config/test-2.yaml
+++ b/config/test-2.yaml
@@ -10,6 +10,7 @@ database:
 # From the project root directory
 storage:
   tmp: 'test2/tmp/'
+  tmp_persistent: 'test2/tmp-persistent/'
   bin: 'test2/bin/'
   avatars: 'test2/avatars/'
   videos: 'test2/videos/'
diff --git a/config/test-3.yaml b/config/test-3.yaml
index 50acf613a..6adec7953 100644
--- a/config/test-3.yaml
+++ b/config/test-3.yaml
@@ -10,6 +10,7 @@ database:
 # From the project root directory
 storage:
   tmp: 'test3/tmp/'
+  tmp_persistent: 'test3/tmp-persistent/'
   bin: 'test3/bin/'
   avatars: 'test3/avatars/'
   videos: 'test3/videos/'
diff --git a/config/test-4.yaml b/config/test-4.yaml
index 615e288b3..f042aee46 100644
--- a/config/test-4.yaml
+++ b/config/test-4.yaml
@@ -10,6 +10,7 @@ database:
 # From the project root directory
 storage:
   tmp: 'test4/tmp/'
+  tmp_persistent: 'test4/tmp-persistent/'
   bin: 'test4/bin/'
   avatars: 'test4/avatars/'
   videos: 'test4/videos/'
diff --git a/config/test-5.yaml b/config/test-5.yaml
index 447e3862a..ad90fec04 100644
--- a/config/test-5.yaml
+++ b/config/test-5.yaml
@@ -10,6 +10,7 @@ database:
 # From the project root directory
 storage:
   tmp: 'test5/tmp/'
+  tmp_persistent: 'test5/tmp-persistent/'
   bin: 'test5/bin/'
   avatars: 'test5/avatars/'
   videos: 'test5/videos/'
diff --git a/config/test-6.yaml b/config/test-6.yaml
index c077d7e38..a579f1f01 100644
--- a/config/test-6.yaml
+++ b/config/test-6.yaml
@@ -10,6 +10,7 @@ database:
 # From the project root directory
 storage:
   tmp: 'test6/tmp/'
+  tmp_persistent: 'test6/tmp-persistent/'
   bin: 'test6/bin/'
   avatars: 'test6/avatars/'
   videos: 'test6/videos/'
diff --git a/server/controllers/api/videos/studio.ts b/server/controllers/api/videos/studio.ts
index 6667532bf..2ccb2fb89 100644
--- a/server/controllers/api/videos/studio.ts
+++ b/server/controllers/api/videos/studio.ts
@@ -1,8 +1,12 @@
+import Bluebird from 'bluebird'
 import express from 'express'
+import { move } from 'fs-extra'
+import { basename, join } from 'path'
 import { createAnyReqFiles } from '@server/helpers/express-utils'
+import { CONFIG } from '@server/initializers/config'
 import { MIMETYPES } from '@server/initializers/constants'
 import { JobQueue } from '@server/lib/job-queue'
-import { buildTaskFileFieldname, getTaskFile } from '@server/lib/video-studio'
+import { buildTaskFileFieldname, getTaskFileFromReq } from '@server/lib/video-studio'
 import {
   HttpStatusCode,
   VideoState,
@@ -68,7 +72,7 @@ async function createEditionTasks (req: express.Request, res: express.Response)
 
   const payload = {
     videoUUID: video.uuid,
-    tasks: body.tasks.map((t, i) => buildTaskPayload(t, i, files))
+    tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files))
   }
 
   JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload })
@@ -77,7 +81,11 @@ async function createEditionTasks (req: express.Request, res: express.Response)
 }
 
 const taskPayloadBuilders: {
-  [id in VideoStudioTask['name']]: (task: VideoStudioTask, indice?: number, files?: Express.Multer.File[]) => VideoStudioTaskPayload
+  [id in VideoStudioTask['name']]: (
+    task: VideoStudioTask,
+    indice?: number,
+    files?: Express.Multer.File[]
+  ) => Promise<VideoStudioTaskPayload>
 } = {
   'add-intro': buildIntroOutroTask,
   'add-outro': buildIntroOutroTask,
@@ -85,34 +93,46 @@ const taskPayloadBuilders: {
   'add-watermark': buildWatermarkTask
 }
 
-function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): VideoStudioTaskPayload {
+function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): Promise<VideoStudioTaskPayload> {
   return taskPayloadBuilders[task.name](task, indice, files)
 }
 
-function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) {
+async function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) {
+  const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path)
+
   return {
     name: task.name,
     options: {
-      file: getTaskFile(files, indice).path
+      file: destination
     }
   }
 }
 
 function buildCutTask (task: VideoStudioTaskCut) {
-  return {
+  return Promise.resolve({
     name: task.name,
     options: {
       start: task.options.start,
       end: task.options.end
     }
-  }
+  })
 }
 
-function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) {
+async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) {
+  const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path)
+
   return {
     name: task.name,
     options: {
-      file: getTaskFile(files, indice).path
+      file: destination
     }
   }
 }
+
+async function moveStudioFileToPersistentTMP (file: string) {
+  const destination = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, basename(file))
+
+  await move(file, destination)
+
+  return destination
+}
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 699dd4704..f2d8f99b5 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -98,6 +98,7 @@ const CONFIG = {
 
   STORAGE: {
     TMP_DIR: buildPath(config.get<string>('storage.tmp')),
+    TMP_PERSISTENT_DIR: buildPath(config.get<string>('storage.tmp_persistent')),
     BIN_DIR: buildPath(config.get<string>('storage.bin')),
     ACTOR_IMAGES: buildPath(config.get<string>('storage.avatars')),
     LOG_DIR: buildPath(config.get<string>('storage.logs')),
diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts
index fbb55a388..5e8dd4f51 100644
--- a/server/lib/job-queue/handlers/video-studio-edition.ts
+++ b/server/lib/job-queue/handlers/video-studio-edition.ts
@@ -12,7 +12,7 @@ import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default
 import { isAbleToUploadVideo } from '@server/lib/user'
 import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
 import { VideoPathManager } from '@server/lib/video-path-manager'
-import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio'
+import { approximateIntroOutroAdditionalSize, safeCleanupStudioTMPFiles } from '@server/lib/video-studio'
 import { UserModel } from '@server/models/user/user'
 import { VideoModel } from '@server/models/video/video'
 import { VideoFileModel } from '@server/models/video/video-file'
@@ -39,63 +39,73 @@ async function processVideoStudioEdition (job: Job) {
 
   logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
 
-  const video = await VideoModel.loadFull(payload.videoUUID)
+  try {
+    const video = await VideoModel.loadFull(payload.videoUUID)
 
-  // No video, maybe deleted?
-  if (!video) {
-    logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
-    return undefined
-  }
+    // No video, maybe deleted?
+    if (!video) {
+      logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
 
-  await checkUserQuotaOrThrow(video, payload)
+      await safeCleanupStudioTMPFiles(payload)
+      return undefined
+    }
 
-  const inputFile = video.getMaxQualityFile()
+    await checkUserQuotaOrThrow(video, payload)
 
-  const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => {
-    let tmpInputFilePath: string
-    let outputPath: string
+    const inputFile = video.getMaxQualityFile()
 
-    for (const task of payload.tasks) {
-      const outputFilename = buildUUID() + inputFile.extname
-      outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
+    const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => {
+      let tmpInputFilePath: string
+      let outputPath: string
 
-      await processTask({
-        inputPath: tmpInputFilePath ?? originalFilePath,
-        video,
-        outputPath,
-        task,
-        lTags
-      })
+      for (const task of payload.tasks) {
+        const outputFilename = buildUUID() + inputFile.extname
+        outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
 
-      if (tmpInputFilePath) await remove(tmpInputFilePath)
+        await processTask({
+          inputPath: tmpInputFilePath ?? originalFilePath,
+          video,
+          outputPath,
+          task,
+          lTags
+        })
 
-      // For the next iteration
-      tmpInputFilePath = outputPath
-    }
+        if (tmpInputFilePath) await remove(tmpInputFilePath)
 
-    return outputPath
-  })
+        // For the next iteration
+        tmpInputFilePath = outputPath
+      }
 
-  logger.info('Video edition ended for video %s.', video.uuid, lTags)
+      return outputPath
+    })
 
-  const newFile = await buildNewFile(video, editionResultPath)
+    logger.info('Video edition ended for video %s.', video.uuid, lTags)
 
-  const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile)
-  await move(editionResultPath, outputPath)
+    const newFile = await buildNewFile(video, editionResultPath)
 
-  await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)
-  await removeAllFiles(video, newFile)
+    const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile)
+    await move(editionResultPath, outputPath)
 
-  await newFile.save()
+    await safeCleanupStudioTMPFiles(payload)
 
-  video.duration = await getVideoStreamDuration(outputPath)
-  await video.save()
+    await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)
+    await removeAllFiles(video, newFile)
 
-  await federateVideoIfNeeded(video, false, undefined)
+    await newFile.save()
 
-  const user = await UserModel.loadByVideoId(video.id)
+    video.duration = await getVideoStreamDuration(outputPath)
+    await video.save()
 
-  await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false })
+    await federateVideoIfNeeded(video, false, undefined)
+
+    const user = await UserModel.loadByVideoId(video.id)
+
+    await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false })
+  } catch (err) {
+    await safeCleanupStudioTMPFiles(payload)
+
+    throw err
+  }
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts
index b392bdb00..beda326a0 100644
--- a/server/lib/video-studio.ts
+++ b/server/lib/video-studio.ts
@@ -1,15 +1,31 @@
+import { logger } from '@server/helpers/logger'
 import { MVideoFullLight } from '@server/types/models'
 import { getVideoStreamDuration } from '@shared/ffmpeg'
-import { VideoStudioTask } from '@shared/models'
+import { VideoStudioEditionPayload, VideoStudioTask } from '@shared/models'
+import { remove } from 'fs-extra'
 
 function buildTaskFileFieldname (indice: number, fieldName = 'file') {
   return `tasks[${indice}][options][${fieldName}]`
 }
 
-function getTaskFile (files: Express.Multer.File[], indice: number, fieldName = 'file') {
+function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') {
   return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName))
 }
 
+async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) {
+  for (const task of payload.tasks) {
+    try {
+      if (task.name === 'add-intro' || task.name === 'add-outro') {
+        await remove(task.options.file)
+      } else if (task.name === 'add-watermark') {
+        await remove(task.options.file)
+      }
+    } catch (err) {
+      logger.error('Cannot remove studio file', { err })
+    }
+  }
+}
+
 async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) {
   let additionalDuration = 0
 
@@ -28,5 +44,6 @@ async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, task
 export {
   approximateIntroOutroAdditionalSize,
   buildTaskFileFieldname,
-  getTaskFile
+  getTaskFileFromReq,
+  safeCleanupStudioTMPFiles
 }
diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts
index 4397e887e..7a68f88e5 100644
--- a/server/middlewares/validators/videos/video-studio.ts
+++ b/server/middlewares/validators/videos/video-studio.ts
@@ -9,7 +9,7 @@ import {
 } from '@server/helpers/custom-validators/video-studio'
 import { cleanUpReqFiles } from '@server/helpers/express-utils'
 import { CONFIG } from '@server/initializers/config'
-import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio'
+import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio'
 import { isAudioFile } from '@shared/ffmpeg'
 import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models'
 import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared'
@@ -49,7 +49,7 @@ const videoStudioAddEditionValidator = [
       }
 
       if (task.name === 'add-intro' || task.name === 'add-outro') {
-        const filePath = getTaskFile(files, i).path
+        const filePath = getTaskFileFromReq(files, i).path
 
         // Our concat filter needs a video stream
         if (await isAudioFile(filePath)) {
@@ -79,7 +79,7 @@ const videoStudioAddEditionValidator = [
     if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
 
     // Try to make an approximation of bytes added by the intro/outro
-    const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFile(files, i).path)
+    const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFileFromReq(files, i).path)
     if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req)
 
     return next()
diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts
index ab08e8fb6..30f72e6e9 100644
--- a/server/tests/api/transcoding/video-studio.ts
+++ b/server/tests/api/transcoding/video-studio.ts
@@ -1,5 +1,5 @@
 import { expect } from 'chai'
-import { expectStartWith } from '@server/tests/shared'
+import { checkPersistentTmpIsEmpty, expectStartWith } from '@server/tests/shared'
 import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils'
 import { VideoStudioTask } from '@shared/models'
 import {
@@ -356,6 +356,29 @@ describe('Test video studio', function () {
     })
   })
 
+  describe('Server restart', function () {
+
+    it('Should still be able to run video edition after a server restart', async function () {
+      this.timeout(240_000)
+
+      await renewVideo()
+      await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks: VideoStudioCommand.getComplexTask() })
+
+      await servers[0].kill()
+      await servers[0].run()
+
+      await waitJobs(servers)
+
+      for (const server of servers) {
+        await checkDuration(server, 9)
+      }
+    })
+
+    it('Should have an empty persistent tmp directory', async function () {
+      await checkPersistentTmpIsEmpty(servers[0])
+    })
+  })
+
   after(async function () {
     await cleanupTests(servers)
   })
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts
index 90d534a06..a614cef7c 100644
--- a/server/tests/shared/directories.ts
+++ b/server/tests/shared/directories.ts
@@ -12,6 +12,10 @@ async function checkTmpIsEmpty (server: PeerTubeServer) {
   }
 }
 
+async function checkPersistentTmpIsEmpty (server: PeerTubeServer) {
+  await checkDirectoryIsEmpty(server, 'tmp-persistent')
+}
+
 async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) {
   const directoryPath = server.getDirectoryPath(directory)
 
@@ -26,5 +30,6 @@ async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string,
 
 export {
   checkTmpIsEmpty,
+  checkPersistentTmpIsEmpty,
   checkDirectoryIsEmpty
 }
diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index f68b81367..c8f6a0c5b 100644
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -364,6 +364,7 @@ export class PeerTubeServer {
       },
       storage: {
         tmp: this.getDirectoryPath('tmp') + '/',
+        tmp_persistent: this.getDirectoryPath('tmp-persistent') + '/',
         bin: this.getDirectoryPath('bin') + '/',
         avatars: this.getDirectoryPath('avatars') + '/',
         videos: this.getDirectoryPath('videos') + '/',
diff --git a/support/docker/production/config/production.yaml b/support/docker/production/config/production.yaml
index f1d698580..e3f6247d8 100644
--- a/support/docker/production/config/production.yaml
+++ b/support/docker/production/config/production.yaml
@@ -44,6 +44,7 @@ redis:
 # From the project root directory
 storage:
   tmp: '../data/tmp/' # Use to download data (imports etc), store uploaded files before and during processing...
+  tmp_persistent: '../data/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts
   bin: '../data/bin/'
   avatars: '../data/avatars/'
   videos: '../data/videos/'