Implement remote runner jobs in server

Move ffmpeg functions to @shared

diff --git a/config/default.yaml b/config/default.yaml
index dfa43a0aab331255968b0a90f8903a805276edf8..986b2e999c50f87dfc5281c44bd7fbf894784272 100644 (file)
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -375,6 +375,12 @@ feeds:
     # Default number of comments displayed in feeds
     count: 20
 
     # Default number of comments displayed in feeds
     count: 20
 

+remote_runners:
+  # Consider jobs that are processed by a remote runner as stalled after this period of time without any update
+  stalled_jobs:
+    live: '30 seconds'
+    vod: '2 minutes'
+

 cache:
   previews:
     size: 500 # Max number of previews you want to cache
 cache:
   previews:
     size: 500 # Max number of previews you want to cache


@@ -433,12 +439,18 @@ transcoding:
   # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file
   allow_audio_files: true
 
   # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file
   allow_audio_files: true
 

-  # Amount of threads used by ffmpeg for 1 transcoding job
+  # Enable remote runners to transcode your videos
+  # If enabled, your instance won't transcode the videos itself
+  # At least 1 remote runner must be configured to transcode your videos
+  remote_runners:
+    enabled: false
+
+  # Amount of threads used by ffmpeg for 1 local transcoding job

   threads: 1
   threads: 1

-  # Amount of transcoding jobs to execute in parallel
+  # Amount of local transcoding jobs to execute in parallel

   concurrency: 1
 
   concurrency: 1
 

-  # Choose the transcoding profile
+  # Choose the local transcoding profile

   # New profiles can be added by plugins
   # Available in core PeerTube: 'default'
   profile: 'default'
   # New profiles can be added by plugins
   # Available in core PeerTube: 'default'
   profile: 'default'


@@ -533,9 +545,17 @@ live:
   # Allow to transcode the live streaming in multiple live resolutions
   transcoding:
     enabled: true
   # Allow to transcode the live streaming in multiple live resolutions
   transcoding:
     enabled: true

+
+    # Enable remote runners to transcode your videos
+    # If enabled, your instance won't transcode the videos itself
+    # At least 1 remote runner must be configured to transcode your videos
+    remote_runners:
+      enabled: false
+
+    # Amount of threads used by ffmpeg per live when using local transcoding

     threads: 2
 
     threads: 2
 

-    # Choose the transcoding profile
+    # Choose the local transcoding profile

     # New profiles can be added by plugins
     # Available in core PeerTube: 'default'
     profile: 'default'
     # New profiles can be added by plugins
     # Available in core PeerTube: 'default'
     profile: 'default'


@@ -754,7 +774,7 @@ search:
   search_index:
     enabled: false
     # URL of the search index, that should use the same search API and routes
   search_index:
     enabled: false
     # URL of the search index, that should use the same search API and routes

-    # than PeerTube: https://docs.joinpeertube.org/api/rest-reference.html
+    # than PeerTube: https://docs.joinpeertube.org/api-rest-reference.html

     # You should deploy your own with https://framagit.org/framasoft/peertube/search-index,
     # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index
     url: ''
     # You should deploy your own with https://framagit.org/framasoft/peertube/search-index,
     # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index
     url: ''

diff --git a/config/production.yaml.example b/config/production.yaml.example
index 0fb6ababcd01e7e92857d3420488a8d4d18332ea..bd01375cd32996018503b6ef83e0ec0e4e085ea2 100644 (file)
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -373,6 +373,12 @@ feeds:
     # Default number of comments displayed in feeds
     count: 20
 
     # Default number of comments displayed in feeds
     count: 20
 

+remote_runners:
+  # Consider jobs that are processed by a remote runner as stalled after this period of time without any update
+  stalled_jobs:
+    live: '30 seconds'
+    vod: '2 minutes'
+

 ###############################################################################
 #
 # From this point, almost all following keys can be overridden by the web interface
 ###############################################################################
 #
 # From this point, almost all following keys can be overridden by the web interface


@@ -443,12 +449,18 @@ transcoding:
   # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file
   allow_audio_files: true
 
   # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file
   allow_audio_files: true
 

-  # Amount of threads used by ffmpeg for 1 transcoding job
+  # Enable remote runners to transcode your videos
+  # If enabled, your instance won't transcode the videos itself
+  # At least 1 remote runner must be configured to transcode your videos
+  remote_runners:
+    enabled: false
+
+  # Amount of threads used by ffmpeg for 1 local transcoding job

   threads: 1
   threads: 1

-  # Amount of transcoding jobs to execute in parallel
+  # Amount of local transcoding jobs to execute in parallel

   concurrency: 1
 
   concurrency: 1
 

-  # Choose the transcoding profile
+  # Choose the local transcoding profile

   # New profiles can be added by plugins
   # Available in core PeerTube: 'default'
   profile: 'default'
   # New profiles can be added by plugins
   # Available in core PeerTube: 'default'
   profile: 'default'


@@ -543,9 +555,17 @@ live:
   # Allow to transcode the live streaming in multiple live resolutions
   transcoding:
     enabled: true
   # Allow to transcode the live streaming in multiple live resolutions
   transcoding:
     enabled: true

+
+    # Enable remote runners to transcode your videos
+    # If enabled, your instance won't transcode the videos itself
+    # At least 1 remote runner must be configured to transcode your videos
+    remote_runners:
+      enabled: false
+
+    # Amount of threads used by ffmpeg per live when using local transcoding

     threads: 2
 
     threads: 2
 

-    # Choose the transcoding profile
+    # Choose the local transcoding profile

     # New profiles can be added by plugins
     # Available in core PeerTube: 'default'
     profile: 'default'
     # New profiles can be added by plugins
     # Available in core PeerTube: 'default'
     profile: 'default'


@@ -607,7 +627,7 @@ import:
       # See https://docs.joinpeertube.org/maintain/configuration#security for more information
       enabled: false
 
       # See https://docs.joinpeertube.org/maintain/configuration#security for more information
       enabled: false
 

-  # Add ability for your users to synchronize their channels with external channels, playlists, etc.
+  # Add ability for your users to synchronize their channels with external channels, playlists, etc

   video_channel_synchronization:
     enabled: false
 
   video_channel_synchronization:
     enabled: false
 


@@ -768,9 +788,9 @@ search:
     # You should deploy your own with https://framagit.org/framasoft/peertube/search-index,
     # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index
     url: ''
     # You should deploy your own with https://framagit.org/framasoft/peertube/search-index,
     # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index
     url: ''

-    # You can disable local search, so users only use the search index
+    # You can disable local search in the client, so users only use the search index

     disable_local_search: false
     disable_local_search: false

-    # If you did not disable local search, you can decide to use the search index by default
+    # If you did not disable local search in the client, you can decide to use the search index by default

     is_default_search: false
 
 # PeerTube client/interface configuration
     is_default_search: false
 
 # PeerTube client/interface configuration

diff --git a/server.ts b/server.ts
index 7bab18b0cdcdc4370341e2449a222b4e71f7aa58..a7a723b2400cc622ffbf9fa8c118764cb2a4fc4e 100644 (file)
--- a/server.ts
+++ b/server.ts
@@ -133,6 +133,7 @@ import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-in
 import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
 import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
 import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'
 import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
 import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
 import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'

+import { RunnerJobWatchDogScheduler } from './server/lib/schedulers/runner-job-watch-dog-scheduler'

 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
 import { PeerTubeSocket } from './server/lib/peertube-socket'
 import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
 import { PeerTubeSocket } from './server/lib/peertube-socket'
 import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'


@@ -331,6 +332,7 @@ async function startApplication () {
   VideoChannelSyncLatestScheduler.Instance.enable()
   VideoViewsBufferScheduler.Instance.enable()
   GeoIPUpdateScheduler.Instance.enable()
   VideoChannelSyncLatestScheduler.Instance.enable()
   VideoViewsBufferScheduler.Instance.enable()
   GeoIPUpdateScheduler.Instance.enable()

+  RunnerJobWatchDogScheduler.Instance.enable()

 
   OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })
 
 
   OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })
 

diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 60d168d1218ceab5f1fcb80453d51ad653a54a65..0b9aaffdab09e849c01454180f2f2baee204d28e 100644 (file)
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -217,6 +217,9 @@ function customConfig (): CustomConfig {
     },
     transcoding: {
       enabled: CONFIG.TRANSCODING.ENABLED,
     },
     transcoding: {
       enabled: CONFIG.TRANSCODING.ENABLED,

+      remoteRunners: {
+        enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED
+      },

       allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS,
       allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES,
       threads: CONFIG.TRANSCODING.THREADS,
       allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS,
       allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES,
       threads: CONFIG.TRANSCODING.THREADS,


@@ -252,6 +255,9 @@ function customConfig (): CustomConfig {
       maxUserLives: CONFIG.LIVE.MAX_USER_LIVES,
       transcoding: {
         enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
       maxUserLives: CONFIG.LIVE.MAX_USER_LIVES,
       transcoding: {
         enabled: CONFIG.LIVE.TRANSCODING.ENABLED,

+        remoteRunners: {
+          enabled: CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED
+        },

         threads: CONFIG.LIVE.TRANSCODING.THREADS,
         profile: CONFIG.LIVE.TRANSCODING.PROFILE,
         resolutions: {
         threads: CONFIG.LIVE.TRANSCODING.THREADS,
         profile: CONFIG.LIVE.TRANSCODING.PROFILE,
         resolutions: {

diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts
index e1d197c8a4ea43c96b1d1175e5c04bf9c3bc5b26..646f9597e79ba1cc21ffb325c52f605722bed0e1 100644 (file)
--- a/server/controllers/api/index.ts
+++ b/server/controllers/api/index.ts
@@ -15,6 +15,7 @@ import { metricsRouter } from './metrics'
 import { oauthClientsRouter } from './oauth-clients'
 import { overviewsRouter } from './overviews'
 import { pluginRouter } from './plugins'
 import { oauthClientsRouter } from './oauth-clients'
 import { overviewsRouter } from './overviews'
 import { pluginRouter } from './plugins'

+import { runnersRouter } from './runners'

 import { searchRouter } from './search'
 import { serverRouter } from './server'
 import { usersRouter } from './users'
 import { searchRouter } from './search'
 import { serverRouter } from './server'
 import { usersRouter } from './users'


@@ -55,6 +56,7 @@ apiRouter.use('/overviews', overviewsRouter)
 apiRouter.use('/plugins', pluginRouter)
 apiRouter.use('/custom-pages', customPageRouter)
 apiRouter.use('/blocklist', blocklistRouter)
 apiRouter.use('/plugins', pluginRouter)
 apiRouter.use('/custom-pages', customPageRouter)
 apiRouter.use('/blocklist', blocklistRouter)

+apiRouter.use('/runners', runnersRouter)

 apiRouter.use('/ping', pong)
 apiRouter.use('/*', badRequest)
 
 apiRouter.use('/ping', pong)
 apiRouter.use('/*', badRequest)
 

diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts
index 6a53e308363e522f4b4aed3da9b4d3b1faa629db..b63e2f962cf6ad37d9d0b4bfdd7954d1c3f4ab6d 100644 (file)
--- a/server/controllers/api/jobs.ts
+++ b/server/controllers/api/jobs.ts
@@ -93,6 +93,9 @@ async function formatJob (job: BullJob, state?: JobState): Promise<Job> {
     state: state || await job.getState(),
     type: job.queueName as JobType,
     data: job.data,
     state: state || await job.getState(),
     type: job.queueName as JobType,
     data: job.data,

+    parent: job.parent
+      ? { id: job.parent.id }
+      : undefined,

     progress: job.progress as number,
     priority: job.opts.priority,
     error,
     progress: job.progress as number,
     priority: job.opts.priority,
     error,

diff --git a/server/controllers/api/runners/index.ts b/server/controllers/api/runners/index.ts
new file mode 100644 (file)
index 0000000..c98ded3
--- /dev/null
+++ b/server/controllers/api/runners/index.ts
@@ -0,0 +1,18 @@
+import express from 'express'
+import { runnerJobsRouter } from './jobs'
+import { runnerJobFilesRouter } from './jobs-files'
+import { manageRunnersRouter } from './manage-runners'
+import { runnerRegistrationTokensRouter } from './registration-tokens'
+
+const runnersRouter = express.Router()
+
+runnersRouter.use('/', manageRunnersRouter)
+runnersRouter.use('/', runnerJobsRouter)
+runnersRouter.use('/', runnerJobFilesRouter)
+runnersRouter.use('/', runnerRegistrationTokensRouter)
+
+// ---------------------------------------------------------------------------
+
+export {
+  runnersRouter
+}

diff --git a/server/controllers/api/runners/jobs-files.ts b/server/controllers/api/runners/jobs-files.ts
new file mode 100644 (file)
index 0000000..e43ce35
--- /dev/null
+++ b/server/controllers/api/runners/jobs-files.ts
@@ -0,0 +1,84 @@
+import express from 'express'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { asyncMiddleware } from '@server/middlewares'
+import { jobOfRunnerGetValidator } from '@server/middlewares/validators/runners'
+import { runnerJobGetVideoTranscodingFileValidator } from '@server/middlewares/validators/runners/job-files'
+import { VideoStorage } from '@shared/models'
+
+const lTags = loggerTagsFactory('api', 'runner')
+
+const runnerJobFilesRouter = express.Router()
+
+runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/max-quality',
+  asyncMiddleware(jobOfRunnerGetValidator),
+  asyncMiddleware(runnerJobGetVideoTranscodingFileValidator),
+  asyncMiddleware(getMaxQualityVideoFile)
+)
+
+runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/previews/max-quality',
+  asyncMiddleware(jobOfRunnerGetValidator),
+  asyncMiddleware(runnerJobGetVideoTranscodingFileValidator),
+  getMaxQualityVideoPreview
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  runnerJobFilesRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function getMaxQualityVideoFile (req: express.Request, res: express.Response) {
+  const runnerJob = res.locals.runnerJob
+  const runner = runnerJob.Runner
+  const video = res.locals.videoAll
+
+  logger.info(
+    'Get max quality file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name,
+    lTags(runner.name, runnerJob.id, runnerJob.type)
+  )
+
+  const file = video.getMaxQualityFile()
+
+  if (file.storage === VideoStorage.OBJECT_STORAGE) {
+    if (file.isHLS()) {
+      return proxifyHLS({
+        req,
+        res,
+        filename: file.filename,
+        playlist: video.getHLSPlaylist(),
+        reinjectVideoFileToken: false,
+        video
+      })
+    }
+
+    // Web video
+    return proxifyWebTorrentFile({
+      req,
+      res,
+      filename: file.filename
+    })
+  }
+
+  return VideoPathManager.Instance.makeAvailableVideoFile(file, videoPath => {
+    return res.sendFile(videoPath)
+  })
+}
+
+function getMaxQualityVideoPreview (req: express.Request, res: express.Response) {
+  const runnerJob = res.locals.runnerJob
+  const runner = runnerJob.Runner
+  const video = res.locals.videoAll
+
+  logger.info(
+    'Get max quality preview file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name,
+    lTags(runner.name, runnerJob.id, runnerJob.type)
+  )
+
+  const file = video.getPreview()
+
+  return res.sendFile(file.getPath())
+}

diff --git a/server/controllers/api/runners/jobs.ts b/server/controllers/api/runners/jobs.ts
new file mode 100644 (file)
index 0000000..7d488ec
--- /dev/null
+++ b/server/controllers/api/runners/jobs.ts
@@ -0,0 +1,352 @@
+import express, { UploadFiles } from 'express'
+import { createReqFiles } from '@server/helpers/express-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { generateRunnerJobToken } from '@server/helpers/token-generator'
+import { MIMETYPES } from '@server/initializers/constants'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { getRunnerJobHandlerClass, updateLastRunnerContact } from '@server/lib/runners'
+import {
+  asyncMiddleware,
+  authenticate,
+  ensureUserHasRight,
+  paginationValidator,
+  runnerJobsSortValidator,
+  setDefaultPagination,
+  setDefaultSort
+} from '@server/middlewares'
+import {
+  abortRunnerJobValidator,
+  acceptRunnerJobValidator,
+  errorRunnerJobValidator,
+  getRunnerFromTokenValidator,
+  jobOfRunnerGetValidator,
+  runnerJobGetValidator,
+  successRunnerJobValidator,
+  updateRunnerJobValidator
+} from '@server/middlewares/validators/runners'
+import { RunnerModel } from '@server/models/runner/runner'
+import { RunnerJobModel } from '@server/models/runner/runner-job'
+import {
+  AbortRunnerJobBody,
+  AcceptRunnerJobResult,
+  ErrorRunnerJobBody,
+  HttpStatusCode,
+  ListRunnerJobsQuery,
+  LiveRTMPHLSTranscodingUpdatePayload,
+  RequestRunnerJobResult,
+  RunnerJobState,
+  RunnerJobSuccessBody,
+  RunnerJobSuccessPayload,
+  RunnerJobType,
+  RunnerJobUpdateBody,
+  RunnerJobUpdatePayload,
+  UserRight,
+  VODAudioMergeTranscodingSuccess,
+  VODHLSTranscodingSuccess,
+  VODWebVideoTranscodingSuccess
+} from '@shared/models'
+
+const postRunnerJobSuccessVideoFiles = createReqFiles(
+  [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]' ],
+  { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT }
+)
+
+const runnerJobUpdateVideoFiles = createReqFiles(
+  [ 'payload[videoChunkFile]', 'payload[resolutionPlaylistFile]', 'payload[masterPlaylistFile]' ],
+  { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT }
+)
+
+const lTags = loggerTagsFactory('api', 'runner')
+
+const runnerJobsRouter = express.Router()
+
+// ---------------------------------------------------------------------------
+// Controllers for runners
+// ---------------------------------------------------------------------------
+
+runnerJobsRouter.post('/jobs/request',
+  asyncMiddleware(getRunnerFromTokenValidator),
+  asyncMiddleware(requestRunnerJob)
+)
+
+runnerJobsRouter.post('/jobs/:jobUUID/accept',
+  asyncMiddleware(runnerJobGetValidator),
+  acceptRunnerJobValidator,
+  asyncMiddleware(getRunnerFromTokenValidator),
+  asyncMiddleware(acceptRunnerJob)
+)
+
+runnerJobsRouter.post('/jobs/:jobUUID/abort',
+  asyncMiddleware(jobOfRunnerGetValidator),
+  abortRunnerJobValidator,
+  asyncMiddleware(abortRunnerJob)
+)
+
+runnerJobsRouter.post('/jobs/:jobUUID/update',
+  runnerJobUpdateVideoFiles,
+  asyncMiddleware(jobOfRunnerGetValidator),
+  updateRunnerJobValidator,
+  asyncMiddleware(updateRunnerJobController)
+)
+
+runnerJobsRouter.post('/jobs/:jobUUID/error',
+  asyncMiddleware(jobOfRunnerGetValidator),
+  errorRunnerJobValidator,
+  asyncMiddleware(errorRunnerJob)
+)
+
+runnerJobsRouter.post('/jobs/:jobUUID/success',
+  postRunnerJobSuccessVideoFiles,
+  asyncMiddleware(jobOfRunnerGetValidator),
+  successRunnerJobValidator,
+  asyncMiddleware(postRunnerJobSuccess)
+)
+
+// ---------------------------------------------------------------------------
+// Controllers for admins
+// ---------------------------------------------------------------------------
+
+runnerJobsRouter.post('/jobs/:jobUUID/cancel',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_RUNNERS),
+  asyncMiddleware(runnerJobGetValidator),
+  asyncMiddleware(cancelRunnerJob)
+)
+
+runnerJobsRouter.get('/jobs',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_RUNNERS),
+  paginationValidator,
+  runnerJobsSortValidator,
+  setDefaultSort,
+  setDefaultPagination,
+  asyncMiddleware(listRunnerJobs)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  runnerJobsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+// ---------------------------------------------------------------------------
+// Controllers for runners
+// ---------------------------------------------------------------------------
+
+async function requestRunnerJob (req: express.Request, res: express.Response) {
+  const runner = res.locals.runner
+  const availableJobs = await RunnerJobModel.listAvailableJobs()
+
+  logger.debug('Runner %s requests for a job.', runner.name, { availableJobs, ...lTags(runner.name) })
+
+  const result: RequestRunnerJobResult = {
+    availableJobs: availableJobs.map(j => ({
+      uuid: j.uuid,
+      type: j.type,
+      payload: j.payload
+    }))
+  }
+
+  updateLastRunnerContact(req, runner)
+
+  return res.json(result)
+}
+
+async function acceptRunnerJob (req: express.Request, res: express.Response) {
+  const runner = res.locals.runner
+  const runnerJob = res.locals.runnerJob
+
+  runnerJob.state = RunnerJobState.PROCESSING
+  runnerJob.processingJobToken = generateRunnerJobToken()
+  runnerJob.startedAt = new Date()
+  runnerJob.runnerId = runner.id
+
+  const newRunnerJob = await sequelizeTypescript.transaction(transaction => {
+    return runnerJob.save({ transaction })
+  })
+  newRunnerJob.Runner = runner as RunnerModel
+
+  const result: AcceptRunnerJobResult = {
+    job: {
+      ...newRunnerJob.toFormattedJSON(),
+
+      jobToken: newRunnerJob.processingJobToken
+    }
+  }
+
+  updateLastRunnerContact(req, runner)
+
+  logger.info(
+    'Remote runner %s has accepted job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type,
+    lTags(runner.name, runnerJob.uuid, runnerJob.type)
+  )
+
+  return res.json(result)
+}
+
+async function abortRunnerJob (req: express.Request, res: express.Response) {
+  const runnerJob = res.locals.runnerJob
+  const runner = runnerJob.Runner
+  const body: AbortRunnerJobBody = req.body
+
+  logger.info(
+    'Remote runner %s is aborting job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type,
+    { reason: body.reason, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) }
+  )
+
+  const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
+  await new RunnerJobHandler().abort({ runnerJob })
+
+  updateLastRunnerContact(req, runnerJob.Runner)
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+async function errorRunnerJob (req: express.Request, res: express.Response) {
+  const runnerJob = res.locals.runnerJob
+  const runner = runnerJob.Runner
+  const body: ErrorRunnerJobBody = req.body
+
+  runnerJob.failures += 1
+
+  logger.error(
+    'Remote runner %s had an error with job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type,
+    { errorMessage: body.message, totalFailures: runnerJob.failures, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) }
+  )
+
+  const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
+  await new RunnerJobHandler().error({ runnerJob, message: body.message })
+
+  updateLastRunnerContact(req, runnerJob.Runner)
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+// ---------------------------------------------------------------------------
+
+const jobUpdateBuilders: {
+  [id in RunnerJobType]?: (payload: RunnerJobUpdatePayload, files?: UploadFiles) => RunnerJobUpdatePayload
+} = {
+  'live-rtmp-hls-transcoding': (payload: LiveRTMPHLSTranscodingUpdatePayload, files) => {
+    return {
+      ...payload,
+
+      masterPlaylistFile: files['payload[masterPlaylistFile]']?.[0].path,
+      resolutionPlaylistFile: files['payload[resolutionPlaylistFile]']?.[0].path,
+      videoChunkFile: files['payload[videoChunkFile]']?.[0].path
+    }
+  }
+}
+
+async function updateRunnerJobController (req: express.Request, res: express.Response) {
+  const runnerJob = res.locals.runnerJob
+  const runner = runnerJob.Runner
+  const body: RunnerJobUpdateBody = req.body
+
+  const payloadBuilder = jobUpdateBuilders[runnerJob.type]
+  const updatePayload = payloadBuilder
+    ? payloadBuilder(body.payload, req.files as UploadFiles)
+    : undefined
+
+  logger.debug(
+    'Remote runner %s is updating job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type,
+    { body, updatePayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) }
+  )
+
+  const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
+  await new RunnerJobHandler().update({
+    runnerJob,
+    progress: req.body.progress,
+    updatePayload
+  })
+
+  updateLastRunnerContact(req, runnerJob.Runner)
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+// ---------------------------------------------------------------------------
+
+const jobSuccessPayloadBuilders: {
+  [id in RunnerJobType]: (payload: RunnerJobSuccessPayload, files?: UploadFiles) => RunnerJobSuccessPayload
+} = {
+  'vod-web-video-transcoding': (payload: VODWebVideoTranscodingSuccess, files) => {
+    return {
+      ...payload,
+
+      videoFile: files['payload[videoFile]'][0].path
+    }
+  },
+
+  'vod-hls-transcoding': (payload: VODHLSTranscodingSuccess, files) => {
+    return {
+      ...payload,
+
+      videoFile: files['payload[videoFile]'][0].path,
+      resolutionPlaylistFile: files['payload[resolutionPlaylistFile]'][0].path
+    }
+  },
+
+  'vod-audio-merge-transcoding': (payload: VODAudioMergeTranscodingSuccess, files) => {
+    return {
+      ...payload,
+
+      videoFile: files['payload[videoFile]'][0].path
+    }
+  },
+
+  'live-rtmp-hls-transcoding': () => ({})
+}
+
+async function postRunnerJobSuccess (req: express.Request, res: express.Response) {
+  const runnerJob = res.locals.runnerJob
+  const runner = runnerJob.Runner
+  const body: RunnerJobSuccessBody = req.body
+
+  const resultPayload = jobSuccessPayloadBuilders[runnerJob.type](body.payload, req.files as UploadFiles)
+
+  logger.info(
+    'Remote runner %s is sending success result for job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type,
+    { resultPayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) }
+  )
+
+  const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
+  await new RunnerJobHandler().complete({ runnerJob, resultPayload })
+
+  updateLastRunnerContact(req, runnerJob.Runner)
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+// ---------------------------------------------------------------------------
+// Controllers for admins
+// ---------------------------------------------------------------------------
+
+async function cancelRunnerJob (req: express.Request, res: express.Response) {
+  const runnerJob = res.locals.runnerJob
+
+  logger.info('Cancelling job %s (%s)', runnerJob.type, lTags(runnerJob.uuid, runnerJob.type))
+
+  const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
+  await new RunnerJobHandler().cancel({ runnerJob })
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+async function listRunnerJobs (req: express.Request, res: express.Response) {
+  const query: ListRunnerJobsQuery = req.query
+
+  const resultList = await RunnerJobModel.listForApi({
+    start: query.start,
+    count: query.count,
+    sort: query.sort,
+    search: query.search
+  })
+
+  return res.json({
+    total: resultList.total,
+    data: resultList.data.map(d => d.toFormattedAdminJSON())
+  })
+}

diff --git a/server/controllers/api/runners/manage-runners.ts b/server/controllers/api/runners/manage-runners.ts
new file mode 100644 (file)
index 0000000..eb08c4b
--- /dev/null
+++ b/server/controllers/api/runners/manage-runners.ts
@@ -0,0 +1,107 @@
+import express from 'express'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { generateRunnerToken } from '@server/helpers/token-generator'
+import {
+  asyncMiddleware,
+  authenticate,
+  ensureUserHasRight,
+  paginationValidator,
+  runnersSortValidator,
+  setDefaultPagination,
+  setDefaultSort
+} from '@server/middlewares'
+import { deleteRunnerValidator, getRunnerFromTokenValidator, registerRunnerValidator } from '@server/middlewares/validators/runners'
+import { RunnerModel } from '@server/models/runner/runner'
+import { HttpStatusCode, ListRunnersQuery, RegisterRunnerBody, UserRight } from '@shared/models'
+
+const lTags = loggerTagsFactory('api', 'runner')
+
+const manageRunnersRouter = express.Router()
+
+manageRunnersRouter.post('/register',
+  asyncMiddleware(registerRunnerValidator),
+  asyncMiddleware(registerRunner)
+)
+manageRunnersRouter.post('/unregister',
+  asyncMiddleware(getRunnerFromTokenValidator),
+  asyncMiddleware(unregisterRunner)
+)
+
+manageRunnersRouter.delete('/:runnerId',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_RUNNERS),
+  asyncMiddleware(deleteRunnerValidator),
+  asyncMiddleware(deleteRunner)
+)
+
+manageRunnersRouter.get('/',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_RUNNERS),
+  paginationValidator,
+  runnersSortValidator,
+  setDefaultSort,
+  setDefaultPagination,
+  asyncMiddleware(listRunners)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  manageRunnersRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function registerRunner (req: express.Request, res: express.Response) {
+  const body: RegisterRunnerBody = req.body
+
+  const runnerToken = generateRunnerToken()
+
+  const runner = new RunnerModel({
+    runnerToken,
+    name: body.name,
+    description: body.description,
+    lastContact: new Date(),
+    ip: req.ip,
+    runnerRegistrationTokenId: res.locals.runnerRegistrationToken.id
+  })
+
+  await runner.save()
+
+  logger.info('Registered new runner %s', runner.name, { ...lTags(runner.name) })
+
+  return res.json({ id: runner.id, runnerToken })
+}
+async function unregisterRunner (req: express.Request, res: express.Response) {
+  const runner = res.locals.runner
+  await runner.destroy()
+
+  logger.info('Unregistered runner %s', runner.name, { ...lTags(runner.name) })
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+async function deleteRunner (req: express.Request, res: express.Response) {
+  const runner = res.locals.runner
+
+  await runner.destroy()
+
+  logger.info('Deleted runner %s', runner.name, { ...lTags(runner.name) })
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+async function listRunners (req: express.Request, res: express.Response) {
+  const query: ListRunnersQuery = req.query
+
+  const resultList = await RunnerModel.listForApi({
+    start: query.start,
+    count: query.count,
+    sort: query.sort
+  })
+
+  return res.json({
+    total: resultList.total,
+    data: resultList.data.map(d => d.toFormattedJSON())
+  })
+}

diff --git a/server/controllers/api/runners/registration-tokens.ts b/server/controllers/api/runners/registration-tokens.ts
new file mode 100644 (file)
index 0000000..5ac3773
--- /dev/null
+++ b/server/controllers/api/runners/registration-tokens.ts
@@ -0,0 +1,87 @@
+import express from 'express'
+import { generateRunnerRegistrationToken } from '@server/helpers/token-generator'
+import {
+  asyncMiddleware,
+  authenticate,
+  ensureUserHasRight,
+  paginationValidator,
+  runnerRegistrationTokensSortValidator,
+  setDefaultPagination,
+  setDefaultSort
+} from '@server/middlewares'
+import { deleteRegistrationTokenValidator } from '@server/middlewares/validators/runners'
+import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
+import { HttpStatusCode, ListRunnerRegistrationTokensQuery, UserRight } from '@shared/models'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+
+const lTags = loggerTagsFactory('api', 'runner')
+
+const runnerRegistrationTokensRouter = express.Router()
+
+runnerRegistrationTokensRouter.post('/registration-tokens/generate',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_RUNNERS),
+  asyncMiddleware(generateRegistrationToken)
+)
+
+runnerRegistrationTokensRouter.delete('/registration-tokens/:id',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_RUNNERS),
+  asyncMiddleware(deleteRegistrationTokenValidator),
+  asyncMiddleware(deleteRegistrationToken)
+)
+
+runnerRegistrationTokensRouter.get('/registration-tokens',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_RUNNERS),
+  paginationValidator,
+  runnerRegistrationTokensSortValidator,
+  setDefaultSort,
+  setDefaultPagination,
+  asyncMiddleware(listRegistrationTokens)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  runnerRegistrationTokensRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function generateRegistrationToken (req: express.Request, res: express.Response) {
+  logger.info('Generating new runner registration token.', lTags())
+
+  const registrationToken = new RunnerRegistrationTokenModel({
+    registrationToken: generateRunnerRegistrationToken()
+  })
+
+  await registrationToken.save()
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+async function deleteRegistrationToken (req: express.Request, res: express.Response) {
+  logger.info('Removing runner registration token.', lTags())
+
+  const runnerRegistrationToken = res.locals.runnerRegistrationToken
+
+  await runnerRegistrationToken.destroy()
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
+async function listRegistrationTokens (req: express.Request, res: express.Response) {
+  const query: ListRunnerRegistrationTokensQuery = req.query
+
+  const resultList = await RunnerRegistrationTokenModel.listForApi({
+    start: query.start,
+    count: query.count,
+    sort: query.sort
+  })
+
+  return res.json({
+    total: resultList.total,
+    data: resultList.data.map(d => d.toFormattedJSON())
+  })
+}

diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts
index 8c9a5322b78a29b7f15efd3aa6cba59eec5950fc..54f484b2b83fa16988e469efd241a74959b599cf 100644 (file)
--- a/server/controllers/api/videos/transcoding.ts
+++ b/server/controllers/api/videos/transcoding.ts
@@ -1,10 +1,8 @@
-import Bluebird from 'bluebird'

 import express from 'express'
 import express from 'express'

-import { computeResolutionsToTranscode } from '@server/helpers/ffmpeg'

 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { logger, loggerTagsFactory } from '@server/helpers/logger'

-import { JobQueue } from '@server/lib/job-queue'

 import { Hooks } from '@server/lib/plugins/hooks'
 import { Hooks } from '@server/lib/plugins/hooks'

-import { buildTranscodingJob } from '@server/lib/video'
+import { createTranscodingJobs } from '@server/lib/transcoding/create-transcoding-job'
+import { computeResolutionsToTranscode } from '@server/lib/transcoding/transcoding-resolutions'

 import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models'
 import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares'
 
 import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models'
 import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares'
 


@@ -47,82 +45,13 @@ async function createTranscoding (req: express.Request, res: express.Response) {
   video.state = VideoState.TO_TRANSCODE
   await video.save()
 
   video.state = VideoState.TO_TRANSCODE
   await video.save()
 

-  const childrenResolutions = resolutions.filter(r => r !== maxResolution)
-
-  logger.info('Manually creating transcoding jobs for %s.', body.transcodingType, { childrenResolutions, maxResolution })
-
-  const children = await Bluebird.mapSeries(childrenResolutions, resolution => {
-    if (body.transcodingType === 'hls') {
-      return buildHLSJobOption({
-        videoUUID: video.uuid,
-        hasAudio,
-        resolution,
-        isMaxQuality: false
-      })
-    }
-
-    if (body.transcodingType === 'webtorrent') {
-      return buildWebTorrentJobOption({
-        videoUUID: video.uuid,
-        hasAudio,
-        resolution
-      })
-    }
-  })
-
-  const parent = body.transcodingType === 'hls'
-    ? await buildHLSJobOption({
-      videoUUID: video.uuid,
-      hasAudio,
-      resolution: maxResolution,
-      isMaxQuality: false
-    })
-    : await buildWebTorrentJobOption({
-      videoUUID: video.uuid,
-      hasAudio,
-      resolution: maxResolution
-    })
-
-  // Porcess the last resolution after the other ones to prevent concurrency issue
-  // Because low resolutions use the biggest one as ffmpeg input
-  await JobQueue.Instance.createJobWithChildren(parent, children)
-
-  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
-}
-
-function buildHLSJobOption (options: {
-  videoUUID: string
-  hasAudio: boolean
-  resolution: number
-  isMaxQuality: boolean
-}) {
-  const { videoUUID, hasAudio, resolution, isMaxQuality } = options
-
-  return buildTranscodingJob({
-    type: 'new-resolution-to-hls',
-    videoUUID,
-    resolution,
-    hasAudio,
-    copyCodecs: false,
+  await createTranscodingJobs({
+    video,
+    resolutions,
+    transcodingType: body.transcodingType,

     isNewVideo: false,
     isNewVideo: false,

-    autoDeleteWebTorrentIfNeeded: false,
-    isMaxQuality
+    user: null // Don't specify priority since these transcoding jobs are fired by the admin

   })
   })

-}

 
 

-function buildWebTorrentJobOption (options: {
-  videoUUID: string
-  hasAudio: boolean
-  resolution: number
-}) {
-  const { videoUUID, hasAudio, resolution } = options
-
-  return buildTranscodingJob({
-    type: 'new-resolution-to-webtorrent',
-    videoUUID,
-    isNewVideo: false,
-    resolution,
-    hasAudio,
-    createHLSIfNeeded: false
-  })
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)

 }
 }

diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 43313a143f2e2b7ea918132180ff0097e1c552bb..885ac8b81925d0d263a18a63147f0fcaffbc1e75 100644 (file)
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -3,28 +3,20 @@ import { move } from 'fs-extra'
 import { basename } from 'path'
 import { getResumableUploadPath } from '@server/helpers/upload'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
 import { basename } from 'path'
 import { getResumableUploadPath } from '@server/helpers/upload'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'

-import { JobQueue } from '@server/lib/job-queue'
-import { generateWebTorrentVideoFilename } from '@server/lib/paths'
+import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue'

 import { Redis } from '@server/lib/redis'
 import { uploadx } from '@server/lib/uploadx'
 import { Redis } from '@server/lib/redis'
 import { uploadx } from '@server/lib/uploadx'

-import {
-  buildLocalVideoFromReq,
-  buildMoveToObjectStorageJob,
-  buildOptimizeOrMergeAudioJob,
-  buildVideoThumbnailsFromReq,
-  setVideoTags
-} from '@server/lib/video'
+import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
+import { buildNewFile } from '@server/lib/video-file'

 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { openapiOperationDoc } from '@server/middlewares/doc'
 import { VideoSourceModel } from '@server/models/video/video-source'
 import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { openapiOperationDoc } from '@server/middlewares/doc'
 import { VideoSourceModel } from '@server/models/video/video-source'
 import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'

-import { getLowercaseExtension } from '@shared/core-utils'
-import { isAudioFile, uuidToShort } from '@shared/extra-utils'
-import { HttpStatusCode, VideoCreate, VideoResolution, VideoState } from '@shared/models'
+import { uuidToShort } from '@shared/extra-utils'
+import { HttpStatusCode, VideoCreate, VideoState } from '@shared/models'

 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
 import { createReqFiles } from '../../../helpers/express-utils'
 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
 import { createReqFiles } from '../../../helpers/express-utils'

-import { buildFileMetadata, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '../../../helpers/ffmpeg'

 import { logger, loggerTagsFactory } from '../../../helpers/logger'
 import { MIMETYPES } from '../../../initializers/constants'
 import { sequelizeTypescript } from '../../../initializers/database'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'
 import { MIMETYPES } from '../../../initializers/constants'
 import { sequelizeTypescript } from '../../../initializers/database'


@@ -41,7 +33,6 @@ import {
 } from '../../../middlewares'
 import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
 import { VideoModel } from '../../../models/video/video'
 } from '../../../middlewares'
 import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
 import { VideoModel } from '../../../models/video/video'

-import { VideoFileModel } from '../../../models/video/video-file'

 
 const lTags = loggerTagsFactory('api', 'video')
 const auditLogger = auditLoggerFactory('videos')
 
 const lTags = loggerTagsFactory('api', 'video')
 const auditLogger = auditLoggerFactory('videos')


@@ -148,7 +139,7 @@ async function addVideo (options: {
   video.VideoChannel = videoChannel
   video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
 
   video.VideoChannel = videoChannel
   video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
 

-  const videoFile = await buildNewFile(videoPhysicalFile)
+  const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' })

   const originalFilename = videoPhysicalFile.originalname
 
   // Move physical file
   const originalFilename = videoPhysicalFile.originalname
 
   // Move physical file


@@ -227,30 +218,8 @@ async function addVideo (options: {
   }
 }
 
   }
 }
 

-async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) {
-  const videoFile = new VideoFileModel({
-    extname: getLowercaseExtension(videoPhysicalFile.filename),
-    size: videoPhysicalFile.size,
-    videoStreamingPlaylistId: null,
-    metadata: await buildFileMetadata(videoPhysicalFile.path)
-  })
-
-  const probe = await ffprobePromise(videoPhysicalFile.path)
-
-  if (await isAudioFile(videoPhysicalFile.path, probe)) {
-    videoFile.resolution = VideoResolution.H_NOVIDEO
-  } else {
-    videoFile.fps = await getVideoStreamFPS(videoPhysicalFile.path, probe)
-    videoFile.resolution = (await getVideoStreamDimensionsInfo(videoPhysicalFile.path, probe)).resolution
-  }
-
-  videoFile.filename = generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname)
-
-  return videoFile
-}
-

 async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) {
 async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) {

-  return JobQueue.Instance.createSequentialJobFlow(
+  const jobs: (CreateJobArgument & CreateJobOptions)[] = [

     {
       type: 'manage-video-torrent' as 'manage-video-torrent',
       payload: {
     {
       type: 'manage-video-torrent' as 'manage-video-torrent',
       payload: {


@@ -274,16 +243,26 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
         videoUUID: video.uuid,
         isNewVideo: true
       }
         videoUUID: video.uuid,
         isNewVideo: true
       }

-    },
+    }
+  ]

 
 

-    video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
-      ? await buildMoveToObjectStorageJob({ video, previousVideoState: undefined })
-      : undefined,
+  if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
+    jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined }))
+  }
+
+  if (video.state === VideoState.TO_TRANSCODE) {
+    jobs.push({
+      type: 'transcoding-job-builder' as 'transcoding-job-builder',
+      payload: {
+        videoUUID: video.uuid,
+        optimizeJob: {
+          isNewVideo: true
+        }
+      }
+    })
+  }

 
 

-    video.state === VideoState.TO_TRANSCODE
-      ? await buildOptimizeOrMergeAudioJob({ video, videoFile, user })
-      : undefined
-  )
+  return JobQueue.Instance.createSequentialJobFlow(...jobs)

 }
 
 async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) {
 }
 
 async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) {

diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts
index a5ce1d79f2671026d794be31468a15c4e471711e..2b825a73093e7e0b88d21472ca10134b967ac8dd 100644 (file)
--- a/server/controllers/bots.ts
+++ b/server/controllers/bots.ts
@@ -1,8 +1,8 @@
-import { getServerActor } from '@server/models/application/application'
-import { logger } from '@uploadx/core'

 import express from 'express'
 import { truncate } from 'lodash'
 import express from 'express'
 import { truncate } from 'lodash'

-import { SitemapStream, streamToPromise, ErrorLevel } from 'sitemap'
+import { ErrorLevel, SitemapStream, streamToPromise } from 'sitemap'
+import { logger } from '@server/helpers/logger'
+import { getServerActor } from '@server/models/application/application'

 import { buildNSFWFilter } from '../helpers/express-utils'
 import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
 import { asyncMiddleware } from '../middlewares'
 import { buildNSFWFilter } from '../helpers/express-utils'
 import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
 import { asyncMiddleware } from '../middlewares'

diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts
index c530b57f83ed8b586855c3493fb72c90f771f255..8e2cc4af985592b25a440d6cf3c3adbe42967c33 100644 (file)
--- a/server/controllers/object-storage-proxy.ts
+++ b/server/controllers/object-storage-proxy.ts
@@ -1,11 +1,7 @@
 import cors from 'cors'
 import express from 'express'
 import cors from 'cors'
 import express from 'express'

-import { PassThrough, pipeline } from 'stream'
-import { logger } from '@server/helpers/logger'
-import { StreamReplacer } from '@server/helpers/stream-replacer'

 import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
 import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'

-import { injectQueryToPlaylistUrls } from '@server/lib/hls'
-import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
+import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage'

 import {
   asyncMiddleware,
   ensureCanAccessPrivateVideoHLSFiles,
 import {
   asyncMiddleware,
   ensureCanAccessPrivateVideoHLSFiles,


@@ -13,9 +9,7 @@ import {
   ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate
 } from '@server/middlewares'
   ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate
 } from '@server/middlewares'

-import { HttpStatusCode } from '@shared/models'
-import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist'
-import { GetObjectCommandOutput } from '@aws-sdk/client-s3'
+import { doReinjectVideoFileToken } from './shared/m3u8-playlist'

 
 const objectStorageProxyRouter = express.Router()
 
 
 const objectStorageProxyRouter = express.Router()
 


@@ -25,14 +19,14 @@ objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':file
   ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate,
   asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
   ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate,
   asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),

-  asyncMiddleware(proxifyWebTorrent)
+  asyncMiddleware(proxifyWebTorrentController)

 )
 
 objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
   ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate,
   asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
 )
 
 objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
   ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate,
   asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),

-  asyncMiddleware(proxifyHLS)
+  asyncMiddleware(proxifyHLSController)

 )
 
 // ---------------------------------------------------------------------------
 )
 
 // ---------------------------------------------------------------------------


@@ -41,76 +35,25 @@ export {
   objectStorageProxyRouter
 }
 
   objectStorageProxyRouter
 }
 

-async function proxifyWebTorrent (req: express.Request, res: express.Response) {
+function proxifyWebTorrentController (req: express.Request, res: express.Response) {

   const filename = req.params.filename
 
   const filename = req.params.filename
 

-  logger.debug('Proxifying WebTorrent file %s from object storage.', filename)
-
-  try {
-    const { response: s3Response, stream } = await getWebTorrentFileReadStream({
-      filename,
-      rangeHeader: req.header('range')
-    })
-
-    setS3Headers(res, s3Response)
-
-    return stream.pipe(res)
-  } catch (err) {
-    return handleObjectStorageFailure(res, err)
-  }
+  return proxifyWebTorrentFile({ req, res, filename })

 }
 
 }
 

-async function proxifyHLS (req: express.Request, res: express.Response) {
+function proxifyHLSController (req: express.Request, res: express.Response) {

   const playlist = res.locals.videoStreamingPlaylist
   const video = res.locals.onlyVideo
   const filename = req.params.filename
 
   const playlist = res.locals.videoStreamingPlaylist
   const video = res.locals.onlyVideo
   const filename = req.params.filename
 

-  logger.debug('Proxifying HLS file %s from object storage.', filename)
-
-  try {
-    const { response: s3Response, stream } = await getHLSFileReadStream({
-      playlist: playlist.withVideo(video),
-      filename,
-      rangeHeader: req.header('range')
-    })
-
-    setS3Headers(res, s3Response)
-
-    const streamReplacer = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req)
-      ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8'))))
-      : new PassThrough()
+  const reinjectVideoFileToken = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req)

 
 

-    return pipeline(
-      stream,
-      streamReplacer,
-      res,
-      err => {
-        if (!err) return
-
-        handleObjectStorageFailure(res, err)
-      }
-    )
-  } catch (err) {
-    return handleObjectStorageFailure(res, err)
-  }
-}
-
-function handleObjectStorageFailure (res: express.Response, err: Error) {
-  if (err.name === 'NoSuchKey') {
-    logger.debug('Could not find key in object storage to proxify private HLS video file.', { err })
-    return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
-  }
-
-  return res.fail({
-    status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
-    message: err.message,
-    type: err.name
+  return proxifyHLS({
+    req,
+    res,
+    playlist,
+    video,
+    filename,
+    reinjectVideoFileToken

   })
 }
   })
 }

-
-function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) {
-  if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) {
-    res.setHeader('Content-Range', s3Response.ContentRange)
-    res.status(HttpStatusCode.PARTIAL_CONTENT_206)
-  }
-}

diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 73bd994c17c7618a69db3eb65de751681f8ff6aa..242c49e896090249c3e1d6cda61a5e05ba2cd519 100644 (file)
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -11,6 +11,7 @@ import { truncate } from 'lodash'
 import { pipeline } from 'stream'
 import { URL } from 'url'
 import { promisify } from 'util'
 import { pipeline } from 'stream'
 import { URL } from 'url'
 import { promisify } from 'util'

+import { promisify1, promisify2, promisify3 } from '@shared/core-utils'

 
 const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
   if (!oldObject || typeof oldObject !== 'object') {
 
 const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
   if (!oldObject || typeof oldObject !== 'object') {


@@ -229,18 +230,6 @@ function execShell (command: string, options?: ExecOptions) {
 
 // ---------------------------------------------------------------------------
 
 
 // ---------------------------------------------------------------------------
 

-function isOdd (num: number) {
-  return (num % 2) !== 0
-}
-
-function toEven (num: number) {
-  if (isOdd(num)) return num + 1
-
-  return num
-}
-
-// ---------------------------------------------------------------------------
-

 function generateRSAKeyPairPromise (size: number) {
   return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
     const options: RSAKeyPairOptions<'pem', 'pem'> = {
 function generateRSAKeyPairPromise (size: number) {
   return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
     const options: RSAKeyPairOptions<'pem', 'pem'> = {


@@ -286,40 +275,6 @@ function generateED25519KeyPairPromise () {
 
 // ---------------------------------------------------------------------------
 
 
 // ---------------------------------------------------------------------------
 

-function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
-  return function promisified (): Promise<A> {
-    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
-      func.apply(null, [ (err: any, res: A) => err ? reject(err) : resolve(res) ])
-    })
-  }
-}
-
-// Thanks to https://gist.github.com/kumasento/617daa7e46f13ecdd9b2
-function promisify1<T, A> (func: (arg: T, cb: (err: any, result: A) => void) => void): (arg: T) => Promise<A> {
-  return function promisified (arg: T): Promise<A> {
-    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
-      func.apply(null, [ arg, (err: any, res: A) => err ? reject(err) : resolve(res) ])
-    })
-  }
-}
-
-function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> {
-  return function promisified (arg1: T, arg2: U): Promise<A> {
-    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
-      func.apply(null, [ arg1, arg2, (err: any, res: A) => err ? reject(err) : resolve(res) ])
-    })
-  }
-}
-
-// eslint-disable-next-line max-len
-function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> {
-  return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> {
-    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
-      func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ])
-    })
-  }
-}
-

 const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
 const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
 const execPromise2 = promisify2<string, any, string>(exec)
 const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
 const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
 const execPromise2 = promisify2<string, any, string>(exec)


@@ -345,10 +300,6 @@ export {
   pageToStartAndCount,
   peertubeTruncate,
 
   pageToStartAndCount,
   peertubeTruncate,
 

-  promisify0,
-  promisify1,
-  promisify2,
-

   scryptPromise,
 
   randomBytesPromise,
   scryptPromise,
 
   randomBytesPromise,


@@ -360,8 +311,5 @@ export {
   execPromise,
   pipelinePromise,
 
   execPromise,
   pipelinePromise,
 

-  parseSemVersion,
-
-  isOdd,
-  toEven
+  parseSemVersion

 }
 }

diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index ebab4c6b244ba1277a0424c089e535fd9faac131..fa0f469f609c81546d40aa720e6b835a46529b77 100644 (file)
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -15,6 +15,10 @@ function isSafePath (p: string) {
     })
 }
 
     })
 }
 

+function isSafeFilename (filename: string, extension: string) {
+  return typeof filename === 'string' && !!filename.match(new RegExp(`^[a-z0-9-]+\\.${extension}$`))
+}
+

 function isSafePeerTubeFilenameWithoutExtension (filename: string) {
   return filename.match(/^[a-z0-9-]+$/)
 }
 function isSafePeerTubeFilenameWithoutExtension (filename: string) {
   return filename.match(/^[a-z0-9-]+$/)
 }


@@ -177,5 +181,6 @@ export {
   toIntArray,
   isFileValid,
   isSafePeerTubeFilenameWithoutExtension,
   toIntArray,
   isFileValid,
   isSafePeerTubeFilenameWithoutExtension,

+  isSafeFilename,

   checkMimetypeRegex
 }
   checkMimetypeRegex
 }

diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts
new file mode 100644 (file)
index 0000000..5f755d5
--- /dev/null
+++ b/server/helpers/custom-validators/runners/jobs.ts
@@ -0,0 +1,166 @@
+import { UploadFilesForCheck } from 'express'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
+import {
+  LiveRTMPHLSTranscodingSuccess,
+  RunnerJobSuccessPayload,
+  RunnerJobType,
+  RunnerJobUpdatePayload,
+  VODAudioMergeTranscodingSuccess,
+  VODHLSTranscodingSuccess,
+  VODWebVideoTranscodingSuccess
+} from '@shared/models'
+import { exists, isFileValid, isSafeFilename } from '../misc'
+
+const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS
+
+const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ])
+function isRunnerJobTypeValid (value: RunnerJobType) {
+  return runnerJobTypes.has(value)
+}
+
+function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) {
+  return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) ||
+    isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
+    isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
+    isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type)
+}
+
+// ---------------------------------------------------------------------------
+
+function isRunnerJobProgressValid (value: string) {
+  return validator.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS)
+}
+
+function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) {
+  return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) ||
+    isRunnerJobVODHLSUpdatePayloadValid(value, type, files) ||
+    isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) ||
+    isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files)
+}
+
+// ---------------------------------------------------------------------------
+
+function isRunnerJobTokenValid (value: string) {
+  return exists(value) && validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+function isRunnerJobAbortReasonValid (value: string) {
+  return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON)
+}
+
+function isRunnerJobErrorMessageValid (value: string) {
+  return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isRunnerJobTypeValid,
+  isRunnerJobSuccessPayloadValid,
+  isRunnerJobUpdatePayloadValid,
+  isRunnerJobTokenValid,
+  isRunnerJobErrorMessageValid,
+  isRunnerJobProgressValid,
+  isRunnerJobAbortReasonValid
+}
+
+// ---------------------------------------------------------------------------
+
+function isRunnerJobVODWebVideoResultPayloadValid (
+  _value: VODWebVideoTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-web-video-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobVODHLSResultPayloadValid (
+  _value: VODHLSTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-hls-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) &&
+    isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobVODAudioMergeResultPayloadValid (
+  _value: VODAudioMergeTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-audio-merge-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobLiveRTMPHLSResultPayloadValid (
+  value: LiveRTMPHLSTranscodingSuccess,
+  type: RunnerJobType
+) {
+  return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+// ---------------------------------------------------------------------------
+
+function isRunnerJobVODWebVideoUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-web-video-transcoding' &&
+    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobVODHLSUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-hls-transcoding' &&
+    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobVODAudioMergeUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-audio-merge-transcoding' &&
+    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobLiveRTMPHLSUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files
+
+  result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true })
+
+  result &&= isFileValid({
+    files,
+    field: 'payload[resolutionPlaylistFile]',
+    mimeTypeRegex: null,
+    maxSize: null,
+    optional: !value.resolutionPlaylistFilename
+  })
+
+  if (files['payload[resolutionPlaylistFile]']) {
+    result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8')
+  }
+
+  return result &&
+    isSafeFilename(value.videoChunkFilename, 'ts') &&
+    (
+      (
+        value.type === 'remove-chunk'
+      ) ||
+      (
+        value.type === 'add-chunk' &&
+        isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null })
+      )
+    )
+}

diff --git a/server/helpers/custom-validators/runners/runners.ts b/server/helpers/custom-validators/runners/runners.ts
new file mode 100644 (file)
index 0000000..953fac3
--- /dev/null
+++ b/server/helpers/custom-validators/runners/runners.ts
@@ -0,0 +1,30 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
+import { exists } from '../misc'
+
+const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS
+
+function isRunnerRegistrationTokenValid (value: string) {
+  return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+function isRunnerTokenValid (value: string) {
+  return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+function isRunnerNameValid (value: string) {
+  return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME)
+}
+
+function isRunnerDescriptionValid (value: string) {
+  return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isRunnerRegistrationTokenValid,
+  isRunnerTokenValid,
+  isRunnerNameValid,
+  isRunnerDescriptionValid
+}

diff --git a/server/helpers/debounce.ts b/server/helpers/debounce.ts
new file mode 100644 (file)
index 0000000..77d99a8
--- /dev/null
+++ b/server/helpers/debounce.ts
@@ -0,0 +1,16 @@
+export function Debounce (config: { timeoutMS: number }) {
+  let timeoutRef: NodeJS.Timeout
+
+  return function (_target, _key, descriptor: PropertyDescriptor) {
+    const original = descriptor.value
+
+    descriptor.value = function (...args: any[]) {
+      clearTimeout(timeoutRef)
+
+      timeoutRef = setTimeout(() => {
+        original.apply(this, args)
+
+      }, config.timeoutMS)
+    }
+  }
+}

diff --git a/server/helpers/ffmpeg/codecs.ts b/server/helpers/ffmpeg/codecs.ts
new file mode 100644 (file)
index 0000000..3bd7db3
--- /dev/null
+++ b/server/helpers/ffmpeg/codecs.ts
@@ -0,0 +1,64 @@
+import { FfprobeData } from 'fluent-ffmpeg'
+import { getAudioStream, getVideoStream } from '@shared/ffmpeg'
+import { logger } from '../logger'
+import { forceNumber } from '@shared/core-utils'
+
+export async function getVideoStreamCodec (path: string) {
+  const videoStream = await getVideoStream(path)
+  if (!videoStream) return ''
+
+  const videoCodec = videoStream.codec_tag_string
+
+  if (videoCodec === 'vp09') return 'vp09.00.50.08'
+  if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0'
+
+  const baseProfileMatrix = {
+    avc1: {
+      High: '6400',
+      Main: '4D40',
+      Baseline: '42E0'
+    },
+    av01: {
+      High: '1',
+      Main: '0',
+      Professional: '2'
+    }
+  }
+
+  let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile]
+  if (!baseProfile) {
+    logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
+    baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback
+  }
+
+  if (videoCodec === 'av01') {
+    let level = videoStream.level.toString()
+    if (level.length === 1) level = `0${level}`
+
+    // Guess the tier indicator and bit depth
+    return `${videoCodec}.${baseProfile}.${level}M.08`
+  }
+
+  let level = forceNumber(videoStream.level).toString(16)
+  if (level.length === 1) level = `0${level}`
+
+  // Default, h264 codec
+  return `${videoCodec}.${baseProfile}${level}`
+}
+
+export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
+  const { audioStream } = await getAudioStream(path, existingProbe)
+
+  if (!audioStream) return ''
+
+  const audioCodecName = audioStream.codec_name
+
+  if (audioCodecName === 'opus') return 'opus'
+  if (audioCodecName === 'vorbis') return 'vorbis'
+  if (audioCodecName === 'aac') return 'mp4a.40.2'
+  if (audioCodecName === 'mp3') return 'mp4a.40.34'
+
+  logger.warn('Cannot get audio codec of %s.', path, { audioStream })
+
+  return 'mp4a.40.2' // Fallback
+}

diff --git a/server/helpers/ffmpeg/ffmpeg-commons.ts b/server/helpers/ffmpeg/ffmpeg-commons.ts
deleted file mode 100644 (file)
index 3906a20..0000000
--- a/server/helpers/ffmpeg/ffmpeg-commons.ts
+++ /dev/null
@@ -1,114 +0,0 @@
-import { Job } from 'bullmq'
-import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg'
-import { execPromise } from '@server/helpers/core-utils'
-import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { CONFIG } from '@server/initializers/config'
-import { FFMPEG_NICE } from '@server/initializers/constants'
-import { EncoderOptions } from '@shared/models'
-
-const lTags = loggerTagsFactory('ffmpeg')
-
-type StreamType = 'audio' | 'video'
-
-function getFFmpeg (input: string, type: 'live' | 'vod') {
-  // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems
-  const command = ffmpeg(input, {
-    niceness: type === 'live' ? FFMPEG_NICE.LIVE : FFMPEG_NICE.VOD,
-    cwd: CONFIG.STORAGE.TMP_DIR
-  })
-
-  const threads = type === 'live'
-    ? CONFIG.LIVE.TRANSCODING.THREADS
-    : CONFIG.TRANSCODING.THREADS
-
-  if (threads > 0) {
-    // If we don't set any threads ffmpeg will chose automatically
-    command.outputOption('-threads ' + threads)
-  }
-
-  return command
-}
-
-function getFFmpegVersion () {
-  return new Promise<string>((res, rej) => {
-    (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => {
-      if (err) return rej(err)
-      if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path'))
-
-      return execPromise(`${ffmpegPath} -version`)
-        .then(stdout => {
-          const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/)
-          if (!parsed?.[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`))
-
-          // Fix ffmpeg version that does not include patch version (4.4 for example)
-          let version = parsed[1]
-          if (version.match(/^\d+\.\d+$/)) {
-            version += '.0'
-          }
-
-          return res(version)
-        })
-        .catch(err => rej(err))
-    })
-  })
-}
-
-async function runCommand (options: {
-  command: FfmpegCommand
-  silent?: boolean // false by default
-  job?: Job
-}) {
-  const { command, silent = false, job } = options
-
-  return new Promise<void>((res, rej) => {
-    let shellCommand: string
-
-    command.on('start', cmdline => { shellCommand = cmdline })
-
-    command.on('error', (err, stdout, stderr) => {
-      if (silent !== true) logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...lTags() })
-
-      rej(err)
-    })
-
-    command.on('end', (stdout, stderr) => {
-      logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...lTags() })
-
-      res()
-    })
-
-    if (job) {
-      command.on('progress', progress => {
-        if (!progress.percent) return
-
-        job.updateProgress(Math.round(progress.percent))
-          .catch(err => logger.warn('Cannot set ffmpeg job progress.', { err, ...lTags() }))
-      })
-    }
-
-    command.run()
-  })
-}
-
-function buildStreamSuffix (base: string, streamNum?: number) {
-  if (streamNum !== undefined) {
-    return `${base}:${streamNum}`
-  }
-
-  return base
-}
-
-function getScaleFilter (options: EncoderOptions): string {
-  if (options.scaleFilter) return options.scaleFilter.name
-
-  return 'scale'
-}
-
-export {
-  getFFmpeg,
-  getFFmpegVersion,
-  runCommand,
-  StreamType,
-  buildStreamSuffix,
-  getScaleFilter
-}

diff --git a/server/helpers/ffmpeg/ffmpeg-edition.ts b/server/helpers/ffmpeg/ffmpeg-edition.ts
deleted file mode 100644 (file)
index 02c5ea8..0000000
--- a/server/helpers/ffmpeg/ffmpeg-edition.ts
+++ /dev/null
@@ -1,258 +0,0 @@
-import { FilterSpecification } from 'fluent-ffmpeg'
-import { VIDEO_FILTERS } from '@server/initializers/constants'
-import { AvailableEncoders } from '@shared/models'
-import { logger, loggerTagsFactory } from '../logger'
-import { getFFmpeg, runCommand } from './ffmpeg-commons'
-import { presetVOD } from './ffmpeg-presets'
-import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe-utils'
-
-const lTags = loggerTagsFactory('ffmpeg')
-
-async function cutVideo (options: {
-  inputPath: string
-  outputPath: string
-  start?: number
-  end?: number
-
-  availableEncoders: AvailableEncoders
-  profile: string
-}) {
-  const { inputPath, outputPath, availableEncoders, profile } = options
-
-  logger.debug('Will cut the video.', { options, ...lTags() })
-
-  const mainProbe = await ffprobePromise(inputPath)
-  const fps = await getVideoStreamFPS(inputPath, mainProbe)
-  const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe)
-
-  let command = getFFmpeg(inputPath, 'vod')
-    .output(outputPath)
-
-  command = await presetVOD({
-    command,
-    input: inputPath,
-    availableEncoders,
-    profile,
-    resolution,
-    fps,
-    canCopyAudio: false,
-    canCopyVideo: false
-  })
-
-  if (options.start) {
-    command.outputOption('-ss ' + options.start)
-  }
-
-  if (options.end) {
-    command.outputOption('-to ' + options.end)
-  }
-
-  await runCommand({ command })
-}
-
-async function addWatermark (options: {
-  inputPath: string
-  watermarkPath: string
-  outputPath: string
-
-  availableEncoders: AvailableEncoders
-  profile: string
-}) {
-  const { watermarkPath, inputPath, outputPath, availableEncoders, profile } = options
-
-  logger.debug('Will add watermark to the video.', { options, ...lTags() })
-
-  const videoProbe = await ffprobePromise(inputPath)
-  const fps = await getVideoStreamFPS(inputPath, videoProbe)
-  const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe)
-
-  let command = getFFmpeg(inputPath, 'vod')
-    .output(outputPath)
-  command.input(watermarkPath)
-
-  command = await presetVOD({
-    command,
-    input: inputPath,
-    availableEncoders,
-    profile,
-    resolution,
-    fps,
-    canCopyAudio: true,
-    canCopyVideo: false
-  })
-
-  const complexFilter: FilterSpecification[] = [
-    // Scale watermark
-    {
-      inputs: [ '[1]', '[0]' ],
-      filter: 'scale2ref',
-      options: {
-        w: 'oh*mdar',
-        h: `ih*${VIDEO_FILTERS.WATERMARK.SIZE_RATIO}`
-      },
-      outputs: [ '[watermark]', '[video]' ]
-    },
-
-    {
-      inputs: [ '[video]', '[watermark]' ],
-      filter: 'overlay',
-      options: {
-        x: `main_w - overlay_w - (main_h * ${VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO})`,
-        y: `main_h * ${VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO}`
-      }
-    }
-  ]
-
-  command.complexFilter(complexFilter)
-
-  await runCommand({ command })
-}
-
-async function addIntroOutro (options: {
-  inputPath: string
-  introOutroPath: string
-  outputPath: string
-  type: 'intro' | 'outro'
-
-  availableEncoders: AvailableEncoders
-  profile: string
-}) {
-  const { introOutroPath, inputPath, outputPath, availableEncoders, profile, type } = options
-
-  logger.debug('Will add intro/outro to the video.', { options, ...lTags() })
-
-  const mainProbe = await ffprobePromise(inputPath)
-  const fps = await getVideoStreamFPS(inputPath, mainProbe)
-  const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe)
-  const mainHasAudio = await hasAudioStream(inputPath, mainProbe)
-
-  const introOutroProbe = await ffprobePromise(introOutroPath)
-  const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe)
-
-  let command = getFFmpeg(inputPath, 'vod')
-    .output(outputPath)
-
-  command.input(introOutroPath)
-
-  if (!introOutroHasAudio && mainHasAudio) {
-    const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe)
-
-    command.input('anullsrc')
-    command.withInputFormat('lavfi')
-    command.withInputOption('-t ' + duration)
-  }
-
-  command = await presetVOD({
-    command,
-    input: inputPath,
-    availableEncoders,
-    profile,
-    resolution,
-    fps,
-    canCopyAudio: false,
-    canCopyVideo: false
-  })
-
-  // Add black background to correctly scale intro/outro with padding
-  const complexFilter: FilterSpecification[] = [
-    {
-      inputs: [ '1', '0' ],
-      filter: 'scale2ref',
-      options: {
-        w: 'iw',
-        h: `ih`
-      },
-      outputs: [ 'intro-outro', 'main' ]
-    },
-    {
-      inputs: [ 'intro-outro', 'main' ],
-      filter: 'scale2ref',
-      options: {
-        w: 'iw',
-        h: `ih`
-      },
-      outputs: [ 'to-scale', 'main' ]
-    },
-    {
-      inputs: 'to-scale',
-      filter: 'drawbox',
-      options: {
-        t: 'fill'
-      },
-      outputs: [ 'to-scale-bg' ]
-    },
-    {
-      inputs: [ '1', 'to-scale-bg' ],
-      filter: 'scale2ref',
-      options: {
-        w: 'iw',
-        h: 'ih',
-        force_original_aspect_ratio: 'decrease',
-        flags: 'spline'
-      },
-      outputs: [ 'to-scale', 'to-scale-bg' ]
-    },
-    {
-      inputs: [ 'to-scale-bg', 'to-scale' ],
-      filter: 'overlay',
-      options: {
-        x: '(main_w - overlay_w)/2',
-        y: '(main_h - overlay_h)/2'
-      },
-      outputs: 'intro-outro-resized'
-    }
-  ]
-
-  const concatFilter = {
-    inputs: [],
-    filter: 'concat',
-    options: {
-      n: 2,
-      v: 1,
-      unsafe: 1
-    },
-    outputs: [ 'v' ]
-  }
-
-  const introOutroFilterInputs = [ 'intro-outro-resized' ]
-  const mainFilterInputs = [ 'main' ]
-
-  if (mainHasAudio) {
-    mainFilterInputs.push('0:a')
-
-    if (introOutroHasAudio) {
-      introOutroFilterInputs.push('1:a')
-    } else {
-      // Silent input
-      introOutroFilterInputs.push('2:a')
-    }
-  }
-
-  if (type === 'intro') {
-    concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ]
-  } else {
-    concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ]
-  }
-
-  if (mainHasAudio) {
-    concatFilter.options['a'] = 1
-    concatFilter.outputs.push('a')
-
-    command.outputOption('-map [a]')
-  }
-
-  command.outputOption('-map [v]')
-
-  complexFilter.push(concatFilter)
-  command.complexFilter(complexFilter)
-
-  await runCommand({ command })
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  cutVideo,
-  addIntroOutro,
-  addWatermark
-}

diff --git a/server/helpers/ffmpeg/ffmpeg-encoders.ts b/server/helpers/ffmpeg/ffmpeg-encoders.ts
deleted file mode 100644 (file)
index 5bd80ba..0000000
--- a/server/helpers/ffmpeg/ffmpeg-encoders.ts
+++ /dev/null
@@ -1,116 +0,0 @@
-import { getAvailableEncoders } from 'fluent-ffmpeg'
-import { pick } from '@shared/core-utils'
-import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models'
-import { promisify0 } from '../core-utils'
-import { logger, loggerTagsFactory } from '../logger'
-
-const lTags = loggerTagsFactory('ffmpeg')
-
-// Detect supported encoders by ffmpeg
-let supportedEncoders: Map<string, boolean>
-async function checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise<Map<string, boolean>> {
-  if (supportedEncoders !== undefined) {
-    return supportedEncoders
-  }
-
-  const getAvailableEncodersPromise = promisify0(getAvailableEncoders)
-  const availableFFmpegEncoders = await getAvailableEncodersPromise()
-
-  const searchEncoders = new Set<string>()
-  for (const type of [ 'live', 'vod' ]) {
-    for (const streamType of [ 'audio', 'video' ]) {
-      for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) {
-        searchEncoders.add(encoder)
-      }
-    }
-  }
-
-  supportedEncoders = new Map<string, boolean>()
-
-  for (const searchEncoder of searchEncoders) {
-    supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined)
-  }
-
-  logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...lTags() })
-
-  return supportedEncoders
-}
-
-function resetSupportedEncoders () {
-  supportedEncoders = undefined
-}
-
-// Run encoder builder depending on available encoders
-// Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one
-// If the default one does not exist, check the next encoder
-async function getEncoderBuilderResult (options: EncoderOptionsBuilderParams & {
-  streamType: 'video' | 'audio'
-  input: string
-
-  availableEncoders: AvailableEncoders
-  profile: string
-
-  videoType: 'vod' | 'live'
-}) {
-  const { availableEncoders, profile, streamType, videoType } = options
-
-  const encodersToTry = availableEncoders.encodersToTry[videoType][streamType]
-  const encoders = availableEncoders.available[videoType]
-
-  for (const encoder of encodersToTry) {
-    if (!(await checkFFmpegEncoders(availableEncoders)).get(encoder)) {
-      logger.debug('Encoder %s not available in ffmpeg, skipping.', encoder, lTags())
-      continue
-    }
-
-    if (!encoders[encoder]) {
-      logger.debug('Encoder %s not available in peertube encoders, skipping.', encoder, lTags())
-      continue
-    }
-
-    // An object containing available profiles for this encoder
-    const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = encoders[encoder]
-    let builder = builderProfiles[profile]
-
-    if (!builder) {
-      logger.debug('Profile %s for encoder %s not available. Fallback to default.', profile, encoder, lTags())
-      builder = builderProfiles.default
-
-      if (!builder) {
-        logger.debug('Default profile for encoder %s not available. Try next available encoder.', encoder, lTags())
-        continue
-      }
-    }
-
-    const result = await builder(
-      pick(options, [
-        'input',
-        'canCopyAudio',
-        'canCopyVideo',
-        'resolution',
-        'inputBitrate',
-        'fps',
-        'inputRatio',
-        'streamNum'
-      ])
-    )
-
-    return {
-      result,
-
-      // If we don't have output options, then copy the input stream
-      encoder: result.copy === true
-        ? 'copy'
-        : encoder
-    }
-  }
-
-  return null
-}
-
-export {
-  checkFFmpegEncoders,
-  resetSupportedEncoders,
-
-  getEncoderBuilderResult
-}

diff --git a/server/helpers/ffmpeg/ffmpeg-image.ts b/server/helpers/ffmpeg/ffmpeg-image.ts
new file mode 100644 (file)
index 0000000..0bb0ff2
--- /dev/null
+++ b/server/helpers/ffmpeg/ffmpeg-image.ts
@@ -0,0 +1,14 @@
+import { FFmpegImage } from '@shared/ffmpeg'
+import { getFFmpegCommandWrapperOptions } from './ffmpeg-options'
+
+export function processGIF (options: Parameters<FFmpegImage['processGIF']>[0]) {
+  return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options)
+}
+
+export function generateThumbnailFromVideo (options: Parameters<FFmpegImage['generateThumbnailFromVideo']>[0]) {
+  return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options)
+}
+
+export function convertWebPToJPG (options: Parameters<FFmpegImage['convertWebPToJPG']>[0]) {
+  return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options)
+}

diff --git a/server/helpers/ffmpeg/ffmpeg-images.ts b/server/helpers/ffmpeg/ffmpeg-images.ts
deleted file mode 100644 (file)
index 7f64c6d..0000000
--- a/server/helpers/ffmpeg/ffmpeg-images.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import ffmpeg from 'fluent-ffmpeg'
-import { FFMPEG_NICE } from '@server/initializers/constants'
-import { runCommand } from './ffmpeg-commons'
-
-function convertWebPToJPG (path: string, destination: string): Promise<void> {
-  const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL })
-    .output(destination)
-
-  return runCommand({ command, silent: true })
-}
-
-function processGIF (
-  path: string,
-  destination: string,
-  newSize: { width: number, height: number }
-): Promise<void> {
-  const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL })
-    .fps(20)
-    .size(`${newSize.width}x${newSize.height}`)
-    .output(destination)
-
-  return runCommand({ command })
-}
-
-async function generateThumbnailFromVideo (fromPath: string, folder: string, imageName: string) {
-  const pendingImageName = 'pending-' + imageName
-
-  const options = {
-    filename: pendingImageName,
-    count: 1,
-    folder
-  }
-
-  return new Promise<string>((res, rej) => {
-    ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
-      .on('error', rej)
-      .on('end', () => res(imageName))
-      .thumbnail(options)
-  })
-}
-
-export {
-  convertWebPToJPG,
-  processGIF,
-  generateThumbnailFromVideo
-}

diff --git a/server/helpers/ffmpeg/ffmpeg-live.ts b/server/helpers/ffmpeg/ffmpeg-live.ts
deleted file mode 100644 (file)
index 379d7b1..0000000
--- a/server/helpers/ffmpeg/ffmpeg-live.ts
+++ /dev/null
@@ -1,204 +0,0 @@
-import { FfmpegCommand, FilterSpecification } from 'fluent-ffmpeg'
-import { join } from 'path'
-import { VIDEO_LIVE } from '@server/initializers/constants'
-import { AvailableEncoders, LiveVideoLatencyMode } from '@shared/models'
-import { logger, loggerTagsFactory } from '../logger'
-import { buildStreamSuffix, getFFmpeg, getScaleFilter, StreamType } from './ffmpeg-commons'
-import { getEncoderBuilderResult } from './ffmpeg-encoders'
-import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './ffmpeg-presets'
-import { computeFPS } from './ffprobe-utils'
-
-const lTags = loggerTagsFactory('ffmpeg')
-
-async function getLiveTranscodingCommand (options: {
-  inputUrl: string
-
-  outPath: string
-  masterPlaylistName: string
-  latencyMode: LiveVideoLatencyMode
-
-  resolutions: number[]
-
-  // Input information
-  fps: number
-  bitrate: number
-  ratio: number
-  hasAudio: boolean
-
-  availableEncoders: AvailableEncoders
-  profile: string
-}) {
-  const {
-    inputUrl,
-    outPath,
-    resolutions,
-    fps,
-    bitrate,
-    availableEncoders,
-    profile,
-    masterPlaylistName,
-    ratio,
-    latencyMode,
-    hasAudio
-  } = options
-
-  const command = getFFmpeg(inputUrl, 'live')
-
-  const varStreamMap: string[] = []
-
-  const complexFilter: FilterSpecification[] = [
-    {
-      inputs: '[v:0]',
-      filter: 'split',
-      options: resolutions.length,
-      outputs: resolutions.map(r => `vtemp${r}`)
-    }
-  ]
-
-  command.outputOption('-sc_threshold 0')
-
-  addDefaultEncoderGlobalParams(command)
-
-  for (let i = 0; i < resolutions.length; i++) {
-    const streamMap: string[] = []
-    const resolution = resolutions[i]
-    const resolutionFPS = computeFPS(fps, resolution)
-
-    const baseEncoderBuilderParams = {
-      input: inputUrl,
-
-      availableEncoders,
-      profile,
-
-      canCopyAudio: true,
-      canCopyVideo: true,
-
-      inputBitrate: bitrate,
-      inputRatio: ratio,
-
-      resolution,
-      fps: resolutionFPS,
-
-      streamNum: i,
-      videoType: 'live' as 'live'
-    }
-
-    {
-      const streamType: StreamType = 'video'
-      const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType })
-      if (!builderResult) {
-        throw new Error('No available live video encoder found')
-      }
-
-      command.outputOption(`-map [vout${resolution}]`)
-
-      addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
-
-      logger.debug(
-        'Apply ffmpeg live video params from %s using %s profile.', builderResult.encoder, profile,
-        { builderResult, fps: resolutionFPS, resolution, ...lTags() }
-      )
-
-      command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`)
-      applyEncoderOptions(command, builderResult.result)
-
-      complexFilter.push({
-        inputs: `vtemp${resolution}`,
-        filter: getScaleFilter(builderResult.result),
-        options: `w=-2:h=${resolution}`,
-        outputs: `vout${resolution}`
-      })
-
-      streamMap.push(`v:${i}`)
-    }
-
-    if (hasAudio) {
-      const streamType: StreamType = 'audio'
-      const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType })
-      if (!builderResult) {
-        throw new Error('No available live audio encoder found')
-      }
-
-      command.outputOption('-map a:0')
-
-      addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
-
-      logger.debug(
-        'Apply ffmpeg live audio params from %s using %s profile.', builderResult.encoder, profile,
-        { builderResult, fps: resolutionFPS, resolution, ...lTags() }
-      )
-
-      command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`)
-      applyEncoderOptions(command, builderResult.result)
-
-      streamMap.push(`a:${i}`)
-    }
-
-    varStreamMap.push(streamMap.join(','))
-  }
-
-  command.complexFilter(complexFilter)
-
-  addDefaultLiveHLSParams({ command, outPath, masterPlaylistName, latencyMode })
-
-  command.outputOption('-var_stream_map', varStreamMap.join(' '))
-
-  return command
-}
-
-function getLiveMuxingCommand (options: {
-  inputUrl: string
-  outPath: string
-  masterPlaylistName: string
-  latencyMode: LiveVideoLatencyMode
-}) {
-  const { inputUrl, outPath, masterPlaylistName, latencyMode } = options
-
-  const command = getFFmpeg(inputUrl, 'live')
-
-  command.outputOption('-c:v copy')
-  command.outputOption('-c:a copy')
-  command.outputOption('-map 0:a?')
-  command.outputOption('-map 0:v?')
-
-  addDefaultLiveHLSParams({ command, outPath, masterPlaylistName, latencyMode })
-
-  return command
-}
-
-function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) {
-  if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) {
-    return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY
-  }
-
-  return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  getLiveSegmentTime,
-
-  getLiveTranscodingCommand,
-  getLiveMuxingCommand
-}
-
-// ---------------------------------------------------------------------------
-
-function addDefaultLiveHLSParams (options: {
-  command: FfmpegCommand
-  outPath: string
-  masterPlaylistName: string
-  latencyMode: LiveVideoLatencyMode
-}) {
-  const { command, outPath, masterPlaylistName, latencyMode } = options
-
-  command.outputOption('-hls_time ' + getLiveSegmentTime(latencyMode))
-  command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
-  command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time')
-  command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
-  command.outputOption('-master_pl_name ' + masterPlaylistName)
-  command.outputOption(`-f hls`)
-
-  command.output(join(outPath, '%v.m3u8'))
-}

diff --git a/server/helpers/ffmpeg/ffmpeg-options.ts b/server/helpers/ffmpeg/ffmpeg-options.ts
new file mode 100644 (file)
index 0000000..db6350d
--- /dev/null
+++ b/server/helpers/ffmpeg/ffmpeg-options.ts
@@ -0,0 +1,45 @@
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { FFMPEG_NICE } from '@server/initializers/constants'
+import { FFmpegCommandWrapperOptions } from '@shared/ffmpeg'
+import { AvailableEncoders } from '@shared/models'
+
+type CommandType = 'live' | 'vod' | 'thumbnail'
+
+export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions {
+  return {
+    availableEncoders,
+    profile: getProfile(type),
+
+    niceness: FFMPEG_NICE[type],
+    tmpDirectory: CONFIG.STORAGE.TMP_DIR,
+    threads: getThreads(type),
+
+    logger: {
+      debug: logger.debug.bind(logger),
+      info: logger.info.bind(logger),
+      warn: logger.warn.bind(logger),
+      error: logger.error.bind(logger)
+    },
+    lTags: { tags: [ 'ffmpeg' ] }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function getThreads (type: CommandType) {
+  if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS
+  if (type === 'vod') return CONFIG.TRANSCODING.THREADS
+
+  // Auto
+  return 0
+}
+
+function getProfile (type: CommandType) {
+  if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE
+  if (type === 'vod') return CONFIG.TRANSCODING.PROFILE
+
+  return undefined
+}

diff --git a/server/helpers/ffmpeg/ffmpeg-presets.ts b/server/helpers/ffmpeg/ffmpeg-presets.ts
deleted file mode 100644 (file)
index d1160a4..0000000
--- a/server/helpers/ffmpeg/ffmpeg-presets.ts
+++ /dev/null
@@ -1,156 +0,0 @@
-import { FfmpegCommand } from 'fluent-ffmpeg'
-import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { pick } from '@shared/core-utils'
-import { AvailableEncoders, EncoderOptions } from '@shared/models'
-import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-commons'
-import { getEncoderBuilderResult } from './ffmpeg-encoders'
-import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from './ffprobe-utils'
-
-const lTags = loggerTagsFactory('ffmpeg')
-
-// ---------------------------------------------------------------------------
-
-function addDefaultEncoderGlobalParams (command: FfmpegCommand) {
-  // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375
-  command.outputOption('-max_muxing_queue_size 1024')
-         // strip all metadata
-         .outputOption('-map_metadata -1')
-         // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
-         .outputOption('-pix_fmt yuv420p')
-}
-
-function addDefaultEncoderParams (options: {
-  command: FfmpegCommand
-  encoder: 'libx264' | string
-  fps: number
-
-  streamNum?: number
-}) {
-  const { command, encoder, fps, streamNum } = options
-
-  if (encoder === 'libx264') {
-    // 3.1 is the minimal resource allocation for our highest supported resolution
-    command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1')
-
-    if (fps) {
-      // Keyframe interval of 2 seconds for faster seeking and resolution switching.
-      // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
-      // https://superuser.com/a/908325
-      command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2))
-    }
-  }
-}
-
-// ---------------------------------------------------------------------------
-
-async function presetVOD (options: {
-  command: FfmpegCommand
-  input: string
-
-  availableEncoders: AvailableEncoders
-  profile: string
-
-  canCopyAudio: boolean
-  canCopyVideo: boolean
-
-  resolution: number
-  fps: number
-
-  scaleFilterValue?: string
-}) {
-  const { command, input, profile, resolution, fps, scaleFilterValue } = options
-
-  let localCommand = command
-    .format('mp4')
-    .outputOption('-movflags faststart')
-
-  addDefaultEncoderGlobalParams(command)
-
-  const probe = await ffprobePromise(input)
-
-  // Audio encoder
-  const bitrate = await getVideoStreamBitrate(input, probe)
-  const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe)
-
-  let streamsToProcess: StreamType[] = [ 'audio', 'video' ]
-
-  if (!await hasAudioStream(input, probe)) {
-    localCommand = localCommand.noAudio()
-    streamsToProcess = [ 'video' ]
-  }
-
-  for (const streamType of streamsToProcess) {
-    const builderResult = await getEncoderBuilderResult({
-      ...pick(options, [ 'availableEncoders', 'canCopyAudio', 'canCopyVideo' ]),
-
-      input,
-      inputBitrate: bitrate,
-      inputRatio: videoStreamDimensions?.ratio || 0,
-
-      profile,
-      resolution,
-      fps,
-      streamType,
-
-      videoType: 'vod' as 'vod'
-    })
-
-    if (!builderResult) {
-      throw new Error('No available encoder found for stream ' + streamType)
-    }
-
-    logger.debug(
-      'Apply ffmpeg params from %s for %s stream of input %s using %s profile.',
-      builderResult.encoder, streamType, input, profile,
-      { builderResult, resolution, fps, ...lTags() }
-    )
-
-    if (streamType === 'video') {
-      localCommand.videoCodec(builderResult.encoder)
-
-      if (scaleFilterValue) {
-        localCommand.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`)
-      }
-    } else if (streamType === 'audio') {
-      localCommand.audioCodec(builderResult.encoder)
-    }
-
-    applyEncoderOptions(localCommand, builderResult.result)
-    addDefaultEncoderParams({ command: localCommand, encoder: builderResult.encoder, fps })
-  }
-
-  return localCommand
-}
-
-function presetCopy (command: FfmpegCommand): FfmpegCommand {
-  return command
-    .format('mp4')
-    .videoCodec('copy')
-    .audioCodec('copy')
-}
-
-function presetOnlyAudio (command: FfmpegCommand): FfmpegCommand {
-  return command
-    .format('mp4')
-    .audioCodec('copy')
-    .noVideo()
-}
-
-function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions): FfmpegCommand {
-  return command
-    .inputOptions(options.inputOptions ?? [])
-    .outputOptions(options.outputOptions ?? [])
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  presetVOD,
-  presetCopy,
-  presetOnlyAudio,
-
-  addDefaultEncoderGlobalParams,
-  addDefaultEncoderParams,
-
-  applyEncoderOptions
-}

diff --git a/server/helpers/ffmpeg/ffmpeg-vod.ts b/server/helpers/ffmpeg/ffmpeg-vod.ts
deleted file mode 100644 (file)
index d84703e..0000000
--- a/server/helpers/ffmpeg/ffmpeg-vod.ts
+++ /dev/null
@@ -1,267 +0,0 @@
-import { MutexInterface } from 'async-mutex'
-import { Job } from 'bullmq'
-import { FfmpegCommand } from 'fluent-ffmpeg'
-import { readFile, writeFile } from 'fs-extra'
-import { dirname } from 'path'
-import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
-import { pick } from '@shared/core-utils'
-import { AvailableEncoders, VideoResolution } from '@shared/models'
-import { logger, loggerTagsFactory } from '../logger'
-import { getFFmpeg, runCommand } from './ffmpeg-commons'
-import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets'
-import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils'
-
-const lTags = loggerTagsFactory('ffmpeg')
-
-// ---------------------------------------------------------------------------
-
-type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
-
-interface BaseTranscodeVODOptions {
-  type: TranscodeVODOptionsType
-
-  inputPath: string
-  outputPath: string
-
-  // Will be released after the ffmpeg started
-  // To prevent a bug where the input file does not exist anymore when running ffmpeg
-  inputFileMutexReleaser: MutexInterface.Releaser
-
-  availableEncoders: AvailableEncoders
-  profile: string
-
-  resolution: number
-
-  job?: Job
-}
-
-interface HLSTranscodeOptions extends BaseTranscodeVODOptions {
-  type: 'hls'
-  copyCodecs: boolean
-  hlsPlaylist: {
-    videoFilename: string
-  }
-}
-
-interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions {
-  type: 'hls-from-ts'
-
-  isAAC: boolean
-
-  hlsPlaylist: {
-    videoFilename: string
-  }
-}
-
-interface QuickTranscodeOptions extends BaseTranscodeVODOptions {
-  type: 'quick-transcode'
-}
-
-interface VideoTranscodeOptions extends BaseTranscodeVODOptions {
-  type: 'video'
-}
-
-interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions {
-  type: 'merge-audio'
-  audioPath: string
-}
-
-interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions {
-  type: 'only-audio'
-}
-
-type TranscodeVODOptions =
-  HLSTranscodeOptions
-  | HLSFromTSTranscodeOptions
-  | VideoTranscodeOptions
-  | MergeAudioTranscodeOptions
-  | OnlyAudioTranscodeOptions
-  | QuickTranscodeOptions
-
-// ---------------------------------------------------------------------------
-
-const builders: {
-  [ type in TranscodeVODOptionsType ]: (c: FfmpegCommand, o?: TranscodeVODOptions) => Promise<FfmpegCommand> | FfmpegCommand
-} = {
-  'quick-transcode': buildQuickTranscodeCommand,
-  'hls': buildHLSVODCommand,
-  'hls-from-ts': buildHLSVODFromTSCommand,
-  'merge-audio': buildAudioMergeCommand,
-  'only-audio': buildOnlyAudioCommand,
-  'video': buildVODCommand
-}
-
-async function transcodeVOD (options: TranscodeVODOptions) {
-  logger.debug('Will run transcode.', { options, ...lTags() })
-
-  let command = getFFmpeg(options.inputPath, 'vod')
-    .output(options.outputPath)
-
-  command = await builders[options.type](command, options)
-
-  command.on('start', () => {
-    setTimeout(() => {
-      options.inputFileMutexReleaser()
-    }, 1000)
-  })
-
-  await runCommand({ command, job: options.job })
-
-  await fixHLSPlaylistIfNeeded(options)
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  transcodeVOD,
-
-  buildVODCommand,
-
-  TranscodeVODOptions,
-  TranscodeVODOptionsType
-}
-
-// ---------------------------------------------------------------------------
-
-async function buildVODCommand (command: FfmpegCommand, options: TranscodeVODOptions) {
-  const probe = await ffprobePromise(options.inputPath)
-
-  let fps = await getVideoStreamFPS(options.inputPath, probe)
-  fps = computeFPS(fps, options.resolution)
-
-  let scaleFilterValue: string
-
-  if (options.resolution !== undefined) {
-    const videoStreamInfo = await getVideoStreamDimensionsInfo(options.inputPath, probe)
-
-    scaleFilterValue = videoStreamInfo?.isPortraitMode === true
-      ? `w=${options.resolution}:h=-2`
-      : `w=-2:h=${options.resolution}`
-  }
-
-  command = await presetVOD({
-    ...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]),
-
-    command,
-    input: options.inputPath,
-    canCopyAudio: true,
-    canCopyVideo: true,
-    fps,
-    scaleFilterValue
-  })
-
-  return command
-}
-
-function buildQuickTranscodeCommand (command: FfmpegCommand) {
-  command = presetCopy(command)
-
-  command = command.outputOption('-map_metadata -1') // strip all metadata
-                   .outputOption('-movflags faststart')
-
-  return command
-}
-
-// ---------------------------------------------------------------------------
-// Audio transcoding
-// ---------------------------------------------------------------------------
-
-async function buildAudioMergeCommand (command: FfmpegCommand, options: MergeAudioTranscodeOptions) {
-  command = command.loop(undefined)
-
-  const scaleFilterValue = getMergeAudioScaleFilterValue()
-  command = await presetVOD({
-    ...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]),
-
-    command,
-    input: options.audioPath,
-    canCopyAudio: true,
-    canCopyVideo: true,
-    fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE,
-    scaleFilterValue
-  })
-
-  command.outputOption('-preset:v veryfast')
-
-  command = command.input(options.audioPath)
-                   .outputOption('-tune stillimage')
-                   .outputOption('-shortest')
-
-  return command
-}
-
-function buildOnlyAudioCommand (command: FfmpegCommand, _options: OnlyAudioTranscodeOptions) {
-  command = presetOnlyAudio(command)
-
-  return command
-}
-
-// ---------------------------------------------------------------------------
-// HLS transcoding
-// ---------------------------------------------------------------------------
-
-async function buildHLSVODCommand (command: FfmpegCommand, options: HLSTranscodeOptions) {
-  const videoPath = getHLSVideoPath(options)
-
-  if (options.copyCodecs) command = presetCopy(command)
-  else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
-  else command = await buildVODCommand(command, options)
-
-  addCommonHLSVODCommandOptions(command, videoPath)
-
-  return command
-}
-
-function buildHLSVODFromTSCommand (command: FfmpegCommand, options: HLSFromTSTranscodeOptions) {
-  const videoPath = getHLSVideoPath(options)
-
-  command.outputOption('-c copy')
-
-  if (options.isAAC) {
-    // Required for example when copying an AAC stream from an MPEG-TS
-    // Since it's a bitstream filter, we don't need to reencode the audio
-    command.outputOption('-bsf:a aac_adtstoasc')
-  }
-
-  addCommonHLSVODCommandOptions(command, videoPath)
-
-  return command
-}
-
-function addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) {
-  return command.outputOption('-hls_time 4')
-                .outputOption('-hls_list_size 0')
-                .outputOption('-hls_playlist_type vod')
-                .outputOption('-hls_segment_filename ' + outputPath)
-                .outputOption('-hls_segment_type fmp4')
-                .outputOption('-f hls')
-                .outputOption('-hls_flags single_file')
-}
-
-async function fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) {
-  if (options.type !== 'hls' && options.type !== 'hls-from-ts') return
-
-  const fileContent = await readFile(options.outputPath)
-
-  const videoFileName = options.hlsPlaylist.videoFilename
-  const videoFilePath = getHLSVideoPath(options)
-
-  // Fix wrong mapping with some ffmpeg versions
-  const newContent = fileContent.toString()
-                                .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
-
-  await writeFile(options.outputPath, newContent)
-}
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-function getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) {
-  return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
-}
-
-// Avoid "height not divisible by 2" error
-function getMergeAudioScaleFilterValue () {
-  return 'trunc(iw/2)*2:trunc(ih/2)*2'
-}

diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts
deleted file mode 100644 (file)
index fb270b3..0000000
--- a/server/helpers/ffmpeg/ffprobe-utils.ts
+++ /dev/null
@@ -1,254 +0,0 @@
-import { FfprobeData } from 'fluent-ffmpeg'
-import { getMaxBitrate } from '@shared/core-utils'
-import {
-  buildFileMetadata,
-  ffprobePromise,
-  getAudioStream,
-  getMaxAudioBitrate,
-  getVideoStream,
-  getVideoStreamBitrate,
-  getVideoStreamDimensionsInfo,
-  getVideoStreamDuration,
-  getVideoStreamFPS,
-  hasAudioStream
-} from '@shared/extra-utils/ffprobe'
-import { VideoResolution, VideoTranscodingFPS } from '@shared/models'
-import { CONFIG } from '../../initializers/config'
-import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants'
-import { toEven } from '../core-utils'
-import { logger } from '../logger'
-
-/**
- *
- * Helpers to run ffprobe and extract data from the JSON output
- *
- */
-
-// ---------------------------------------------------------------------------
-// Codecs
-// ---------------------------------------------------------------------------
-
-async function getVideoStreamCodec (path: string) {
-  const videoStream = await getVideoStream(path)
-  if (!videoStream) return ''
-
-  const videoCodec = videoStream.codec_tag_string
-
-  if (videoCodec === 'vp09') return 'vp09.00.50.08'
-  if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0'
-
-  const baseProfileMatrix = {
-    avc1: {
-      High: '6400',
-      Main: '4D40',
-      Baseline: '42E0'
-    },
-    av01: {
-      High: '1',
-      Main: '0',
-      Professional: '2'
-    }
-  }
-
-  let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile]
-  if (!baseProfile) {
-    logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
-    baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback
-  }
-
-  if (videoCodec === 'av01') {
-    let level = videoStream.level.toString()
-    if (level.length === 1) level = `0${level}`
-
-    // Guess the tier indicator and bit depth
-    return `${videoCodec}.${baseProfile}.${level}M.08`
-  }
-
-  let level = videoStream.level.toString(16)
-  if (level.length === 1) level = `0${level}`
-
-  // Default, h264 codec
-  return `${videoCodec}.${baseProfile}${level}`
-}
-
-async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
-  const { audioStream } = await getAudioStream(path, existingProbe)
-
-  if (!audioStream) return ''
-
-  const audioCodecName = audioStream.codec_name
-
-  if (audioCodecName === 'opus') return 'opus'
-  if (audioCodecName === 'vorbis') return 'vorbis'
-  if (audioCodecName === 'aac') return 'mp4a.40.2'
-  if (audioCodecName === 'mp3') return 'mp4a.40.34'
-
-  logger.warn('Cannot get audio codec of %s.', path, { audioStream })
-
-  return 'mp4a.40.2' // Fallback
-}
-
-// ---------------------------------------------------------------------------
-// Resolutions
-// ---------------------------------------------------------------------------
-
-function computeResolutionsToTranscode (options: {
-  input: number
-  type: 'vod' | 'live'
-  includeInput: boolean
-  strictLower: boolean
-  hasAudio: boolean
-}) {
-  const { input, type, includeInput, strictLower, hasAudio } = options
-
-  const configResolutions = type === 'vod'
-    ? CONFIG.TRANSCODING.RESOLUTIONS
-    : CONFIG.LIVE.TRANSCODING.RESOLUTIONS
-
-  const resolutionsEnabled = new Set<number>()
-
-  // Put in the order we want to proceed jobs
-  const availableResolutions: VideoResolution[] = [
-    VideoResolution.H_NOVIDEO,
-    VideoResolution.H_480P,
-    VideoResolution.H_360P,
-    VideoResolution.H_720P,
-    VideoResolution.H_240P,
-    VideoResolution.H_144P,
-    VideoResolution.H_1080P,
-    VideoResolution.H_1440P,
-    VideoResolution.H_4K
-  ]
-
-  for (const resolution of availableResolutions) {
-    // Resolution not enabled
-    if (configResolutions[resolution + 'p'] !== true) continue
-    // Too big resolution for input file
-    if (input < resolution) continue
-    // We only want lower resolutions than input file
-    if (strictLower && input === resolution) continue
-    // Audio resolutio but no audio in the video
-    if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue
-
-    resolutionsEnabled.add(resolution)
-  }
-
-  if (includeInput) {
-    // Always use an even resolution to avoid issues with ffmpeg
-    resolutionsEnabled.add(toEven(input))
-  }
-
-  return Array.from(resolutionsEnabled)
-}
-
-// ---------------------------------------------------------------------------
-// Can quick transcode
-// ---------------------------------------------------------------------------
-
-async function canDoQuickTranscode (path: string): Promise<boolean> {
-  if (CONFIG.TRANSCODING.PROFILE !== 'default') return false
-
-  const probe = await ffprobePromise(path)
-
-  return await canDoQuickVideoTranscode(path, probe) &&
-         await canDoQuickAudioTranscode(path, probe)
-}
-
-async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
-  const parsedAudio = await getAudioStream(path, probe)
-
-  if (!parsedAudio.audioStream) return true
-
-  if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
-
-  const audioBitrate = parsedAudio.bitrate
-  if (!audioBitrate) return false
-
-  const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate)
-  if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false
-
-  const channelLayout = parsedAudio.audioStream['channel_layout']
-  // Causes playback issues with Chrome
-  if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false
-
-  return true
-}
-
-async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
-  const videoStream = await getVideoStream(path, probe)
-  const fps = await getVideoStreamFPS(path, probe)
-  const bitRate = await getVideoStreamBitrate(path, probe)
-  const resolutionData = await getVideoStreamDimensionsInfo(path, probe)
-
-  // If ffprobe did not manage to guess the bitrate
-  if (!bitRate) return false
-
-  // check video params
-  if (!videoStream) return false
-  if (videoStream['codec_name'] !== 'h264') return false
-  if (videoStream['pix_fmt'] !== 'yuv420p') return false
-  if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
-  if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false
-
-  return true
-}
-
-// ---------------------------------------------------------------------------
-// Framerate
-// ---------------------------------------------------------------------------
-
-function getClosestFramerateStandard <K extends keyof Pick<VideoTranscodingFPS, 'HD_STANDARD' | 'STANDARD'>> (fps: number, type: K) {
-  return VIDEO_TRANSCODING_FPS[type].slice(0)
-                                    .sort((a, b) => fps % a - fps % b)[0]
-}
-
-function computeFPS (fpsArg: number, resolution: VideoResolution) {
-  let fps = fpsArg
-
-  if (
-    // On small/medium resolutions, limit FPS
-    resolution !== undefined &&
-    resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
-    fps > VIDEO_TRANSCODING_FPS.AVERAGE
-  ) {
-    // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
-    fps = getClosestFramerateStandard(fps, 'STANDARD')
-  }
-
-  // Hard FPS limits
-  if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
-
-  if (fps < VIDEO_TRANSCODING_FPS.MIN) {
-    throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`)
-  }
-
-  return fps
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  // Re export ffprobe utils
-  getVideoStreamDimensionsInfo,
-  buildFileMetadata,
-  getMaxAudioBitrate,
-  getVideoStream,
-  getVideoStreamDuration,
-  getAudioStream,
-  hasAudioStream,
-  getVideoStreamFPS,
-  ffprobePromise,
-  getVideoStreamBitrate,
-
-  getVideoStreamCodec,
-  getAudioStreamCodec,
-
-  computeFPS,
-  getClosestFramerateStandard,
-
-  computeResolutionsToTranscode,
-
-  canDoQuickTranscode,
-  canDoQuickVideoTranscode,
-  canDoQuickAudioTranscode
-}

diff --git a/server/helpers/ffmpeg/framerate.ts b/server/helpers/ffmpeg/framerate.ts
new file mode 100644 (file)
index 0000000..18cb0e0
--- /dev/null
+++ b/server/helpers/ffmpeg/framerate.ts
@@ -0,0 +1,44 @@
+import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
+import { VideoResolution } from '@shared/models'
+
+export function computeOutputFPS (options: {
+  inputFPS: number
+  resolution: VideoResolution
+}) {
+  const { resolution } = options
+
+  let fps = options.inputFPS
+
+  if (
+    // On small/medium resolutions, limit FPS
+    resolution !== undefined &&
+    resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
+    fps > VIDEO_TRANSCODING_FPS.AVERAGE
+  ) {
+    // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
+    fps = getClosestFramerateStandard({ fps, type: 'STANDARD' })
+  }
+
+  // Hard FPS limits
+  if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' })
+
+  if (fps < VIDEO_TRANSCODING_FPS.MIN) {
+    throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`)
+  }
+
+  return fps
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function getClosestFramerateStandard (options: {
+  fps: number
+  type: 'HD_STANDARD' | 'STANDARD'
+}) {
+  const { fps, type } = options
+
+  return VIDEO_TRANSCODING_FPS[type].slice(0)
+                                    .sort((a, b) => fps % a - fps % b)[0]
+}

diff --git a/server/helpers/ffmpeg/index.ts b/server/helpers/ffmpeg/index.ts
index e3bb2013f4fbc91111a9fbf06754a3289ae6a3d1..bf1c73fb6528367abaae8d7fe3f5d98bb9448a69 100644 (file)
--- a/server/helpers/ffmpeg/index.ts
+++ b/server/helpers/ffmpeg/index.ts
@@ -1,8 +1,4 @@
-export * from './ffmpeg-commons'
-export * from './ffmpeg-edition'
-export * from './ffmpeg-encoders'
-export * from './ffmpeg-images'
-export * from './ffmpeg-live'
-export * from './ffmpeg-presets'
-export * from './ffmpeg-vod'
-export * from './ffprobe-utils'
+export * from './codecs'
+export * from './ffmpeg-image'
+export * from './ffmpeg-options'
+export * from './framerate'

diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts
index bbd4692ef31b47919647c9ef15c2e99f4a46d569..05b258d8ab0ca7897fe8e5324ac8e57d208109a3 100644 (file)
--- a/server/helpers/image-utils.ts
+++ b/server/helpers/image-utils.ts
@@ -3,7 +3,7 @@ import Jimp, { read as jimpRead } from 'jimp'
 import { join } from 'path'
 import { getLowercaseExtension } from '@shared/core-utils'
 import { buildUUID } from '@shared/extra-utils'
 import { join } from 'path'
 import { getLowercaseExtension } from '@shared/core-utils'
 import { buildUUID } from '@shared/extra-utils'

-import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg/ffmpeg-images'
+import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg'

 import { logger, loggerTagsFactory } from './logger'
 
 const lTags = loggerTagsFactory('image-utils')
 import { logger, loggerTagsFactory } from './logger'
 
 const lTags = loggerTagsFactory('image-utils')


@@ -30,7 +30,7 @@ async function processImage (options: {
 
   // Use FFmpeg to process GIF
   if (extension === '.gif') {
 
   // Use FFmpeg to process GIF
   if (extension === '.gif') {

-    await processGIF(path, destination, newSize)
+    await processGIF({ path, destination, newSize })

   } else {
     await jimpProcessor(path, destination, newSize, extension)
   }
   } else {
     await jimpProcessor(path, destination, newSize, extension)
   }


@@ -50,7 +50,7 @@ async function generateImageFromVideoFile (options: {
   const pendingImagePath = join(folder, pendingImageName)
 
   try {
   const pendingImagePath = join(folder, pendingImageName)
 
   try {

-    await generateThumbnailFromVideo(fromPath, folder, imageName)
+    await generateThumbnailFromVideo({ fromPath, folder, imageName })

 
     const destination = join(folder, imageName)
     await processImage({ path: pendingImagePath, destination, newSize: size })
 
     const destination = join(folder, imageName)
     await processImage({ path: pendingImagePath, destination, newSize: size })


@@ -99,7 +99,7 @@ async function jimpProcessor (path: string, destination: string, newSize: { widt
     logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err })
 
     const newName = path + '.jpg'
     logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err })
 
     const newName = path + '.jpg'

-    await convertWebPToJPG(path, newName)
+    await convertWebPToJPG({ path, destination: newName })

     await rename(newName, path)
 
     sourceImage = await jimpRead(path)
     await rename(newName, path)
 
     sourceImage = await jimpRead(path)

diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts
index ae7d11800b431888b54a1634eaeaa78480fc54ae..95e78a9048ec34a0f31d6cb86ea0be579417f635 100644 (file)
--- a/server/helpers/peertube-crypto.ts
+++ b/server/helpers/peertube-crypto.ts
@@ -2,10 +2,11 @@ import { compare, genSalt, hash } from 'bcrypt'
 import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto'
 import { Request } from 'express'
 import { cloneDeep } from 'lodash'
 import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto'
 import { Request } from 'express'
 import { cloneDeep } from 'lodash'

+import { promisify1, promisify2 } from '@shared/core-utils'

 import { sha256 } from '@shared/extra-utils'
 import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
 import { MActor } from '../types/models'
 import { sha256 } from '@shared/extra-utils'
 import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
 import { MActor } from '../types/models'

-import { generateRSAKeyPairPromise, promisify1, promisify2, randomBytesPromise, scryptPromise } from './core-utils'
+import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils'

 import { jsonld } from './custom-jsonld-signature'
 import { logger } from './logger'
 
 import { jsonld } from './custom-jsonld-signature'
 import { logger } from './logger'
 

diff --git a/server/helpers/token-generator.ts b/server/helpers/token-generator.ts
new file mode 100644 (file)
index 0000000..16313b8
--- /dev/null
+++ b/server/helpers/token-generator.ts
@@ -0,0 +1,19 @@
+import { buildUUID } from '@shared/extra-utils'
+
+function generateRunnerRegistrationToken () {
+  return 'ptrrt-' + buildUUID()
+}
+
+function generateRunnerToken () {
+  return 'ptrt-' + buildUUID()
+}
+
+function generateRunnerJobToken () {
+  return 'ptrjt-' + buildUUID()
+}
+
+export {
+  generateRunnerRegistrationToken,
+  generateRunnerToken,
+  generateRunnerJobToken
+}

diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index a3c93e6fe101c1abf4bd693ebd29505f2464b38a..e690e3890eb831878b86a7d9f5a254c0b652587c 100644 (file)
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -13,9 +13,9 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
 import { MVideo } from '@server/types/models/video/video'
 import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file'
 import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist'
 import { MVideo } from '@server/types/models/video/video'
 import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file'
 import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist'

+import { promisify2 } from '@shared/core-utils'

 import { sha1 } from '@shared/extra-utils'
 import { CONFIG } from '../initializers/config'
 import { sha1 } from '@shared/extra-utils'
 import { CONFIG } from '../initializers/config'

-import { promisify2 } from './core-utils'

 import { logger } from './logger'
 import { generateVideoImportTmpPath } from './utils'
 import { extractVideo } from './video'
 import { logger } from './logger'
 import { generateVideoImportTmpPath } from './utils'
 import { extractVideo } from './video'

diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index 14ed82cb4e0b900444198a7a91071be647aae9e8..68dea909d271cf357fc7507ffc76f094de86253b 100644 (file)
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -1,7 +1,7 @@
 import config from 'config'
 import { URL } from 'url'
 import config from 'config'
 import { URL } from 'url'

-import { getFFmpegVersion } from '@server/helpers/ffmpeg'

 import { uniqify } from '@shared/core-utils'
 import { uniqify } from '@shared/core-utils'

+import { getFFmpegVersion } from '@shared/ffmpeg'

 import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
 import { RecentlyAddedStrategy } from '../../shared/models/redundancy'
 import { isProdInstance, parseBytes, parseSemVersion } from '../helpers/core-utils'
 import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
 import { RecentlyAddedStrategy } from '../../shared/models/redundancy'
 import { isProdInstance, parseBytes, parseSemVersion } from '../helpers/core-utils'

diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 49010c0592f4b60c8f0f228919fc1385653e9cec..2361aa1eb285861ea0816d79f363c8108f262344 100644 (file)
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -1,5 +1,6 @@
 import { IConfig } from 'config'
 import { IConfig } from 'config'

-import { parseSemVersion, promisify0 } from '../helpers/core-utils'
+import { promisify0 } from '@shared/core-utils'
+import { parseSemVersion } from '../helpers/core-utils'

 import { logger } from '../helpers/logger'
 
 // Special behaviour for config because we can reload it
 import { logger } from '../helpers/logger'
 
 // Special behaviour for config because we can reload it


@@ -36,7 +37,9 @@ function checkMissedConfig () {
     'transcoding.profile', 'transcoding.concurrency',
     'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p',
     'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
     'transcoding.profile', 'transcoding.concurrency',
     'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p',
     'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',

-    'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
+    'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled',
+    'video_studio.enabled',
+    'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live',

     'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
     'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
     'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
     'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
     'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
     'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',


@@ -74,7 +77,8 @@ function checkMissedConfig () {
     'live.transcoding.enabled', 'live.transcoding.threads', 'live.transcoding.profile',
     'live.transcoding.resolutions.144p', 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p',
     'live.transcoding.resolutions.480p', 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p',
     'live.transcoding.enabled', 'live.transcoding.threads', 'live.transcoding.profile',
     'live.transcoding.resolutions.144p', 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p',
     'live.transcoding.resolutions.480p', 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p',

-    'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution'
+    'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution',
+    'live.transcoding.remote_runners.enabled'

   ]
 
   const requiredAlternatives = [
   ]
 
   const requiredAlternatives = [

diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index e2442213cec17aaef9eb313dbeb0456b569974e7..699dd4704cdca10dc6d80506a98584cf2cef0474 100644 (file)
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -304,6 +304,12 @@ const CONFIG = {
       COUNT: config.get<number>('feeds.comments.count')
     }
   },
       COUNT: config.get<number>('feeds.comments.count')
     }
   },

+  REMOTE_RUNNERS: {
+    STALLED_JOBS: {
+      LIVE: parseDurationToMs(config.get<string>('remote_runners.stalled_jobs.live')),
+      VOD: parseDurationToMs(config.get<string>('remote_runners.stalled_jobs.vod'))
+    }
+  },

   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },


@@ -359,6 +365,9 @@ const CONFIG = {
     },
     WEBTORRENT: {
       get ENABLED () { return config.get<boolean>('transcoding.webtorrent.enabled') }
     },
     WEBTORRENT: {
       get ENABLED () { return config.get<boolean>('transcoding.webtorrent.enabled') }

+    },
+    REMOTE_RUNNERS: {
+      get ENABLED () { return config.get<boolean>('transcoding.remote_runners.enabled') }

     }
   },
   LIVE: {
     }
   },
   LIVE: {


@@ -406,6 +415,9 @@ const CONFIG = {
         get '1080p' () { return config.get<boolean>('live.transcoding.resolutions.1080p') },
         get '1440p' () { return config.get<boolean>('live.transcoding.resolutions.1440p') },
         get '2160p' () { return config.get<boolean>('live.transcoding.resolutions.2160p') }
         get '1080p' () { return config.get<boolean>('live.transcoding.resolutions.1080p') },
         get '1440p' () { return config.get<boolean>('live.transcoding.resolutions.1440p') },
         get '2160p' () { return config.get<boolean>('live.transcoding.resolutions.2160p') }

+      },
+      REMOTE_RUNNERS: {
+        get ENABLED () { return config.get<boolean>('live.transcoding.remote_runners.enabled') }

       }
     }
   },
       }
     }
   },

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6cad4eb235d9c37a4059b29a38b983958ea60625..279e7742167a3bd9e77aa66618c91637a994f90b 100644 (file)
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
 import {
   AbuseState,
   JobType,
 import {
   AbuseState,
   JobType,

+  RunnerJobState,

   UserRegistrationState,
   VideoChannelSyncState,
   VideoImportState,
   UserRegistrationState,
   VideoChannelSyncState,
   VideoImportState,


@@ -26,7 +27,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
 
 // ---------------------------------------------------------------------------
 

-const LAST_MIGRATION_VERSION = 760
+const LAST_MIGRATION_VERSION = 765

 
 // ---------------------------------------------------------------------------
 
 
 // ---------------------------------------------------------------------------
 


@@ -81,6 +82,10 @@ const SORTABLE_COLUMNS = {
 
   USER_REGISTRATIONS: [ 'createdAt', 'state' ],
 
 
   USER_REGISTRATIONS: [ 'createdAt', 'state' ],
 

+  RUNNERS: [ 'createdAt' ],
+  RUNNER_REGISTRATION_TOKENS: [ 'createdAt' ],
+  RUNNER_JOBS: [ 'updatedAt', 'createdAt', 'priority', 'state', 'progress' ],
+

   VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ],
 
   // Don't forget to update peertube-search-index with the same values
   VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ],
 
   // Don't forget to update peertube-search-index with the same values


@@ -139,6 +144,8 @@ const REMOTE_SCHEME = {
   WS: 'wss'
 }
 
   WS: 'wss'
 }
 

+// ---------------------------------------------------------------------------
+

 const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'activitypub-http-broadcast': 1,
   'activitypub-http-broadcast-parallel': 1,
 const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'activitypub-http-broadcast': 1,
   'activitypub-http-broadcast-parallel': 1,


@@ -160,6 +167,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'video-channel-import': 1,
   'after-video-channel-import': 1,
   'move-to-object-storage': 3,
   'video-channel-import': 1,
   'after-video-channel-import': 1,
   'move-to-object-storage': 3,

+  'transcoding-job-builder': 1,

   'notify': 1,
   'federate-video': 1
 }
   'notify': 1,
   'federate-video': 1
 }


@@ -183,6 +191,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
   'move-to-object-storage': 1,
   'video-channel-import': 1,
   'after-video-channel-import': 1,
   'move-to-object-storage': 1,
   'video-channel-import': 1,
   'after-video-channel-import': 1,

+  'transcoding-job-builder': 1,

   'notify': 5,
   'federate-video': 3
 }
   'notify': 5,
   'federate-video': 3
 }


@@ -207,6 +216,7 @@ const JOB_TTL: { [id in JobType]: number } = {
   'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
   'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
   'after-video-channel-import': 60000 * 5, // 5 minutes
   'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
   'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
   'after-video-channel-import': 60000 * 5, // 5 minutes

+  'transcoding-job-builder': 60000, // 1 minute

   'notify': 60000 * 5, // 5 minutes
   'federate-video': 60000 * 5 // 5 minutes
 }
   'notify': 60000 * 5, // 5 minutes
   'federate-video': 60000 * 5 // 5 minutes
 }


@@ -222,21 +232,6 @@ const JOB_PRIORITY = {
   TRANSCODING: 100
 }
 
   TRANSCODING: 100
 }
 

-const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
-const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
-
-const AP_CLEANER = {
-  CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job
-  UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource
-  PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS
-}
-
-const REQUEST_TIMEOUTS = {
-  DEFAULT: 7000, // 7 seconds
-  FILE: 30000, // 30 seconds
-  REDUNDANCY: JOB_TTL['video-redundancy']
-}
-

 const JOB_REMOVAL_OPTIONS = {
   COUNT: 10000, // Max jobs to store
 
 const JOB_REMOVAL_OPTIONS = {
   COUNT: 10000, // Max jobs to store
 


@@ -256,7 +251,29 @@ const JOB_REMOVAL_OPTIONS = {
 
 const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9)
 
 
 const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9)
 

+const RUNNER_JOBS = {
+  MAX_FAILURES: 5
+}
+
+// ---------------------------------------------------------------------------
+
+const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
+const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
+
+const AP_CLEANER = {
+  CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job
+  UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource
+  PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS
+}
+
+const REQUEST_TIMEOUTS = {
+  DEFAULT: 7000, // 7 seconds
+  FILE: 30000, // 30 seconds
+  REDUNDANCY: JOB_TTL['video-redundancy']
+}
+

 const SCHEDULER_INTERVALS_MS = {
 const SCHEDULER_INTERVALS_MS = {

+  RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE),

   ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour
   REMOVE_OLD_JOBS: 60000 * 60, // 1 hour
   UPDATE_VIDEOS: 60000, // 1 minute
   ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour
   REMOVE_OLD_JOBS: 60000 * 60, // 1 hour
   UPDATE_VIDEOS: 60000, // 1 minute


@@ -410,6 +427,17 @@ const CONSTRAINTS_FIELDS = {
     CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length
     CLIENT_META: { min: 1, max: 5000 }, // Length
     CLIENT_USER_AGENT: { min: 1, max: 200 } // Length
     CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length
     CLIENT_META: { min: 1, max: 5000 }, // Length
     CLIENT_USER_AGENT: { min: 1, max: 200 } // Length

+  },
+  RUNNERS: {
+    TOKEN: { min: 1, max: 1000 }, // Length
+    NAME: { min: 1, max: 100 }, // Length
+    DESCRIPTION: { min: 1, max: 1000 } // Length
+  },
+  RUNNER_JOBS: {
+    TOKEN: { min: 1, max: 1000 }, // Length
+    REASON: { min: 1, max: 5000 }, // Length
+    ERROR_MESSAGE: { min: 1, max: 5000 }, // Length
+    PROGRESS: { min: 0, max: 100 } // Value

   }
 }
 
   }
 }
 


@@ -540,6 +568,17 @@ const VIDEO_PLAYLIST_TYPES: { [ id in VideoPlaylistType ]: string } = {
   [VideoPlaylistType.WATCH_LATER]: 'Watch later'
 }
 
   [VideoPlaylistType.WATCH_LATER]: 'Watch later'
 }
 

+const RUNNER_JOB_STATES: { [ id in RunnerJobState ]: string } = {
+  [RunnerJobState.PROCESSING]: 'Processing',
+  [RunnerJobState.COMPLETED]: 'Completed',
+  [RunnerJobState.PENDING]: 'Pending',
+  [RunnerJobState.ERRORED]: 'Errored',
+  [RunnerJobState.WAITING_FOR_PARENT_JOB]: 'Waiting for parent job to finish',
+  [RunnerJobState.CANCELLED]: 'Cancelled',
+  [RunnerJobState.PARENT_ERRORED]: 'Parent job failed',
+  [RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled'
+}
+

 const MIMETYPES = {
   AUDIO: {
     MIMETYPE_EXT: {
 const MIMETYPES = {
   AUDIO: {
     MIMETYPE_EXT: {


@@ -594,6 +633,11 @@ const MIMETYPES = {
     MIMETYPE_EXT: {
       'application/x-bittorrent': '.torrent'
     }
     MIMETYPE_EXT: {
       'application/x-bittorrent': '.torrent'
     }

+  },
+  M3U8: {
+    MIMETYPE_EXT: {
+      'application/vnd.apple.mpegurl': '.m3u8'
+    }

   }
 }
 MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT)
   }
 }
 MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT)


@@ -1027,6 +1071,7 @@ export {
   SEARCH_INDEX,
   DIRECTORIES,
   RESUMABLE_UPLOAD_SESSION_LIFETIME,
   SEARCH_INDEX,
   DIRECTORIES,
   RESUMABLE_UPLOAD_SESSION_LIFETIME,

+  RUNNER_JOB_STATES,

   P2P_MEDIA_LOADER_PEER_VERSION,
   ACTOR_IMAGES_SIZE,
   ACCEPT_HEADERS,
   P2P_MEDIA_LOADER_PEER_VERSION,
   ACTOR_IMAGES_SIZE,
   ACCEPT_HEADERS,


@@ -1085,6 +1130,7 @@ export {
   USER_REGISTRATION_STATES,
   LRU_CACHE,
   REQUEST_TIMEOUTS,
   USER_REGISTRATION_STATES,
   LRU_CACHE,
   REQUEST_TIMEOUTS,

+  RUNNER_JOBS,

   MAX_LOCAL_VIEWER_WATCH_SECTIONS,
   USER_PASSWORD_RESET_LIFETIME,
   USER_PASSWORD_CREATE_LIFETIME,
   MAX_LOCAL_VIEWER_WATCH_SECTIONS,
   USER_PASSWORD_RESET_LIFETIME,
   USER_PASSWORD_CREATE_LIFETIME,

diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 3f31099edb5ba6647238c0b443f8e6dfd27bfdd1..14dd8c3790ae5db788f9e004e41699d99b2e61d3 100644 (file)
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -1,6 +1,9 @@
 import { QueryTypes, Transaction } from 'sequelize'
 import { Sequelize as SequelizeTypescript } from 'sequelize-typescript'
 import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
 import { QueryTypes, Transaction } from 'sequelize'
 import { Sequelize as SequelizeTypescript } from 'sequelize-typescript'
 import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'

+import { RunnerModel } from '@server/models/runner/runner'
+import { RunnerJobModel } from '@server/models/runner/runner-job'
+import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'

 import { TrackerModel } from '@server/models/server/tracker'
 import { VideoTrackerModel } from '@server/models/server/video-tracker'
 import { UserModel } from '@server/models/user/user'
 import { TrackerModel } from '@server/models/server/tracker'
 import { VideoTrackerModel } from '@server/models/server/video-tracker'
 import { UserModel } from '@server/models/user/user'


@@ -9,6 +12,7 @@ import { UserRegistrationModel } from '@server/models/user/user-registration'
 import { UserVideoHistoryModel } from '@server/models/user/user-video-history'
 import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { UserVideoHistoryModel } from '@server/models/user/user-video-history'
 import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'

+import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'

 import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
 import { VideoSourceModel } from '@server/models/video/video-source'
 import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
 import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
 import { VideoSourceModel } from '@server/models/video/video-source'
 import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'


@@ -52,7 +56,6 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
 import { VideoTagModel } from '../models/video/video-tag'
 import { VideoViewModel } from '../models/view/video-view'
 import { CONFIG } from './config'
 import { VideoTagModel } from '../models/video/video-tag'
 import { VideoViewModel } from '../models/view/video-view'
 import { CONFIG } from './config'

-import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'

 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 


@@ -159,7 +162,10 @@ async function initDatabaseModels (silent: boolean) {
     ActorCustomPageModel,
     VideoJobInfoModel,
     VideoChannelSyncModel,
     ActorCustomPageModel,
     VideoJobInfoModel,
     VideoChannelSyncModel,

-    UserRegistrationModel
+    UserRegistrationModel,
+    RunnerRegistrationTokenModel,
+    RunnerModel,
+    RunnerJobModel

   ])
 
   // Check extensions exist in the database
   ])
 
   // Check extensions exist in the database

diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index f48f348a7bc118a91937b774b7eb1ae4fc92ae46..2406a59367249da2c61f61d10b43dc51c6dfb3b5 100644 (file)
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -2,7 +2,9 @@ import { ensureDir, readdir, remove } from 'fs-extra'
 import passwordGenerator from 'password-generator'
 import { join } from 'path'
 import { isTestOrDevInstance } from '@server/helpers/core-utils'
 import passwordGenerator from 'password-generator'
 import { join } from 'path'
 import { isTestOrDevInstance } from '@server/helpers/core-utils'

+import { generateRunnerRegistrationToken } from '@server/helpers/token-generator'

 import { getNodeABIVersion } from '@server/helpers/version'
 import { getNodeABIVersion } from '@server/helpers/version'

+import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'

 import { UserRole } from '@shared/models'
 import { logger } from '../helpers/logger'
 import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user'
 import { UserRole } from '@shared/models'
 import { logger } from '../helpers/logger'
 import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user'


@@ -22,7 +24,8 @@ async function installApplication () {
           return Promise.all([
             createApplicationIfNotExist(),
             createOAuthClientIfNotExist(),
           return Promise.all([
             createApplicationIfNotExist(),
             createOAuthClientIfNotExist(),

-            createOAuthAdminIfNotExist()
+            createOAuthAdminIfNotExist(),
+            createRunnerRegistrationTokenIfNotExist()

           ])
         }),
 
           ])
         }),
 


@@ -183,3 +186,14 @@ async function createApplicationIfNotExist () {
 
   return createApplicationActor(application.id)
 }
 
   return createApplicationActor(application.id)
 }

+
+async function createRunnerRegistrationTokenIfNotExist () {
+  const total = await RunnerRegistrationTokenModel.countTotal()
+  if (total !== 0) return undefined
+
+  const token = new RunnerRegistrationTokenModel({
+    registrationToken: generateRunnerRegistrationToken()
+  })
+
+  await token.save()
+}

diff --git a/server/initializers/migrations/0765-remote-transcoding.ts b/server/initializers/migrations/0765-remote-transcoding.ts
new file mode 100644 (file)
index 0000000..40cca03
--- /dev/null
+++ b/server/initializers/migrations/0765-remote-transcoding.ts
@@ -0,0 +1,78 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+  {
+    const query = `
+    CREATE TABLE IF NOT EXISTS "runnerRegistrationToken"(
+      "id" serial,
+      "registrationToken" varchar(255) NOT NULL,
+      "createdAt" timestamp with time zone NOT NULL,
+      "updatedAt" timestamp with time zone NOT NULL,
+      PRIMARY KEY ("id")
+    );
+    `
+
+    await utils.sequelize.query(query, { transaction : utils.transaction })
+  }
+
+  {
+    const query = `
+    CREATE TABLE IF NOT EXISTS "runner"(
+      "id" serial,
+      "runnerToken" varchar(255) NOT NULL,
+      "name" varchar(255) NOT NULL,
+      "description" varchar(1000),
+      "lastContact" timestamp with time zone NOT NULL,
+      "ip" varchar(255) NOT NULL,
+      "runnerRegistrationTokenId" integer REFERENCES "runnerRegistrationToken"("id") ON DELETE CASCADE ON UPDATE CASCADE,
+      "createdAt" timestamp with time zone NOT NULL,
+      "updatedAt" timestamp with time zone NOT NULL,
+      PRIMARY KEY ("id")
+    );
+    `
+
+    await utils.sequelize.query(query, { transaction : utils.transaction })
+  }
+
+  {
+    const query = `
+    CREATE TABLE IF NOT EXISTS "runnerJob"(
+      "id" serial,
+      "uuid" uuid NOT NULL,
+      "type" varchar(255) NOT NULL,
+      "payload" jsonb NOT NULL,
+      "privatePayload" jsonb NOT NULL,
+      "state" integer NOT NULL,
+      "failures" integer NOT NULL DEFAULT 0,
+      "error" varchar(5000),
+      "priority" integer NOT NULL,
+      "processingJobToken" varchar(255),
+      "progress" integer,
+      "startedAt" timestamp with time zone,
+      "finishedAt" timestamp with time zone,
+      "dependsOnRunnerJobId" integer REFERENCES "runnerJob"("id") ON DELETE CASCADE ON UPDATE CASCADE,
+      "runnerId" integer REFERENCES "runner"("id") ON DELETE SET NULL ON UPDATE CASCADE,
+      "createdAt" timestamp with time zone NOT NULL,
+      "updatedAt" timestamp with time zone NOT NULL,
+      PRIMARY KEY ("id")
+    );
+
+
+    `
+
+    await utils.sequelize.query(query, { transaction : utils.transaction })
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 053b5d3262f50f59fc9307e158690a87e97cb4af..fc1d7e1b0cc613f3707083a0ae6706ed42cddf76 100644 (file)
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -3,10 +3,11 @@ import { flatten } from 'lodash'
 import PQueue from 'p-queue'
 import { basename, dirname, join } from 'path'
 import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
 import PQueue from 'p-queue'
 import { basename, dirname, join } from 'path'
 import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'

-import { uniqify } from '@shared/core-utils'
+import { uniqify, uuidRegex } from '@shared/core-utils'

 import { sha256 } from '@shared/extra-utils'
 import { sha256 } from '@shared/extra-utils'

+import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg'

 import { VideoStorage } from '@shared/models'
 import { VideoStorage } from '@shared/models'

-import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg'
+import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg'

 import { logger } from '../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
 import { generateRandomString } from '../helpers/utils'
 import { logger } from '../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
 import { generateRandomString } from '../helpers/utils'


@@ -234,6 +235,16 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
 
 // ---------------------------------------------------------------------------
 
 
 // ---------------------------------------------------------------------------
 

+async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) {
+  const content = await readFile(playlistPath, 'utf8')
+
+  const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename)
+
+  await writeFile(playlistPath, newContent, 'utf8')
+}
+
+// ---------------------------------------------------------------------------
+

 function injectQueryToPlaylistUrls (content: string, queryString: string) {
   return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString)
 }
 function injectQueryToPlaylistUrls (content: string, queryString: string) {
   return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString)
 }


@@ -247,7 +258,8 @@ export {
   downloadPlaylistSegments,
   updateStreamingPlaylistsInfohashesIfNeeded,
   updatePlaylistAfterFileChange,
   downloadPlaylistSegments,
   updateStreamingPlaylistsInfohashesIfNeeded,
   updatePlaylistAfterFileChange,

-  injectQueryToPlaylistUrls
+  injectQueryToPlaylistUrls,
+  renameVideoFileInPlaylist

 }
 
 // ---------------------------------------------------------------------------
 }
 
 // ---------------------------------------------------------------------------

diff --git a/server/lib/job-queue/handlers/transcoding-job-builder.ts b/server/lib/job-queue/handlers/transcoding-job-builder.ts
new file mode 100644 (file)
index 0000000..8b4a877
--- /dev/null
+++ b/server/lib/job-queue/handlers/transcoding-job-builder.ts
@@ -0,0 +1,47 @@
+import { Job } from 'bullmq'
+import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job'
+import { UserModel } from '@server/models/user/user'
+import { VideoModel } from '@server/models/video/video'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info'
+import { pick } from '@shared/core-utils'
+import { TranscodingJobBuilderPayload } from '@shared/models'
+import { logger } from '../../../helpers/logger'
+import { JobQueue } from '../job-queue'
+
+async function processTranscodingJobBuilder (job: Job) {
+  const payload = job.data as TranscodingJobBuilderPayload
+
+  logger.info('Processing transcoding job builder in job %s.', job.id)
+
+  if (payload.optimizeJob) {
+    const video = await VideoModel.loadFull(payload.videoUUID)
+    const user = await UserModel.loadByVideoId(video.id)
+    const videoFile = video.getMaxQualityFile()
+
+    await createOptimizeOrMergeAudioJobs({
+      ...pick(payload.optimizeJob, [ 'isNewVideo' ]),
+
+      video,
+      videoFile,
+      user
+    })
+  }
+
+  for (const job of (payload.jobs || [])) {
+    await JobQueue.Instance.createJob(job)
+
+    await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
+  }
+
+  for (const sequentialJobs of (payload.sequentialJobs || [])) {
+    await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs)
+
+    await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.length)
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processTranscodingJobBuilder
+}

diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index d950f64072e20db266a2989641322358db7e9bef..9a4550e4de06415f6d57e20b365774bc226519a4 100644 (file)
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -10,8 +10,8 @@ import { VideoModel } from '@server/models/video/video'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { MVideoFullLight } from '@server/types/models'
 import { getLowercaseExtension } from '@shared/core-utils'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { MVideoFullLight } from '@server/types/models'
 import { getLowercaseExtension } from '@shared/core-utils'

+import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'

 import { VideoFileImportPayload, VideoStorage } from '@shared/models'
 import { VideoFileImportPayload, VideoStorage } from '@shared/models'

-import { getVideoStreamFPS, getVideoStreamDimensionsInfo } from '../../../helpers/ffmpeg'

 import { logger } from '../../../helpers/logger'
 import { JobQueue } from '../job-queue'
 
 import { logger } from '../../../helpers/logger'
 import { JobQueue } from '../job-queue'
 

diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 4d361c7b915762101ea2c7c3a776670733a4b5ff..2a063282cf2aef546c587bc012ff2b75a17488fe 100644 (file)
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -7,15 +7,16 @@ import { isPostImportVideoAccepted } from '@server/lib/moderation'
 import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import { Hooks } from '@server/lib/plugins/hooks'
 import { ServerConfigManager } from '@server/lib/server-config-manager'
 import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import { Hooks } from '@server/lib/plugins/hooks'
 import { ServerConfigManager } from '@server/lib/server-config-manager'

+import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job'

 import { isAbleToUploadVideo } from '@server/lib/user'
 import { isAbleToUploadVideo } from '@server/lib/user'

-import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
+import { buildMoveToObjectStorageJob } from '@server/lib/video'

 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
 import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
 import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
 import { getLowercaseExtension } from '@shared/core-utils'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
 import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
 import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
 import { getLowercaseExtension } from '@shared/core-utils'

-import { isAudioFile } from '@shared/extra-utils'
+import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg'

 import {
   ThumbnailType,
   VideoImportPayload,
 import {
   ThumbnailType,
   VideoImportPayload,


@@ -28,7 +29,6 @@ import {
   VideoResolution,
   VideoState
 } from '@shared/models'
   VideoResolution,
   VideoState
 } from '@shared/models'

-import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '../../../helpers/ffmpeg'

 import { logger } from '../../../helpers/logger'
 import { getSecureTorrentName } from '../../../helpers/utils'
 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
 import { logger } from '../../../helpers/logger'
 import { getSecureTorrentName } from '../../../helpers/utils'
 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'


@@ -137,7 +137,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
 
     const { resolution } = await isAudioFile(tempVideoPath, probe)
       ? { resolution: VideoResolution.H_NOVIDEO }
 
     const { resolution } = await isAudioFile(tempVideoPath, probe)
       ? { resolution: VideoResolution.H_NOVIDEO }

-      : await getVideoStreamDimensionsInfo(tempVideoPath)
+      : await getVideoStreamDimensionsInfo(tempVideoPath, probe)

 
     const fps = await getVideoStreamFPS(tempVideoPath, probe)
     const duration = await getVideoStreamDuration(tempVideoPath, probe)
 
     const fps = await getVideoStreamFPS(tempVideoPath, probe)
     const duration = await getVideoStreamDuration(tempVideoPath, probe)


@@ -313,9 +313,7 @@ async function afterImportSuccess (options: {
   }
 
   if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
   }
 
   if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?

-    await JobQueue.Instance.createJob(
-      await buildOptimizeOrMergeAudioJob({ video, videoFile, user })
-    )
+    await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user })

   }
 }
 
   }
 }
 

diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 2f3a971bdae909b988f32993899878c2efb2f3fc..1bf43f5928c1727dab769556262ec58b48642042 100644 (file)
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -1,25 +1,25 @@
 import { Job } from 'bullmq'
 import { readdir, remove } from 'fs-extra'
 import { join } from 'path'
 import { Job } from 'bullmq'
 import { readdir, remove } from 'fs-extra'
 import { join } from 'path'

-import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg'

 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
 import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
 import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
 import { generateVideoMiniature } from '@server/lib/thumbnail'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
 import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
 import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
 import { generateVideoMiniature } from '@server/lib/thumbnail'

-import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding'
+import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding'
+import { VideoPathManager } from '@server/lib/video-path-manager'

 import { moveToNextState } from '@server/lib/video-state'
 import { VideoModel } from '@server/models/video/video'
 import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { VideoLiveModel } from '@server/models/video/video-live'
 import { moveToNextState } from '@server/lib/video-state'
 import { VideoModel } from '@server/models/video/video'
 import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { VideoLiveModel } from '@server/models/video/video-live'

+import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'

 import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
 import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models'
 import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
 import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models'

+import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'

 import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'
 import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'

-import { VideoPathManager } from '@server/lib/video-path-manager'
-import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'

 
 const lTags = loggerTagsFactory('live', 'job')
 
 
 const lTags = loggerTagsFactory('live', 'job')
 


@@ -224,6 +224,7 @@ async function assignReplayFilesToVideo (options: {
     const probe = await ffprobePromise(concatenatedTsFilePath)
     const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
     const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
     const probe = await ffprobePromise(concatenatedTsFilePath)
     const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
     const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)

+    const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe)

 
     try {
       await generateHlsPlaylistResolutionFromTS({
 
     try {
       await generateHlsPlaylistResolutionFromTS({


@@ -231,6 +232,7 @@ async function assignReplayFilesToVideo (options: {
         inputFileMutexReleaser,
         concatenatedTsFilePath,
         resolution,
         inputFileMutexReleaser,
         concatenatedTsFilePath,
         resolution,

+        fps,

         isAAC: audioStream?.codec_name === 'aac'
       })
     } catch (err) {
         isAAC: audioStream?.codec_name === 'aac'
       })
     } catch (err) {

diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts
index 3e208d83df4c818115d2c73db42f4fa8a97feca8..991d11ef153498c7b58d38d132d5e895cec6636d 100644 (file)
--- a/server/lib/job-queue/handlers/video-studio-edition.ts
+++ b/server/lib/job-queue/handlers/video-studio-edition.ts
@@ -1,15 +1,16 @@
 import { Job } from 'bullmq'
 import { move, remove } from 'fs-extra'
 import { join } from 'path'
 import { Job } from 'bullmq'
 import { move, remove } from 'fs-extra'
 import { join } from 'path'

-import { addIntroOutro, addWatermark, cutVideo } from '@server/helpers/ffmpeg'
+import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'

 import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent'
 import { CONFIG } from '@server/initializers/config'
 import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent'
 import { CONFIG } from '@server/initializers/config'

+import { VIDEO_FILTERS } from '@server/initializers/constants'

 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
 import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
 import { generateWebTorrentVideoFilename } from '@server/lib/paths'

+import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job'

 import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles'
 import { isAbleToUploadVideo } from '@server/lib/user'
 import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles'
 import { isAbleToUploadVideo } from '@server/lib/user'

-import { buildOptimizeOrMergeAudioJob } from '@server/lib/video'
-import { removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
+import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'

 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio'
 import { UserModel } from '@server/models/user/user'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio'
 import { UserModel } from '@server/models/user/user'


@@ -17,15 +18,8 @@ import { VideoModel } from '@server/models/video/video'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models'
 import { getLowercaseExtension, pick } from '@shared/core-utils'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models'
 import { getLowercaseExtension, pick } from '@shared/core-utils'

-import {
-  buildFileMetadata,
-  buildUUID,
-  ffprobePromise,
-  getFileSize,
-  getVideoStreamDimensionsInfo,
-  getVideoStreamDuration,
-  getVideoStreamFPS
-} from '@shared/extra-utils'
+import { buildUUID, getFileSize } from '@shared/extra-utils'
+import { FFmpegEdition, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg'

 import {
   VideoStudioEditionPayload,
   VideoStudioTask,
 import {
   VideoStudioEditionPayload,
   VideoStudioTask,


@@ -36,7 +30,6 @@ import {
   VideoStudioTaskWatermarkPayload
 } from '@shared/models'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'
   VideoStudioTaskWatermarkPayload
 } from '@shared/models'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'

-import { JobQueue } from '../job-queue'

 
 const lTagsBase = loggerTagsFactory('video-edition')
 
 
 const lTagsBase = loggerTagsFactory('video-edition')
 


@@ -102,9 +95,7 @@ async function processVideoStudioEdition (job: Job) {
 
   const user = await UserModel.loadByVideoId(video.id)
 
 
   const user = await UserModel.loadByVideoId(video.id)
 

-  await JobQueue.Instance.createJob(
-    await buildOptimizeOrMergeAudioJob({ video, videoFile: newFile, user, isNewVideo: false })
-  )
+  await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user })

 }
 
 // ---------------------------------------------------------------------------
 }
 
 // ---------------------------------------------------------------------------


@@ -131,9 +122,9 @@ const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessor
 }
 
 async function processTask (options: TaskProcessorOptions) {
 }
 
 async function processTask (options: TaskProcessorOptions) {

-  const { video, task } = options
+  const { video, task, lTags } = options

 
 

-  logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...options.lTags })
+  logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags })

 
   const processor = taskProcessors[options.task.name]
   if (!process) throw new Error('Unknown task ' + task.name)
 
   const processor = taskProcessors[options.task.name]
   if (!process) throw new Error('Unknown task ' + task.name)


@@ -142,48 +133,53 @@ async function processTask (options: TaskProcessorOptions) {
 }
 
 function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {
 }
 
 function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {

-  const { task } = options
+  const { task, lTags } = options
+
+  logger.debug('Will add intro/outro to the video.', { options, ...lTags })

 
 

-  return addIntroOutro({
+  return buildFFmpegEdition().addIntroOutro({

     ...pick(options, [ 'inputPath', 'outputPath' ]),
 
     introOutroPath: task.options.file,
     type: task.name === 'add-intro'
       ? 'intro'
     ...pick(options, [ 'inputPath', 'outputPath' ]),
 
     introOutroPath: task.options.file,
     type: task.name === 'add-intro'
       ? 'intro'

-      : 'outro',
-
-    availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-    profile: CONFIG.TRANSCODING.PROFILE
+      : 'outro'

   })
 }
 
 function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
   })
 }
 
 function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {

-  const { task } = options
+  const { task, lTags } = options

 
 

-  return cutVideo({
+  logger.debug('Will cut the video.', { options, ...lTags })
+
+  return buildFFmpegEdition().cutVideo({

     ...pick(options, [ 'inputPath', 'outputPath' ]),
 
     start: task.options.start,
     ...pick(options, [ 'inputPath', 'outputPath' ]),
 
     start: task.options.start,

-    end: task.options.end,
-
-    availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-    profile: CONFIG.TRANSCODING.PROFILE
+    end: task.options.end

   })
 }
 
 function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {
   })
 }
 
 function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {

-  const { task } = options
+  const { task, lTags } = options
+
+  logger.debug('Will add watermark to the video.', { options, ...lTags })

 
 

-  return addWatermark({
+  return buildFFmpegEdition().addWatermark({

     ...pick(options, [ 'inputPath', 'outputPath' ]),
 
     watermarkPath: task.options.file,
 
     ...pick(options, [ 'inputPath', 'outputPath' ]),
 
     watermarkPath: task.options.file,
 

-    availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-    profile: CONFIG.TRANSCODING.PROFILE
+    videoFilters: {
+      watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO,
+      horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO,
+      verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO
+    }

   })
 }
 
   })
 }
 

+// ---------------------------------------------------------------------------
+

 async function buildNewFile (video: MVideoId, path: string) {
   const videoFile = new VideoFileModel({
     extname: getLowercaseExtension(path),
 async function buildNewFile (video: MVideoId, path: string) {
   const videoFile = new VideoFileModel({
     extname: getLowercaseExtension(path),


@@ -223,3 +219,7 @@ async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStud
     throw new Error('Quota exceeded for this user to edit the video')
   }
 }
     throw new Error('Quota exceeded for this user to edit the video')
   }
 }

+
+function buildFFmpegEdition () {
+  return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))
+}

diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 3e6d2336339a7552514bb801f8c86af72aabbd32..17b717275fefdd3296b3c3f1eb28ad0cc084584a 100644 (file)
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -1,13 +1,13 @@
 import { Job } from 'bullmq'
 import { Job } from 'bullmq'

-import { TranscodeVODOptionsType } from '@server/helpers/ffmpeg'
-import { Hooks } from '@server/lib/plugins/hooks'
-import { buildTranscodingJob, getTranscodingJobPriority } from '@server/lib/video'
+import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding'
+import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding'
+import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebTorrentResolution } from '@server/lib/transcoding/web-transcoding'
+import { removeAllWebTorrentFiles } from '@server/lib/video-file'

 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { VideoPathManager } from '@server/lib/video-path-manager'

-import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state'
+import { moveToFailedTranscodingState } from '@server/lib/video-state'

 import { UserModel } from '@server/models/user/user'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { UserModel } from '@server/models/user/user'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'

-import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models'
-import { pick } from '@shared/core-utils'
+import { MUser, MUserId, MVideoFullLight } from '@server/types/models'

 import {
   HLSTranscodingPayload,
   MergeAudioTranscodingPayload,
 import {
   HLSTranscodingPayload,
   MergeAudioTranscodingPayload,


@@ -15,18 +15,8 @@ import {
   OptimizeTranscodingPayload,
   VideoTranscodingPayload
 } from '@shared/models'
   OptimizeTranscodingPayload,
   VideoTranscodingPayload
 } from '@shared/models'

-import { retryTransactionWrapper } from '../../../helpers/database-utils'
-import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg'

 import { logger, loggerTagsFactory } from '../../../helpers/logger'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'

-import { CONFIG } from '../../../initializers/config'

 import { VideoModel } from '../../../models/video/video'
 import { VideoModel } from '../../../models/video/video'

-import {
-  generateHlsPlaylistResolution,
-  mergeAudioVideofile,
-  optimizeOriginalVideofile,
-  transcodeNewWebTorrentResolution
-} from '../../transcoding/transcoding'
-import { JobQueue } from '../job-queue'

 
 type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
 
 
 type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
 


@@ -84,260 +74,72 @@ export {
 // Job handlers
 // ---------------------------------------------------------------------------
 
 // Job handlers
 // ---------------------------------------------------------------------------
 

-async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MVideoFullLight, user: MUser) {
-  logger.info('Handling HLS transcoding job for %s.', video.uuid, lTags(video.uuid))
-
-  const videoFileInput = payload.copyCodecs
-    ? video.getWebTorrentFile(payload.resolution)
-    : video.getMaxQualityFile()
-
-  const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
-
-  const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
-
-  try {
-    await videoFileInput.getVideo().reload()
-
-    await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
-      return generateHlsPlaylistResolution({
-        video,
-        videoInputPath,
-        inputFileMutexReleaser,
-        resolution: payload.resolution,
-        copyCodecs: payload.copyCodecs,
-        job
-      })
-    })
-  } finally {
-    inputFileMutexReleaser()
-  }
-
-  logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid))
-
-  await onHlsPlaylistGeneration(video, user, payload)
-}
-
-async function handleNewWebTorrentResolutionJob (
-  job: Job,
-  payload: NewWebTorrentResolutionTranscodingPayload,
-  video: MVideoFullLight,
-  user: MUserId
-) {
-  logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid))
-
-  await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, job })
-
-  logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid))
-
-  await onNewWebTorrentFileResolution(video, user, payload)
-}
-

 async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) {
   logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid))
 
 async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) {
   logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid))
 

-  await mergeAudioVideofile({ video, resolution: payload.resolution, job })
+  await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })

 
   logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid))
 
 
   logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid))
 

-  await onVideoFirstWebTorrentTranscoding(video, payload, 'video', user)
+  await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })

 }
 
 async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
   logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid))
 
 }
 
 async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
   logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid))
 

-  const { transcodeType } = await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), job })
+  await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job })

 
   logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid))
 
 
   logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid))
 

-  await onVideoFirstWebTorrentTranscoding(video, payload, transcodeType, user)
+  await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })

 }
 
 }
 

-// ---------------------------------------------------------------------------
-
-async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) {
-  if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
-    // Remove webtorrent files if not enabled
-    for (const file of video.VideoFiles) {
-      await video.removeWebTorrentFile(file)
-      await file.destroy()
-    }
-
-    video.VideoFiles = []
-
-    // Create HLS new resolution jobs
-    await createLowerResolutionsJobs({
-      video,
-      user,
-      videoFileResolution: payload.resolution,
-      hasAudio: payload.hasAudio,
-      isNewVideo: payload.isNewVideo ?? true,
-      type: 'hls'
-    })
-  }
-
-  await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
-  await retryTransactionWrapper(moveToNextState, { video, isNewVideo: payload.isNewVideo })
-}
+async function handleNewWebTorrentResolutionJob (job: Job, payload: NewWebTorrentResolutionTranscodingPayload, video: MVideoFullLight) {
+  logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid))

 
 

-async function onVideoFirstWebTorrentTranscoding (
-  videoArg: MVideoWithFile,
-  payload: OptimizeTranscodingPayload | MergeAudioTranscodingPayload,
-  transcodeType: TranscodeVODOptionsType,
-  user: MUserId
-) {
-  const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
+  await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, fps: payload.fps, job })

 
 

-  try {
-    // Maybe the video changed in database, refresh it
-    const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
-    // Video does not exist anymore
-    if (!videoDatabase) return undefined
-
-    const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile()
-
-    // Generate HLS version of the original file
-    const originalFileHLSPayload = {
-      ...payload,
-
-      hasAudio: !!audioStream,
-      resolution: videoDatabase.getMaxQualityFile().resolution,
-      // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
-      copyCodecs: transcodeType !== 'quick-transcode',
-      isMaxQuality: true
-    }
-    const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
-    const hasNewResolutions = await createLowerResolutionsJobs({
-      video: videoDatabase,
-      user,
-      videoFileResolution: resolution,
-      hasAudio: !!audioStream,
-      type: 'webtorrent',
-      isNewVideo: payload.isNewVideo ?? true
-    })
-
-    await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
+  logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid))

 
 

-    // Move to next state if there are no other resolutions to generate
-    if (!hasHls && !hasNewResolutions) {
-      await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
-    }
-  } finally {
-    mutexReleaser()
-  }
+  await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })

 }
 
 }
 

-async function onNewWebTorrentFileResolution (
-  video: MVideo,
-  user: MUserId,
-  payload: NewWebTorrentResolutionTranscodingPayload | MergeAudioTranscodingPayload
-) {
-  if (payload.createHLSIfNeeded) {
-    await createHlsJobIfEnabled(user, { hasAudio: true, copyCodecs: true, isMaxQuality: false, ...payload })
-  }
-
-  await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
+async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MVideoFullLight) {
+  logger.info('Handling HLS transcoding job for %s.', video.uuid, lTags(video.uuid))

 
 

-  await retryTransactionWrapper(moveToNextState, { video, isNewVideo: payload.isNewVideo })
-}
+  const videoFileInput = payload.copyCodecs
+    ? video.getWebTorrentFile(payload.resolution)
+    : video.getMaxQualityFile()

 
 

-// ---------------------------------------------------------------------------
+  const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()

 
 

-async function createHlsJobIfEnabled (user: MUserId, payload: {
-  videoUUID: string
-  resolution: number
-  hasAudio: boolean
-  copyCodecs: boolean
-  isMaxQuality: boolean
-  isNewVideo?: boolean
-}) {
-  if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false
-
-  const jobOptions = {
-    priority: await getTranscodingJobPriority(user)
-  }
+  const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)

 
 

-  const hlsTranscodingPayload: HLSTranscodingPayload = {
-    type: 'new-resolution-to-hls',
-    autoDeleteWebTorrentIfNeeded: true,
+  try {
+    await videoFileInput.getVideo().reload()

 
 

-    ...pick(payload, [ 'videoUUID', 'resolution', 'copyCodecs', 'isMaxQuality', 'isNewVideo', 'hasAudio' ])
+    await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
+      return generateHlsPlaylistResolution({
+        video,
+        videoInputPath,
+        inputFileMutexReleaser,
+        resolution: payload.resolution,
+        fps: payload.fps,
+        copyCodecs: payload.copyCodecs,
+        job
+      })
+    })
+  } finally {
+    inputFileMutexReleaser()

   }
 
   }
 

-  await JobQueue.Instance.createJob(await buildTranscodingJob(hlsTranscodingPayload, jobOptions))
-
-  return true
-}
-
-async function createLowerResolutionsJobs (options: {
-  video: MVideoFullLight
-  user: MUserId
-  videoFileResolution: number
-  hasAudio: boolean
-  isNewVideo: boolean
-  type: 'hls' | 'webtorrent'
-}) {
-  const { video, user, videoFileResolution, isNewVideo, hasAudio, type } = options
-
-  // Create transcoding jobs if there are enabled resolutions
-  const resolutionsEnabled = await Hooks.wrapObject(
-    computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
-    'filter:transcoding.auto.resolutions-to-transcode.result',
-    options
-  )
-
-  const resolutionCreated: string[] = []
-
-  for (const resolution of resolutionsEnabled) {
-    let dataInput: VideoTranscodingPayload
-
-    if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED && type === 'webtorrent') {
-      // WebTorrent will create subsequent HLS job
-      dataInput = {
-        type: 'new-resolution-to-webtorrent',
-        videoUUID: video.uuid,
-        resolution,
-        hasAudio,
-        createHLSIfNeeded: true,
-        isNewVideo
-      }
-
-      resolutionCreated.push('webtorrent-' + resolution)
-    }
-
-    if (CONFIG.TRANSCODING.HLS.ENABLED && type === 'hls') {
-      dataInput = {
-        type: 'new-resolution-to-hls',
-        videoUUID: video.uuid,
-        resolution,
-        hasAudio,
-        copyCodecs: false,
-        isMaxQuality: false,
-        autoDeleteWebTorrentIfNeeded: true,
-        isNewVideo
-      }
-
-      resolutionCreated.push('hls-' + resolution)
-    }
-
-    if (!dataInput) continue
-
-    const jobOptions = {
-      priority: await getTranscodingJobPriority(user)
-    }
-
-    await JobQueue.Instance.createJob(await buildTranscodingJob(dataInput, jobOptions))
-  }
+  logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid))

 
 

-  if (resolutionCreated.length === 0) {
-    logger.info('No transcoding jobs created for video %s (no resolutions).', video.uuid, lTags(video.uuid))
+  if (payload.deleteWebTorrentFiles === true) {
+    logger.info('Removing WebTorrent files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid))

 
 

-    return false
+    await removeAllWebTorrentFiles(video)

   }
 
   }
 

-  logger.info(
-    'New resolutions %s transcoding jobs created for video %s and origin file resolution of %d.', type, video.uuid, videoFileResolution,
-    { resolutionCreated, ...lTags(video.uuid) }
-  )
-
-  return true
+  await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })

 }
 }

diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index cc6be0bd8d1fb3b48f9765d326d31f727e294ea7..21bf0f22651eb10fbbc03d465cf1737d24e4c997 100644 (file)
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -31,6 +31,7 @@ import {
   MoveObjectStoragePayload,
   NotifyPayload,
   RefreshPayload,
   MoveObjectStoragePayload,
   NotifyPayload,
   RefreshPayload,

+  TranscodingJobBuilderPayload,

   VideoChannelImportPayload,
   VideoFileImportPayload,
   VideoImportPayload,
   VideoChannelImportPayload,
   VideoFileImportPayload,
   VideoImportPayload,


@@ -56,6 +57,7 @@ import { processFederateVideo } from './handlers/federate-video'
 import { processManageVideoTorrent } from './handlers/manage-video-torrent'
 import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
 import { processNotify } from './handlers/notify'
 import { processManageVideoTorrent } from './handlers/manage-video-torrent'
 import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
 import { processNotify } from './handlers/notify'

+import { processTranscodingJobBuilder } from './handlers/transcoding-job-builder'

 import { processVideoChannelImport } from './handlers/video-channel-import'
 import { processVideoFileImport } from './handlers/video-file-import'
 import { processVideoImport } from './handlers/video-import'
 import { processVideoChannelImport } from './handlers/video-channel-import'
 import { processVideoFileImport } from './handlers/video-file-import'
 import { processVideoImport } from './handlers/video-import'


@@ -69,11 +71,12 @@ export type CreateJobArgument =
   { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } |
   { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } |
   { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } |
   { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } |
   { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } |
   { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } |

-  { type: 'activitypub-http-cleaner', payload: {} } |
+  { type: 'activitypub-cleaner', payload: {} } |

   { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
   { type: 'video-file-import', payload: VideoFileImportPayload } |
   { type: 'video-transcoding', payload: VideoTranscodingPayload } |
   { type: 'email', payload: EmailPayload } |
   { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
   { type: 'video-file-import', payload: VideoFileImportPayload } |
   { type: 'video-transcoding', payload: VideoTranscodingPayload } |
   { type: 'email', payload: EmailPayload } |

+  { type: 'transcoding-job-builder', payload: TranscodingJobBuilderPayload } |

   { type: 'video-import', payload: VideoImportPayload } |
   { type: 'activitypub-refresher', payload: RefreshPayload } |
   { type: 'videos-views-stats', payload: {} } |
   { type: 'video-import', payload: VideoImportPayload } |
   { type: 'activitypub-refresher', payload: RefreshPayload } |
   { type: 'videos-views-stats', payload: {} } |


@@ -96,28 +99,29 @@ export type CreateJobOptions = {
 }
 
 const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
 }
 
 const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {

-  'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast,
-  'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast,
-  'activitypub-http-unicast': processActivityPubHttpUnicast,
-  'activitypub-http-fetcher': processActivityPubHttpFetcher,

   'activitypub-cleaner': processActivityPubCleaner,
   'activitypub-follow': processActivityPubFollow,
   'activitypub-cleaner': processActivityPubCleaner,
   'activitypub-follow': processActivityPubFollow,

-  'video-file-import': processVideoFileImport,
-  'video-transcoding': processVideoTranscoding,
+  'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast,
+  'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast,
+  'activitypub-http-fetcher': processActivityPubHttpFetcher,
+  'activitypub-http-unicast': processActivityPubHttpUnicast,
+  'activitypub-refresher': refreshAPObject,
+  'actor-keys': processActorKeys,
+  'after-video-channel-import': processAfterVideoChannelImport,

   'email': processEmail,
   'email': processEmail,

+  'federate-video': processFederateVideo,
+  'transcoding-job-builder': processTranscodingJobBuilder,
+  'manage-video-torrent': processManageVideoTorrent,
+  'move-to-object-storage': processMoveToObjectStorage,
+  'notify': processNotify,
+  'video-channel-import': processVideoChannelImport,
+  'video-file-import': processVideoFileImport,

   'video-import': processVideoImport,
   'video-import': processVideoImport,

-  'videos-views-stats': processVideosViewsStats,
-  'activitypub-refresher': refreshAPObject,

   'video-live-ending': processVideoLiveEnding,
   'video-live-ending': processVideoLiveEnding,

-  'actor-keys': processActorKeys,

   'video-redundancy': processVideoRedundancy,
   'video-redundancy': processVideoRedundancy,

-  'move-to-object-storage': processMoveToObjectStorage,
-  'manage-video-torrent': processManageVideoTorrent,

   'video-studio-edition': processVideoStudioEdition,
   'video-studio-edition': processVideoStudioEdition,

-  'video-channel-import': processVideoChannelImport,
-  'after-video-channel-import': processAfterVideoChannelImport,
-  'notify': processNotify,
-  'federate-video': processFederateVideo
+  'video-transcoding': processVideoTranscoding,
+  'videos-views-stats': processVideosViewsStats

 }
 
 const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
 }
 
 const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {


@@ -125,28 +129,29 @@ const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> }
 }
 
 const jobTypes: JobType[] = [
 }
 
 const jobTypes: JobType[] = [

+  'activitypub-cleaner',

   'activitypub-follow',
   'activitypub-follow',

-  'activitypub-http-broadcast',

   'activitypub-http-broadcast-parallel',
   'activitypub-http-broadcast-parallel',

+  'activitypub-http-broadcast',

   'activitypub-http-fetcher',
   'activitypub-http-unicast',
   'activitypub-http-fetcher',
   'activitypub-http-unicast',

-  'activitypub-cleaner',
+  'activitypub-refresher',
+  'actor-keys',
+  'after-video-channel-import',

   'email',
   'email',

-  'video-transcoding',
+  'federate-video',
+  'transcoding-job-builder',
+  'manage-video-torrent',
+  'move-to-object-storage',
+  'notify',
+  'video-channel-import',

   'video-file-import',
   'video-import',
   'video-file-import',
   'video-import',

-  'videos-views-stats',
-  'activitypub-refresher',
-  'video-redundancy',
-  'actor-keys',

   'video-live-ending',
   'video-live-ending',

-  'move-to-object-storage',
-  'manage-video-torrent',
+  'video-redundancy',

   'video-studio-edition',
   'video-studio-edition',

-  'video-channel-import',
-  'after-video-channel-import',
-  'notify',
-  'federate-video'
+  'video-transcoding',
+  'videos-views-stats'

 ]
 
 const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ])
 ]
 
 const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ])

diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 05274955d9f4d9408d7322ffeb21868d4394cc1a..aa32a9d522d292e0ccc02b6d34057333f7b7e48b 100644 (file)
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -2,36 +2,30 @@ import { readdir, readFile } from 'fs-extra'
 import { createServer, Server } from 'net'
 import { join } from 'path'
 import { createServer as createServerTLS, Server as ServerTLS } from 'tls'
 import { createServer, Server } from 'net'
 import { join } from 'path'
 import { createServer as createServerTLS, Server as ServerTLS } from 'tls'

-import {
-  computeResolutionsToTranscode,
-  ffprobePromise,
-  getLiveSegmentTime,
-  getVideoStreamBitrate,
-  getVideoStreamDimensionsInfo,
-  getVideoStreamFPS,
-  hasAudioStream
-} from '@server/helpers/ffmpeg'

 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
 import { VIDEO_LIVE } from '@server/initializers/constants'
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
 import { VIDEO_LIVE } from '@server/initializers/constants'

+import { sequelizeTypescript } from '@server/initializers/database'

 import { UserModel } from '@server/models/user/user'
 import { VideoModel } from '@server/models/video/video'
 import { VideoLiveModel } from '@server/models/video/video-live'
 import { UserModel } from '@server/models/user/user'
 import { VideoModel } from '@server/models/video/video'
 import { VideoLiveModel } from '@server/models/video/video-live'

+import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'

 import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
 import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models'
 import { pick, wait } from '@shared/core-utils'
 import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
 import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models'
 import { pick, wait } from '@shared/core-utils'

+import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg'

 import { LiveVideoError, VideoState } from '@shared/models'
 import { federateVideoIfNeeded } from '../activitypub/videos'
 import { JobQueue } from '../job-queue'
 import { getLiveReplayBaseDirectory } from '../paths'
 import { PeerTubeSocket } from '../peertube-socket'
 import { Hooks } from '../plugins/hooks'
 import { LiveVideoError, VideoState } from '@shared/models'
 import { federateVideoIfNeeded } from '../activitypub/videos'
 import { JobQueue } from '../job-queue'
 import { getLiveReplayBaseDirectory } from '../paths'
 import { PeerTubeSocket } from '../peertube-socket'
 import { Hooks } from '../plugins/hooks'

+import { computeResolutionsToTranscode } from '../transcoding/transcoding-resolutions'

 import { LiveQuotaStore } from './live-quota-store'
 import { LiveQuotaStore } from './live-quota-store'

-import { cleanupAndDestroyPermanentLive } from './live-utils'
+import { cleanupAndDestroyPermanentLive, getLiveSegmentTime } from './live-utils'

 import { MuxingSession } from './shared'
 import { MuxingSession } from './shared'

-import { sequelizeTypescript } from '@server/initializers/database'
-import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
+import { RunnerJobModel } from '@server/models/runner/runner-job'

 
 const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
 const context = require('node-media-server/src/node_core_ctx')
 
 const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
 const context = require('node-media-server/src/node_core_ctx')


@@ -57,7 +51,7 @@ class LiveManager {
   private static instance: LiveManager
 
   private readonly muxingSessions = new Map<string, MuxingSession>()
   private static instance: LiveManager
 
   private readonly muxingSessions = new Map<string, MuxingSession>()

-  private readonly videoSessions = new Map<number, string>()
+  private readonly videoSessions = new Map<string, string>()

 
   private rtmpServer: Server
   private rtmpsServer: ServerTLS
 
   private rtmpServer: Server
   private rtmpsServer: ServerTLS


@@ -177,14 +171,19 @@ class LiveManager {
     return !!this.rtmpServer
   }
 
     return !!this.rtmpServer
   }
 

-  stopSessionOf (videoId: number, error: LiveVideoError | null) {
-    const sessionId = this.videoSessions.get(videoId)
-    if (!sessionId) return
+  stopSessionOf (videoUUID: string, error: LiveVideoError | null) {
+    const sessionId = this.videoSessions.get(videoUUID)
+    if (!sessionId) {
+      logger.debug('No live session to stop for video %s', videoUUID, lTags(sessionId, videoUUID))
+      return
+    }

 
 

-    this.saveEndingSession(videoId, error)
-      .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
+    logger.info('Stopping live session of video %s', videoUUID, { error, ...lTags(sessionId, videoUUID) })

 
 

-    this.videoSessions.delete(videoId)
+    this.saveEndingSession(videoUUID, error)
+      .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId, videoUUID) }))
+
+    this.videoSessions.delete(videoUUID)

     this.abortSession(sessionId)
   }
 
     this.abortSession(sessionId)
   }
 


@@ -221,6 +220,11 @@ class LiveManager {
       return this.abortSession(sessionId)
     }
 
       return this.abortSession(sessionId)
     }
 

+    if (this.videoSessions.has(video.uuid)) {
+      logger.warn('Video %s has already a live session. Refusing stream %s.', video.uuid, streamKey, lTags(sessionId, video.uuid))
+      return this.abortSession(sessionId)
+    }
+

     // Cleanup old potential live (could happen with a permanent live)
     const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
     if (oldStreamingPlaylist) {
     // Cleanup old potential live (could happen with a permanent live)
     const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
     if (oldStreamingPlaylist) {


@@ -229,7 +233,7 @@ class LiveManager {
       await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist)
     }
 
       await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist)
     }
 

-    this.videoSessions.set(video.id, sessionId)
+    this.videoSessions.set(video.uuid, sessionId)

 
     const now = Date.now()
     const probe = await ffprobePromise(inputUrl)
 
     const now = Date.now()
     const probe = await ffprobePromise(inputUrl)


@@ -253,7 +257,7 @@ class LiveManager {
     )
 
     logger.info(
     )
 
     logger.info(

-      'Will mux/transcode live video of original resolution %d.', resolution,
+      'Handling live video of original resolution %d.', resolution,

       { allResolutions, ...lTags(sessionId, video.uuid) }
     )
 
       { allResolutions, ...lTags(sessionId, video.uuid) }
     )
 


@@ -301,44 +305,44 @@ class LiveManager {
 
     muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags))
 
 
     muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags))
 

-    muxingSession.on('bad-socket-health', ({ videoId }) => {
+    muxingSession.on('bad-socket-health', ({ videoUUID }) => {

       logger.error(
         'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
         ' Stopping session of video %s.', videoUUID,
         localLTags
       )
 
       logger.error(
         'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
         ' Stopping session of video %s.', videoUUID,
         localLTags
       )
 

-      this.stopSessionOf(videoId, LiveVideoError.BAD_SOCKET_HEALTH)
+      this.stopSessionOf(videoUUID, LiveVideoError.BAD_SOCKET_HEALTH)

     })
 
     })
 

-    muxingSession.on('duration-exceeded', ({ videoId }) => {
+    muxingSession.on('duration-exceeded', ({ videoUUID }) => {

       logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags)
 
       logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags)
 

-      this.stopSessionOf(videoId, LiveVideoError.DURATION_EXCEEDED)
+      this.stopSessionOf(videoUUID, LiveVideoError.DURATION_EXCEEDED)

     })
 
     })
 

-    muxingSession.on('quota-exceeded', ({ videoId }) => {
+    muxingSession.on('quota-exceeded', ({ videoUUID }) => {

       logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags)
 
       logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags)
 

-      this.stopSessionOf(videoId, LiveVideoError.QUOTA_EXCEEDED)
+      this.stopSessionOf(videoUUID, LiveVideoError.QUOTA_EXCEEDED)

     })
 
     })
 

-    muxingSession.on('ffmpeg-error', ({ videoId }) => {
-      this.stopSessionOf(videoId, LiveVideoError.FFMPEG_ERROR)
+    muxingSession.on('transcoding-error', ({ videoUUID }) => {
+      this.stopSessionOf(videoUUID, LiveVideoError.FFMPEG_ERROR)

     })
 
     })
 

-    muxingSession.on('ffmpeg-end', ({ videoId }) => {
-      this.onMuxingFFmpegEnd(videoId, sessionId)
+    muxingSession.on('transcoding-end', ({ videoUUID }) => {
+      this.onMuxingFFmpegEnd(videoUUID, sessionId)

     })
 
     })
 

-    muxingSession.on('after-cleanup', ({ videoId }) => {
+    muxingSession.on('after-cleanup', ({ videoUUID }) => {

       this.muxingSessions.delete(sessionId)
 
       LiveQuotaStore.Instance.removeLive(user.id, videoLive.id)
 
       muxingSession.destroy()
 
       this.muxingSessions.delete(sessionId)
 
       LiveQuotaStore.Instance.removeLive(user.id, videoLive.id)
 
       muxingSession.destroy()
 

-      return this.onAfterMuxingCleanup({ videoId, liveSession })
+      return this.onAfterMuxingCleanup({ videoUUID, liveSession })

         .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags }))
     })
 
         .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags }))
     })
 


@@ -379,22 +383,24 @@ class LiveManager {
     }
   }
 
     }
   }
 

-  private onMuxingFFmpegEnd (videoId: number, sessionId: string) {
-    this.videoSessions.delete(videoId)
+  private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) {
+    this.videoSessions.delete(videoUUID)

 
 

-    this.saveEndingSession(videoId, null)
+    this.saveEndingSession(videoUUID, null)

       .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
   }
 
   private async onAfterMuxingCleanup (options: {
       .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) }))
   }
 
   private async onAfterMuxingCleanup (options: {

-    videoId: number | string
+    videoUUID: string

     liveSession?: MVideoLiveSession
     cleanupNow?: boolean // Default false
   }) {
     liveSession?: MVideoLiveSession
     cleanupNow?: boolean // Default false
   }) {

-    const { videoId, liveSession: liveSessionArg, cleanupNow = false } = options
+    const { videoUUID, liveSession: liveSessionArg, cleanupNow = false } = options
+
+    logger.debug('Live of video %s has been cleaned up. Moving to its next state.', videoUUID, lTags(videoUUID))

 
     try {
 
     try {

-      const fullVideo = await VideoModel.loadFull(videoId)
+      const fullVideo = await VideoModel.loadFull(videoUUID)

       if (!fullVideo) return
 
       const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
       if (!fullVideo) return
 
       const live = await VideoLiveModel.loadByVideoId(fullVideo.id)


@@ -437,15 +443,17 @@ class LiveManager {
 
       await federateVideoIfNeeded(fullVideo, false)
     } catch (err) {
 
       await federateVideoIfNeeded(fullVideo, false)
     } catch (err) {

-      logger.error('Cannot save/federate new video state of live streaming of video %d.', videoId, { err, ...lTags(videoId + '') })
+      logger.error('Cannot save/federate new video state of live streaming of video %s.', videoUUID, { err, ...lTags(videoUUID) })

     }
   }
 
   private async handleBrokenLives () {
     }
   }
 
   private async handleBrokenLives () {

+    await RunnerJobModel.cancelAllJobs({ type: 'live-rtmp-hls-transcoding' })
+

     const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
 
     for (const uuid of videoUUIDs) {
     const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
 
     for (const uuid of videoUUIDs) {

-      await this.onAfterMuxingCleanup({ videoId: uuid, cleanupNow: true })
+      await this.onAfterMuxingCleanup({ videoUUID: uuid, cleanupNow: true })

     }
   }
 
     }
   }
 


@@ -494,8 +502,8 @@ class LiveManager {
     })
   }
 
     })
   }
 

-  private async saveEndingSession (videoId: number, error: LiveVideoError | null) {
-    const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoId)
+  private async saveEndingSession (videoUUID: string, error: LiveVideoError | null) {
+    const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoUUID)

     if (!liveSession) return
 
     liveSession.endDate = new Date()
     if (!liveSession) return
 
     liveSession.endDate = new Date()

diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts
index 4d03754a9dc3944b782ee117005325736cc9d545..251301141c69562398d8db1e9d6c2e1f95748506 100644 (file)
--- a/server/lib/live/live-segment-sha-store.ts
+++ b/server/lib/live/live-segment-sha-store.ts
@@ -52,7 +52,10 @@ class LiveSegmentShaStore {
     logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID))
 
     if (!this.segmentsSha256.has(segmentName)) {
     logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID))
 
     if (!this.segmentsSha256.has(segmentName)) {

-      logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID))
+      logger.warn(
+        'Unknown segment in live segment hash store for video %s and segment %s.',
+        this.videoUUID, segmentPath, lTags(this.videoUUID)
+      )

       return
     }
 
       return
     }
 

diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
index c0dec9829b8cfe1a3b1b6d2f8dcf34283a4d2a4d..3fb3ce1cee998886f96f8f3dce6d0888b678f0dc 100644 (file)
--- a/server/lib/live/live-utils.ts
+++ b/server/lib/live/live-utils.ts
@@ -1,8 +1,9 @@
 import { pathExists, readdir, remove } from 'fs-extra'
 import { basename, join } from 'path'
 import { logger } from '@server/helpers/logger'
 import { pathExists, readdir, remove } from 'fs-extra'
 import { basename, join } from 'path'
 import { logger } from '@server/helpers/logger'

+import { VIDEO_LIVE } from '@server/initializers/constants'

 import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
 import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'

-import { VideoStorage } from '@shared/models'
+import { LiveVideoLatencyMode, VideoStorage } from '@shared/models'

 import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage'
 import { getLiveDirectory } from '../paths'
 
 import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage'
 import { getLiveDirectory } from '../paths'
 


@@ -37,10 +38,19 @@ async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreaming
   await cleanupTMPLiveFilesFromFilesystem(video)
 }
 
   await cleanupTMPLiveFilesFromFilesystem(video)
 }
 

+function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) {
+  if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) {
+    return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY
+  }
+
+  return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY
+}
+

 export {
   cleanupAndDestroyPermanentLive,
   cleanupUnsavedNormalLive,
   cleanupTMPLiveFiles,
 export {
   cleanupAndDestroyPermanentLive,
   cleanupUnsavedNormalLive,
   cleanupTMPLiveFiles,

+  getLiveSegmentTime,

   buildConcatenatedName
 }
 
   buildConcatenatedName
 }
 

diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index 2727fc4a767973213dcb69c96a48f3cdf19e993f..f3f8fc8863fc9c65ef1f8ed8c6f959159aef9223 100644 (file)
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -1,11 +1,10 @@
 import { mapSeries } from 'bluebird'
 import { FSWatcher, watch } from 'chokidar'
 import { mapSeries } from 'bluebird'
 import { FSWatcher, watch } from 'chokidar'

-import { FfmpegCommand } from 'fluent-ffmpeg'
+import { EventEmitter } from 'events'

 import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
 import PQueue from 'p-queue'
 import { basename, join } from 'path'
 import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
 import PQueue from 'p-queue'
 import { basename, join } from 'path'

-import { EventEmitter } from 'stream'
-import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg'
+import { computeOutputFPS } from '@server/helpers/ffmpeg'

 import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
 import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants'
 import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
 import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants'


@@ -20,24 +19,24 @@ import {
   getLiveDirectory,
   getLiveReplayBaseDirectory
 } from '../../paths'
   getLiveDirectory,
   getLiveReplayBaseDirectory
 } from '../../paths'

-import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles'

 import { isAbleToUploadVideo } from '../../user'
 import { LiveQuotaStore } from '../live-quota-store'
 import { LiveSegmentShaStore } from '../live-segment-sha-store'
 import { isAbleToUploadVideo } from '../../user'
 import { LiveQuotaStore } from '../live-quota-store'
 import { LiveSegmentShaStore } from '../live-segment-sha-store'

-import { buildConcatenatedName } from '../live-utils'
+import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils'
+import { AbstractTranscodingWrapper, FFmpegTranscodingWrapper, RemoteTranscodingWrapper } from './transcoding-wrapper'

 
 import memoizee = require('memoizee')
 interface MuxingSessionEvents {
 
 import memoizee = require('memoizee')
 interface MuxingSessionEvents {

-  'live-ready': (options: { videoId: number }) => void
+  'live-ready': (options: { videoUUID: string }) => void

 
 

-  'bad-socket-health': (options: { videoId: number }) => void
-  'duration-exceeded': (options: { videoId: number }) => void
-  'quota-exceeded': (options: { videoId: number }) => void
+  'bad-socket-health': (options: { videoUUID: string }) => void
+  'duration-exceeded': (options: { videoUUID: string }) => void
+  'quota-exceeded': (options: { videoUUID: string }) => void

 
 

-  'ffmpeg-end': (options: { videoId: number }) => void
-  'ffmpeg-error': (options: { videoId: number }) => void
+  'transcoding-end': (options: { videoUUID: string }) => void
+  'transcoding-error': (options: { videoUUID: string }) => void

 
 

-  'after-cleanup': (options: { videoId: number }) => void
+  'after-cleanup': (options: { videoUUID: string }) => void

 }
 
 declare interface MuxingSession {
 }
 
 declare interface MuxingSession {


@@ -52,7 +51,7 @@ declare interface MuxingSession {
 
 class MuxingSession extends EventEmitter {
 
 
 class MuxingSession extends EventEmitter {
 

-  private ffmpegCommand: FfmpegCommand
+  private transcodingWrapper: AbstractTranscodingWrapper

 
   private readonly context: any
   private readonly user: MUserId
 
   private readonly context: any
   private readonly user: MUserId


@@ -67,7 +66,6 @@ class MuxingSession extends EventEmitter {
 
   private readonly hasAudio: boolean
 
 
   private readonly hasAudio: boolean
 

-  private readonly videoId: number

   private readonly videoUUID: string
   private readonly saveReplay: boolean
 
   private readonly videoUUID: string
   private readonly saveReplay: boolean
 


@@ -126,7 +124,6 @@ class MuxingSession extends EventEmitter {
 
     this.allResolutions = options.allResolutions
 
 
     this.allResolutions = options.allResolutions
 

-    this.videoId = this.videoLive.Video.id

     this.videoUUID = this.videoLive.Video.uuid
 
     this.saveReplay = this.videoLive.saveReplay
     this.videoUUID = this.videoLive.Video.uuid
 
     this.saveReplay = this.videoLive.saveReplay


@@ -145,63 +142,23 @@ class MuxingSession extends EventEmitter {
 
     await this.prepareDirectories()
 
 
     await this.prepareDirectories()
 

-    this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
-      ? await getLiveTranscodingCommand({
-        inputUrl: this.inputUrl,
+    this.transcodingWrapper = this.buildTranscodingWrapper()

 
 

-        outPath: this.outDirectory,
-        masterPlaylistName: this.streamingPlaylist.playlistFilename,
+    this.transcodingWrapper.on('end', () => this.onTranscodedEnded())
+    this.transcodingWrapper.on('error', () => this.onTranscodingError())

 
 

-        latencyMode: this.videoLive.latencyMode,
-
-        resolutions: this.allResolutions,
-        fps: this.fps,
-        bitrate: this.bitrate,
-        ratio: this.ratio,
-
-        hasAudio: this.hasAudio,
-
-        availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-        profile: CONFIG.LIVE.TRANSCODING.PROFILE
-      })
-      : getLiveMuxingCommand({
-        inputUrl: this.inputUrl,
-        outPath: this.outDirectory,
-        masterPlaylistName: this.streamingPlaylist.playlistFilename,
-        latencyMode: this.videoLive.latencyMode
-      })
-
-    logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags())
+    await this.transcodingWrapper.run()

 
     this.watchMasterFile()
     this.watchTSFiles()
     this.watchM3U8File()
 
     this.watchMasterFile()
     this.watchTSFiles()
     this.watchM3U8File()

-
-    let ffmpegShellCommand: string
-    this.ffmpegCommand.on('start', cmdline => {
-      ffmpegShellCommand = cmdline
-
-      logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() })
-    })
-
-    this.ffmpegCommand.on('error', (err, stdout, stderr) => {
-      this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand })
-    })
-
-    this.ffmpegCommand.on('end', () => {
-      this.emit('ffmpeg-end', ({ videoId: this.videoId }))
-
-      this.onFFmpegEnded()
-    })
-
-    this.ffmpegCommand.run()

   }
 
   abort () {
   }
 
   abort () {

-    if (!this.ffmpegCommand) return
+    if (!this.transcodingWrapper) return

 
     this.aborted = true
 
     this.aborted = true

-    this.ffmpegCommand.kill('SIGINT')
+    this.transcodingWrapper.abort()

   }
 
   destroy () {
   }
 
   destroy () {


@@ -210,48 +167,6 @@ class MuxingSession extends EventEmitter {
     this.hasClientSocketInBadHealthWithCache.clear()
   }
 
     this.hasClientSocketInBadHealthWithCache.clear()
   }
 

-  private onFFmpegError (options: {
-    err: any
-    stdout: string
-    stderr: string
-    ffmpegShellCommand: string
-  }) {
-    const { err, stdout, stderr, ffmpegShellCommand } = options
-
-    this.onFFmpegEnded()
-
-    // Don't care that we killed the ffmpeg process
-    if (err?.message?.includes('Exiting normally')) return
-
-    logger.error('Live transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() })
-
-    this.emit('ffmpeg-error', ({ videoId: this.videoId }))
-  }
-
-  private onFFmpegEnded () {
-    logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags())
-
-    setTimeout(() => {
-      // Wait latest segments generation, and close watchers
-
-      Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ])
-        .then(() => {
-          // Process remaining segments hash
-          for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
-            this.processSegments(this.segmentsToProcessPerPlaylist[key])
-          }
-        })
-        .catch(err => {
-          logger.error(
-            'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory,
-            { err, ...this.lTags() }
-          )
-        })
-
-      this.emit('after-cleanup', { videoId: this.videoId })
-    }, 1000)
-  }
-

   private watchMasterFile () {
     this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename)
 
   private watchMasterFile () {
     this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename)
 


@@ -272,6 +187,8 @@ class MuxingSession extends EventEmitter {
 
       this.masterPlaylistCreated = true
 
 
       this.masterPlaylistCreated = true
 

+      logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags())
+

       this.masterWatcher.close()
         .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() }))
     })
       this.masterWatcher.close()
         .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() }))
     })


@@ -318,19 +235,19 @@ class MuxingSession extends EventEmitter {
       this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
 
       if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) {
       this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
 
       if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) {

-        this.emit('bad-socket-health', { videoId: this.videoId })
+        this.emit('bad-socket-health', { videoUUID: this.videoUUID })

         return
       }
 
       // Duration constraint check
       if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
         return
       }
 
       // Duration constraint check
       if (this.isDurationConstraintValid(startStreamDateTime) !== true) {

-        this.emit('duration-exceeded', { videoId: this.videoId })
+        this.emit('duration-exceeded', { videoUUID: this.videoUUID })

         return
       }
 
       // Check user quota if the user enabled replay saving
       if (await this.isQuotaExceeded(segmentPath) === true) {
         return
       }
 
       // Check user quota if the user enabled replay saving
       if (await this.isQuotaExceeded(segmentPath) === true) {

-        this.emit('quota-exceeded', { videoId: this.videoId })
+        this.emit('quota-exceeded', { videoUUID: this.videoUUID })

       }
     }
 
       }
     }
 


@@ -438,10 +355,40 @@ class MuxingSession extends EventEmitter {
     if (this.masterPlaylistCreated && !this.liveReady) {
       this.liveReady = true
 
     if (this.masterPlaylistCreated && !this.liveReady) {
       this.liveReady = true
 

-      this.emit('live-ready', { videoId: this.videoId })
+      this.emit('live-ready', { videoUUID: this.videoUUID })

     }
   }
 
     }
   }
 

+  private onTranscodingError () {
+    this.emit('transcoding-error', ({ videoUUID: this.videoUUID }))
+  }
+
+  private onTranscodedEnded () {
+    this.emit('transcoding-end', ({ videoUUID: this.videoUUID }))
+
+    logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags())
+
+    setTimeout(() => {
+      // Wait latest segments generation, and close watchers
+
+      Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ])
+        .then(() => {
+          // Process remaining segments hash
+          for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
+            this.processSegments(this.segmentsToProcessPerPlaylist[key])
+          }
+        })
+        .catch(err => {
+          logger.error(
+            'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory,
+            { err, ...this.lTags() }
+          )
+        })
+
+      this.emit('after-cleanup', { videoUUID: this.videoUUID })
+    }, 1000)
+  }
+

   private hasClientSocketInBadHealth (sessionId: string) {
     const rtmpSession = this.context.sessions.get(sessionId)
 
   private hasClientSocketInBadHealth (sessionId: string) {
     const rtmpSession = this.context.sessions.get(sessionId)
 


@@ -503,6 +450,36 @@ class MuxingSession extends EventEmitter {
       sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED
     })
   }
       sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED
     })
   }

+
+  private buildTranscodingWrapper () {
+    const options = {
+      streamingPlaylist: this.streamingPlaylist,
+      videoLive: this.videoLive,
+
+      lTags: this.lTags,
+
+      inputUrl: this.inputUrl,
+
+      toTranscode: this.allResolutions.map(resolution => ({
+        resolution,
+        fps: computeOutputFPS({ inputFPS: this.fps, resolution })
+      })),
+
+      fps: this.fps,
+      bitrate: this.bitrate,
+      ratio: this.ratio,
+      hasAudio: this.hasAudio,
+
+      segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE,
+      segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode),
+
+      outDirectory: this.outDirectory
+    }
+
+    return CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED
+      ? new RemoteTranscodingWrapper(options)
+      : new FFmpegTranscodingWrapper(options)
+  }

 }
 
 // ---------------------------------------------------------------------------
 }
 
 // ---------------------------------------------------------------------------

diff --git a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts
new file mode 100644 (file)
index 0000000..226ba45
--- /dev/null
+++ b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts
@@ -0,0 +1,101 @@
+import EventEmitter from 'events'
+import { LoggerTagsFn } from '@server/helpers/logger'
+import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models'
+import { LiveVideoError } from '@shared/models'
+
+interface TranscodingWrapperEvents {
+  'end': () => void
+
+  'error': (options: { err: Error }) => void
+}
+
+declare interface AbstractTranscodingWrapper {
+  on<U extends keyof TranscodingWrapperEvents>(
+    event: U, listener: TranscodingWrapperEvents[U]
+  ): this
+
+  emit<U extends keyof TranscodingWrapperEvents>(
+    event: U, ...args: Parameters<TranscodingWrapperEvents[U]>
+  ): boolean
+}
+
+interface AbstractTranscodingWrapperOptions {
+  streamingPlaylist: MStreamingPlaylistVideo
+  videoLive: MVideoLiveVideo
+
+  lTags: LoggerTagsFn
+
+  inputUrl: string
+  fps: number
+  toTranscode: {
+    resolution: number
+    fps: number
+  }[]
+
+  bitrate: number
+  ratio: number
+  hasAudio: boolean
+
+  segmentListSize: number
+  segmentDuration: number
+
+  outDirectory: string
+}
+
+abstract class AbstractTranscodingWrapper extends EventEmitter {
+  protected readonly videoLive: MVideoLiveVideo
+
+  protected readonly toTranscode: {
+    resolution: number
+    fps: number
+  }[]
+
+  protected readonly inputUrl: string
+  protected readonly fps: number
+  protected readonly bitrate: number
+  protected readonly ratio: number
+  protected readonly hasAudio: boolean
+
+  protected readonly segmentListSize: number
+  protected readonly segmentDuration: number
+
+  protected readonly videoUUID: string
+
+  protected readonly outDirectory: string
+
+  protected readonly lTags: LoggerTagsFn
+
+  protected readonly streamingPlaylist: MStreamingPlaylistVideo
+
+  constructor (options: AbstractTranscodingWrapperOptions) {
+    super()
+
+    this.lTags = options.lTags
+
+    this.videoLive = options.videoLive
+    this.videoUUID = options.videoLive.Video.uuid
+    this.streamingPlaylist = options.streamingPlaylist
+
+    this.inputUrl = options.inputUrl
+    this.fps = options.fps
+    this.toTranscode = options.toTranscode
+
+    this.bitrate = options.bitrate
+    this.ratio = options.ratio
+    this.hasAudio = options.hasAudio
+
+    this.segmentListSize = options.segmentListSize
+    this.segmentDuration = options.segmentDuration
+
+    this.outDirectory = options.outDirectory
+  }
+
+  abstract run (): Promise<void>
+
+  abstract abort (error?: LiveVideoError): void
+}
+
+export {
+  AbstractTranscodingWrapper,
+  AbstractTranscodingWrapperOptions
+}

diff --git a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts
new file mode 100644 (file)
index 0000000..1f4c12b
--- /dev/null
+++ b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts
@@ -0,0 +1,95 @@
+import { FfmpegCommand } from 'fluent-ffmpeg'
+import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { VIDEO_LIVE } from '@server/initializers/constants'
+import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles'
+import { FFmpegLive } from '@shared/ffmpeg'
+import { getLiveSegmentTime } from '../../live-utils'
+import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper'
+
+export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {
+  private ffmpegCommand: FfmpegCommand
+  private ended = false
+
+  async run () {
+    this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
+      ? await this.buildFFmpegLive().getLiveTranscodingCommand({
+        inputUrl: this.inputUrl,
+
+        outPath: this.outDirectory,
+        masterPlaylistName: this.streamingPlaylist.playlistFilename,
+
+        segmentListSize: this.segmentListSize,
+        segmentDuration: this.segmentDuration,
+
+        toTranscode: this.toTranscode,
+
+        bitrate: this.bitrate,
+        ratio: this.ratio,
+
+        hasAudio: this.hasAudio
+      })
+      : this.buildFFmpegLive().getLiveMuxingCommand({
+        inputUrl: this.inputUrl,
+        outPath: this.outDirectory,
+
+        masterPlaylistName: this.streamingPlaylist.playlistFilename,
+
+        segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE,
+        segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode)
+      })
+
+    logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags())
+
+    this.ffmpegCommand.run()
+
+    let ffmpegShellCommand: string
+    this.ffmpegCommand.on('start', cmdline => {
+      ffmpegShellCommand = cmdline
+
+      logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() })
+    })
+
+    this.ffmpegCommand.on('error', (err, stdout, stderr) => {
+      this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand })
+    })
+
+    this.ffmpegCommand.on('end', () => {
+      this.onFFmpegEnded()
+    })
+
+    this.ffmpegCommand.run()
+  }
+
+  abort () {
+    // Nothing to do, ffmpeg will automatically exit
+  }
+
+  private onFFmpegError (options: {
+    err: any
+    stdout: string
+    stderr: string
+    ffmpegShellCommand: string
+  }) {
+    const { err, stdout, stderr, ffmpegShellCommand } = options
+
+    // Don't care that we killed the ffmpeg process
+    if (err?.message?.includes('Exiting normally')) return
+
+    logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() })
+
+    this.emit('error', { err })
+  }
+
+  private onFFmpegEnded () {
+    if (this.ended) return
+
+    this.ended = true
+    this.emit('end')
+  }
+
+  private buildFFmpegLive () {
+    return new FFmpegLive(getFFmpegCommandWrapperOptions('live', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))
+  }
+}

diff --git a/server/lib/live/shared/transcoding-wrapper/index.ts b/server/lib/live/shared/transcoding-wrapper/index.ts
new file mode 100644 (file)
index 0000000..ae28fa1
--- /dev/null
+++ b/server/lib/live/shared/transcoding-wrapper/index.ts
@@ -0,0 +1,3 @@
+export * from './abstract-transcoding-wrapper'
+export * from './ffmpeg-transcoding-wrapper'
+export * from './remote-transcoding-wrapper'

diff --git a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts
new file mode 100644 (file)
index 0000000..345eaf4
--- /dev/null
+++ b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts
@@ -0,0 +1,20 @@
+import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners'
+import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper'
+
+export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper {
+  async run () {
+    await new LiveRTMPHLSTranscodingJobHandler().create({
+      rtmpUrl: this.inputUrl,
+      toTranscode: this.toTranscode,
+      video: this.videoLive.Video,
+      outputDirectory: this.outDirectory,
+      playlist: this.streamingPlaylist,
+      segmentListSize: this.segmentListSize,
+      segmentDuration: this.segmentDuration
+    })
+  }
+
+  abort () {
+    this.emit('end')
+  }
+}

diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts
index 8b413a40ef0fe3ab94e1cc59035e14b2971ac542..6525f8dfb2ba2ce8ff1673ba837707dc2063c388 100644 (file)
--- a/server/lib/object-storage/index.ts
+++ b/server/lib/object-storage/index.ts
@@ -1,3 +1,4 @@
 export * from './keys'
 export * from './keys'

+export * from './proxy'

 export * from './urls'
 export * from './videos'
 export * from './urls'
 export * from './videos'

diff --git a/server/lib/object-storage/proxy.ts b/server/lib/object-storage/proxy.ts
new file mode 100644 (file)
index 0000000..c782a8a
--- /dev/null
+++ b/server/lib/object-storage/proxy.ts
@@ -0,0 +1,97 @@
+import express from 'express'
+import { PassThrough, pipeline } from 'stream'
+import { GetObjectCommandOutput } from '@aws-sdk/client-s3'
+import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist'
+import { logger } from '@server/helpers/logger'
+import { StreamReplacer } from '@server/helpers/stream-replacer'
+import { MStreamingPlaylist, MVideo } from '@server/types/models'
+import { HttpStatusCode } from '@shared/models'
+import { injectQueryToPlaylistUrls } from '../hls'
+import { getHLSFileReadStream, getWebTorrentFileReadStream } from './videos'
+
+export async function proxifyWebTorrentFile (options: {
+  req: express.Request
+  res: express.Response
+  filename: string
+}) {
+  const { req, res, filename } = options
+
+  logger.debug('Proxifying WebTorrent file %s from object storage.', filename)
+
+  try {
+    const { response: s3Response, stream } = await getWebTorrentFileReadStream({
+      filename,
+      rangeHeader: req.header('range')
+    })
+
+    setS3Headers(res, s3Response)
+
+    return stream.pipe(res)
+  } catch (err) {
+    return handleObjectStorageFailure(res, err)
+  }
+}
+
+export async function proxifyHLS (options: {
+  req: express.Request
+  res: express.Response
+  playlist: MStreamingPlaylist
+  video: MVideo
+  filename: string
+  reinjectVideoFileToken: boolean
+}) {
+  const { req, res, playlist, video, filename, reinjectVideoFileToken } = options
+
+  logger.debug('Proxifying HLS file %s from object storage.', filename)
+
+  try {
+    const { response: s3Response, stream } = await getHLSFileReadStream({
+      playlist: playlist.withVideo(video),
+      filename,
+      rangeHeader: req.header('range')
+    })
+
+    setS3Headers(res, s3Response)
+
+    const streamReplacer = reinjectVideoFileToken
+      ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8'))))
+      : new PassThrough()
+
+    return pipeline(
+      stream,
+      streamReplacer,
+      res,
+      err => {
+        if (!err) return
+
+        handleObjectStorageFailure(res, err)
+      }
+    )
+  } catch (err) {
+    return handleObjectStorageFailure(res, err)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function handleObjectStorageFailure (res: express.Response, err: Error) {
+  if (err.name === 'NoSuchKey') {
+    logger.debug('Could not find key in object storage to proxify private HLS video file.', { err })
+    return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
+  }
+
+  return res.fail({
+    status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
+    message: err.message,
+    type: err.name
+  })
+}
+
+function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) {
+  if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) {
+    res.setHeader('Content-Range', s3Response.ContentRange)
+    res.status(HttpStatusCode.PARTIAL_CONTENT_206)
+  }
+}

diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts
index 0398ca61dbf8f71e68c52be44cbed4fe2023dd7b..ded7e97433a8a547767981e71d13963a33e16690 100644 (file)
--- a/server/lib/peertube-socket.ts
+++ b/server/lib/peertube-socket.ts
@@ -2,10 +2,12 @@ import { Server as HTTPServer } from 'http'
 import { Namespace, Server as SocketServer, Socket } from 'socket.io'
 import { isIdValid } from '@server/helpers/custom-validators/misc'
 import { MVideo, MVideoImmutable } from '@server/types/models'
 import { Namespace, Server as SocketServer, Socket } from 'socket.io'
 import { isIdValid } from '@server/helpers/custom-validators/misc'
 import { MVideo, MVideoImmutable } from '@server/types/models'

+import { MRunner } from '@server/types/models/runners'

 import { UserNotificationModelForApi } from '@server/types/models/user'
 import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models'
 import { logger } from '../helpers/logger'
 import { UserNotificationModelForApi } from '@server/types/models/user'
 import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models'
 import { logger } from '../helpers/logger'

-import { authenticateSocket } from '../middlewares'
+import { authenticateRunnerSocket, authenticateSocket } from '../middlewares'
+import { Debounce } from '@server/helpers/debounce'

 
 class PeerTubeSocket {
 
 
 class PeerTubeSocket {
 


@@ -13,6 +15,7 @@ class PeerTubeSocket {
 
   private userNotificationSockets: { [ userId: number ]: Socket[] } = {}
   private liveVideosNamespace: Namespace
 
   private userNotificationSockets: { [ userId: number ]: Socket[] } = {}
   private liveVideosNamespace: Namespace

+  private readonly runnerSockets = new Set<Socket>()

 
   private constructor () {}
 
 
   private constructor () {}
 


@@ -24,7 +27,7 @@ class PeerTubeSocket {
       .on('connection', socket => {
         const userId = socket.handshake.auth.user.id
 
       .on('connection', socket => {
         const userId = socket.handshake.auth.user.id
 

-        logger.debug('User %d connected on the notification system.', userId)
+        logger.debug('User %d connected to the notification system.', userId)

 
         if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = []
 
 
         if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = []
 


@@ -53,6 +56,22 @@ class PeerTubeSocket {
           socket.leave(videoId)
         })
       })
           socket.leave(videoId)
         })
       })

+
+    io.of('/runners')
+      .use(authenticateRunnerSocket)
+      .on('connection', socket => {
+        const runner: MRunner = socket.handshake.auth.runner
+
+        logger.debug(`New runner "${runner.name}" connected to the notification system.`)
+
+        this.runnerSockets.add(socket)
+
+        socket.on('disconnect', () => {
+          logger.debug(`Runner "${runner.name}" disconnected from the notification system.`)
+
+          this.runnerSockets.delete(socket)
+        })
+      })

   }
 
   sendNotification (userId: number, notification: UserNotificationModelForApi) {
   }
 
   sendNotification (userId: number, notification: UserNotificationModelForApi) {


@@ -89,6 +108,15 @@ class PeerTubeSocket {
       .emit(type, data)
   }
 
       .emit(type, data)
   }
 

+  @Debounce({ timeoutMS: 1000 })
+  sendAvailableJobsPingToRunners () {
+    logger.debug(`Sending available-jobs notification to ${this.runnerSockets.size} runner sockets`)
+
+    for (const runners of this.runnerSockets) {
+      runners.emit('available-jobs')
+    }
+  }
+

   static get Instance () {
     return this.instance || (this.instance = new this())
   }
   static get Instance () {
     return this.instance || (this.instance = new this())
   }

diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts
index 66383af46a0b462e497dd9120d13e6b409b92d30..92ef87cca717d7ea9d991bfe9b78c8208fa69e89 100644 (file)
--- a/server/lib/plugins/plugin-helpers-builder.ts
+++ b/server/lib/plugins/plugin-helpers-builder.ts
@@ -1,7 +1,6 @@
 import express from 'express'
 import { Server } from 'http'
 import { join } from 'path'
 import express from 'express'
 import { Server } from 'http'
 import { join } from 'path'

-import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils'

 import { buildLogger } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
 import { WEBSERVER } from '@server/initializers/constants'
 import { buildLogger } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
 import { WEBSERVER } from '@server/initializers/constants'


@@ -16,6 +15,7 @@ import { VideoModel } from '@server/models/video/video'
 import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
 import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models'
 import { PeerTubeHelpers } from '@server/types/plugins'
 import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
 import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models'
 import { PeerTubeHelpers } from '@server/types/plugins'

+import { ffprobePromise } from '@shared/ffmpeg'

 import { VideoBlacklistCreate, VideoStorage } from '@shared/models'
 import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist'
 import { PeerTubeSocket } from '../peertube-socket'
 import { VideoBlacklistCreate, VideoStorage } from '@shared/models'
 import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist'
 import { PeerTubeSocket } from '../peertube-socket'

diff --git a/server/lib/runners/index.ts b/server/lib/runners/index.ts
new file mode 100644 (file)
index 0000000..a737c7b
--- /dev/null
+++ b/server/lib/runners/index.ts
@@ -0,0 +1,3 @@
+export * from './job-handlers'
+export * from './runner'
+export * from './runner-urls'

diff --git a/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/lib/runners/job-handlers/abstract-job-handler.ts
new file mode 100644 (file)
index 0000000..73fc145
--- /dev/null
+++ b/server/lib/runners/job-handlers/abstract-job-handler.ts
@@ -0,0 +1,271 @@
+import { retryTransactionWrapper } from '@server/helpers/database-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { RUNNER_JOBS } from '@server/initializers/constants'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { PeerTubeSocket } from '@server/lib/peertube-socket'
+import { RunnerJobModel } from '@server/models/runner/runner-job'
+import { setAsUpdated } from '@server/models/shared'
+import { MRunnerJob } from '@server/types/models/runners'
+import { pick } from '@shared/core-utils'
+import {
+  RunnerJobLiveRTMPHLSTranscodingPayload,
+  RunnerJobLiveRTMPHLSTranscodingPrivatePayload,
+  RunnerJobState,
+  RunnerJobSuccessPayload,
+  RunnerJobType,
+  RunnerJobUpdatePayload,
+  RunnerJobVODAudioMergeTranscodingPayload,
+  RunnerJobVODAudioMergeTranscodingPrivatePayload,
+  RunnerJobVODHLSTranscodingPayload,
+  RunnerJobVODHLSTranscodingPrivatePayload,
+  RunnerJobVODWebVideoTranscodingPayload,
+  RunnerJobVODWebVideoTranscodingPrivatePayload
+} from '@shared/models'
+
+type CreateRunnerJobArg =
+  {
+    type: Extract<RunnerJobType, 'vod-web-video-transcoding'>
+    payload: RunnerJobVODWebVideoTranscodingPayload
+    privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload
+  } |
+  {
+    type: Extract<RunnerJobType, 'vod-hls-transcoding'>
+    payload: RunnerJobVODHLSTranscodingPayload
+    privatePayload: RunnerJobVODHLSTranscodingPrivatePayload
+  } |
+  {
+    type: Extract<RunnerJobType, 'vod-audio-merge-transcoding'>
+    payload: RunnerJobVODAudioMergeTranscodingPayload
+    privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload
+  } |
+  {
+    type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'>
+    payload: RunnerJobLiveRTMPHLSTranscodingPayload
+    privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload
+  }
+
+export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {
+
+  protected readonly lTags = loggerTagsFactory('runner')
+
+  // ---------------------------------------------------------------------------
+
+  abstract create (options: C): Promise<MRunnerJob>
+
+  protected async createRunnerJob (options: CreateRunnerJobArg & {
+    jobUUID: string
+    priority: number
+    dependsOnRunnerJob?: MRunnerJob
+  }): Promise<MRunnerJob> {
+    const { priority, dependsOnRunnerJob } = options
+
+    const runnerJob = new RunnerJobModel({
+      ...pick(options, [ 'type', 'payload', 'privatePayload' ]),
+
+      uuid: options.jobUUID,
+
+      state: dependsOnRunnerJob
+        ? RunnerJobState.WAITING_FOR_PARENT_JOB
+        : RunnerJobState.PENDING,
+
+      dependsOnRunnerJobId: dependsOnRunnerJob?.id,
+
+      priority
+    })
+
+    const job = await sequelizeTypescript.transaction(async transaction => {
+      return runnerJob.save({ transaction })
+    })
+
+    if (runnerJob.state === RunnerJobState.PENDING) {
+      PeerTubeSocket.Instance.sendAvailableJobsPingToRunners()
+    }
+
+    return job
+  }
+
+  // ---------------------------------------------------------------------------
+
+  protected abstract specificUpdate (options: {
+    runnerJob: MRunnerJob
+    updatePayload?: U
+  }): Promise<void> | void
+
+  async update (options: {
+    runnerJob: MRunnerJob
+    progress?: number
+    updatePayload?: U
+  }) {
+    const { runnerJob, progress } = options
+
+    await this.specificUpdate(options)
+
+    if (progress) runnerJob.progress = progress
+
+    await retryTransactionWrapper(() => {
+      return sequelizeTypescript.transaction(async transaction => {
+        if (runnerJob.changed()) {
+          return runnerJob.save({ transaction })
+        }
+
+        // Don't update the job too often
+        if (new Date().getTime() - runnerJob.updatedAt.getTime() > 2000) {
+          await setAsUpdated({ sequelize: sequelizeTypescript, table: 'runnerJob', id: runnerJob.id, transaction })
+        }
+      })
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async complete (options: {
+    runnerJob: MRunnerJob
+    resultPayload: S
+  }) {
+    const { runnerJob } = options
+
+    try {
+      await this.specificComplete(options)
+
+      runnerJob.state = RunnerJobState.COMPLETED
+    } catch (err) {
+      logger.error('Cannot complete runner job', { err, ...this.lTags(runnerJob.id, runnerJob.type) })
+
+      runnerJob.state = RunnerJobState.ERRORED
+      runnerJob.error = err.message
+    }
+
+    runnerJob.progress = null
+    runnerJob.finishedAt = new Date()
+
+    await retryTransactionWrapper(() => {
+      return sequelizeTypescript.transaction(async transaction => {
+        await runnerJob.save({ transaction })
+      })
+    })
+
+    const [ affectedCount ] = await RunnerJobModel.updateDependantJobsOf(runnerJob)
+
+    if (affectedCount !== 0) PeerTubeSocket.Instance.sendAvailableJobsPingToRunners()
+  }
+
+  protected abstract specificComplete (options: {
+    runnerJob: MRunnerJob
+    resultPayload: S
+  }): Promise<void> | void
+
+  // ---------------------------------------------------------------------------
+
+  async cancel (options: {
+    runnerJob: MRunnerJob
+    fromParent?: boolean
+  }) {
+    const { runnerJob, fromParent } = options
+
+    await this.specificCancel(options)
+
+    const cancelState = fromParent
+      ? RunnerJobState.PARENT_CANCELLED
+      : RunnerJobState.CANCELLED
+
+    runnerJob.setToErrorOrCancel(cancelState)
+
+    await retryTransactionWrapper(() => {
+      return sequelizeTypescript.transaction(async transaction => {
+        await runnerJob.save({ transaction })
+      })
+    })
+
+    const children = await RunnerJobModel.listChildrenOf(runnerJob)
+    for (const child of children) {
+      logger.info(`Cancelling child job ${child.uuid} of ${runnerJob.uuid} because of parent cancel`, this.lTags(child.uuid))
+
+      await this.cancel({ runnerJob: child, fromParent: true })
+    }
+  }
+
+  protected abstract specificCancel (options: {
+    runnerJob: MRunnerJob
+  }): Promise<void> | void
+
+  // ---------------------------------------------------------------------------
+
+  protected abstract isAbortSupported (): boolean
+
+  async abort (options: {
+    runnerJob: MRunnerJob
+  }) {
+    const { runnerJob } = options
+
+    if (this.isAbortSupported() !== true) {
+      return this.error({ runnerJob, message: 'Job has been aborted but it is not supported by this job type' })
+    }
+
+    await this.specificAbort(options)
+
+    runnerJob.resetToPending()
+
+    await retryTransactionWrapper(() => {
+      return sequelizeTypescript.transaction(async transaction => {
+        await runnerJob.save({ transaction })
+      })
+    })
+  }
+
+  protected setAbortState (runnerJob: MRunnerJob) {
+    runnerJob.resetToPending()
+  }
+
+  protected abstract specificAbort (options: {
+    runnerJob: MRunnerJob
+  }): Promise<void> | void
+
+  // ---------------------------------------------------------------------------
+
+  async error (options: {
+    runnerJob: MRunnerJob
+    message: string
+    fromParent?: boolean
+  }) {
+    const { runnerJob, message, fromParent } = options
+
+    const errorState = fromParent
+      ? RunnerJobState.PARENT_ERRORED
+      : RunnerJobState.ERRORED
+
+    const nextState = errorState === RunnerJobState.ERRORED && this.isAbortSupported() && runnerJob.failures < RUNNER_JOBS.MAX_FAILURES
+      ? RunnerJobState.PENDING
+      : errorState
+
+    await this.specificError({ ...options, nextState })
+
+    if (nextState === errorState) {
+      runnerJob.setToErrorOrCancel(nextState)
+      runnerJob.error = message
+    } else {
+      runnerJob.resetToPending()
+    }
+
+    await retryTransactionWrapper(() => {
+      return sequelizeTypescript.transaction(async transaction => {
+        await runnerJob.save({ transaction })
+      })
+    })
+
+    if (runnerJob.state === errorState) {
+      const children = await RunnerJobModel.listChildrenOf(runnerJob)
+
+      for (const child of children) {
+        logger.info(`Erroring child job ${child.uuid} of ${runnerJob.uuid} because of parent error`, this.lTags(child.uuid))
+
+        await this.error({ runnerJob: child, message: 'Parent error', fromParent: true })
+      }
+    }
+  }
+
+  protected abstract specificError (options: {
+    runnerJob: MRunnerJob
+    message: string
+    nextState: RunnerJobState
+  }): Promise<void> | void
+}

diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts
new file mode 100644 (file)
index 0000000..5176458
--- /dev/null
+++ b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts
@@ -0,0 +1,71 @@
+
+import { retryTransactionWrapper } from '@server/helpers/database-utils'
+import { logger } from '@server/helpers/logger'
+import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info'
+import { MRunnerJob } from '@server/types/models/runners'
+import {
+  LiveRTMPHLSTranscodingUpdatePayload,
+  RunnerJobSuccessPayload,
+  RunnerJobUpdatePayload,
+  RunnerJobVODPrivatePayload
+} from '@shared/models'
+import { AbstractJobHandler } from './abstract-job-handler'
+import { loadTranscodingRunnerVideo } from './shared'
+
+// eslint-disable-next-line max-len
+export abstract class AbstractVODTranscodingJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> extends AbstractJobHandler<C, U, S> {
+
+  // ---------------------------------------------------------------------------
+
+  protected isAbortSupported () {
+    return true
+  }
+
+  protected specificUpdate (_options: {
+    runnerJob: MRunnerJob
+    updatePayload?: LiveRTMPHLSTranscodingUpdatePayload
+  }) {
+    // empty
+  }
+
+  protected specificAbort (_options: {
+    runnerJob: MRunnerJob
+  }) {
+    // empty
+  }
+
+  protected async specificError (options: {
+    runnerJob: MRunnerJob
+  }) {
+    const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags)
+    if (!video) return
+
+    await moveToFailedTranscodingState(video)
+
+    await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
+  }
+
+  protected async specificCancel (options: {
+    runnerJob: MRunnerJob
+  }) {
+    const { runnerJob } = options
+
+    const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags)
+    if (!video) return
+
+    const pending = await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
+
+    logger.debug(`Pending transcode decreased to ${pending} after cancel`, this.lTags(video.uuid))
+
+    if (pending === 0) {
+      logger.info(
+        `All transcoding jobs of ${video.uuid} have been processed or canceled, moving it to its next state`,
+        this.lTags(video.uuid)
+      )
+
+      const privatePayload = runnerJob.privatePayload as RunnerJobVODPrivatePayload
+      await retryTransactionWrapper(moveToNextState, { video, isNewVideo: privatePayload.isNewVideo })
+    }
+  }
+}

diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts
new file mode 100644 (file)
index 0000000..0fca72b
--- /dev/null
+++ b/server/lib/runners/job-handlers/index.ts
@@ -0,0 +1,6 @@
+export * from './abstract-job-handler'
+export * from './live-rtmp-hls-transcoding-job-handler'
+export * from './vod-audio-merge-transcoding-job-handler'
+export * from './vod-hls-transcoding-job-handler'
+export * from './vod-web-video-transcoding-job-handler'
+export * from './runner-job-handlers'

diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts
new file mode 100644 (file)
index 0000000..c3d0e42
--- /dev/null
+++ b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts
@@ -0,0 +1,170 @@
+import { move, remove } from 'fs-extra'
+import { join } from 'path'
+import { logger } from '@server/helpers/logger'
+import { JOB_PRIORITY } from '@server/initializers/constants'
+import { LiveManager } from '@server/lib/live'
+import { MStreamingPlaylist, MVideo } from '@server/types/models'
+import { MRunnerJob } from '@server/types/models/runners'
+import { buildUUID } from '@shared/extra-utils'
+import {
+  LiveRTMPHLSTranscodingSuccess,
+  LiveRTMPHLSTranscodingUpdatePayload,
+  LiveVideoError,
+  RunnerJobLiveRTMPHLSTranscodingPayload,
+  RunnerJobLiveRTMPHLSTranscodingPrivatePayload,
+  RunnerJobState
+} from '@shared/models'
+import { AbstractJobHandler } from './abstract-job-handler'
+
+type CreateOptions = {
+  video: MVideo
+  playlist: MStreamingPlaylist
+
+  rtmpUrl: string
+
+  toTranscode: {
+    resolution: number
+    fps: number
+  }[]
+
+  segmentListSize: number
+  segmentDuration: number
+
+  outputDirectory: string
+}
+
+// eslint-disable-next-line max-len
+export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateOptions, LiveRTMPHLSTranscodingUpdatePayload, LiveRTMPHLSTranscodingSuccess> {
+
+  async create (options: CreateOptions) {
+    const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory } = options
+
+    const jobUUID = buildUUID()
+    const payload: RunnerJobLiveRTMPHLSTranscodingPayload = {
+      input: {
+        rtmpUrl
+      },
+      output: {
+        toTranscode,
+        segmentListSize,
+        segmentDuration
+      }
+    }
+
+    const privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload = {
+      videoUUID: video.uuid,
+      masterPlaylistName: playlist.playlistFilename,
+      outputDirectory
+    }
+
+    const job = await this.createRunnerJob({
+      type: 'live-rtmp-hls-transcoding',
+      jobUUID,
+      payload,
+      privatePayload,
+      priority: JOB_PRIORITY.TRANSCODING
+    })
+
+    return job
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async specificUpdate (options: {
+    runnerJob: MRunnerJob
+    updatePayload: LiveRTMPHLSTranscodingUpdatePayload
+  }) {
+    const { runnerJob, updatePayload } = options
+
+    const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload
+    const outputDirectory = privatePayload.outputDirectory
+    const videoUUID = privatePayload.videoUUID
+
+    if (updatePayload.type === 'add-chunk') {
+      await move(
+        updatePayload.videoChunkFile as string,
+        join(outputDirectory, updatePayload.videoChunkFilename),
+        { overwrite: true }
+      )
+    } else if (updatePayload.type === 'remove-chunk') {
+      await remove(join(outputDirectory, updatePayload.videoChunkFilename))
+    }
+
+    if (updatePayload.resolutionPlaylistFile && updatePayload.resolutionPlaylistFilename) {
+      await move(
+        updatePayload.resolutionPlaylistFile as string,
+        join(outputDirectory, updatePayload.resolutionPlaylistFilename),
+        { overwrite: true }
+      )
+    }
+
+    if (updatePayload.masterPlaylistFile) {
+      await move(updatePayload.masterPlaylistFile as string, join(outputDirectory, privatePayload.masterPlaylistName), { overwrite: true })
+    }
+
+    logger.info(
+      'Runner live RTMP to HLS job %s for %s updated.',
+      runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) }
+    )
+  }
+
+  // ---------------------------------------------------------------------------
+
+  protected specificComplete (options: {
+    runnerJob: MRunnerJob
+  }) {
+    return this.stopLive({
+      runnerJob: options.runnerJob,
+      type: 'ended'
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  protected isAbortSupported () {
+    return false
+  }
+
+  protected specificAbort () {
+    throw new Error('Not implemented')
+  }
+
+  protected specificError (options: {
+    runnerJob: MRunnerJob
+    nextState: RunnerJobState
+  }) {
+    return this.stopLive({
+      runnerJob: options.runnerJob,
+      type: 'errored'
+    })
+  }
+
+  protected specificCancel (options: {
+    runnerJob: MRunnerJob
+  }) {
+    return this.stopLive({
+      runnerJob: options.runnerJob,
+      type: 'cancelled'
+    })
+  }
+
+  private stopLive (options: {
+    runnerJob: MRunnerJob
+    type: 'ended' | 'errored' | 'cancelled'
+  }) {
+    const { runnerJob, type } = options
+
+    const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload
+    const videoUUID = privatePayload.videoUUID
+
+    const errorType = {
+      ended: null,
+      errored: LiveVideoError.RUNNER_JOB_ERROR,
+      cancelled: LiveVideoError.RUNNER_JOB_CANCEL
+    }
+
+    LiveManager.Instance.stopSessionOf(privatePayload.videoUUID, errorType[type])
+
+    logger.info('Runner live RTMP to HLS job %s for video %s %s.', runnerJob.uuid, videoUUID, type, this.lTags(runnerJob.uuid, videoUUID))
+  }
+}

diff --git a/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/lib/runners/job-handlers/runner-job-handlers.ts
new file mode 100644 (file)
index 0000000..7bad1bc
--- /dev/null
+++ b/server/lib/runners/job-handlers/runner-job-handlers.ts
@@ -0,0 +1,18 @@
+import { MRunnerJob } from '@server/types/models/runners'
+import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models'
+import { AbstractJobHandler } from './abstract-job-handler'
+import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler'
+import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler'
+import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler'
+import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler'
+
+const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, RunnerJobUpdatePayload, RunnerJobSuccessPayload>> = {
+  'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler,
+  'vod-hls-transcoding': VODHLSTranscodingJobHandler,
+  'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler,
+  'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler
+}
+
+export function getRunnerJobHandlerClass (job: MRunnerJob) {
+  return processors[job.type]
+}

diff --git a/server/lib/runners/job-handlers/shared/index.ts b/server/lib/runners/job-handlers/shared/index.ts
new file mode 100644 (file)
index 0000000..348273a
--- /dev/null
+++ b/server/lib/runners/job-handlers/shared/index.ts
@@ -0,0 +1 @@
+export * from './vod-helpers'

diff --git a/server/lib/runners/job-handlers/shared/vod-helpers.ts b/server/lib/runners/job-handlers/shared/vod-helpers.ts
new file mode 100644 (file)
index 0000000..93ae89f
--- /dev/null
+++ b/server/lib/runners/job-handlers/shared/vod-helpers.ts
@@ -0,0 +1,44 @@
+import { move } from 'fs-extra'
+import { dirname, join } from 'path'
+import { logger, LoggerTagsFn } from '@server/helpers/logger'
+import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding'
+import { onWebTorrentVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding'
+import { buildNewFile } from '@server/lib/video-file'
+import { VideoModel } from '@server/models/video/video'
+import { MVideoFullLight } from '@server/types/models'
+import { MRunnerJob } from '@server/types/models/runners'
+import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@shared/models'
+
+export async function onVODWebVideoOrAudioMergeTranscodingJob (options: {
+  video: MVideoFullLight
+  videoFilePath: string
+  privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload
+}) {
+  const { video, videoFilePath, privatePayload } = options
+
+  const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video' })
+  videoFile.videoId = video.id
+
+  const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename)
+  await move(videoFilePath, newVideoFilePath)
+
+  await onWebTorrentVideoFileTranscoding({
+    video,
+    videoFile,
+    videoOutputPath: newVideoFilePath
+  })
+
+  await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video })
+}
+
+export async function loadTranscodingRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) {
+  const videoUUID = runnerJob.privatePayload.videoUUID
+
+  const video = await VideoModel.loadFull(videoUUID)
+  if (!video) {
+    logger.info('Video %s does not exist anymore after transcoding runner job.', videoUUID, lTags(videoUUID))
+    return undefined
+  }
+
+  return video
+}

diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts
new file mode 100644 (file)
index 0000000..a7b33f8
--- /dev/null
+++ b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts
@@ -0,0 +1,97 @@
+import { pick } from 'lodash'
+import { logger } from '@server/helpers/logger'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info'
+import { MVideo } from '@server/types/models'
+import { MRunnerJob } from '@server/types/models/runners'
+import { buildUUID } from '@shared/extra-utils'
+import { getVideoStreamDuration } from '@shared/ffmpeg'
+import {
+  RunnerJobUpdatePayload,
+  RunnerJobVODAudioMergeTranscodingPayload,
+  RunnerJobVODWebVideoTranscodingPrivatePayload,
+  VODAudioMergeTranscodingSuccess
+} from '@shared/models'
+import { generateRunnerTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoPreviewFileUrl } from '../runner-urls'
+import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler'
+import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared'
+
+type CreateOptions = {
+  video: MVideo
+  isNewVideo: boolean
+  resolution: number
+  fps: number
+  priority: number
+  dependsOnRunnerJob?: MRunnerJob
+}
+
+// eslint-disable-next-line max-len
+export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODAudioMergeTranscodingSuccess> {
+
+  async create (options: CreateOptions) {
+    const { video, resolution, fps, priority, dependsOnRunnerJob } = options
+
+    const jobUUID = buildUUID()
+    const payload: RunnerJobVODAudioMergeTranscodingPayload = {
+      input: {
+        audioFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid),
+        previewFileUrl: generateRunnerTranscodingVideoPreviewFileUrl(jobUUID, video.uuid)
+      },
+      output: {
+        resolution,
+        fps
+      }
+    }
+
+    const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = {
+      ...pick(options, [ 'isNewVideo' ]),
+
+      videoUUID: video.uuid
+    }
+
+    const job = await this.createRunnerJob({
+      type: 'vod-audio-merge-transcoding',
+      jobUUID,
+      payload,
+      privatePayload,
+      priority,
+      dependsOnRunnerJob
+    })
+
+    await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
+
+    return job
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async specificComplete (options: {
+    runnerJob: MRunnerJob
+    resultPayload: VODAudioMergeTranscodingSuccess
+  }) {
+    const { runnerJob, resultPayload } = options
+    const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload
+
+    const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags)
+    if (!video) return
+
+    const videoFilePath = resultPayload.videoFile as string
+
+    // ffmpeg generated a new video file, so update the video duration
+    // See https://trac.ffmpeg.org/ticket/5456
+    video.duration = await getVideoStreamDuration(videoFilePath)
+    await video.save()
+
+    // We can remove the old audio file
+    const oldAudioFile = video.VideoFiles[0]
+    await video.removeWebTorrentFile(oldAudioFile)
+    await oldAudioFile.destroy()
+    video.VideoFiles = []
+
+    await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload })
+
+    logger.info(
+      'Runner VOD audio merge transcoding job %s for %s ended.',
+      runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid)
+    )
+  }
+}

diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts
new file mode 100644 (file)
index 0000000..02566b9
--- /dev/null
+++ b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts
@@ -0,0 +1,114 @@
+import { move } from 'fs-extra'
+import { dirname, join } from 'path'
+import { logger } from '@server/helpers/logger'
+import { renameVideoFileInPlaylist } from '@server/lib/hls'
+import { getHlsResolutionPlaylistFilename } from '@server/lib/paths'
+import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding'
+import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding'
+import { buildNewFile, removeAllWebTorrentFiles } from '@server/lib/video-file'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info'
+import { MVideo } from '@server/types/models'
+import { MRunnerJob } from '@server/types/models/runners'
+import { pick } from '@shared/core-utils'
+import { buildUUID } from '@shared/extra-utils'
+import {
+  RunnerJobUpdatePayload,
+  RunnerJobVODHLSTranscodingPayload,
+  RunnerJobVODHLSTranscodingPrivatePayload,
+  VODHLSTranscodingSuccess
+} from '@shared/models'
+import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls'
+import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler'
+import { loadTranscodingRunnerVideo } from './shared'
+
+type CreateOptions = {
+  video: MVideo
+  isNewVideo: boolean
+  deleteWebVideoFiles: boolean
+  resolution: number
+  fps: number
+  priority: number
+  dependsOnRunnerJob?: MRunnerJob
+}
+
+// eslint-disable-next-line max-len
+export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODHLSTranscodingSuccess> {
+
+  async create (options: CreateOptions) {
+    const { video, resolution, fps, dependsOnRunnerJob, priority } = options
+
+    const jobUUID = buildUUID()
+
+    const payload: RunnerJobVODHLSTranscodingPayload = {
+      input: {
+        videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid)
+      },
+      output: {
+        resolution,
+        fps
+      }
+    }
+
+    const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = {
+      ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]),
+
+      videoUUID: video.uuid
+    }
+
+    const job = await this.createRunnerJob({
+      type: 'vod-hls-transcoding',
+      jobUUID,
+      payload,
+      privatePayload,
+      priority,
+      dependsOnRunnerJob
+    })
+
+    await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
+
+    return job
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async specificComplete (options: {
+    runnerJob: MRunnerJob
+    resultPayload: VODHLSTranscodingSuccess
+  }) {
+    const { runnerJob, resultPayload } = options
+    const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload
+
+    const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags)
+    if (!video) return
+
+    const videoFilePath = resultPayload.videoFile as string
+    const resolutionPlaylistFilePath = resultPayload.resolutionPlaylistFile as string
+
+    const videoFile = await buildNewFile({ path: videoFilePath, mode: 'hls' })
+    const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename)
+    await move(videoFilePath, newVideoFilePath)
+
+    const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFile.filename)
+    const newResolutionPlaylistFilePath = join(dirname(resolutionPlaylistFilePath), resolutionPlaylistFilename)
+    await move(resolutionPlaylistFilePath, newResolutionPlaylistFilePath)
+
+    await renameVideoFileInPlaylist(newResolutionPlaylistFilePath, videoFile.filename)
+
+    await onHLSVideoFileTranscoding({
+      video,
+      videoFile,
+      m3u8OutputPath: newResolutionPlaylistFilePath,
+      videoOutputPath: newVideoFilePath
+    })
+
+    await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video })
+
+    if (privatePayload.deleteWebVideoFiles === true) {
+      logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid))
+
+      await removeAllWebTorrentFiles(video)
+    }
+
+    logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid))
+  }
+}

diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts
new file mode 100644 (file)
index 0000000..57761a7
--- /dev/null
+++ b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts
@@ -0,0 +1,84 @@
+import { pick } from 'lodash'
+import { logger } from '@server/helpers/logger'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info'
+import { MVideo } from '@server/types/models'
+import { MRunnerJob } from '@server/types/models/runners'
+import { buildUUID } from '@shared/extra-utils'
+import {
+  RunnerJobUpdatePayload,
+  RunnerJobVODWebVideoTranscodingPayload,
+  RunnerJobVODWebVideoTranscodingPrivatePayload,
+  VODWebVideoTranscodingSuccess
+} from '@shared/models'
+import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls'
+import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler'
+import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared'
+
+type CreateOptions = {
+  video: MVideo
+  isNewVideo: boolean
+  resolution: number
+  fps: number
+  priority: number
+  dependsOnRunnerJob?: MRunnerJob
+}
+
+// eslint-disable-next-line max-len
+export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODWebVideoTranscodingSuccess> {
+
+  async create (options: CreateOptions) {
+    const { video, resolution, fps, priority, dependsOnRunnerJob } = options
+
+    const jobUUID = buildUUID()
+    const payload: RunnerJobVODWebVideoTranscodingPayload = {
+      input: {
+        videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid)
+      },
+      output: {
+        resolution,
+        fps
+      }
+    }
+
+    const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = {
+      ...pick(options, [ 'isNewVideo' ]),
+
+      videoUUID: video.uuid
+    }
+
+    const job = await this.createRunnerJob({
+      type: 'vod-web-video-transcoding',
+      jobUUID,
+      payload,
+      privatePayload,
+      dependsOnRunnerJob,
+      priority
+    })
+
+    await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
+
+    return job
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async specificComplete (options: {
+    runnerJob: MRunnerJob
+    resultPayload: VODWebVideoTranscodingSuccess
+  }) {
+    const { runnerJob, resultPayload } = options
+    const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload
+
+    const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags)
+    if (!video) return
+
+    const videoFilePath = resultPayload.videoFile as string
+
+    await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload })
+
+    logger.info(
+      'Runner VOD web video transcoding job %s for %s ended.',
+      runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid)
+    )
+  }
+}

diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts
new file mode 100644 (file)
index 0000000..329fb11
--- /dev/null
+++ b/server/lib/runners/runner-urls.ts
@@ -0,0 +1,9 @@
+import { WEBSERVER } from '@server/initializers/constants'
+
+export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string) {
+  return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/max-quality'
+}
+
+export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) {
+  return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality'
+}

diff --git a/server/lib/runners/runner.ts b/server/lib/runners/runner.ts
new file mode 100644 (file)
index 0000000..74c814b
--- /dev/null
+++ b/server/lib/runners/runner.ts
@@ -0,0 +1,36 @@
+import express from 'express'
+import { retryTransactionWrapper } from '@server/helpers/database-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { MRunner } from '@server/types/models/runners'
+
+const lTags = loggerTagsFactory('runner')
+
+const updatingRunner = new Set<number>()
+
+function updateLastRunnerContact (req: express.Request, runner: MRunner) {
+  const now = new Date()
+
+  // Don't update last runner contact too often
+  if (now.getTime() - runner.lastContact.getTime() < 2000) return
+  if (updatingRunner.has(runner.id)) return
+
+  updatingRunner.add(runner.id)
+
+  runner.lastContact = now
+  runner.ip = req.ip
+
+  logger.debug('Updating last runner contact for %s', runner.name, lTags(runner.name))
+
+  retryTransactionWrapper(() => {
+    return sequelizeTypescript.transaction(async transaction => {
+      return runner.save({ transaction })
+    })
+  })
+  .catch(err => logger.error('Cannot update last runner contact for %s', runner.name, { err, ...lTags(runner.name) }))
+  .finally(() => updatingRunner.delete(runner.id))
+}
+
+export {
+  updateLastRunnerContact
+}

diff --git a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts
new file mode 100644 (file)
index 0000000..f7a26d2
--- /dev/null
+++ b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts
@@ -0,0 +1,42 @@
+import { CONFIG } from '@server/initializers/config'
+import { RunnerJobModel } from '@server/models/runner/runner-job'
+import { logger, loggerTagsFactory } from '../../helpers/logger'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { getRunnerJobHandlerClass } from '../runners'
+import { AbstractScheduler } from './abstract-scheduler'
+
+const lTags = loggerTagsFactory('runner')
+
+export class RunnerJobWatchDogScheduler extends AbstractScheduler {
+
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG
+
+  private constructor () {
+    super()
+  }
+
+  protected async internalExecute () {
+    const vodStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD,
+      types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ]
+    })
+
+    const liveStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE,
+      types: [ 'live-rtmp-hls-transcoding' ]
+    })
+
+    for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) {
+      logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type))
+
+      const Handler = getRunnerJobHandlerClass(stalled)
+      await new Handler().abort({ runnerJob: stalled })
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts
index e87e2854f6d8b5cda66df964cacbe4c3187c1593..ba791636330eae65312bd2b692c3d2a6b0901894 100644 (file)
--- a/server/lib/server-config-manager.ts
+++ b/server/lib/server-config-manager.ts
@@ -126,11 +126,14 @@ class ServerConfigManager {
       serverVersion: PEERTUBE_VERSION,
       serverCommit: this.serverCommit,
       transcoding: {
       serverVersion: PEERTUBE_VERSION,
       serverCommit: this.serverCommit,
       transcoding: {

+        remoteRunners: {
+          enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED
+        },

         hls: {
         hls: {

-          enabled: CONFIG.TRANSCODING.HLS.ENABLED
+          enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED

         },
         webtorrent: {
         },
         webtorrent: {

-          enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED
+          enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEBTORRENT.ENABLED

         },
         enabledResolutions: this.getEnabledResolutions('vod'),
         profile: CONFIG.TRANSCODING.PROFILE,
         },
         enabledResolutions: this.getEnabledResolutions('vod'),
         profile: CONFIG.TRANSCODING.PROFILE,


@@ -150,6 +153,9 @@ class ServerConfigManager {
 
         transcoding: {
           enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
 
         transcoding: {
           enabled: CONFIG.LIVE.TRANSCODING.ENABLED,

+          remoteRunners: {
+            enabled: CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED
+          },

           enabledResolutions: this.getEnabledResolutions('live'),
           profile: CONFIG.LIVE.TRANSCODING.PROFILE,
           availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live')
           enabledResolutions: this.getEnabledResolutions('live'),
           profile: CONFIG.LIVE.TRANSCODING.PROFILE,
           availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live')

diff --git a/server/lib/transcoding/create-transcoding-job.ts b/server/lib/transcoding/create-transcoding-job.ts
new file mode 100644 (file)
index 0000000..46831a9
--- /dev/null
+++ b/server/lib/transcoding/create-transcoding-job.ts
@@ -0,0 +1,36 @@
+import { CONFIG } from '@server/initializers/config'
+import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
+import { TranscodingJobQueueBuilder, TranscodingRunnerJobBuilder } from './shared'
+
+export function createOptimizeOrMergeAudioJobs (options: {
+  video: MVideoFullLight
+  videoFile: MVideoFile
+  isNewVideo: boolean
+  user: MUserId
+}) {
+  return getJobBuilder().createOptimizeOrMergeAudioJobs(options)
+}
+
+// ---------------------------------------------------------------------------
+
+export function createTranscodingJobs (options: {
+  transcodingType: 'hls' | 'webtorrent'
+  video: MVideoFullLight
+  resolutions: number[]
+  isNewVideo: boolean
+  user: MUserId
+}) {
+  return getJobBuilder().createTranscodingJobs(options)
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function getJobBuilder () {
+  if (CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED === true) {
+    return new TranscodingRunnerJobBuilder()
+  }
+
+  return new TranscodingJobQueueBuilder()
+}

diff --git a/server/lib/transcoding/default-transcoding-profiles.ts b/server/lib/transcoding/default-transcoding-profiles.ts
index f4771881926cf5a6c963106ff9c72b7ce691a7d9..5251784ac874eb81c6449fcfd7ca7fb8214faa4c 100644 (file)
--- a/server/lib/transcoding/default-transcoding-profiles.ts
+++ b/server/lib/transcoding/default-transcoding-profiles.ts
@@ -1,15 +1,9 @@
 
 import { logger } from '@server/helpers/logger'
 import { getAverageBitrate, getMinLimitBitrate } from '@shared/core-utils'
 
 import { logger } from '@server/helpers/logger'
 import { getAverageBitrate, getMinLimitBitrate } from '@shared/core-utils'

-import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '../../../shared/models/videos'
-import {
-  buildStreamSuffix,
-  canDoQuickAudioTranscode,
-  ffprobePromise,
-  getAudioStream,
-  getMaxAudioBitrate,
-  resetSupportedEncoders
-} from '../../helpers/ffmpeg'
+import { buildStreamSuffix, FFmpegCommandWrapper, ffprobePromise, getAudioStream, getMaxAudioBitrate } from '@shared/ffmpeg'
+import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '@shared/models'
+import { canDoQuickAudioTranscode } from './transcoding-quick-transcode'

 
 /**
  *
 
 /**
  *


@@ -184,14 +178,14 @@ class VideoTranscodingProfilesManager {
   addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) {
     this.encodersPriorities[type][streamType].push({ name: encoder, priority })
 
   addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) {
     this.encodersPriorities[type][streamType].push({ name: encoder, priority })
 

-    resetSupportedEncoders()
+    FFmpegCommandWrapper.resetSupportedEncoders()

   }
 
   removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) {
     this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType]
                                                     .filter(o => o.name !== encoder && o.priority !== priority)
 
   }
 
   removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) {
     this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType]
                                                     .filter(o => o.name !== encoder && o.priority !== priority)
 

-    resetSupportedEncoders()
+    FFmpegCommandWrapper.resetSupportedEncoders()

   }
 
   private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') {
   }
 
   private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') {

diff --git a/server/lib/transcoding/ended-transcoding.ts b/server/lib/transcoding/ended-transcoding.ts
new file mode 100644 (file)
index 0000000..d31674e
--- /dev/null
+++ b/server/lib/transcoding/ended-transcoding.ts
@@ -0,0 +1,18 @@
+import { retryTransactionWrapper } from '@server/helpers/database-utils'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info'
+import { MVideo } from '@server/types/models'
+import { moveToNextState } from '../video-state'
+
+export async function onTranscodingEnded (options: {
+  video: MVideo
+  isNewVideo: boolean
+  moveVideoToNextState: boolean
+}) {
+  const { video, isNewVideo, moveVideoToNextState } = options
+
+  await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
+
+  if (moveVideoToNextState) {
+    await retryTransactionWrapper(moveToNextState, { video, isNewVideo })
+  }
+}

diff --git a/server/lib/transcoding/hls-transcoding.ts b/server/lib/transcoding/hls-transcoding.ts
new file mode 100644 (file)
index 0000000..cffa859
--- /dev/null
+++ b/server/lib/transcoding/hls-transcoding.ts
@@ -0,0 +1,181 @@
+import { MutexInterface } from 'async-mutex'
+import { Job } from 'bullmq'
+import { ensureDir, move, stat } from 'fs-extra'
+import { basename, extname as extnameUtil, join } from 'path'
+import { retryTransactionWrapper } from '@server/helpers/database-utils'
+import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { MVideo, MVideoFile } from '@server/types/models'
+import { pick } from '@shared/core-utils'
+import { getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg'
+import { VideoResolution } from '@shared/models'
+import { CONFIG } from '../../initializers/config'
+import { VideoFileModel } from '../../models/video/video-file'
+import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
+import { updatePlaylistAfterFileChange } from '../hls'
+import { generateHLSVideoFilename, getHlsResolutionPlaylistFilename } from '../paths'
+import { buildFileMetadata } from '../video-file'
+import { VideoPathManager } from '../video-path-manager'
+import { buildFFmpegVOD } from './shared'
+
+// Concat TS segments from a live video to a fragmented mp4 HLS playlist
+export async function generateHlsPlaylistResolutionFromTS (options: {
+  video: MVideo
+  concatenatedTsFilePath: string
+  resolution: VideoResolution
+  fps: number
+  isAAC: boolean
+  inputFileMutexReleaser: MutexInterface.Releaser
+}) {
+  return generateHlsPlaylistCommon({
+    type: 'hls-from-ts' as 'hls-from-ts',
+    inputPath: options.concatenatedTsFilePath,
+
+    ...pick(options, [ 'video', 'resolution', 'fps', 'inputFileMutexReleaser', 'isAAC' ])
+  })
+}
+
+// Generate an HLS playlist from an input file, and update the master playlist
+export function generateHlsPlaylistResolution (options: {
+  video: MVideo
+  videoInputPath: string
+  resolution: VideoResolution
+  fps: number
+  copyCodecs: boolean
+  inputFileMutexReleaser: MutexInterface.Releaser
+  job?: Job
+}) {
+  return generateHlsPlaylistCommon({
+    type: 'hls' as 'hls',
+    inputPath: options.videoInputPath,
+
+    ...pick(options, [ 'video', 'resolution', 'fps', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
+  })
+}
+
+export async function onHLSVideoFileTranscoding (options: {
+  video: MVideo
+  videoFile: MVideoFile
+  videoOutputPath: string
+  m3u8OutputPath: string
+}) {
+  const { video, videoFile, videoOutputPath, m3u8OutputPath } = options
+
+  // Create or update the playlist
+  const playlist = await retryTransactionWrapper(() => {
+    return sequelizeTypescript.transaction(async transaction => {
+      return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
+    })
+  })
+  videoFile.videoStreamingPlaylistId = playlist.id
+
+  const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+
+  try {
+    // VOD transcoding is a long task, refresh video attributes
+    await video.reload()
+
+    const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, videoFile)
+    await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
+
+    // Move playlist file
+    const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, basename(m3u8OutputPath))
+    await move(m3u8OutputPath, resolutionPlaylistPath, { overwrite: true })
+    // Move video file
+    await move(videoOutputPath, videoFilePath, { overwrite: true })
+
+    // Update video duration if it was not set (in case of a live for example)
+    if (!video.duration) {
+      video.duration = await getVideoStreamDuration(videoFilePath)
+      await video.save()
+    }
+
+    const stats = await stat(videoFilePath)
+
+    videoFile.size = stats.size
+    videoFile.fps = await getVideoStreamFPS(videoFilePath)
+    videoFile.metadata = await buildFileMetadata(videoFilePath)
+
+    await createTorrentAndSetInfoHash(playlist, videoFile)
+
+    const oldFile = await VideoFileModel.loadHLSFile({
+      playlistId: playlist.id,
+      fps: videoFile.fps,
+      resolution: videoFile.resolution
+    })
+
+    if (oldFile) {
+      await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
+      await oldFile.destroy()
+    }
+
+    const savedVideoFile = await VideoFileModel.customUpsert(videoFile, 'streaming-playlist', undefined)
+
+    await updatePlaylistAfterFileChange(video, playlist)
+
+    return { resolutionPlaylistPath, videoFile: savedVideoFile }
+  } finally {
+    mutexReleaser()
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+async function generateHlsPlaylistCommon (options: {
+  type: 'hls' | 'hls-from-ts'
+  video: MVideo
+  inputPath: string
+
+  resolution: VideoResolution
+  fps: number
+
+  inputFileMutexReleaser: MutexInterface.Releaser
+
+  copyCodecs?: boolean
+  isAAC?: boolean
+
+  job?: Job
+}) {
+  const { type, video, inputPath, resolution, fps, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
+  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
+
+  const videoTranscodedBasePath = join(transcodeDirectory, type)
+  await ensureDir(videoTranscodedBasePath)
+
+  const videoFilename = generateHLSVideoFilename(resolution)
+  const videoOutputPath = join(videoTranscodedBasePath, videoFilename)
+
+  const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
+  const m3u8OutputPath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
+
+  const transcodeOptions = {
+    type,
+
+    inputPath,
+    outputPath: m3u8OutputPath,
+
+    resolution,
+    fps,
+    copyCodecs,
+
+    isAAC,
+
+    inputFileMutexReleaser,
+
+    hlsPlaylist: {
+      videoFilename
+    }
+  }
+
+  await buildFFmpegVOD(job).transcode(transcodeOptions)
+
+  const newVideoFile = new VideoFileModel({
+    resolution,
+    extname: extnameUtil(videoFilename),
+    size: 0,
+    filename: videoFilename,
+    fps: -1
+  })
+
+  await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath })
+}

diff --git a/server/lib/transcoding/shared/ffmpeg-builder.ts b/server/lib/transcoding/shared/ffmpeg-builder.ts
new file mode 100644 (file)
index 0000000..441445e
--- /dev/null
+++ b/server/lib/transcoding/shared/ffmpeg-builder.ts
@@ -0,0 +1,18 @@
+import { Job } from 'bullmq'
+import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
+import { logger } from '@server/helpers/logger'
+import { FFmpegVOD } from '@shared/ffmpeg'
+import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles'
+
+export function buildFFmpegVOD (job?: Job) {
+  return new FFmpegVOD({
+    ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()),
+
+    updateJobProgress: progress => {
+      if (!job) return
+
+      job.updateProgress(progress)
+        .catch(err => logger.error('Cannot update ffmpeg job progress', { err }))
+    }
+  })
+}

diff --git a/server/lib/transcoding/shared/index.ts b/server/lib/transcoding/shared/index.ts
new file mode 100644 (file)
index 0000000..f0b45bc
--- /dev/null
+++ b/server/lib/transcoding/shared/index.ts
@@ -0,0 +1,2 @@
+export * from './job-builders'
+export * from './ffmpeg-builder'

diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts
new file mode 100644 (file)
index 0000000..f1e9efd
--- /dev/null
+++ b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts
@@ -0,0 +1,38 @@
+
+import { JOB_PRIORITY } from '@server/initializers/constants'
+import { VideoModel } from '@server/models/video/video'
+import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
+
+export abstract class AbstractJobBuilder {
+
+  abstract createOptimizeOrMergeAudioJobs (options: {
+    video: MVideoFullLight
+    videoFile: MVideoFile
+    isNewVideo: boolean
+    user: MUserId
+  }): Promise<any>
+
+  abstract createTranscodingJobs (options: {
+    transcodingType: 'hls' | 'webtorrent'
+    video: MVideoFullLight
+    resolutions: number[]
+    isNewVideo: boolean
+    user: MUserId | null
+  }): Promise<any>
+
+  protected async getTranscodingJobPriority (options: {
+    user: MUserId
+    fallback: number
+  }) {
+    const { user, fallback } = options
+
+    if (!user) return fallback
+
+    const now = new Date()
+    const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
+
+    const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
+
+    return JOB_PRIORITY.TRANSCODING + videoUploadedByUser
+  }
+}

diff --git a/server/lib/transcoding/shared/job-builders/index.ts b/server/lib/transcoding/shared/job-builders/index.ts
new file mode 100644 (file)
index 0000000..9b1c82a
--- /dev/null
+++ b/server/lib/transcoding/shared/job-builders/index.ts
@@ -0,0 +1,2 @@
+export * from './transcoding-job-queue-builder'
+export * from './transcoding-runner-job-builder'

diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
new file mode 100644 (file)
index 0000000..7c89271
--- /dev/null
+++ b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
@@ -0,0 +1,308 @@
+import Bluebird from 'bluebird'
+import { computeOutputFPS } from '@server/helpers/ffmpeg'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
+import { CreateJobArgument, JobQueue } from '@server/lib/job-queue'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info'
+import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models'
+import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg'
+import {
+  HLSTranscodingPayload,
+  MergeAudioTranscodingPayload,
+  NewWebTorrentResolutionTranscodingPayload,
+  OptimizeTranscodingPayload,
+  VideoTranscodingPayload
+} from '@shared/models'
+import { canDoQuickTranscode } from '../../transcoding-quick-transcode'
+import { computeResolutionsToTranscode } from '../../transcoding-resolutions'
+import { AbstractJobBuilder } from './abstract-job-builder'
+
+export class TranscodingJobQueueBuilder extends AbstractJobBuilder {
+
+  async createOptimizeOrMergeAudioJobs (options: {
+    video: MVideoFullLight
+    videoFile: MVideoFile
+    isNewVideo: boolean
+    user: MUserId
+  }) {
+    const { video, videoFile, isNewVideo, user } = options
+
+    let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload
+    let nextTranscodingSequentialJobPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = []
+
+    const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+
+    try {
+      await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => {
+        const probe = await ffprobePromise(videoFilePath)
+
+        const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe)
+        const hasAudio = await hasAudioStream(videoFilePath, probe)
+        const quickTranscode = await canDoQuickTranscode(videoFilePath, probe)
+        const inputFPS = videoFile.isAudio()
+          ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value
+          : await getVideoStreamFPS(videoFilePath, probe)
+
+        const maxResolution = await isAudioFile(videoFilePath, probe)
+          ? DEFAULT_AUDIO_RESOLUTION
+          : resolution
+
+        if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
+          nextTranscodingSequentialJobPayloads.push([
+            this.buildHLSJobPayload({
+              deleteWebTorrentFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false,
+
+              // We had some issues with a web video quick transcoded while producing a HLS version of it
+              copyCodecs: !quickTranscode,
+
+              resolution: maxResolution,
+              fps: computeOutputFPS({ inputFPS, resolution: maxResolution }),
+              videoUUID: video.uuid,
+              isNewVideo
+            })
+          ])
+        }
+
+        const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({
+          video,
+          inputVideoResolution: maxResolution,
+          inputVideoFPS: inputFPS,
+          hasAudio,
+          isNewVideo
+        })
+
+        nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ]
+
+        mergeOrOptimizePayload = videoFile.isAudio()
+          ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo })
+          : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode })
+      })
+    } finally {
+      mutexReleaser()
+    }
+
+    const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => {
+      return Bluebird.mapSeries(payloads, payload => {
+        return this.buildTranscodingJob({ payload, user })
+      })
+    })
+
+    const transcodingJobBuilderJob: CreateJobArgument = {
+      type: 'transcoding-job-builder',
+      payload: {
+        videoUUID: video.uuid,
+        sequentialJobs: nextTranscodingSequentialJobs
+      }
+    }
+
+    const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user })
+
+    return JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ])
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async createTranscodingJobs (options: {
+    transcodingType: 'hls' | 'webtorrent'
+    video: MVideoFullLight
+    resolutions: number[]
+    isNewVideo: boolean
+    user: MUserId | null
+  }) {
+    const { video, transcodingType, resolutions, isNewVideo } = options
+
+    const maxResolution = Math.max(...resolutions)
+    const childrenResolutions = resolutions.filter(r => r !== maxResolution)
+
+    logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution })
+
+    const { fps: inputFPS } = await video.probeMaxQualityFile()
+
+    const children = childrenResolutions.map(resolution => {
+      const fps = computeOutputFPS({ inputFPS, resolution })
+
+      if (transcodingType === 'hls') {
+        return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo })
+      }
+
+      if (transcodingType === 'webtorrent') {
+        return this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo })
+      }
+
+      throw new Error('Unknown transcoding type')
+    })
+
+    const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
+
+    const parent = transcodingType === 'hls'
+      ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo })
+      : this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo })
+
+    // Process the last resolution after the other ones to prevent concurrency issue
+    // Because low resolutions use the biggest one as ffmpeg input
+    await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async createTranscodingJobsWithChildren (options: {
+    videoUUID: string
+    parent: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)
+    children: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)[]
+    user: MUserId | null
+  }) {
+    const { videoUUID, parent, children, user } = options
+
+    const parentJob = await this.buildTranscodingJob({ payload: parent, user })
+    const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user }))
+
+    await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs)
+
+    await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length)
+  }
+
+  private async buildTranscodingJob (options: {
+    payload: VideoTranscodingPayload
+    user: MUserId | null // null means we don't want priority
+  }) {
+    const { user, payload } = options
+
+    return {
+      type: 'video-transcoding' as 'video-transcoding',
+      priority: await this.getTranscodingJobPriority({ user, fallback: undefined }),
+      payload
+    }
+  }
+
+  private async buildLowerResolutionJobPayloads (options: {
+    video: MVideoWithFileThumbnail
+    inputVideoResolution: number
+    inputVideoFPS: number
+    hasAudio: boolean
+    isNewVideo: boolean
+  }) {
+    const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options
+
+    // Create transcoding jobs if there are enabled resolutions
+    const resolutionsEnabled = await Hooks.wrapObject(
+      computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
+      'filter:transcoding.auto.resolutions-to-transcode.result',
+      options
+    )
+
+    const sequentialPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = []
+
+    for (const resolution of resolutionsEnabled) {
+      const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution })
+
+      if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) {
+        const payloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[] = [
+          this.buildWebTorrentJobPayload({
+            videoUUID: video.uuid,
+            resolution,
+            fps,
+            isNewVideo
+          })
+        ]
+
+        // Create a subsequent job to create HLS resolution that will just copy web video codecs
+        if (CONFIG.TRANSCODING.HLS.ENABLED) {
+          payloads.push(
+            this.buildHLSJobPayload({
+              videoUUID: video.uuid,
+              resolution,
+              fps,
+              isNewVideo,
+              copyCodecs: true
+            })
+          )
+        }
+
+        sequentialPayloads.push(payloads)
+      } else if (CONFIG.TRANSCODING.HLS.ENABLED) {
+        sequentialPayloads.push([
+          this.buildHLSJobPayload({
+            videoUUID: video.uuid,
+            resolution,
+            fps,
+            copyCodecs: false,
+            isNewVideo
+          })
+        ])
+      }
+    }
+
+    return sequentialPayloads
+  }
+
+  private buildHLSJobPayload (options: {
+    videoUUID: string
+    resolution: number
+    fps: number
+    isNewVideo: boolean
+    deleteWebTorrentFiles?: boolean // default false
+    copyCodecs?: boolean // default false
+  }): HLSTranscodingPayload {
+    const { videoUUID, resolution, fps, isNewVideo, deleteWebTorrentFiles = false, copyCodecs = false } = options
+
+    return {
+      type: 'new-resolution-to-hls',
+      videoUUID,
+      resolution,
+      fps,
+      copyCodecs,
+      isNewVideo,
+      deleteWebTorrentFiles
+    }
+  }
+
+  private buildWebTorrentJobPayload (options: {
+    videoUUID: string
+    resolution: number
+    fps: number
+    isNewVideo: boolean
+  }): NewWebTorrentResolutionTranscodingPayload {
+    const { videoUUID, resolution, fps, isNewVideo } = options
+
+    return {
+      type: 'new-resolution-to-webtorrent',
+      videoUUID,
+      isNewVideo,
+      resolution,
+      fps
+    }
+  }
+
+  private buildMergeAudioPayload (options: {
+    videoUUID: string
+    isNewVideo: boolean
+  }): MergeAudioTranscodingPayload {
+    const { videoUUID, isNewVideo } = options
+
+    return {
+      type: 'merge-audio-to-webtorrent',
+      resolution: DEFAULT_AUDIO_RESOLUTION,
+      fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE,
+      videoUUID,
+      isNewVideo
+    }
+  }
+
+  private buildOptimizePayload (options: {
+    videoUUID: string
+    quickTranscode: boolean
+    isNewVideo: boolean
+  }): OptimizeTranscodingPayload {
+    const { videoUUID, quickTranscode, isNewVideo } = options
+
+    return {
+      type: 'optimize-to-webtorrent',
+      videoUUID,
+      isNewVideo,
+      quickTranscode
+    }
+  }
+}

diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
new file mode 100644 (file)
index 0000000..c7a63d2
--- /dev/null
+++ b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
@@ -0,0 +1,189 @@
+import { computeOutputFPS } from '@server/helpers/ffmpeg'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models'
+import { MRunnerJob } from '@server/types/models/runners'
+import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg'
+import { computeResolutionsToTranscode } from '../../transcoding-resolutions'
+import { AbstractJobBuilder } from './abstract-job-builder'
+
+/**
+ *
+ * Class to build transcoding job in the local job queue
+ *
+ */
+
+const lTags = loggerTagsFactory('transcoding')
+
+export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
+
+  async createOptimizeOrMergeAudioJobs (options: {
+    video: MVideoFullLight
+    videoFile: MVideoFile
+    isNewVideo: boolean
+    user: MUserId
+  }) {
+    const { video, videoFile, isNewVideo, user } = options
+
+    const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+
+    try {
+      await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => {
+        const probe = await ffprobePromise(videoFilePath)
+
+        const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe)
+        const hasAudio = await hasAudioStream(videoFilePath, probe)
+        const inputFPS = videoFile.isAudio()
+          ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value
+          : await getVideoStreamFPS(videoFilePath, probe)
+
+        const maxResolution = await isAudioFile(videoFilePath, probe)
+          ? DEFAULT_AUDIO_RESOLUTION
+          : resolution
+
+        const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
+        const priority = await this.getTranscodingJobPriority({ user, fallback: 0 })
+
+        const mainRunnerJob = videoFile.isAudio()
+          ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority })
+          : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority })
+
+        if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
+          await new VODHLSTranscodingJobHandler().create({
+            video,
+            deleteWebVideoFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false,
+            resolution: maxResolution,
+            fps,
+            isNewVideo,
+            dependsOnRunnerJob: mainRunnerJob,
+            priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
+          })
+        }
+
+        await this.buildLowerResolutionJobPayloads({
+          video,
+          inputVideoResolution: maxResolution,
+          inputVideoFPS: inputFPS,
+          hasAudio,
+          isNewVideo,
+          mainRunnerJob,
+          user
+        })
+      })
+    } finally {
+      mutexReleaser()
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async createTranscodingJobs (options: {
+    transcodingType: 'hls' | 'webtorrent'
+    video: MVideoFullLight
+    resolutions: number[]
+    isNewVideo: boolean
+    user: MUserId | null
+  }) {
+    const { video, transcodingType, resolutions, isNewVideo, user } = options
+
+    const maxResolution = Math.max(...resolutions)
+    const { fps: inputFPS } = await video.probeMaxQualityFile()
+    const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution })
+    const priority = await this.getTranscodingJobPriority({ user, fallback: 0 })
+
+    const childrenResolutions = resolutions.filter(r => r !== maxResolution)
+
+    logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution })
+
+    // Process the last resolution before the other ones to prevent concurrency issue
+    // Because low resolutions use the biggest one as ffmpeg input
+    const mainJob = transcodingType === 'hls'
+      // eslint-disable-next-line max-len
+      ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority })
+      : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority })
+
+    for (const resolution of childrenResolutions) {
+      const dependsOnRunnerJob = mainJob
+      const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
+
+      if (transcodingType === 'hls') {
+        await new VODHLSTranscodingJobHandler().create({
+          video,
+          resolution,
+          fps,
+          isNewVideo,
+          deleteWebVideoFiles: false,
+          dependsOnRunnerJob,
+          priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
+        })
+        continue
+      }
+
+      if (transcodingType === 'webtorrent') {
+        await new VODWebVideoTranscodingJobHandler().create({
+          video,
+          resolution,
+          fps,
+          isNewVideo,
+          dependsOnRunnerJob,
+          priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
+        })
+        continue
+      }
+
+      throw new Error('Unknown transcoding type')
+    }
+  }
+
+  private async buildLowerResolutionJobPayloads (options: {
+    mainRunnerJob: MRunnerJob
+    video: MVideoWithFileThumbnail
+    inputVideoResolution: number
+    inputVideoFPS: number
+    hasAudio: boolean
+    isNewVideo: boolean
+    user: MUserId
+  }) {
+    const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options
+
+    // Create transcoding jobs if there are enabled resolutions
+    const resolutionsEnabled = await Hooks.wrapObject(
+      computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
+      'filter:transcoding.auto.resolutions-to-transcode.result',
+      options
+    )
+
+    logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) })
+
+    for (const resolution of resolutionsEnabled) {
+      const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution })
+
+      if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) {
+        await new VODWebVideoTranscodingJobHandler().create({
+          video,
+          resolution,
+          fps,
+          isNewVideo,
+          dependsOnRunnerJob: mainRunnerJob,
+          priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
+        })
+      }
+
+      if (CONFIG.TRANSCODING.HLS.ENABLED) {
+        await new VODHLSTranscodingJobHandler().create({
+          video,
+          resolution,
+          fps,
+          isNewVideo,
+          deleteWebVideoFiles: false,
+          dependsOnRunnerJob: mainRunnerJob,
+          priority: await this.getTranscodingJobPriority({ user, fallback: 0 })
+        })
+      }
+    }
+  }
+}

diff --git a/server/lib/transcoding/transcoding-quick-transcode.ts b/server/lib/transcoding/transcoding-quick-transcode.ts
new file mode 100644 (file)
index 0000000..b7f9218
--- /dev/null
+++ b/server/lib/transcoding/transcoding-quick-transcode.ts
@@ -0,0 +1,61 @@
+import { FfprobeData } from 'fluent-ffmpeg'
+import { CONFIG } from '@server/initializers/config'
+import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
+import { getMaxBitrate } from '@shared/core-utils'
+import {
+  ffprobePromise,
+  getAudioStream,
+  getMaxAudioBitrate,
+  getVideoStream,
+  getVideoStreamBitrate,
+  getVideoStreamDimensionsInfo,
+  getVideoStreamFPS
+} from '@shared/ffmpeg'
+
+export async function canDoQuickTranscode (path: string, existingProbe?: FfprobeData): Promise<boolean> {
+  if (CONFIG.TRANSCODING.PROFILE !== 'default') return false
+
+  const probe = existingProbe || await ffprobePromise(path)
+
+  return await canDoQuickVideoTranscode(path, probe) &&
+         await canDoQuickAudioTranscode(path, probe)
+}
+
+export async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
+  const parsedAudio = await getAudioStream(path, probe)
+
+  if (!parsedAudio.audioStream) return true
+
+  if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
+
+  const audioBitrate = parsedAudio.bitrate
+  if (!audioBitrate) return false
+
+  const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate)
+  if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false
+
+  const channelLayout = parsedAudio.audioStream['channel_layout']
+  // Causes playback issues with Chrome
+  if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false
+
+  return true
+}
+
+export async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
+  const videoStream = await getVideoStream(path, probe)
+  const fps = await getVideoStreamFPS(path, probe)
+  const bitRate = await getVideoStreamBitrate(path, probe)
+  const resolutionData = await getVideoStreamDimensionsInfo(path, probe)
+
+  // If ffprobe did not manage to guess the bitrate
+  if (!bitRate) return false
+
+  // check video params
+  if (!videoStream) return false
+  if (videoStream['codec_name'] !== 'h264') return false
+  if (videoStream['pix_fmt'] !== 'yuv420p') return false
+  if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
+  if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false
+
+  return true
+}

diff --git a/server/lib/transcoding/transcoding-resolutions.ts b/server/lib/transcoding/transcoding-resolutions.ts
new file mode 100644 (file)
index 0000000..91f4d18
--- /dev/null
+++ b/server/lib/transcoding/transcoding-resolutions.ts
@@ -0,0 +1,52 @@
+import { CONFIG } from '@server/initializers/config'
+import { toEven } from '@shared/core-utils'
+import { VideoResolution } from '@shared/models'
+
+export function computeResolutionsToTranscode (options: {
+  input: number
+  type: 'vod' | 'live'
+  includeInput: boolean
+  strictLower: boolean
+  hasAudio: boolean
+}) {
+  const { input, type, includeInput, strictLower, hasAudio } = options
+
+  const configResolutions = type === 'vod'
+    ? CONFIG.TRANSCODING.RESOLUTIONS
+    : CONFIG.LIVE.TRANSCODING.RESOLUTIONS
+
+  const resolutionsEnabled = new Set<number>()
+
+  // Put in the order we want to proceed jobs
+  const availableResolutions: VideoResolution[] = [
+    VideoResolution.H_NOVIDEO,
+    VideoResolution.H_480P,
+    VideoResolution.H_360P,
+    VideoResolution.H_720P,
+    VideoResolution.H_240P,
+    VideoResolution.H_144P,
+    VideoResolution.H_1080P,
+    VideoResolution.H_1440P,
+    VideoResolution.H_4K
+  ]
+
+  for (const resolution of availableResolutions) {
+    // Resolution not enabled
+    if (configResolutions[resolution + 'p'] !== true) continue
+    // Too big resolution for input file
+    if (input < resolution) continue
+    // We only want lower resolutions than input file
+    if (strictLower && input === resolution) continue
+    // Audio resolutio but no audio in the video
+    if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue
+
+    resolutionsEnabled.add(resolution)
+  }
+
+  if (includeInput) {
+    // Always use an even resolution to avoid issues with ffmpeg
+    resolutionsEnabled.add(toEven(input))
+  }
+
+  return Array.from(resolutionsEnabled)
+}

diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts
deleted file mode 100644 (file)
index c7b61e9..0000000
--- a/server/lib/transcoding/transcoding.ts
+++ /dev/null
@@ -1,465 +0,0 @@
-import { MutexInterface } from 'async-mutex'
-import { Job } from 'bullmq'
-import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
-import { basename, extname as extnameUtil, join } from 'path'
-import { toEven } from '@server/helpers/core-utils'
-import { retryTransactionWrapper } from '@server/helpers/database-utils'
-import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
-import { sequelizeTypescript } from '@server/initializers/database'
-import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
-import { pick } from '@shared/core-utils'
-import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
-import {
-  buildFileMetadata,
-  canDoQuickTranscode,
-  computeResolutionsToTranscode,
-  ffprobePromise,
-  getVideoStreamDuration,
-  getVideoStreamFPS,
-  transcodeVOD,
-  TranscodeVODOptions,
-  TranscodeVODOptionsType
-} from '../../helpers/ffmpeg'
-import { CONFIG } from '../../initializers/config'
-import { VideoFileModel } from '../../models/video/video-file'
-import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
-import { updatePlaylistAfterFileChange } from '../hls'
-import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths'
-import { VideoPathManager } from '../video-path-manager'
-import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
-
-/**
- *
- * Functions that run transcoding functions, update the database, cleanup files, create torrent files...
- * Mainly called by the job queue
- *
- */
-
-// Optimize the original video file and replace it. The resolution is not changed.
-async function optimizeOriginalVideofile (options: {
-  video: MVideoFullLight
-  inputVideoFile: MVideoFile
-  job: Job
-}) {
-  const { video, inputVideoFile, job } = options
-
-  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
-  const newExtname = '.mp4'
-
-  // Will be released by our transcodeVOD function once ffmpeg is ran
-  const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
-
-  try {
-    await video.reload()
-
-    const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
-
-    const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
-      const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
-
-      const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
-        ? 'quick-transcode'
-        : 'video'
-
-      const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
-
-      const transcodeOptions: TranscodeVODOptions = {
-        type: transcodeType,
-
-        inputPath: videoInputPath,
-        outputPath: videoTranscodedPath,
-
-        inputFileMutexReleaser,
-
-        availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-        profile: CONFIG.TRANSCODING.PROFILE,
-
-        resolution,
-
-        job
-      }
-
-      // Could be very long!
-      await transcodeVOD(transcodeOptions)
-
-      // Important to do this before getVideoFilename() to take in account the new filename
-      inputVideoFile.resolution = resolution
-      inputVideoFile.extname = newExtname
-      inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
-      inputVideoFile.storage = VideoStorage.FILE_SYSTEM
-
-      const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
-      await remove(videoInputPath)
-
-      return { transcodeType, videoFile }
-    })
-
-    return result
-  } finally {
-    inputFileMutexReleaser()
-  }
-}
-
-// Transcode the original video file to a lower resolution compatible with WebTorrent
-async function transcodeNewWebTorrentResolution (options: {
-  video: MVideoFullLight
-  resolution: VideoResolution
-  job: Job
-}) {
-  const { video, resolution, job } = options
-
-  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
-  const newExtname = '.mp4'
-
-  const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
-
-  try {
-    await video.reload()
-
-    const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
-
-    const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
-      const newVideoFile = new VideoFileModel({
-        resolution,
-        extname: newExtname,
-        filename: generateWebTorrentVideoFilename(resolution, newExtname),
-        size: 0,
-        videoId: video.id
-      })
-
-      const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
-
-      const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
-        ? {
-          type: 'only-audio' as 'only-audio',
-
-          inputPath: videoInputPath,
-          outputPath: videoTranscodedPath,
-
-          inputFileMutexReleaser,
-
-          availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-          profile: CONFIG.TRANSCODING.PROFILE,
-
-          resolution,
-
-          job
-        }
-        : {
-          type: 'video' as 'video',
-          inputPath: videoInputPath,
-          outputPath: videoTranscodedPath,
-
-          inputFileMutexReleaser,
-
-          availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-          profile: CONFIG.TRANSCODING.PROFILE,
-
-          resolution,
-
-          job
-        }
-
-      await transcodeVOD(transcodeOptions)
-
-      return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
-    })
-
-    return result
-  } finally {
-    inputFileMutexReleaser()
-  }
-}
-
-// Merge an image with an audio file to create a video
-async function mergeAudioVideofile (options: {
-  video: MVideoFullLight
-  resolution: VideoResolution
-  job: Job
-}) {
-  const { video, resolution, job } = options
-
-  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
-  const newExtname = '.mp4'
-
-  const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
-
-  try {
-    await video.reload()
-
-    const inputVideoFile = video.getMinQualityFile()
-
-    const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
-
-    const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
-      const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
-
-      // If the user updates the video preview during transcoding
-      const previewPath = video.getPreview().getPath()
-      const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
-      await copyFile(previewPath, tmpPreviewPath)
-
-      const transcodeOptions = {
-        type: 'merge-audio' as 'merge-audio',
-
-        inputPath: tmpPreviewPath,
-        outputPath: videoTranscodedPath,
-
-        inputFileMutexReleaser,
-
-        availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-        profile: CONFIG.TRANSCODING.PROFILE,
-
-        audioPath: audioInputPath,
-        resolution,
-
-        job
-      }
-
-      try {
-        await transcodeVOD(transcodeOptions)
-
-        await remove(audioInputPath)
-        await remove(tmpPreviewPath)
-      } catch (err) {
-        await remove(tmpPreviewPath)
-        throw err
-      }
-
-      // Important to do this before getVideoFilename() to take in account the new file extension
-      inputVideoFile.extname = newExtname
-      inputVideoFile.resolution = resolution
-      inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
-
-      // ffmpeg generated a new video file, so update the video duration
-      // See https://trac.ffmpeg.org/ticket/5456
-      video.duration = await getVideoStreamDuration(videoTranscodedPath)
-      await video.save()
-
-      return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
-    })
-
-    return result
-  } finally {
-    inputFileMutexReleaser()
-  }
-}
-
-// Concat TS segments from a live video to a fragmented mp4 HLS playlist
-async function generateHlsPlaylistResolutionFromTS (options: {
-  video: MVideo
-  concatenatedTsFilePath: string
-  resolution: VideoResolution
-  isAAC: boolean
-  inputFileMutexReleaser: MutexInterface.Releaser
-}) {
-  return generateHlsPlaylistCommon({
-    type: 'hls-from-ts' as 'hls-from-ts',
-    inputPath: options.concatenatedTsFilePath,
-
-    ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
-  })
-}
-
-// Generate an HLS playlist from an input file, and update the master playlist
-function generateHlsPlaylistResolution (options: {
-  video: MVideo
-  videoInputPath: string
-  resolution: VideoResolution
-  copyCodecs: boolean
-  inputFileMutexReleaser: MutexInterface.Releaser
-  job?: Job
-}) {
-  return generateHlsPlaylistCommon({
-    type: 'hls' as 'hls',
-    inputPath: options.videoInputPath,
-
-    ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
-  })
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  generateHlsPlaylistResolution,
-  generateHlsPlaylistResolutionFromTS,
-  optimizeOriginalVideofile,
-  transcodeNewWebTorrentResolution,
-  mergeAudioVideofile
-}
-
-// ---------------------------------------------------------------------------
-
-async function onWebTorrentVideoFileTranscoding (
-  video: MVideoFullLight,
-  videoFile: MVideoFile,
-  transcodingPath: string,
-  newVideoFile: MVideoFile
-) {
-  const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
-
-  try {
-    await video.reload()
-
-    const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
-
-    const stats = await stat(transcodingPath)
-
-    const probe = await ffprobePromise(transcodingPath)
-    const fps = await getVideoStreamFPS(transcodingPath, probe)
-    const metadata = await buildFileMetadata(transcodingPath, probe)
-
-    await move(transcodingPath, outputPath, { overwrite: true })
-
-    videoFile.size = stats.size
-    videoFile.fps = fps
-    videoFile.metadata = metadata
-
-    await createTorrentAndSetInfoHash(video, videoFile)
-
-    const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
-    if (oldFile) await video.removeWebTorrentFile(oldFile)
-
-    await VideoFileModel.customUpsert(videoFile, 'video', undefined)
-    video.VideoFiles = await video.$get('VideoFiles')
-
-    return { video, videoFile }
-  } finally {
-    mutexReleaser()
-  }
-}
-
-async function generateHlsPlaylistCommon (options: {
-  type: 'hls' | 'hls-from-ts'
-  video: MVideo
-  inputPath: string
-  resolution: VideoResolution
-
-  inputFileMutexReleaser: MutexInterface.Releaser
-
-  copyCodecs?: boolean
-  isAAC?: boolean
-
-  job?: Job
-}) {
-  const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
-  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
-
-  const videoTranscodedBasePath = join(transcodeDirectory, type)
-  await ensureDir(videoTranscodedBasePath)
-
-  const videoFilename = generateHLSVideoFilename(resolution)
-  const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
-  const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
-
-  const transcodeOptions = {
-    type,
-
-    inputPath,
-    outputPath: resolutionPlaylistFileTranscodePath,
-
-    availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
-    profile: CONFIG.TRANSCODING.PROFILE,
-
-    resolution,
-    copyCodecs,
-
-    isAAC,
-
-    inputFileMutexReleaser,
-
-    hlsPlaylist: {
-      videoFilename
-    },
-
-    job
-  }
-
-  await transcodeVOD(transcodeOptions)
-
-  // Create or update the playlist
-  const playlist = await retryTransactionWrapper(() => {
-    return sequelizeTypescript.transaction(async transaction => {
-      return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
-    })
-  })
-
-  const newVideoFile = new VideoFileModel({
-    resolution,
-    extname: extnameUtil(videoFilename),
-    size: 0,
-    filename: videoFilename,
-    fps: -1,
-    videoStreamingPlaylistId: playlist.id
-  })
-
-  const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
-
-  try {
-    // VOD transcoding is a long task, refresh video attributes
-    await video.reload()
-
-    const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
-    await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
-
-    // Move playlist file
-    const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
-    await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
-    // Move video file
-    await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
-
-    // Update video duration if it was not set (in case of a live for example)
-    if (!video.duration) {
-      video.duration = await getVideoStreamDuration(videoFilePath)
-      await video.save()
-    }
-
-    const stats = await stat(videoFilePath)
-
-    newVideoFile.size = stats.size
-    newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
-    newVideoFile.metadata = await buildFileMetadata(videoFilePath)
-
-    await createTorrentAndSetInfoHash(playlist, newVideoFile)
-
-    const oldFile = await VideoFileModel.loadHLSFile({
-      playlistId: playlist.id,
-      fps: newVideoFile.fps,
-      resolution: newVideoFile.resolution
-    })
-
-    if (oldFile) {
-      await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
-      await oldFile.destroy()
-    }
-
-    const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
-
-    await updatePlaylistAfterFileChange(video, playlist)
-
-    return { resolutionPlaylistPath, videoFile: savedVideoFile }
-  } finally {
-    mutexReleaser()
-  }
-}
-
-function buildOriginalFileResolution (inputResolution: number) {
-  if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) {
-    return toEven(inputResolution)
-  }
-
-  const resolutions = computeResolutionsToTranscode({
-    input: inputResolution,
-    type: 'vod',
-    includeInput: false,
-    strictLower: false,
-    // We don't really care about the audio resolution in this context
-    hasAudio: true
-  })
-
-  if (resolutions.length === 0) {
-    return toEven(inputResolution)
-  }
-
-  return Math.max(...resolutions)
-}

diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts
new file mode 100644 (file)
index 0000000..d43d03b
--- /dev/null
+++ b/server/lib/transcoding/web-transcoding.ts
@@ -0,0 +1,273 @@
+import { Job } from 'bullmq'
+import { copyFile, move, remove, stat } from 'fs-extra'
+import { basename, join } from 'path'
+import { computeOutputFPS } from '@server/helpers/ffmpeg'
+import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
+import { MVideoFile, MVideoFullLight } from '@server/types/models'
+import { toEven } from '@shared/core-utils'
+import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVODOptionsType } from '@shared/ffmpeg'
+import { VideoResolution, VideoStorage } from '@shared/models'
+import { CONFIG } from '../../initializers/config'
+import { VideoFileModel } from '../../models/video/video-file'
+import { generateWebTorrentVideoFilename } from '../paths'
+import { buildFileMetadata } from '../video-file'
+import { VideoPathManager } from '../video-path-manager'
+import { buildFFmpegVOD } from './shared'
+import { computeResolutionsToTranscode } from './transcoding-resolutions'
+
+// Optimize the original video file and replace it. The resolution is not changed.
+export async function optimizeOriginalVideofile (options: {
+  video: MVideoFullLight
+  inputVideoFile: MVideoFile
+  quickTranscode: boolean
+  job: Job
+}) {
+  const { video, inputVideoFile, quickTranscode, job } = options
+
+  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
+  const newExtname = '.mp4'
+
+  // Will be released by our transcodeVOD function once ffmpeg is ran
+  const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+
+  try {
+    await video.reload()
+
+    const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
+
+    const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
+      const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
+
+      const transcodeType: TranscodeVODOptionsType = quickTranscode
+        ? 'quick-transcode'
+        : 'video'
+
+      const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
+      const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution })
+
+      // Could be very long!
+      await buildFFmpegVOD(job).transcode({
+        type: transcodeType,
+
+        inputPath: videoInputPath,
+        outputPath: videoOutputPath,
+
+        inputFileMutexReleaser,
+
+        resolution,
+        fps
+      })
+
+      // Important to do this before getVideoFilename() to take in account the new filename
+      inputVideoFile.resolution = resolution
+      inputVideoFile.extname = newExtname
+      inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
+      inputVideoFile.storage = VideoStorage.FILE_SYSTEM
+
+      const { videoFile } = await onWebTorrentVideoFileTranscoding({
+        video,
+        videoFile: inputVideoFile,
+        videoOutputPath
+      })
+
+      await remove(videoInputPath)
+
+      return { transcodeType, videoFile }
+    })
+
+    return result
+  } finally {
+    inputFileMutexReleaser()
+  }
+}
+
+// Transcode the original video file to a lower resolution compatible with WebTorrent
+export async function transcodeNewWebTorrentResolution (options: {
+  video: MVideoFullLight
+  resolution: VideoResolution
+  fps: number
+  job: Job
+}) {
+  const { video, resolution, fps, job } = options
+
+  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
+  const newExtname = '.mp4'
+
+  const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+
+  try {
+    await video.reload()
+
+    const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
+
+    const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
+      const newVideoFile = new VideoFileModel({
+        resolution,
+        extname: newExtname,
+        filename: generateWebTorrentVideoFilename(resolution, newExtname),
+        size: 0,
+        videoId: video.id
+      })
+
+      const videoOutputPath = join(transcodeDirectory, newVideoFile.filename)
+
+      const transcodeOptions = {
+        type: 'video' as 'video',
+
+        inputPath: videoInputPath,
+        outputPath: videoOutputPath,
+
+        inputFileMutexReleaser,
+
+        resolution,
+        fps
+      }
+
+      await buildFFmpegVOD(job).transcode(transcodeOptions)
+
+      return onWebTorrentVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath })
+    })
+
+    return result
+  } finally {
+    inputFileMutexReleaser()
+  }
+}
+
+// Merge an image with an audio file to create a video
+export async function mergeAudioVideofile (options: {
+  video: MVideoFullLight
+  resolution: VideoResolution
+  fps: number
+  job: Job
+}) {
+  const { video, resolution, fps, job } = options
+
+  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
+  const newExtname = '.mp4'
+
+  const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+
+  try {
+    await video.reload()
+
+    const inputVideoFile = video.getMinQualityFile()
+
+    const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
+
+    const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
+      const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
+
+      // If the user updates the video preview during transcoding
+      const previewPath = video.getPreview().getPath()
+      const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
+      await copyFile(previewPath, tmpPreviewPath)
+
+      const transcodeOptions = {
+        type: 'merge-audio' as 'merge-audio',
+
+        inputPath: tmpPreviewPath,
+        outputPath: videoOutputPath,
+
+        inputFileMutexReleaser,
+
+        audioPath: audioInputPath,
+        resolution,
+        fps
+      }
+
+      try {
+        await buildFFmpegVOD(job).transcode(transcodeOptions)
+
+        await remove(audioInputPath)
+        await remove(tmpPreviewPath)
+      } catch (err) {
+        await remove(tmpPreviewPath)
+        throw err
+      }
+
+      // Important to do this before getVideoFilename() to take in account the new file extension
+      inputVideoFile.extname = newExtname
+      inputVideoFile.resolution = resolution
+      inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
+
+      // ffmpeg generated a new video file, so update the video duration
+      // See https://trac.ffmpeg.org/ticket/5456
+      video.duration = await getVideoStreamDuration(videoOutputPath)
+      await video.save()
+
+      return onWebTorrentVideoFileTranscoding({
+        video,
+        videoFile: inputVideoFile,
+        videoOutputPath
+      })
+    })
+
+    return result
+  } finally {
+    inputFileMutexReleaser()
+  }
+}
+
+export async function onWebTorrentVideoFileTranscoding (options: {
+  video: MVideoFullLight
+  videoFile: MVideoFile
+  videoOutputPath: string
+}) {
+  const { video, videoFile, videoOutputPath } = options
+
+  const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
+
+  try {
+    await video.reload()
+
+    const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
+
+    const stats = await stat(videoOutputPath)
+
+    const probe = await ffprobePromise(videoOutputPath)
+    const fps = await getVideoStreamFPS(videoOutputPath, probe)
+    const metadata = await buildFileMetadata(videoOutputPath, probe)
+
+    await move(videoOutputPath, outputPath, { overwrite: true })
+
+    videoFile.size = stats.size
+    videoFile.fps = fps
+    videoFile.metadata = metadata
+
+    await createTorrentAndSetInfoHash(video, videoFile)
+
+    const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
+    if (oldFile) await video.removeWebTorrentFile(oldFile)
+
+    await VideoFileModel.customUpsert(videoFile, 'video', undefined)
+    video.VideoFiles = await video.$get('VideoFiles')
+
+    return { video, videoFile }
+  } finally {
+    mutexReleaser()
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+function buildOriginalFileResolution (inputResolution: number) {
+  if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) {
+    return toEven(inputResolution)
+  }
+
+  const resolutions = computeResolutionsToTranscode({
+    input: inputResolution,
+    type: 'vod',
+    includeInput: false,
+    strictLower: false,
+    // We don't really care about the audio resolution in this context
+    hasAudio: true
+  })
+
+  if (resolutions.length === 0) {
+    return toEven(inputResolution)
+  }
+
+  return Math.max(...resolutions)
+}

diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts
index 58040cb6d679d5c1adf613242bec5c2ffa4526e9..c7e0eb414e8e501dd8f90d2e6722fd85e46cde2b 100644 (file)
--- a/server/lib/uploadx.ts
+++ b/server/lib/uploadx.ts
@@ -3,6 +3,7 @@ import { buildLogger } from '@server/helpers/logger'
 import { getResumableUploadPath } from '@server/helpers/upload'
 import { CONFIG } from '@server/initializers/config'
 import { LogLevel, Uploadx } from '@uploadx/core'
 import { getResumableUploadPath } from '@server/helpers/upload'
 import { CONFIG } from '@server/initializers/config'
 import { LogLevel, Uploadx } from '@uploadx/core'

+import { extname } from 'path'

 
 const logger = buildLogger('uploadx')
 
 
 const logger = buildLogger('uploadx')
 


@@ -26,7 +27,9 @@ const uploadx = new Uploadx({
     if (!res.locals.oauth) return undefined
 
     return res.locals.oauth.token.user.id + ''
     if (!res.locals.oauth) return undefined
 
     return res.locals.oauth.token.user.id + ''

-  }
+  },
+
+  filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}`

 })
 
 export {
 })
 
 export {

diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts
index fd5837a3a19bd394611586dc07142722beb7fe4c..cb1ea834c46dc32a0b5f52a875999db98fe961f3 100644 (file)
--- a/server/lib/video-blacklist.ts
+++ b/server/lib/video-blacklist.ts
@@ -81,7 +81,7 @@ async function blacklistVideo (videoInstance: MVideoAccountLight, options: Video
   }
 
   if (videoInstance.isLive) {
   }
 
   if (videoInstance.isLive) {

-    LiveManager.Instance.stopSessionOf(videoInstance.id, LiveVideoError.BLACKLISTED)
+    LiveManager.Instance.stopSessionOf(videoInstance.uuid, LiveVideoError.BLACKLISTED)

   }
 
   Notifier.Instance.notifyOnVideoBlacklist(blacklist)
   }
 
   Notifier.Instance.notifyOnVideoBlacklist(blacklist)

diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts
index 2ab7190f1175945bfd96145226bda97fe09bb5c6..8fcc3c2532677eb419e5e89c24bef41358286fdc 100644 (file)
--- a/server/lib/video-file.ts
+++ b/server/lib/video-file.ts
@@ -1,6 +1,44 @@
+import { FfprobeData } from 'fluent-ffmpeg'

 import { logger } from '@server/helpers/logger'
 import { logger } from '@server/helpers/logger'

+import { VideoFileModel } from '@server/models/video/video-file'

 import { MVideoWithAllFiles } from '@server/types/models'
 import { MVideoWithAllFiles } from '@server/types/models'

+import { getLowercaseExtension } from '@shared/core-utils'
+import { getFileSize } from '@shared/extra-utils'
+import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg'
+import { VideoFileMetadata, VideoResolution } from '@shared/models'

 import { lTags } from './object-storage/shared'
 import { lTags } from './object-storage/shared'

+import { generateHLSVideoFilename, generateWebTorrentVideoFilename } from './paths'
+
+async function buildNewFile (options: {
+  path: string
+  mode: 'web-video' | 'hls'
+}) {
+  const { path, mode } = options
+
+  const probe = await ffprobePromise(path)
+  const size = await getFileSize(path)
+
+  const videoFile = new VideoFileModel({
+    extname: getLowercaseExtension(path),
+    size,
+    metadata: await buildFileMetadata(path, probe)
+  })
+
+  if (await isAudioFile(path, probe)) {
+    videoFile.resolution = VideoResolution.H_NOVIDEO
+  } else {
+    videoFile.fps = await getVideoStreamFPS(path, probe)
+    videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution
+  }
+
+  videoFile.filename = mode === 'web-video'
+    ? generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname)
+    : generateHLSVideoFilename(videoFile.resolution)
+
+  return videoFile
+}
+
+// ---------------------------------------------------------------------------

 
 async function removeHLSPlaylist (video: MVideoWithAllFiles) {
   const hls = video.getHLSPlaylist()
 
 async function removeHLSPlaylist (video: MVideoWithAllFiles) {
   const hls = video.getHLSPlaylist()


@@ -61,9 +99,23 @@ async function removeWebTorrentFile (video: MVideoWithAllFiles, fileToDeleteId:
   return video
 }
 
   return video
 }
 

+// ---------------------------------------------------------------------------
+
+async function buildFileMetadata (path: string, existingProbe?: FfprobeData) {
+  const metadata = existingProbe || await ffprobePromise(path)
+
+  return new VideoFileMetadata(metadata)
+}
+
+// ---------------------------------------------------------------------------
+

 export {
 export {

+  buildNewFile,
+

   removeHLSPlaylist,
   removeHLSFile,
   removeAllWebTorrentFiles,
   removeHLSPlaylist,
   removeHLSFile,
   removeAllWebTorrentFiles,

-  removeWebTorrentFile
+  removeWebTorrentFile,
+
+  buildFileMetadata

 }
 }

diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts
index cdacd35f2c15bead905ad3fb0dcec76f213382da..b392bdb0080c975bbcd78bb9c55cb14a467e9fb6 100644 (file)
--- a/server/lib/video-studio.ts
+++ b/server/lib/video-studio.ts
@@ -1,5 +1,5 @@
 import { MVideoFullLight } from '@server/types/models'
 import { MVideoFullLight } from '@server/types/models'

-import { getVideoStreamDuration } from '@shared/extra-utils'
+import { getVideoStreamDuration } from '@shared/ffmpeg'

 import { VideoStudioTask } from '@shared/models'
 
 function buildTaskFileFieldname (indice: number, fieldName = 'file') {
 import { VideoStudioTask } from '@shared/models'
 
 function buildTaskFileFieldname (indice: number, fieldName = 'file') {

diff --git a/server/lib/video.ts b/server/lib/video.ts
index aacc41a7a953fb09b4e2676fa804331f23e1c756..588dc553fafb4fe9922a95cc95d6fa288a293e66 100644 (file)
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -2,14 +2,14 @@ import { UploadFiles } from 'express'
 import memoizee from 'memoizee'
 import { Transaction } from 'sequelize/types'
 import { CONFIG } from '@server/initializers/config'
 import memoizee from 'memoizee'
 import { Transaction } from 'sequelize/types'
 import { CONFIG } from '@server/initializers/config'

-import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY, MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants'
+import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants'

 import { TagModel } from '@server/models/video/tag'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { FilteredModelAttributes } from '@server/types'
 import { TagModel } from '@server/models/video/tag'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { FilteredModelAttributes } from '@server/types'

-import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
-import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
-import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue'
+import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
+import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models'
+import { CreateJobArgument, JobQueue } from './job-queue/job-queue'

 import { updateVideoMiniatureFromExisting } from './thumbnail'
 import { moveFilesIfPrivacyChanged } from './video-privacy'
 
 import { updateVideoMiniatureFromExisting } from './thumbnail'
 import { moveFilesIfPrivacyChanged } from './video-privacy'
 


@@ -87,58 +87,6 @@ async function setVideoTags (options: {
 
 // ---------------------------------------------------------------------------
 
 
 // ---------------------------------------------------------------------------
 

-async function buildOptimizeOrMergeAudioJob (options: {
-  video: MVideoUUID
-  videoFile: MVideoFile
-  user: MUserId
-  isNewVideo?: boolean // Default true
-}) {
-  const { video, videoFile, user, isNewVideo } = options
-
-  let payload: VideoTranscodingPayload
-
-  if (videoFile.isAudio()) {
-    payload = {
-      type: 'merge-audio-to-webtorrent',
-      resolution: DEFAULT_AUDIO_RESOLUTION,
-      videoUUID: video.uuid,
-      createHLSIfNeeded: true,
-      isNewVideo
-    }
-  } else {
-    payload = {
-      type: 'optimize-to-webtorrent',
-      videoUUID: video.uuid,
-      isNewVideo
-    }
-  }
-
-  await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
-
-  return {
-    type: 'video-transcoding' as 'video-transcoding',
-    priority: await getTranscodingJobPriority(user),
-    payload
-  }
-}
-
-async function buildTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions = {}) {
-  await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
-
-  return { type: 'video-transcoding' as 'video-transcoding', payload, ...options }
-}
-
-async function getTranscodingJobPriority (user: MUserId) {
-  const now = new Date()
-  const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
-
-  const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
-
-  return JOB_PRIORITY.TRANSCODING + videoUploadedByUser
-}
-
-// ---------------------------------------------------------------------------
-

 async function buildMoveToObjectStorageJob (options: {
   video: MVideoUUID
   previousVideoState: VideoState
 async function buildMoveToObjectStorageJob (options: {
   video: MVideoUUID
   previousVideoState: VideoState


@@ -235,10 +183,7 @@ export {
   buildLocalVideoFromReq,
   buildVideoThumbnailsFromReq,
   setVideoTags,
   buildLocalVideoFromReq,
   buildVideoThumbnailsFromReq,
   setVideoTags,

-  buildOptimizeOrMergeAudioJob,
-  buildTranscodingJob,

   buildMoveToObjectStorageJob,
   buildMoveToObjectStorageJob,

-  getTranscodingJobPriority,

   addVideoJobsAfterUpdate,
   getCachedVideoDuration
 }
   addVideoJobsAfterUpdate,
   getCachedVideoDuration
 }

diff --git a/server/middlewares/auth.ts b/server/middlewares/auth.ts
index e6025c8ce273bf60a13d20c09e118dd65cd91ec4..0eefa2a8eb7317033c049cdcb43bc1abd2b8f3c7 100644 (file)
--- a/server/middlewares/auth.ts
+++ b/server/middlewares/auth.ts
@@ -1,6 +1,7 @@
 import express from 'express'
 import { Socket } from 'socket.io'
 import { getAccessToken } from '@server/lib/auth/oauth-model'
 import express from 'express'
 import { Socket } from 'socket.io'
 import { getAccessToken } from '@server/lib/auth/oauth-model'

+import { RunnerModel } from '@server/models/runner/runner'

 import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
 import { logger } from '../helpers/logger'
 import { handleOAuthAuthenticate } from '../lib/auth/oauth'
 import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
 import { logger } from '../helpers/logger'
 import { handleOAuthAuthenticate } from '../lib/auth/oauth'


@@ -27,7 +28,7 @@ function authenticate (req: express.Request, res: express.Response, next: expres
 function authenticateSocket (socket: Socket, next: (err?: any) => void) {
   const accessToken = socket.handshake.query['accessToken']
 
 function authenticateSocket (socket: Socket, next: (err?: any) => void) {
   const accessToken = socket.handshake.query['accessToken']
 

-  logger.debug('Checking socket access token %s.', accessToken)
+  logger.debug('Checking access token in runner.')

 
   if (!accessToken) return next(new Error('No access token provided'))
   if (typeof accessToken !== 'string') return next(new Error('Access token is invalid'))
 
   if (!accessToken) return next(new Error('No access token provided'))
   if (typeof accessToken !== 'string') return next(new Error('Access token is invalid'))


@@ -73,9 +74,31 @@ function optionalAuthenticate (req: express.Request, res: express.Response, next
 
 // ---------------------------------------------------------------------------
 
 
 // ---------------------------------------------------------------------------
 

+function authenticateRunnerSocket (socket: Socket, next: (err?: any) => void) {
+  const runnerToken = socket.handshake.auth['runnerToken']
+
+  logger.debug('Checking runner token in socket.')
+
+  if (!runnerToken) return next(new Error('No runner token provided'))
+  if (typeof runnerToken !== 'string') return next(new Error('Runner token is invalid'))
+
+  RunnerModel.loadByToken(runnerToken)
+    .then(runner => {
+      if (!runner) return next(new Error('Invalid runner token.'))
+
+      socket.handshake.auth.runner = runner
+
+      return next()
+    })
+    .catch(err => logger.error('Cannot get runner token.', { err }))
+}
+
+// ---------------------------------------------------------------------------
+

 export {
   authenticate,
   authenticateSocket,
   authenticatePromise,
 export {
   authenticate,
   authenticateSocket,
   authenticatePromise,

-  optionalAuthenticate
+  optionalAuthenticate,
+  authenticateRunnerSocket

 }
 }

diff --git a/server/middlewares/doc.ts b/server/middlewares/doc.ts
index c43f41977c52e6c312294eb9f155f194198028fd..eef76acaa03e4363bcfd125daef7a75d75264995 100644 (file)
--- a/server/middlewares/doc.ts
+++ b/server/middlewares/doc.ts
@@ -5,7 +5,7 @@ function openapiOperationDoc (options: {
   operationId?: string
 }) {
   return (req: express.Request, res: express.Response, next: express.NextFunction) => {
   operationId?: string
 }) {
   return (req: express.Request, res: express.Response, next: express.NextFunction) => {

-    res.locals.docUrl = options.url || 'https://docs.joinpeertube.org/api/rest-reference.html#operation/' + options.operationId
+    res.locals.docUrl = options.url || 'https://docs.joinpeertube.org/api-rest-reference.html#operation/' + options.operationId

 
     if (next) return next()
   }
 
     if (next) return next()
   }

diff --git a/server/middlewares/error.ts b/server/middlewares/error.ts
index 540edaeeb063635c5ee6dbfcd3e5e69f0b5b6ad5..94762e35542bb779b6487d5d91f71d51151a6a2d 100644 (file)
--- a/server/middlewares/error.ts
+++ b/server/middlewares/error.ts
@@ -5,7 +5,7 @@ import { HttpStatusCode } from '@shared/models'
 
 function apiFailMiddleware (req: express.Request, res: express.Response, next: express.NextFunction) {
   res.fail = options => {
 
 function apiFailMiddleware (req: express.Request, res: express.Response, next: express.NextFunction) {
   res.fail = options => {

-    const { status = HttpStatusCode.BAD_REQUEST_400, message, title, type, data, instance } = options
+    const { status = HttpStatusCode.BAD_REQUEST_400, message, title, type, data, instance, tags } = options

 
     const extension = new ProblemDocumentExtension({
       ...data,
 
     const extension = new ProblemDocumentExtension({
       ...data,


@@ -31,11 +31,11 @@ function apiFailMiddleware (req: express.Request, res: express.Response, next: e
       detail: message,
 
       type: type
       detail: message,
 
       type: type

-        ? `https://docs.joinpeertube.org/api/rest-reference.html#section/Errors/${type}`
+        ? `https://docs.joinpeertube.org/api-rest-reference.html#section/Errors/${type}`

         : undefined
     }, extension)
 
         : undefined
     }, extension)
 

-    logger.debug('Bad HTTP request.', { json })
+    logger.debug('Bad HTTP request.', { json, tags })

 
     res.json(json)
   }
 
     res.json(json)
   }

diff --git a/server/middlewares/rate-limiter.ts b/server/middlewares/rate-limiter.ts
index bc95139693192d8d25811031c56d970482169307..1eef8b360d679f7025a001dcff6c9c48f26c08e7 100644 (file)
--- a/server/middlewares/rate-limiter.ts
+++ b/server/middlewares/rate-limiter.ts
@@ -1,10 +1,12 @@
+import express from 'express'
+import RateLimit, { Options as RateLimitHandlerOptions } from 'express-rate-limit'
+import { RunnerModel } from '@server/models/runner/runner'

 import { UserRole } from '@shared/models'
 import { UserRole } from '@shared/models'

-import RateLimit from 'express-rate-limit'

 import { optionalAuthenticate } from './auth'
 
 const whitelistRoles = new Set([ UserRole.ADMINISTRATOR, UserRole.MODERATOR ])
 
 import { optionalAuthenticate } from './auth'
 
 const whitelistRoles = new Set([ UserRole.ADMINISTRATOR, UserRole.MODERATOR ])
 

-function buildRateLimiter (options: {
+export function buildRateLimiter (options: {

   windowMs: number
   max: number
   skipFailedRequests?: boolean
   windowMs: number
   max: number
   skipFailedRequests?: boolean


@@ -15,17 +17,33 @@ function buildRateLimiter (options: {
     skipFailedRequests: options.skipFailedRequests,
 
     handler: (req, res, next, options) => {
     skipFailedRequests: options.skipFailedRequests,
 
     handler: (req, res, next, options) => {

+      // Bypass rate limit for registered runners
+      if (req.body?.runnerToken) {
+        return RunnerModel.loadByToken(req.body.runnerToken)
+          .then(runner => {
+            if (runner) return next()
+
+            return sendRateLimited(res, options)
+          })
+      }
+
+      // Bypass rate limit for admins/moderators

       return optionalAuthenticate(req, res, () => {
         if (res.locals.authenticated === true && whitelistRoles.has(res.locals.oauth.token.User.role)) {
           return next()
         }
 
       return optionalAuthenticate(req, res, () => {
         if (res.locals.authenticated === true && whitelistRoles.has(res.locals.oauth.token.User.role)) {
           return next()
         }
 

-        return res.status(options.statusCode).send(options.message)
+        return sendRateLimited(res, options)

       })
     }
   })
 }
 
       })
     }
   })
 }
 

-export {
-  buildRateLimiter
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function sendRateLimited (res: express.Response, options: RateLimitHandlerOptions) {
+  return res.status(options.statusCode).send(options.message)
+

 }
 }

diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index 4a9d1cb5472cba777ea534461d231d5f77243b79..b3e7e5011ef987f81d6fe77d1b1168cac81f6bcd 100644 (file)
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -54,6 +54,7 @@ const customConfigUpdateValidator = [
   body('transcoding.resolutions.1080p').isBoolean(),
   body('transcoding.resolutions.1440p').isBoolean(),
   body('transcoding.resolutions.2160p').isBoolean(),
   body('transcoding.resolutions.1080p').isBoolean(),
   body('transcoding.resolutions.1440p').isBoolean(),
   body('transcoding.resolutions.2160p').isBoolean(),

+  body('transcoding.remoteRunners.enabled').isBoolean(),

 
   body('transcoding.alwaysTranscodeOriginalResolution').isBoolean(),
 
 
   body('transcoding.alwaysTranscodeOriginalResolution').isBoolean(),
 


@@ -97,6 +98,7 @@ const customConfigUpdateValidator = [
   body('live.transcoding.resolutions.1440p').isBoolean(),
   body('live.transcoding.resolutions.2160p').isBoolean(),
   body('live.transcoding.alwaysTranscodeOriginalResolution').isBoolean(),
   body('live.transcoding.resolutions.1440p').isBoolean(),
   body('live.transcoding.resolutions.2160p').isBoolean(),
   body('live.transcoding.alwaysTranscodeOriginalResolution').isBoolean(),

+  body('live.transcoding.remoteRunners.enabled').isBoolean(),

 
   body('search.remoteUri.users').isBoolean(),
   body('search.remoteUri.anonymous').isBoolean(),
 
   body('search.remoteUri.users').isBoolean(),
   body('search.remoteUri.anonymous').isBoolean(),

diff --git a/server/middlewares/validators/runners/index.ts b/server/middlewares/validators/runners/index.ts
new file mode 100644 (file)
index 0000000..9a9629a
--- /dev/null
+++ b/server/middlewares/validators/runners/index.ts
@@ -0,0 +1,3 @@
+export * from './jobs'
+export * from './registration-token'
+export * from './runners'

diff --git a/server/middlewares/validators/runners/job-files.ts b/server/middlewares/validators/runners/job-files.ts
new file mode 100644 (file)
index 0000000..56afa39
--- /dev/null
+++ b/server/middlewares/validators/runners/job-files.ts
@@ -0,0 +1,27 @@
+import express from 'express'
+import { HttpStatusCode } from '@shared/models'
+import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
+
+const tags = [ 'runner' ]
+
+export const runnerJobGetVideoTranscodingFileValidator = [
+  isValidVideoIdParam('videoId'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res)) return
+
+    if (!await doesVideoExist(req.params.videoId, res, 'all')) return
+
+    const runnerJob = res.locals.runnerJob
+
+    if (runnerJob.privatePayload.videoUUID !== res.locals.videoAll.uuid) {
+      return res.fail({
+        status: HttpStatusCode.FORBIDDEN_403,
+        message: 'Job is not associated to this video',
+        tags: [ ...tags, res.locals.videoAll.uuid ]
+      })
+    }
+
+    return next()
+  }
+]

diff --git a/server/middlewares/validators/runners/jobs.ts b/server/middlewares/validators/runners/jobs.ts
new file mode 100644 (file)
index 0000000..8cb87e9
--- /dev/null
+++ b/server/middlewares/validators/runners/jobs.ts
@@ -0,0 +1,156 @@
+import express from 'express'
+import { body, param } from 'express-validator'
+import { isUUIDValid } from '@server/helpers/custom-validators/misc'
+import {
+  isRunnerJobAbortReasonValid,
+  isRunnerJobErrorMessageValid,
+  isRunnerJobProgressValid,
+  isRunnerJobSuccessPayloadValid,
+  isRunnerJobTokenValid,
+  isRunnerJobUpdatePayloadValid
+} from '@server/helpers/custom-validators/runners/jobs'
+import { isRunnerTokenValid } from '@server/helpers/custom-validators/runners/runners'
+import { cleanUpReqFiles } from '@server/helpers/express-utils'
+import { RunnerJobModel } from '@server/models/runner/runner-job'
+import { HttpStatusCode, RunnerJobState, RunnerJobSuccessBody, RunnerJobUpdateBody, ServerErrorCode } from '@shared/models'
+import { areValidationErrors } from '../shared'
+
+const tags = [ 'runner' ]
+
+export const acceptRunnerJobValidator = [
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (res.locals.runnerJob.state !== RunnerJobState.PENDING) {
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        message: 'This runner job is not in pending state',
+        tags
+      })
+    }
+
+    return next()
+  }
+]
+
+export const abortRunnerJobValidator = [
+  body('reason').custom(isRunnerJobAbortReasonValid),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    return next()
+  }
+]
+
+export const updateRunnerJobValidator = [
+  body('progress').optional().custom(isRunnerJobProgressValid),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req)
+
+    const body = req.body as RunnerJobUpdateBody
+
+    if (isRunnerJobUpdatePayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        message: 'Payload is invalid',
+        tags
+      })
+    }
+
+    return next()
+  }
+]
+
+export const errorRunnerJobValidator = [
+  body('message').custom(isRunnerJobErrorMessageValid),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    return next()
+  }
+]
+
+export const successRunnerJobValidator = [
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    const body = req.body as RunnerJobSuccessBody
+
+    if (isRunnerJobSuccessPayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        message: 'Payload is invalid',
+        tags
+      })
+    }
+
+    return next()
+  }
+]
+
+export const runnerJobGetValidator = [
+  param('jobUUID').custom(isUUIDValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const runnerJob = await RunnerJobModel.loadWithRunner(req.params.jobUUID)
+
+    if (!runnerJob) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Unknown runner job',
+        tags
+      })
+    }
+
+    res.locals.runnerJob = runnerJob
+
+    return next()
+  }
+]
+
+export const jobOfRunnerGetValidator = [
+  param('jobUUID').custom(isUUIDValid),
+
+  body('runnerToken').custom(isRunnerTokenValid),
+  body('jobToken').custom(isRunnerJobTokenValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req)
+
+    const runnerJob = await RunnerJobModel.loadByRunnerAndJobTokensWithRunner({
+      uuid: req.params.jobUUID,
+      runnerToken: req.body.runnerToken,
+      jobToken: req.body.jobToken
+    })
+
+    if (!runnerJob) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Unknown runner job',
+        tags
+      })
+    }
+
+    if (runnerJob.state !== RunnerJobState.PROCESSING) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        type: ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE,
+        message: 'Job is not in "processing" state',
+        tags
+      })
+    }
+
+    res.locals.runnerJob = runnerJob
+
+    return next()
+  }
+]

diff --git a/server/middlewares/validators/runners/registration-token.ts b/server/middlewares/validators/runners/registration-token.ts
new file mode 100644 (file)
index 0000000..cc31d4a
--- /dev/null
+++ b/server/middlewares/validators/runners/registration-token.ts
@@ -0,0 +1,37 @@
+import express from 'express'
+import { param } from 'express-validator'
+import { isIdValid } from '@server/helpers/custom-validators/misc'
+import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
+import { forceNumber } from '@shared/core-utils'
+import { HttpStatusCode } from '@shared/models'
+import { areValidationErrors } from '../shared/utils'
+
+const tags = [ 'runner' ]
+
+const deleteRegistrationTokenValidator = [
+  param('id').custom(isIdValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const registrationToken = await RunnerRegistrationTokenModel.load(forceNumber(req.params.id))
+
+    if (!registrationToken) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Registration token not found',
+        tags
+      })
+    }
+
+    res.locals.runnerRegistrationToken = registrationToken
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  deleteRegistrationTokenValidator
+}

diff --git a/server/middlewares/validators/runners/runners.ts b/server/middlewares/validators/runners/runners.ts
new file mode 100644 (file)
index 0000000..71a1275
--- /dev/null
+++ b/server/middlewares/validators/runners/runners.ts
@@ -0,0 +1,95 @@
+import express from 'express'
+import { body, param } from 'express-validator'
+import { isIdValid } from '@server/helpers/custom-validators/misc'
+import {
+  isRunnerDescriptionValid,
+  isRunnerNameValid,
+  isRunnerRegistrationTokenValid,
+  isRunnerTokenValid
+} from '@server/helpers/custom-validators/runners/runners'
+import { RunnerModel } from '@server/models/runner/runner'
+import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
+import { forceNumber } from '@shared/core-utils'
+import { HttpStatusCode, RegisterRunnerBody, ServerErrorCode } from '@shared/models'
+import { areValidationErrors } from '../shared/utils'
+
+const tags = [ 'runner' ]
+
+const registerRunnerValidator = [
+  body('registrationToken').custom(isRunnerRegistrationTokenValid),
+  body('name').custom(isRunnerNameValid),
+  body('description').optional().custom(isRunnerDescriptionValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const body: RegisterRunnerBody = req.body
+
+    const runnerRegistrationToken = await RunnerRegistrationTokenModel.loadByRegistrationToken(body.registrationToken)
+
+    if (!runnerRegistrationToken) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Registration token is invalid',
+        tags
+      })
+    }
+
+    res.locals.runnerRegistrationToken = runnerRegistrationToken
+
+    return next()
+  }
+]
+
+const deleteRunnerValidator = [
+  param('runnerId').custom(isIdValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const runner = await RunnerModel.load(forceNumber(req.params.runnerId))
+
+    if (!runner) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Runner not found',
+        tags
+      })
+    }
+
+    res.locals.runner = runner
+
+    return next()
+  }
+]
+
+const getRunnerFromTokenValidator = [
+  body('runnerToken').custom(isRunnerTokenValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const runner = await RunnerModel.loadByToken(req.body.runnerToken)
+
+    if (!runner) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Unknown runner token',
+        type: ServerErrorCode.UNKNOWN_RUNNER_TOKEN,
+        tags
+      })
+    }
+
+    res.locals.runner = runner
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  registerRunnerValidator,
+  deleteRunnerValidator,
+  getRunnerFromTokenValidator
+}

diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts
index e6cc46317593772a68c19e22d2071ffd02f3827b..959f663acc21113339e3be0770aff6357bef3300 100644 (file)
--- a/server/middlewares/validators/sort.ts
+++ b/server/middlewares/validators/sort.ts
@@ -34,6 +34,10 @@ export const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COL
 
 export const userRegistrationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_REGISTRATIONS)
 
 
 export const userRegistrationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_REGISTRATIONS)
 

+export const runnersSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNERS)
+export const runnerRegistrationTokensSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_REGISTRATION_TOKENS)
+export const runnerJobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_JOBS)
+

 // ---------------------------------------------------------------------------
 
 function checkSortFactory (columns: string[], tags: string[] = []) {
 // ---------------------------------------------------------------------------
 
 function checkSortFactory (columns: string[], tags: string[] = []) {

diff --git a/server/middlewares/validators/videos/video-live.ts b/server/middlewares/validators/videos/video-live.ts
index e80fe15937b54606d54c5a5b3b225ee47b6d915b..2aff831a8aa139604b36c56e88e683cd64f7982c 100644 (file)
--- a/server/middlewares/validators/videos/video-live.ts
+++ b/server/middlewares/validators/videos/video-live.ts
@@ -115,6 +115,15 @@ const videoLiveAddValidator = getCommonVideoEditAttributes().concat([
       })
     }
 
       })
     }
 

+    if (body.saveReplay && !body.replaySettings?.privacy) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        message: 'Live replay is enabled but privacy replay setting is missing'
+      })
+    }
+

     const user = res.locals.oauth.token.User
     if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req)
 
     const user = res.locals.oauth.token.User
     if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req)
 

diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts
index b3e2d81019c6b40f9034ebf1f98ac051cae61f1b..4397e887e6eec4aee8e5f3b32c3d494c8aad9ee8 100644 (file)
--- a/server/middlewares/validators/videos/video-studio.ts
+++ b/server/middlewares/validators/videos/video-studio.ts
@@ -10,7 +10,7 @@ import {
 import { cleanUpReqFiles } from '@server/helpers/express-utils'
 import { CONFIG } from '@server/initializers/config'
 import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio'
 import { cleanUpReqFiles } from '@server/helpers/express-utils'
 import { CONFIG } from '@server/initializers/config'
 import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio'

-import { isAudioFile } from '@shared/extra-utils'
+import { isAudioFile } from '@shared/ffmpeg'

 import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models'
 import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared'
 
 import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models'
 import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared'
 

diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index d3014e8e735ee47b30e197d2d9c20701ae39a5fd..794e1d4f1efa199b5c55dfed9c202e0ea89ba50a 100644 (file)
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -7,6 +7,7 @@ import { getServerActor } from '@server/models/application/application'
 import { ExpressPromiseHandler } from '@server/types/express-handler'
 import { MUserAccountId, MVideoFullLight } from '@server/types/models'
 import { arrayify, getAllPrivacies } from '@shared/core-utils'
 import { ExpressPromiseHandler } from '@server/types/express-handler'
 import { MUserAccountId, MVideoFullLight } from '@server/types/models'
 import { arrayify, getAllPrivacies } from '@shared/core-utils'

+import { getVideoStreamDuration } from '@shared/ffmpeg'

 import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models'
 import {
   exists,
 import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models'
 import {
   exists,


@@ -37,7 +38,6 @@ import {
   isVideoSupportValid
 } from '../../../helpers/custom-validators/videos'
 import { cleanUpReqFiles } from '../../../helpers/express-utils'
   isVideoSupportValid
 } from '../../../helpers/custom-validators/videos'
 import { cleanUpReqFiles } from '../../../helpers/express-utils'

-import { getVideoStreamDuration } from '../../../helpers/ffmpeg'

 import { logger } from '../../../helpers/logger'
 import { deleteFileAndCatch } from '../../../helpers/utils'
 import { getVideoWithAttributes } from '../../../helpers/video'
 import { logger } from '../../../helpers/logger'
 import { deleteFileAndCatch } from '../../../helpers/utils'
 import { getVideoWithAttributes } from '../../../helpers/video'

diff --git a/server/models/runner/runner-job.ts b/server/models/runner/runner-job.ts
new file mode 100644 (file)
index 0000000..add6f9a
--- /dev/null
+++ b/server/models/runner/runner-job.ts
@@ -0,0 +1,347 @@
+import { FindOptions, Op, Transaction } from 'sequelize'
+import {
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  Default,
+  ForeignKey,
+  IsUUID,
+  Model,
+  Scopes,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
+import { isUUIDValid } from '@server/helpers/custom-validators/misc'
+import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants'
+import { MRunnerJob, MRunnerJobRunner, MRunnerJobRunnerParent } from '@server/types/models/runners'
+import { RunnerJob, RunnerJobAdmin, RunnerJobPayload, RunnerJobPrivatePayload, RunnerJobState, RunnerJobType } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { getSort, searchAttribute } from '../shared'
+import { RunnerModel } from './runner'
+
+enum ScopeNames {
+  WITH_RUNNER = 'WITH_RUNNER',
+  WITH_PARENT = 'WITH_PARENT'
+}
+
+@Scopes(() => ({
+  [ScopeNames.WITH_RUNNER]: {
+    include: [
+      {
+        model: RunnerModel.unscoped(),
+        required: false
+      }
+    ]
+  },
+  [ScopeNames.WITH_PARENT]: {
+    include: [
+      {
+        model: RunnerJobModel.unscoped(),
+        required: false
+      }
+    ]
+  }
+}))
+@Table({
+  tableName: 'runnerJob',
+  indexes: [
+    {
+      fields: [ 'uuid' ],
+      unique: true
+    },
+    {
+      fields: [ 'processingJobToken' ],
+      unique: true
+    },
+    {
+      fields: [ 'runnerId' ]
+    }
+  ]
+})
+export class RunnerJobModel extends Model<Partial<AttributesOnly<RunnerJobModel>>> {
+
+  @AllowNull(false)
+  @IsUUID(4)
+  @Column(DataType.UUID)
+  uuid: string
+
+  @AllowNull(false)
+  @Column
+  type: RunnerJobType
+
+  @AllowNull(false)
+  @Column(DataType.JSONB)
+  payload: RunnerJobPayload
+
+  @AllowNull(false)
+  @Column(DataType.JSONB)
+  privatePayload: RunnerJobPrivatePayload
+
+  @AllowNull(false)
+  @Column
+  state: RunnerJobState
+
+  @AllowNull(false)
+  @Default(0)
+  @Column
+  failures: number
+
+  @AllowNull(true)
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNER_JOBS.ERROR_MESSAGE.max))
+  error: string
+
+  // Less has priority
+  @AllowNull(false)
+  @Column
+  priority: number
+
+  // Used to fetch the appropriate job when the runner wants to post the result
+  @AllowNull(true)
+  @Column
+  processingJobToken: string
+
+  @AllowNull(true)
+  @Column
+  progress: number
+
+  @AllowNull(true)
+  @Column
+  startedAt: Date
+
+  @AllowNull(true)
+  @Column
+  finishedAt: Date
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @ForeignKey(() => RunnerJobModel)
+  @Column
+  dependsOnRunnerJobId: number
+
+  @BelongsTo(() => RunnerJobModel, {
+    foreignKey: {
+      name: 'dependsOnRunnerJobId',
+      allowNull: true
+    },
+    onDelete: 'cascade'
+  })
+  DependsOnRunnerJob: RunnerJobModel
+
+  @ForeignKey(() => RunnerModel)
+  @Column
+  runnerId: number
+
+  @BelongsTo(() => RunnerModel, {
+    foreignKey: {
+      name: 'runnerId',
+      allowNull: true
+    },
+    onDelete: 'SET NULL'
+  })
+  Runner: RunnerModel
+
+  // ---------------------------------------------------------------------------
+
+  static loadWithRunner (uuid: string) {
+    const query = {
+      where: { uuid }
+    }
+
+    return RunnerJobModel.scope(ScopeNames.WITH_RUNNER).findOne<MRunnerJobRunner>(query)
+  }
+
+  static loadByRunnerAndJobTokensWithRunner (options: {
+    uuid: string
+    runnerToken: string
+    jobToken: string
+  }) {
+    const { uuid, runnerToken, jobToken } = options
+
+    const query = {
+      where: {
+        uuid,
+        processingJobToken: jobToken
+      },
+      include: {
+        model: RunnerModel.unscoped(),
+        required: true,
+        where: {
+          runnerToken
+        }
+      }
+    }
+
+    return RunnerJobModel.findOne<MRunnerJobRunner>(query)
+  }
+
+  static listAvailableJobs () {
+    const query = {
+      limit: 10,
+      order: getSort('priority'),
+      where: {
+        state: RunnerJobState.PENDING
+      }
+    }
+
+    return RunnerJobModel.findAll<MRunnerJob>(query)
+  }
+
+  static listStalledJobs (options: {
+    staleTimeMS: number
+    types: RunnerJobType[]
+  }) {
+    const before = new Date(Date.now() - options.staleTimeMS)
+
+    return RunnerJobModel.findAll<MRunnerJob>({
+      where: {
+        type: {
+          [Op.in]: options.types
+        },
+        state: RunnerJobState.PROCESSING,
+        updatedAt: {
+          [Op.lt]: before
+        }
+      }
+    })
+  }
+
+  static listChildrenOf (job: MRunnerJob, transaction?: Transaction) {
+    const query = {
+      where: {
+        dependsOnRunnerJobId: job.id
+      },
+      transaction
+    }
+
+    return RunnerJobModel.findAll<MRunnerJob>(query)
+  }
+
+  static listForApi (options: {
+    start: number
+    count: number
+    sort: string
+    search?: string
+  }) {
+    const { start, count, sort, search } = options
+
+    const query: FindOptions = {
+      offset: start,
+      limit: count,
+      order: getSort(sort)
+    }
+
+    if (search) {
+      if (isUUIDValid(search)) {
+        query.where = { uuid: search }
+      } else {
+        query.where = {
+          [Op.or]: [
+            searchAttribute(search, 'type'),
+            searchAttribute(search, '$Runner.name$')
+          ]
+        }
+      }
+    }
+
+    return Promise.all([
+      RunnerJobModel.scope([ ScopeNames.WITH_RUNNER ]).count(query),
+      RunnerJobModel.scope([ ScopeNames.WITH_RUNNER, ScopeNames.WITH_PARENT ]).findAll<MRunnerJobRunnerParent>(query)
+    ]).then(([ total, data ]) => ({ total, data }))
+  }
+
+  static updateDependantJobsOf (runnerJob: MRunnerJob) {
+    const where = {
+      dependsOnRunnerJobId: runnerJob.id
+    }
+
+    return RunnerJobModel.update({ state: RunnerJobState.PENDING }, { where })
+  }
+
+  static cancelAllJobs (options: { type: RunnerJobType }) {
+    const where = {
+      type: options.type
+    }
+
+    return RunnerJobModel.update({ state: RunnerJobState.CANCELLED }, { where })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  resetToPending () {
+    this.state = RunnerJobState.PENDING
+    this.processingJobToken = null
+    this.progress = null
+    this.startedAt = null
+    this.runnerId = null
+  }
+
+  setToErrorOrCancel (
+    state: RunnerJobState.PARENT_ERRORED | RunnerJobState.ERRORED | RunnerJobState.CANCELLED | RunnerJobState.PARENT_CANCELLED
+  ) {
+    this.state = state
+    this.processingJobToken = null
+    this.finishedAt = new Date()
+  }
+
+  toFormattedJSON (this: MRunnerJobRunnerParent): RunnerJob {
+    const runner = this.Runner
+      ? {
+        id: this.Runner.id,
+        name: this.Runner.name,
+        description: this.Runner.description
+      }
+      : null
+
+    const parent = this.DependsOnRunnerJob
+      ? {
+        id: this.DependsOnRunnerJob.id,
+        uuid: this.DependsOnRunnerJob.uuid,
+        type: this.DependsOnRunnerJob.type,
+        state: {
+          id: this.DependsOnRunnerJob.state,
+          label: RUNNER_JOB_STATES[this.DependsOnRunnerJob.state]
+        }
+      }
+      : undefined
+
+    return {
+      uuid: this.uuid,
+      type: this.type,
+
+      state: {
+        id: this.state,
+        label: RUNNER_JOB_STATES[this.state]
+      },
+
+      progress: this.progress,
+      priority: this.priority,
+      failures: this.failures,
+      error: this.error,
+
+      payload: this.payload,
+
+      startedAt: this.startedAt?.toISOString(),
+      finishedAt: this.finishedAt?.toISOString(),
+
+      createdAt: this.createdAt.toISOString(),
+      updatedAt: this.updatedAt.toISOString(),
+
+      parent,
+      runner
+    }
+  }
+
+  toFormattedAdminJSON (this: MRunnerJobRunnerParent): RunnerJobAdmin {
+    return {
+      ...this.toFormattedJSON(),
+
+      privatePayload: this.privatePayload
+    }
+  }
+}

diff --git a/server/models/runner/runner-registration-token.ts b/server/models/runner/runner-registration-token.ts
new file mode 100644 (file)
index 0000000..b2ae6c9
--- /dev/null
+++ b/server/models/runner/runner-registration-token.ts
@@ -0,0 +1,103 @@
+import { FindOptions, literal } from 'sequelize'
+import { AllowNull, Column, CreatedAt, HasMany, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { MRunnerRegistrationToken } from '@server/types/models/runners'
+import { RunnerRegistrationToken } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { getSort } from '../shared'
+import { RunnerModel } from './runner'
+
+/**
+ *
+ * Tokens used by PeerTube runners to register themselves to the PeerTube instance
+ *
+ */
+
+@Table({
+  tableName: 'runnerRegistrationToken',
+  indexes: [
+    {
+      fields: [ 'registrationToken' ],
+      unique: true
+    }
+  ]
+})
+export class RunnerRegistrationTokenModel extends Model<Partial<AttributesOnly<RunnerRegistrationTokenModel>>> {
+
+  @AllowNull(false)
+  @Column
+  registrationToken: string
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @HasMany(() => RunnerModel, {
+    foreignKey: {
+      allowNull: true
+    },
+    onDelete: 'cascade'
+  })
+  Runners: RunnerModel[]
+
+  static load (id: number) {
+    return RunnerRegistrationTokenModel.findByPk(id)
+  }
+
+  static loadByRegistrationToken (registrationToken: string) {
+    const query = {
+      where: { registrationToken }
+    }
+
+    return RunnerRegistrationTokenModel.findOne(query)
+  }
+
+  static countTotal () {
+    return RunnerRegistrationTokenModel.unscoped().count()
+  }
+
+  static listForApi (options: {
+    start: number
+    count: number
+    sort: string
+  }) {
+    const { start, count, sort } = options
+
+    const query: FindOptions = {
+      attributes: {
+        include: [
+          [
+            literal('(SELECT COUNT(*) FROM "runner" WHERE "runner"."runnerRegistrationTokenId" = "RunnerRegistrationTokenModel"."id")'),
+            'registeredRunnersCount'
+          ]
+        ]
+      },
+      offset: start,
+      limit: count,
+      order: getSort(sort)
+    }
+
+    return Promise.all([
+      RunnerRegistrationTokenModel.count(query),
+      RunnerRegistrationTokenModel.findAll<MRunnerRegistrationToken>(query)
+    ]).then(([ total, data ]) => ({ total, data }))
+  }
+
+  // ---------------------------------------------------------------------------
+
+  toFormattedJSON (this: MRunnerRegistrationToken): RunnerRegistrationToken {
+    const registeredRunnersCount = this.get('registeredRunnersCount') as number
+
+    return {
+      id: this.id,
+
+      registrationToken: this.registrationToken,
+
+      createdAt: this.createdAt,
+      updatedAt: this.updatedAt,
+
+      registeredRunnersCount
+    }
+  }
+}

diff --git a/server/models/runner/runner.ts b/server/models/runner/runner.ts
new file mode 100644 (file)
index 0000000..1ef0018
--- /dev/null
+++ b/server/models/runner/runner.ts
@@ -0,0 +1,112 @@
+import { FindOptions } from 'sequelize'
+import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { MRunner } from '@server/types/models/runners'
+import { Runner } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { getSort } from '../shared'
+import { RunnerRegistrationTokenModel } from './runner-registration-token'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
+
+@Table({
+  tableName: 'runner',
+  indexes: [
+    {
+      fields: [ 'runnerToken' ],
+      unique: true
+    },
+    {
+      fields: [ 'runnerRegistrationTokenId' ]
+    }
+  ]
+})
+export class RunnerModel extends Model<Partial<AttributesOnly<RunnerModel>>> {
+
+  // Used to identify the appropriate runner when it uses the runner REST API
+  @AllowNull(false)
+  @Column
+  runnerToken: string
+
+  @AllowNull(false)
+  @Column
+  name: string
+
+  @AllowNull(true)
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNERS.DESCRIPTION.max))
+  description: string
+
+  @AllowNull(false)
+  @Column
+  lastContact: Date
+
+  @AllowNull(false)
+  @Column
+  ip: string
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @ForeignKey(() => RunnerRegistrationTokenModel)
+  @Column
+  runnerRegistrationTokenId: number
+
+  @BelongsTo(() => RunnerRegistrationTokenModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  RunnerRegistrationToken: RunnerRegistrationTokenModel
+
+  // ---------------------------------------------------------------------------
+
+  static load (id: number) {
+    return RunnerModel.findByPk(id)
+  }
+
+  static loadByToken (runnerToken: string) {
+    const query = {
+      where: { runnerToken }
+    }
+
+    return RunnerModel.findOne(query)
+  }
+
+  static listForApi (options: {
+    start: number
+    count: number
+    sort: string
+  }) {
+    const { start, count, sort } = options
+
+    const query: FindOptions = {
+      offset: start,
+      limit: count,
+      order: getSort(sort)
+    }
+
+    return Promise.all([
+      RunnerModel.count(query),
+      RunnerModel.findAll<MRunner>(query)
+    ]).then(([ total, data ]) => ({ total, data }))
+  }
+
+  // ---------------------------------------------------------------------------
+
+  toFormattedJSON (this: MRunner): Runner {
+    return {
+      id: this.id,
+
+      name: this.name,
+      description: this.description,
+
+      ip: this.ip,
+      lastContact: this.lastContact,
+
+      createdAt: this.createdAt,
+      updatedAt: this.updatedAt
+    }
+  }
+}

diff --git a/server/models/shared/update.ts b/server/models/shared/update.ts
index d02c4535dc5ec02783d6bdd2891c3f01df23a30c..96db437309baf4db289305430c8b4a661b4482ad 100644 (file)
--- a/server/models/shared/update.ts
+++ b/server/models/shared/update.ts
@@ -1,22 +1,32 @@
 import { QueryTypes, Sequelize, Transaction } from 'sequelize'
 
 import { QueryTypes, Sequelize, Transaction } from 'sequelize'
 

+const updating = new Set<string>()
+

 // Sequelize always skip the update if we only update updatedAt field
 // Sequelize always skip the update if we only update updatedAt field

-function setAsUpdated (options: {
+async function setAsUpdated (options: {

   sequelize: Sequelize
   table: string
   id: number
   transaction?: Transaction
 }) {
   const { sequelize, table, id, transaction } = options
   sequelize: Sequelize
   table: string
   id: number
   transaction?: Transaction
 }) {
   const { sequelize, table, id, transaction } = options

+  const key = table + '-' + id
+
+  if (updating.has(key)) return
+  updating.add(key)

 
 

-  return sequelize.query(
-    `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`,
-    {
-      replacements: { table, id, updatedAt: new Date() },
-      type: QueryTypes.UPDATE,
-      transaction
-    }
-  )
+  try {
+    await sequelize.query(
+      `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`,
+      {
+        replacements: { table, id, updatedAt: new Date() },
+        type: QueryTypes.UPDATE,
+        transaction
+      }
+    )
+  } finally {
+    updating.delete(key)
+  }

 }
 
 export {
 }
 
 export {

diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
index 740f6b5c64a94fd9351ed390727f616a254d55e5..5845b8c74b4f0cdfc6e95c07dd41d1383b9f4621 100644 (file)
--- a/server/models/video/video-job-info.ts
+++ b/server/models/video/video-job-info.ts
@@ -1,5 +1,6 @@
 import { Op, QueryTypes, Transaction } from 'sequelize'
 import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript'
 import { Op, QueryTypes, Transaction } from 'sequelize'
 import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript'

+import { forceNumber } from '@shared/core-utils'

 import { AttributesOnly } from '@shared/typescript-utils'
 import { VideoModel } from './video'
 
 import { AttributesOnly } from '@shared/typescript-utils'
 import { VideoModel } from './video'
 


@@ -59,32 +60,33 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
     return VideoJobInfoModel.findOne({ where, transaction })
   }
 
     return VideoJobInfoModel.findOne({ where, transaction })
   }
 

-  static async increaseOrCreate (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> {
+  static async increaseOrCreate (videoUUID: string, column: VideoJobInfoColumnType, amountArg = 1): Promise<number> {

     const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
     const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }

+    const amount = forceNumber(amountArg)

 
 

-    const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
+    const [ result ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`

     INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt")
     SELECT
     INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt")
     SELECT

-      "video"."id" AS "videoId", 1, NOW(), NOW()
+      "video"."id" AS "videoId", ${amount}, NOW(), NOW()

     FROM
       "video"
     WHERE
       "video"."uuid" = $videoUUID
     ON CONFLICT ("videoId") DO UPDATE
     SET
     FROM
       "video"
     WHERE
       "video"."uuid" = $videoUUID
     ON CONFLICT ("videoId") DO UPDATE
     SET

-      "${column}" = "videoJobInfo"."${column}" + 1,
+      "${column}" = "videoJobInfo"."${column}" + ${amount},

       "updatedAt" = NOW()
     RETURNING
       "${column}"
     `, options)
 
       "updatedAt" = NOW()
     RETURNING
       "${column}"
     `, options)
 

-    return pendingMove
+    return result[column]

   }
 
   static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> {
     const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
 
   }
 
   static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> {
     const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
 

-    const result = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
+    const result = await VideoJobInfoModel.sequelize.query(`

     UPDATE
       "videoJobInfo"
     SET
     UPDATE
       "videoJobInfo"
     SET


@@ -99,7 +101,7 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
 
     if (result.length === 0) return undefined
 
 
     if (result.length === 0) return undefined
 

-    return result[0].pendingMove
+    return result[0][column]

   }
 
   static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> {
   }
 
   static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> {

diff --git a/server/models/video/video-live-session.ts b/server/models/video/video-live-session.ts
index dcded787278cd20fc762492fb9e891303c8dbae8..9426f5d11510c18ea754892a501a9c93c90c8c37 100644 (file)
--- a/server/models/video/video-live-session.ts
+++ b/server/models/video/video-live-session.ts
@@ -147,12 +147,21 @@ export class VideoLiveSessionModel extends Model<Partial<AttributesOnly<VideoLiv
     return VideoLiveSessionModel.scope(ScopeNames.WITH_REPLAY).findOne(query)
   }
 
     return VideoLiveSessionModel.scope(ScopeNames.WITH_REPLAY).findOne(query)
   }
 

-  static findCurrentSessionOf (videoId: number) {
+  static findCurrentSessionOf (videoUUID: string) {

     return VideoLiveSessionModel.findOne({
       where: {
     return VideoLiveSessionModel.findOne({
       where: {

-        liveVideoId: videoId,

         endDate: null
       },
         endDate: null
       },

+      include: [
+        {
+          model: VideoModel.unscoped(),
+          as: 'LiveVideo',
+          required: true,
+          where: {
+            uuid: videoUUID
+          }
+        }
+      ],

       order: [ [ 'startDate', 'DESC' ] ]
     })
   }
       order: [ [ 'startDate', 'DESC' ] ]
     })
   }

diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index f817c4a33b8ae4fb33422f22754066633e4080ff..baa8c120a1ddee8eb7d9e7b50093a4376140b9f9 100644 (file)
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -29,12 +29,14 @@ import { LiveManager } from '@server/lib/live/live-manager'
 import { removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
 import { tracer } from '@server/lib/opentelemetry/tracing'
 import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
 import { removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
 import { tracer } from '@server/lib/opentelemetry/tracing'
 import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'

+import { Hooks } from '@server/lib/plugins/hooks'

 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
 import { getServerActor } from '@server/models/application/application'
 import { ModelCache } from '@server/models/shared/model-cache'
 import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
 import { getServerActor } from '@server/models/application/application'
 import { ModelCache } from '@server/models/shared/model-cache'
 import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'

-import { ffprobePromise, getAudioStream, hasAudioStream, uuidToShort } from '@shared/extra-utils'
+import { uuidToShort } from '@shared/extra-utils'
+import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg'

 import {
   ResultList,
   ThumbnailType,
 import {
   ResultList,
   ThumbnailType,


@@ -62,7 +64,6 @@ import {
   isVideoStateValid,
   isVideoSupportValid
 } from '../../helpers/custom-validators/videos'
   isVideoStateValid,
   isVideoSupportValid
 } from '../../helpers/custom-validators/videos'

-import { getVideoStreamDimensionsInfo } from '../../helpers/ffmpeg'

 import { logger } from '../../helpers/logger'
 import { CONFIG } from '../../initializers/config'
 import { ACTIVITY_PUB, API_VERSION, CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
 import { logger } from '../../helpers/logger'
 import { CONFIG } from '../../initializers/config'
 import { ACTIVITY_PUB, API_VERSION, CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants'


@@ -137,7 +138,6 @@ import { VideoShareModel } from './video-share'
 import { VideoSourceModel } from './video-source'
 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
 import { VideoTagModel } from './video-tag'
 import { VideoSourceModel } from './video-source'
 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
 import { VideoTagModel } from './video-tag'

-import { Hooks } from '@server/lib/plugins/hooks'

 
 export enum ScopeNames {
   FOR_API = 'FOR_API',
 
 export enum ScopeNames {
   FOR_API = 'FOR_API',


@@ -798,7 +798,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
 
     logger.info('Stopping live of video %s after video deletion.', instance.uuid)
 
 
     logger.info('Stopping live of video %s after video deletion.', instance.uuid)
 

-    LiveManager.Instance.stopSessionOf(instance.id, null)
+    LiveManager.Instance.stopSessionOf(instance.uuid, null)

   }
 
   @BeforeDestroy
   }
 
   @BeforeDestroy


@@ -1763,10 +1763,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
 
       const { audioStream } = await getAudioStream(originalFilePath, probe)
       const hasAudio = await hasAudioStream(originalFilePath, probe)
 
       const { audioStream } = await getAudioStream(originalFilePath, probe)
       const hasAudio = await hasAudioStream(originalFilePath, probe)

+      const fps = await getVideoStreamFPS(originalFilePath, probe)

 
       return {
         audioStream,
         hasAudio,
 
       return {
         audioStream,
         hasAudio,

+        fps,

 
         ...await getVideoStreamDimensionsInfo(originalFilePath, probe)
       }
 
         ...await getVideoStreamDimensionsInfo(originalFilePath, probe)
       }

diff --git a/server/types/express.d.ts b/server/types/express.d.ts
index a992a9926ce87c9ce937804846e78e1b75e765a0..a8aeabb3a067ac7c9b88465fe2d5e760cd0f3962 100644 (file)
--- a/server/types/express.d.ts
+++ b/server/types/express.d.ts
@@ -44,6 +44,7 @@ import {
   MVideoShareActor,
   MVideoThumbnail
 } from './models'
   MVideoShareActor,
   MVideoThumbnail
 } from './models'

+import { MRunner, MRunnerJobRunner, MRunnerRegistrationToken } from './models/runners'

 import { MVideoSource } from './models/video/video-source'
 
 declare module 'express' {
 import { MVideoSource } from './models/video/video-source'
 
 declare module 'express' {


@@ -102,6 +103,8 @@ declare module 'express' {
       instance?: string
 
       data?: PeerTubeProblemDocumentData
       instance?: string
 
       data?: PeerTubeProblemDocumentData

+
+      tags?: string[]

     }) => void
 
     locals: {
     }) => void
 
     locals: {


@@ -203,6 +206,9 @@ declare module 'express' {
 
       localViewerFull?: MLocalVideoViewerWithWatchSections
 
 
       localViewerFull?: MLocalVideoViewerWithWatchSections
 

+      runner?: MRunner
+      runnerRegistrationToken?: MRunnerRegistrationToken
+      runnerJob?: MRunnerJobRunner

     }
   }
 }
     }
   }
 }

diff --git a/server/types/models/runners/index.ts b/server/types/models/runners/index.ts
new file mode 100644 (file)
index 0000000..e94d479
--- /dev/null
+++ b/server/types/models/runners/index.ts
@@ -0,0 +1,3 @@
+export * from './runner'
+export * from './runner-job'
+export * from './runner-registration-token'

diff --git a/server/types/models/runners/runner-job.ts b/server/types/models/runners/runner-job.ts
new file mode 100644 (file)
index 0000000..ec983ba
--- /dev/null
+++ b/server/types/models/runners/runner-job.ts
@@ -0,0 +1,20 @@
+import { RunnerJobModel } from '@server/models/runner/runner-job'
+import { PickWith } from '@shared/typescript-utils'
+import { MRunner } from './runner'
+
+type Use<K extends keyof RunnerJobModel, M> = PickWith<RunnerJobModel, K, M>
+
+// ############################################################################
+
+export type MRunnerJob = Omit<RunnerJobModel, 'Runner' | 'DependsOnRunnerJob'>
+
+// ############################################################################
+
+export type MRunnerJobRunner =
+  MRunnerJob &
+  Use<'Runner', MRunner>
+
+export type MRunnerJobRunnerParent =
+  MRunnerJob &
+  Use<'Runner', MRunner> &
+  Use<'DependsOnRunnerJob', MRunnerJob>

diff --git a/server/types/models/runners/runner-registration-token.ts b/server/types/models/runners/runner-registration-token.ts
new file mode 100644 (file)
index 0000000..83b8614
--- /dev/null
+++ b/server/types/models/runners/runner-registration-token.ts
@@ -0,0 +1,5 @@
+import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
+
+// ############################################################################
+
+export type MRunnerRegistrationToken = Omit<RunnerRegistrationTokenModel, 'Runners'>

diff --git a/server/types/models/runners/runner.ts b/server/types/models/runners/runner.ts
new file mode 100644 (file)
index 0000000..d353563
--- /dev/null
+++ b/server/types/models/runners/runner.ts
@@ -0,0 +1,5 @@
+import { RunnerModel } from '@server/models/runner/runner'
+
+// ############################################################################
+
+export type MRunner = Omit<RunnerModel, 'RunnerRegistrationToken'>

diff --git a/shared/core-utils/common/number.ts b/shared/core-utils/common/number.ts
index 9a96dcf5c6fd76b2400b23a957594c759eb79a66..ce5a6041a4fad2ec33d7db57cbf6f1d7d355c733 100644 (file)
--- a/shared/core-utils/common/number.ts
+++ b/shared/core-utils/common/number.ts
@@ -1,7 +1,13 @@
-function forceNumber (value: any) {
+export function forceNumber (value: any) {

   return parseInt(value + '')
 }
 
   return parseInt(value + '')
 }
 

-export {
-  forceNumber
+export function isOdd (num: number) {
+  return (num % 2) !== 0
+}
+
+export function toEven (num: number) {
+  if (isOdd(num)) return num + 1
+
+  return num

 }
 }

diff --git a/shared/core-utils/common/promises.ts b/shared/core-utils/common/promises.ts
index f17221b97fb61841f7aeeb86c1d2cd4f6b9b4341..e3792d12e47b5df7d55807c2ba33a7aaecf1a70a 100644 (file)
--- a/shared/core-utils/common/promises.ts
+++ b/shared/core-utils/common/promises.ts
@@ -1,12 +1,12 @@
-function isPromise <T = unknown> (value: T | Promise<T>): value is Promise<T> {
+export function isPromise <T = unknown> (value: T | Promise<T>): value is Promise<T> {

   return value && typeof (value as Promise<T>).then === 'function'
 }
 
   return value && typeof (value as Promise<T>).then === 'function'
 }
 

-function isCatchable (value: any) {
+export function isCatchable (value: any) {

   return value && typeof value.catch === 'function'
 }
 
   return value && typeof value.catch === 'function'
 }
 

-function timeoutPromise <T> (promise: Promise<T>, timeoutMs: number) {
+export function timeoutPromise <T> (promise: Promise<T>, timeoutMs: number) {

   let timer: ReturnType<typeof setTimeout>
 
   return Promise.race([
   let timer: ReturnType<typeof setTimeout>
 
   return Promise.race([


@@ -18,8 +18,41 @@ function timeoutPromise <T> (promise: Promise<T>, timeoutMs: number) {
   ]).finally(() => clearTimeout(timer))
 }
 
   ]).finally(() => clearTimeout(timer))
 }
 

-export {
-  isPromise,
-  isCatchable,
-  timeoutPromise
+export function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
+  return function promisified (): Promise<A> {
+    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
+      // eslint-disable-next-line no-useless-call
+      func.apply(null, [ (err: any, res: A) => err ? reject(err) : resolve(res) ])
+    })
+  }
+}
+
+// Thanks to https://gist.github.com/kumasento/617daa7e46f13ecdd9b2
+export function promisify1<T, A> (func: (arg: T, cb: (err: any, result: A) => void) => void): (arg: T) => Promise<A> {
+  return function promisified (arg: T): Promise<A> {
+    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
+      // eslint-disable-next-line no-useless-call
+      func.apply(null, [ arg, (err: any, res: A) => err ? reject(err) : resolve(res) ])
+    })
+  }
+}
+
+// eslint-disable-next-line max-len
+export function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> {
+  return function promisified (arg1: T, arg2: U): Promise<A> {
+    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
+      // eslint-disable-next-line no-useless-call
+      func.apply(null, [ arg1, arg2, (err: any, res: A) => err ? reject(err) : resolve(res) ])
+    })
+  }
+}
+
+// eslint-disable-next-line max-len
+export function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> {
+  return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> {
+    return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
+      // eslint-disable-next-line no-useless-call
+      func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ])
+    })
+  }

 }
 }

diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts
index e2e161a7b8ea8c948cf5dbca88ebdb00cd96da8a..d4cfcbec832aad678e1514d4ab23036cd94ee532 100644 (file)
--- a/shared/extra-utils/index.ts
+++ b/shared/extra-utils/index.ts
@@ -1,4 +1,3 @@
 export * from './crypto'
 export * from './crypto'

-export * from './ffprobe'

 export * from './file'
 export * from './uuid'
 export * from './file'
 export * from './uuid'

diff --git a/shared/ffmpeg/ffmpeg-command-wrapper.ts b/shared/ffmpeg/ffmpeg-command-wrapper.ts
new file mode 100644 (file)
index 0000000..7a8c19d
--- /dev/null
+++ b/shared/ffmpeg/ffmpeg-command-wrapper.ts
@@ -0,0 +1,234 @@
+import ffmpeg, { FfmpegCommand, getAvailableEncoders } from 'fluent-ffmpeg'
+import { pick, promisify0 } from '@shared/core-utils'
+import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models'
+
+type FFmpegLogger = {
+  info: (msg: string, obj?: any) => void
+  debug: (msg: string, obj?: any) => void
+  warn: (msg: string, obj?: any) => void
+  error: (msg: string, obj?: any) => void
+}
+
+export interface FFmpegCommandWrapperOptions {
+  availableEncoders?: AvailableEncoders
+  profile?: string
+
+  niceness: number
+  tmpDirectory: string
+  threads: number
+
+  logger: FFmpegLogger
+  lTags?: { tags: string[] }
+
+  updateJobProgress?: (progress?: number) => void
+}
+
+export class FFmpegCommandWrapper {
+  private static supportedEncoders: Map<string, boolean>
+
+  private readonly availableEncoders: AvailableEncoders
+  private readonly profile: string
+
+  private readonly niceness: number
+  private readonly tmpDirectory: string
+  private readonly threads: number
+
+  private readonly logger: FFmpegLogger
+  private readonly lTags: { tags: string[] }
+
+  private readonly updateJobProgress: (progress?: number) => void
+
+  private command: FfmpegCommand
+
+  constructor (options: FFmpegCommandWrapperOptions) {
+    this.availableEncoders = options.availableEncoders
+    this.profile = options.profile
+    this.niceness = options.niceness
+    this.tmpDirectory = options.tmpDirectory
+    this.threads = options.threads
+    this.logger = options.logger
+    this.lTags = options.lTags || { tags: [] }
+    this.updateJobProgress = options.updateJobProgress
+  }
+
+  getAvailableEncoders () {
+    return this.availableEncoders
+  }
+
+  getProfile () {
+    return this.profile
+  }
+
+  getCommand () {
+    return this.command
+  }
+
+  // ---------------------------------------------------------------------------
+
+  debugLog (msg: string, meta: any) {
+    this.logger.debug(msg, { ...meta, ...this.lTags })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  buildCommand (input: string) {
+    if (this.command) throw new Error('Command is already built')
+
+    // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems
+    this.command = ffmpeg(input, {
+      niceness: this.niceness,
+      cwd: this.tmpDirectory
+    })
+
+    if (this.threads > 0) {
+      // If we don't set any threads ffmpeg will chose automatically
+      this.command.outputOption('-threads ' + this.threads)
+    }
+
+    return this.command
+  }
+
+  async runCommand (options: {
+    silent?: boolean // false by default
+  } = {}) {
+    const { silent = false } = options
+
+    return new Promise<void>((res, rej) => {
+      let shellCommand: string
+
+      this.command.on('start', cmdline => { shellCommand = cmdline })
+
+      this.command.on('error', (err, stdout, stderr) => {
+        if (silent !== true) this.logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...this.lTags })
+
+        rej(err)
+      })
+
+      this.command.on('end', (stdout, stderr) => {
+        this.logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...this.lTags })
+
+        res()
+      })
+
+      if (this.updateJobProgress) {
+        this.command.on('progress', progress => {
+          if (!progress.percent) return
+
+          // Sometimes ffmpeg returns an invalid progress
+          let percent = Math.round(progress.percent)
+          if (percent < 0) percent = 0
+          if (percent > 100) percent = 100
+
+          this.updateJobProgress(percent)
+        })
+      }
+
+      this.command.run()
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  static resetSupportedEncoders () {
+    FFmpegCommandWrapper.supportedEncoders = undefined
+  }
+
+  // Run encoder builder depending on available encoders
+  // Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one
+  // If the default one does not exist, check the next encoder
+  async getEncoderBuilderResult (options: EncoderOptionsBuilderParams & {
+    streamType: 'video' | 'audio'
+    input: string
+
+    videoType: 'vod' | 'live'
+  }) {
+    if (!this.availableEncoders) {
+      throw new Error('There is no available encoders')
+    }
+
+    const { streamType, videoType } = options
+
+    const encodersToTry = this.availableEncoders.encodersToTry[videoType][streamType]
+    const encoders = this.availableEncoders.available[videoType]
+
+    for (const encoder of encodersToTry) {
+      if (!(await this.checkFFmpegEncoders(this.availableEncoders)).get(encoder)) {
+        this.logger.debug(`Encoder ${encoder} not available in ffmpeg, skipping.`, this.lTags)
+        continue
+      }
+
+      if (!encoders[encoder]) {
+        this.logger.debug(`Encoder ${encoder} not available in peertube encoders, skipping.`, this.lTags)
+        continue
+      }
+
+      // An object containing available profiles for this encoder
+      const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = encoders[encoder]
+      let builder = builderProfiles[this.profile]
+
+      if (!builder) {
+        this.logger.debug(`Profile ${this.profile} for encoder ${encoder} not available. Fallback to default.`, this.lTags)
+        builder = builderProfiles.default
+
+        if (!builder) {
+          this.logger.debug(`Default profile for encoder ${encoder} not available. Try next available encoder.`, this.lTags)
+          continue
+        }
+      }
+
+      const result = await builder(
+        pick(options, [
+          'input',
+          'canCopyAudio',
+          'canCopyVideo',
+          'resolution',
+          'inputBitrate',
+          'fps',
+          'inputRatio',
+          'streamNum'
+        ])
+      )
+
+      return {
+        result,
+
+        // If we don't have output options, then copy the input stream
+        encoder: result.copy === true
+          ? 'copy'
+          : encoder
+      }
+    }
+
+    return null
+  }
+
+  // Detect supported encoders by ffmpeg
+  private async checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise<Map<string, boolean>> {
+    if (FFmpegCommandWrapper.supportedEncoders !== undefined) {
+      return FFmpegCommandWrapper.supportedEncoders
+    }
+
+    const getAvailableEncodersPromise = promisify0(getAvailableEncoders)
+    const availableFFmpegEncoders = await getAvailableEncodersPromise()
+
+    const searchEncoders = new Set<string>()
+    for (const type of [ 'live', 'vod' ]) {
+      for (const streamType of [ 'audio', 'video' ]) {
+        for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) {
+          searchEncoders.add(encoder)
+        }
+      }
+    }
+
+    const supportedEncoders = new Map<string, boolean>()
+
+    for (const searchEncoder of searchEncoders) {
+      supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined)
+    }
+
+    this.logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...this.lTags })
+
+    FFmpegCommandWrapper.supportedEncoders = supportedEncoders
+    return supportedEncoders
+  }
+}

diff --git a/shared/ffmpeg/ffmpeg-edition.ts b/shared/ffmpeg/ffmpeg-edition.ts
new file mode 100644 (file)
index 0000000..724ca1e
--- /dev/null
+++ b/shared/ffmpeg/ffmpeg-edition.ts
@@ -0,0 +1,239 @@
+import { FilterSpecification } from 'fluent-ffmpeg'
+import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper'
+import { presetVOD } from './shared/presets'
+import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe'
+
+export class FFmpegEdition {
+  private readonly commandWrapper: FFmpegCommandWrapper
+
+  constructor (options: FFmpegCommandWrapperOptions) {
+    this.commandWrapper = new FFmpegCommandWrapper(options)
+  }
+
+  async cutVideo (options: {
+    inputPath: string
+    outputPath: string
+    start?: number
+    end?: number
+  }) {
+    const { inputPath, outputPath } = options
+
+    const mainProbe = await ffprobePromise(inputPath)
+    const fps = await getVideoStreamFPS(inputPath, mainProbe)
+    const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe)
+
+    const command = this.commandWrapper.buildCommand(inputPath)
+      .output(outputPath)
+
+    await presetVOD({
+      commandWrapper: this.commandWrapper,
+      input: inputPath,
+      resolution,
+      fps,
+      canCopyAudio: false,
+      canCopyVideo: false
+    })
+
+    if (options.start) {
+      command.outputOption('-ss ' + options.start)
+    }
+
+    if (options.end) {
+      command.outputOption('-to ' + options.end)
+    }
+
+    await this.commandWrapper.runCommand()
+  }
+
+  async addWatermark (options: {
+    inputPath: string
+    watermarkPath: string
+    outputPath: string
+
+    videoFilters: {
+      watermarkSizeRatio: number
+      horitonzalMarginRatio: number
+      verticalMarginRatio: number
+    }
+  }) {
+    const { watermarkPath, inputPath, outputPath, videoFilters } = options
+
+    const videoProbe = await ffprobePromise(inputPath)
+    const fps = await getVideoStreamFPS(inputPath, videoProbe)
+    const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe)
+
+    const command = this.commandWrapper.buildCommand(inputPath)
+      .output(outputPath)
+
+    command.input(watermarkPath)
+
+    await presetVOD({
+      commandWrapper: this.commandWrapper,
+      input: inputPath,
+      resolution,
+      fps,
+      canCopyAudio: true,
+      canCopyVideo: false
+    })
+
+    const complexFilter: FilterSpecification[] = [
+      // Scale watermark
+      {
+        inputs: [ '[1]', '[0]' ],
+        filter: 'scale2ref',
+        options: {
+          w: 'oh*mdar',
+          h: `ih*${videoFilters.watermarkSizeRatio}`
+        },
+        outputs: [ '[watermark]', '[video]' ]
+      },
+
+      {
+        inputs: [ '[video]', '[watermark]' ],
+        filter: 'overlay',
+        options: {
+          x: `main_w - overlay_w - (main_h * ${videoFilters.horitonzalMarginRatio})`,
+          y: `main_h * ${videoFilters.verticalMarginRatio}`
+        }
+      }
+    ]
+
+    command.complexFilter(complexFilter)
+
+    await this.commandWrapper.runCommand()
+  }
+
+  async addIntroOutro (options: {
+    inputPath: string
+    introOutroPath: string
+    outputPath: string
+    type: 'intro' | 'outro'
+  }) {
+    const { introOutroPath, inputPath, outputPath, type } = options
+
+    const mainProbe = await ffprobePromise(inputPath)
+    const fps = await getVideoStreamFPS(inputPath, mainProbe)
+    const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe)
+    const mainHasAudio = await hasAudioStream(inputPath, mainProbe)
+
+    const introOutroProbe = await ffprobePromise(introOutroPath)
+    const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe)
+
+    const command = this.commandWrapper.buildCommand(inputPath)
+      .output(outputPath)
+
+    command.input(introOutroPath)
+
+    if (!introOutroHasAudio && mainHasAudio) {
+      const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe)
+
+      command.input('anullsrc')
+      command.withInputFormat('lavfi')
+      command.withInputOption('-t ' + duration)
+    }
+
+    await presetVOD({
+      commandWrapper: this.commandWrapper,
+      input: inputPath,
+      resolution,
+      fps,
+      canCopyAudio: false,
+      canCopyVideo: false
+    })
+
+    // Add black background to correctly scale intro/outro with padding
+    const complexFilter: FilterSpecification[] = [
+      {
+        inputs: [ '1', '0' ],
+        filter: 'scale2ref',
+        options: {
+          w: 'iw',
+          h: `ih`
+        },
+        outputs: [ 'intro-outro', 'main' ]
+      },
+      {
+        inputs: [ 'intro-outro', 'main' ],
+        filter: 'scale2ref',
+        options: {
+          w: 'iw',
+          h: `ih`
+        },
+        outputs: [ 'to-scale', 'main' ]
+      },
+      {
+        inputs: 'to-scale',
+        filter: 'drawbox',
+        options: {
+          t: 'fill'
+        },
+        outputs: [ 'to-scale-bg' ]
+      },
+      {
+        inputs: [ '1', 'to-scale-bg' ],
+        filter: 'scale2ref',
+        options: {
+          w: 'iw',
+          h: 'ih',
+          force_original_aspect_ratio: 'decrease',
+          flags: 'spline'
+        },
+        outputs: [ 'to-scale', 'to-scale-bg' ]
+      },
+      {
+        inputs: [ 'to-scale-bg', 'to-scale' ],
+        filter: 'overlay',
+        options: {
+          x: '(main_w - overlay_w)/2',
+          y: '(main_h - overlay_h)/2'
+        },
+        outputs: 'intro-outro-resized'
+      }
+    ]
+
+    const concatFilter = {
+      inputs: [],
+      filter: 'concat',
+      options: {
+        n: 2,
+        v: 1,
+        unsafe: 1
+      },
+      outputs: [ 'v' ]
+    }
+
+    const introOutroFilterInputs = [ 'intro-outro-resized' ]
+    const mainFilterInputs = [ 'main' ]
+
+    if (mainHasAudio) {
+      mainFilterInputs.push('0:a')
+
+      if (introOutroHasAudio) {
+        introOutroFilterInputs.push('1:a')
+      } else {
+        // Silent input
+        introOutroFilterInputs.push('2:a')
+      }
+    }
+
+    if (type === 'intro') {
+      concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ]
+    } else {
+      concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ]
+    }
+
+    if (mainHasAudio) {
+      concatFilter.options['a'] = 1
+      concatFilter.outputs.push('a')
+
+      command.outputOption('-map [a]')
+    }
+
+    command.outputOption('-map [v]')
+
+    complexFilter.push(concatFilter)
+    command.complexFilter(complexFilter)
+
+    await this.commandWrapper.runCommand()
+  }
+}

diff --git a/shared/ffmpeg/ffmpeg-images.ts b/shared/ffmpeg/ffmpeg-images.ts
new file mode 100644 (file)
index 0000000..2db63bd
--- /dev/null
+++ b/shared/ffmpeg/ffmpeg-images.ts
@@ -0,0 +1,59 @@
+import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper'
+
+export class FFmpegImage {
+  private readonly commandWrapper: FFmpegCommandWrapper
+
+  constructor (options: FFmpegCommandWrapperOptions) {
+    this.commandWrapper = new FFmpegCommandWrapper(options)
+  }
+
+  convertWebPToJPG (options: {
+    path: string
+    destination: string
+  }): Promise<void> {
+    const { path, destination } = options
+
+    this.commandWrapper.buildCommand(path)
+      .output(destination)
+
+    return this.commandWrapper.runCommand({ silent: true })
+  }
+
+  processGIF (options: {
+    path: string
+    destination: string
+    newSize: { width: number, height: number }
+  }): Promise<void> {
+    const { path, destination, newSize } = options
+
+    this.commandWrapper.buildCommand(path)
+      .fps(20)
+      .size(`${newSize.width}x${newSize.height}`)
+      .output(destination)
+
+    return this.commandWrapper.runCommand()
+  }
+
+  async generateThumbnailFromVideo (options: {
+    fromPath: string
+    folder: string
+    imageName: string
+  }) {
+    const { fromPath, folder, imageName } = options
+
+    const pendingImageName = 'pending-' + imageName
+
+    const thumbnailOptions = {
+      filename: pendingImageName,
+      count: 1,
+      folder
+    }
+
+    return new Promise<string>((res, rej) => {
+      this.commandWrapper.buildCommand(fromPath)
+        .on('error', rej)
+        .on('end', () => res(imageName))
+        .thumbnail(thumbnailOptions)
+    })
+  }
+}

diff --git a/shared/ffmpeg/ffmpeg-live.ts b/shared/ffmpeg/ffmpeg-live.ts
new file mode 100644 (file)
index 0000000..cca4c64
--- /dev/null
+++ b/shared/ffmpeg/ffmpeg-live.ts
@@ -0,0 +1,184 @@
+import { FilterSpecification } from 'fluent-ffmpeg'
+import { join } from 'path'
+import { pick } from '@shared/core-utils'
+import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper'
+import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-utils'
+import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './shared'
+
+export class FFmpegLive {
+  private readonly commandWrapper: FFmpegCommandWrapper
+
+  constructor (options: FFmpegCommandWrapperOptions) {
+    this.commandWrapper = new FFmpegCommandWrapper(options)
+  }
+
+  async getLiveTranscodingCommand (options: {
+    inputUrl: string
+
+    outPath: string
+    masterPlaylistName: string
+
+    toTranscode: {
+      resolution: number
+      fps: number
+    }[]
+
+    // Input information
+    bitrate: number
+    ratio: number
+    hasAudio: boolean
+
+    segmentListSize: number
+    segmentDuration: number
+  }) {
+    const {
+      inputUrl,
+      outPath,
+      toTranscode,
+      bitrate,
+      masterPlaylistName,
+      ratio,
+      hasAudio
+    } = options
+    const command = this.commandWrapper.buildCommand(inputUrl)
+
+    const varStreamMap: string[] = []
+
+    const complexFilter: FilterSpecification[] = [
+      {
+        inputs: '[v:0]',
+        filter: 'split',
+        options: toTranscode.length,
+        outputs: toTranscode.map(t => `vtemp${t.resolution}`)
+      }
+    ]
+
+    command.outputOption('-sc_threshold 0')
+
+    addDefaultEncoderGlobalParams(command)
+
+    for (let i = 0; i < toTranscode.length; i++) {
+      const streamMap: string[] = []
+      const { resolution, fps } = toTranscode[i]
+
+      const baseEncoderBuilderParams = {
+        input: inputUrl,
+
+        canCopyAudio: true,
+        canCopyVideo: true,
+
+        inputBitrate: bitrate,
+        inputRatio: ratio,
+
+        resolution,
+        fps,
+
+        streamNum: i,
+        videoType: 'live' as 'live'
+      }
+
+      {
+        const streamType: StreamType = 'video'
+        const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType })
+        if (!builderResult) {
+          throw new Error('No available live video encoder found')
+        }
+
+        command.outputOption(`-map [vout${resolution}]`)
+
+        addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i })
+
+        this.commandWrapper.debugLog(
+          `Apply ffmpeg live video params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`,
+          { builderResult, fps, toTranscode }
+        )
+
+        command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`)
+        applyEncoderOptions(command, builderResult.result)
+
+        complexFilter.push({
+          inputs: `vtemp${resolution}`,
+          filter: getScaleFilter(builderResult.result),
+          options: `w=-2:h=${resolution}`,
+          outputs: `vout${resolution}`
+        })
+
+        streamMap.push(`v:${i}`)
+      }
+
+      if (hasAudio) {
+        const streamType: StreamType = 'audio'
+        const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType })
+        if (!builderResult) {
+          throw new Error('No available live audio encoder found')
+        }
+
+        command.outputOption('-map a:0')
+
+        addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i })
+
+        this.commandWrapper.debugLog(
+          `Apply ffmpeg live audio params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`,
+          { builderResult, fps, resolution }
+        )
+
+        command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`)
+        applyEncoderOptions(command, builderResult.result)
+
+        streamMap.push(`a:${i}`)
+      }
+
+      varStreamMap.push(streamMap.join(','))
+    }
+
+    command.complexFilter(complexFilter)
+
+    this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName })
+
+    command.outputOption('-var_stream_map', varStreamMap.join(' '))
+
+    return command
+  }
+
+  getLiveMuxingCommand (options: {
+    inputUrl: string
+    outPath: string
+    masterPlaylistName: string
+
+    segmentListSize: number
+    segmentDuration: number
+  }) {
+    const { inputUrl, outPath, masterPlaylistName } = options
+
+    const command = this.commandWrapper.buildCommand(inputUrl)
+
+    command.outputOption('-c:v copy')
+    command.outputOption('-c:a copy')
+    command.outputOption('-map 0:a?')
+    command.outputOption('-map 0:v?')
+
+    this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName })
+
+    return command
+  }
+
+  private addDefaultLiveHLSParams (options: {
+    outPath: string
+    masterPlaylistName: string
+    segmentListSize: number
+    segmentDuration: number
+  }) {
+    const { outPath, masterPlaylistName, segmentListSize, segmentDuration } = options
+
+    const command = this.commandWrapper.getCommand()
+
+    command.outputOption('-hls_time ' + segmentDuration)
+    command.outputOption('-hls_list_size ' + segmentListSize)
+    command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time')
+    command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
+    command.outputOption('-master_pl_name ' + masterPlaylistName)
+    command.outputOption(`-f hls`)
+
+    command.output(join(outPath, '%v.m3u8'))
+  }
+}

diff --git a/shared/ffmpeg/ffmpeg-utils.ts b/shared/ffmpeg/ffmpeg-utils.ts
new file mode 100644 (file)
index 0000000..7d09c32
--- /dev/null
+++ b/shared/ffmpeg/ffmpeg-utils.ts
@@ -0,0 +1,17 @@
+import { EncoderOptions } from '@shared/models'
+
+export type StreamType = 'audio' | 'video'
+
+export function buildStreamSuffix (base: string, streamNum?: number) {
+  if (streamNum !== undefined) {
+    return `${base}:${streamNum}`
+  }
+
+  return base
+}
+
+export function getScaleFilter (options: EncoderOptions): string {
+  if (options.scaleFilter) return options.scaleFilter.name
+
+  return 'scale'
+}

diff --git a/shared/ffmpeg/ffmpeg-version.ts b/shared/ffmpeg/ffmpeg-version.ts
new file mode 100644 (file)
index 0000000..41d9b2d
--- /dev/null
+++ b/shared/ffmpeg/ffmpeg-version.ts
@@ -0,0 +1,24 @@
+import { exec } from 'child_process'
+import ffmpeg from 'fluent-ffmpeg'
+
+export function getFFmpegVersion () {
+  return new Promise<string>((res, rej) => {
+    (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => {
+      if (err) return rej(err)
+      if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path'))
+
+      return exec(`${ffmpegPath} -version`, (err, stdout) => {
+        if (err) return rej(err)
+
+        const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/)
+        if (!parsed?.[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`))
+
+        // Fix ffmpeg version that does not include patch version (4.4 for example)
+        let version = parsed[1]
+        if (version.match(/^\d+\.\d+$/)) {
+          version += '.0'
+        }
+      })
+    })
+  })
+}

diff --git a/shared/ffmpeg/ffmpeg-vod.ts b/shared/ffmpeg/ffmpeg-vod.ts
new file mode 100644 (file)
index 0000000..e40ca0a
--- /dev/null
+++ b/shared/ffmpeg/ffmpeg-vod.ts
@@ -0,0 +1,256 @@
+import { MutexInterface } from 'async-mutex'
+import { FfmpegCommand } from 'fluent-ffmpeg'
+import { readFile, writeFile } from 'fs-extra'
+import { dirname } from 'path'
+import { pick } from '@shared/core-utils'
+import { VideoResolution } from '@shared/models'
+import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper'
+import { ffprobePromise, getVideoStreamDimensionsInfo } from './ffprobe'
+import { presetCopy, presetOnlyAudio, presetVOD } from './shared/presets'
+
+export type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio'
+
+export interface BaseTranscodeVODOptions {
+  type: TranscodeVODOptionsType
+
+  inputPath: string
+  outputPath: string
+
+  // Will be released after the ffmpeg started
+  // To prevent a bug where the input file does not exist anymore when running ffmpeg
+  inputFileMutexReleaser: MutexInterface.Releaser
+
+  resolution: number
+  fps: number
+}
+
+export interface HLSTranscodeOptions extends BaseTranscodeVODOptions {
+  type: 'hls'
+
+  copyCodecs: boolean
+
+  hlsPlaylist: {
+    videoFilename: string
+  }
+}
+
+export interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions {
+  type: 'hls-from-ts'
+
+  isAAC: boolean
+
+  hlsPlaylist: {
+    videoFilename: string
+  }
+}
+
+export interface QuickTranscodeOptions extends BaseTranscodeVODOptions {
+  type: 'quick-transcode'
+}
+
+export interface VideoTranscodeOptions extends BaseTranscodeVODOptions {
+  type: 'video'
+}
+
+export interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions {
+  type: 'merge-audio'
+  audioPath: string
+}
+
+export interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions {
+  type: 'only-audio'
+}
+
+export type TranscodeVODOptions =
+  HLSTranscodeOptions
+  | HLSFromTSTranscodeOptions
+  | VideoTranscodeOptions
+  | MergeAudioTranscodeOptions
+  | OnlyAudioTranscodeOptions
+  | QuickTranscodeOptions
+
+// ---------------------------------------------------------------------------
+
+export class FFmpegVOD {
+  private readonly commandWrapper: FFmpegCommandWrapper
+
+  private ended = false
+
+  constructor (options: FFmpegCommandWrapperOptions) {
+    this.commandWrapper = new FFmpegCommandWrapper(options)
+  }
+
+  async transcode (options: TranscodeVODOptions) {
+    const builders: {
+      [ type in TranscodeVODOptionsType ]: (options: TranscodeVODOptions) => Promise<void> | void
+    } = {
+      'quick-transcode': this.buildQuickTranscodeCommand.bind(this),
+      'hls': this.buildHLSVODCommand.bind(this),
+      'hls-from-ts': this.buildHLSVODFromTSCommand.bind(this),
+      'merge-audio': this.buildAudioMergeCommand.bind(this),
+      // TODO: remove, we merge this in buildWebVideoCommand
+      'only-audio': this.buildOnlyAudioCommand.bind(this),
+      'video': this.buildWebVideoCommand.bind(this)
+    }
+
+    this.commandWrapper.debugLog('Will run transcode.', { options })
+
+    const command = this.commandWrapper.buildCommand(options.inputPath)
+      .output(options.outputPath)
+
+    await builders[options.type](options)
+
+    command.on('start', () => {
+      setTimeout(() => {
+        options.inputFileMutexReleaser()
+      }, 1000)
+    })
+
+    await this.commandWrapper.runCommand()
+
+    await this.fixHLSPlaylistIfNeeded(options)
+
+    this.ended = true
+  }
+
+  isEnded () {
+    return this.ended
+  }
+
+  private async buildWebVideoCommand (options: TranscodeVODOptions) {
+    const { resolution, fps, inputPath } = options
+
+    if (resolution === VideoResolution.H_NOVIDEO) {
+      presetOnlyAudio(this.commandWrapper)
+      return
+    }
+
+    let scaleFilterValue: string
+
+    if (resolution !== undefined) {
+      const probe = await ffprobePromise(inputPath)
+      const videoStreamInfo = await getVideoStreamDimensionsInfo(inputPath, probe)
+
+      scaleFilterValue = videoStreamInfo?.isPortraitMode === true
+        ? `w=${resolution}:h=-2`
+        : `w=-2:h=${resolution}`
+    }
+
+    await presetVOD({
+      commandWrapper: this.commandWrapper,
+
+      resolution,
+      input: inputPath,
+      canCopyAudio: true,
+      canCopyVideo: true,
+      fps,
+      scaleFilterValue
+    })
+  }
+
+  private buildQuickTranscodeCommand (_options: TranscodeVODOptions) {
+    const command = this.commandWrapper.getCommand()
+
+    presetCopy(this.commandWrapper)
+
+    command.outputOption('-map_metadata -1') // strip all metadata
+      .outputOption('-movflags faststart')
+  }
+
+  // ---------------------------------------------------------------------------
+  // Audio transcoding
+  // ---------------------------------------------------------------------------
+
+  private async buildAudioMergeCommand (options: MergeAudioTranscodeOptions) {
+    const command = this.commandWrapper.getCommand()
+
+    command.loop(undefined)
+
+    await presetVOD({
+      ...pick(options, [ 'resolution' ]),
+
+      commandWrapper: this.commandWrapper,
+      input: options.audioPath,
+      canCopyAudio: true,
+      canCopyVideo: true,
+      fps: options.fps,
+      scaleFilterValue: this.getMergeAudioScaleFilterValue()
+    })
+
+    command.outputOption('-preset:v veryfast')
+
+    command.input(options.audioPath)
+      .outputOption('-tune stillimage')
+      .outputOption('-shortest')
+  }
+
+  private buildOnlyAudioCommand (_options: OnlyAudioTranscodeOptions) {
+    presetOnlyAudio(this.commandWrapper)
+  }
+
+  // Avoid "height not divisible by 2" error
+  private getMergeAudioScaleFilterValue () {
+    return 'trunc(iw/2)*2:trunc(ih/2)*2'
+  }
+
+  // ---------------------------------------------------------------------------
+  // HLS transcoding
+  // ---------------------------------------------------------------------------
+
+  private async buildHLSVODCommand (options: HLSTranscodeOptions) {
+    const command = this.commandWrapper.getCommand()
+
+    const videoPath = this.getHLSVideoPath(options)
+
+    if (options.copyCodecs) presetCopy(this.commandWrapper)
+    else if (options.resolution === VideoResolution.H_NOVIDEO) presetOnlyAudio(this.commandWrapper)
+    else await this.buildWebVideoCommand(options)
+
+    this.addCommonHLSVODCommandOptions(command, videoPath)
+  }
+
+  private buildHLSVODFromTSCommand (options: HLSFromTSTranscodeOptions) {
+    const command = this.commandWrapper.getCommand()
+
+    const videoPath = this.getHLSVideoPath(options)
+
+    command.outputOption('-c copy')
+
+    if (options.isAAC) {
+      // Required for example when copying an AAC stream from an MPEG-TS
+      // Since it's a bitstream filter, we don't need to reencode the audio
+      command.outputOption('-bsf:a aac_adtstoasc')
+    }
+
+    this.addCommonHLSVODCommandOptions(command, videoPath)
+  }
+
+  private addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) {
+    return command.outputOption('-hls_time 4')
+                  .outputOption('-hls_list_size 0')
+                  .outputOption('-hls_playlist_type vod')
+                  .outputOption('-hls_segment_filename ' + outputPath)
+                  .outputOption('-hls_segment_type fmp4')
+                  .outputOption('-f hls')
+                  .outputOption('-hls_flags single_file')
+  }
+
+  private async fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) {
+    if (options.type !== 'hls' && options.type !== 'hls-from-ts') return
+
+    const fileContent = await readFile(options.outputPath)
+
+    const videoFileName = options.hlsPlaylist.videoFilename
+    const videoFilePath = this.getHLSVideoPath(options)
+
+    // Fix wrong mapping with some ffmpeg versions
+    const newContent = fileContent.toString()
+                                  .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`)
+
+    await writeFile(options.outputPath, newContent)
+  }
+
+  private getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) {
+    return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
+  }
+}

diff --git a/shared/extra-utils/ffprobe.ts b/shared/ffmpeg/ffprobe.ts
similarity index 91%
rename from shared/extra-utils/ffprobe.ts
rename to shared/ffmpeg/ffprobe.ts
index 7efc58a0d944950e5c8c6ae7287d4a62ceb0af7d..fda08c28e7070d2493fa6d197e76f3267b52cb42 100644 (file)
--- a/shared/extra-utils/ffprobe.ts
+++ b/shared/ffmpeg/ffprobe.ts
@@ -1,6 +1,6 @@
 import { ffprobe, FfprobeData } from 'fluent-ffmpeg'
 import { forceNumber } from '@shared/core-utils'
 import { ffprobe, FfprobeData } from 'fluent-ffmpeg'
 import { forceNumber } from '@shared/core-utils'

-import { VideoFileMetadata, VideoResolution } from '@shared/models/videos'
+import { VideoResolution } from '@shared/models/videos'

 
 /**
  *
 
 /**
  *


@@ -141,35 +141,29 @@ async function getVideoStreamFPS (path: string, existingProbe?: FfprobeData) {
   return 0
 }
 
   return 0
 }
 

-async function buildFileMetadata (path: string, existingProbe?: FfprobeData) {
-  const metadata = existingProbe || await ffprobePromise(path)
-
-  return new VideoFileMetadata(metadata)
-}
-

 async function getVideoStreamBitrate (path: string, existingProbe?: FfprobeData): Promise<number> {
 async function getVideoStreamBitrate (path: string, existingProbe?: FfprobeData): Promise<number> {

-  const metadata = await buildFileMetadata(path, existingProbe)
+  const metadata = existingProbe || await ffprobePromise(path)

 
 

-  let bitrate = metadata.format.bit_rate as number
+  let bitrate = metadata.format.bit_rate

   if (bitrate && !isNaN(bitrate)) return bitrate
 
   const videoStream = await getVideoStream(path, existingProbe)
   if (!videoStream) return undefined
 
   if (bitrate && !isNaN(bitrate)) return bitrate
 
   const videoStream = await getVideoStream(path, existingProbe)
   if (!videoStream) return undefined
 

-  bitrate = videoStream?.bit_rate
+  bitrate = forceNumber(videoStream?.bit_rate)

   if (bitrate && !isNaN(bitrate)) return bitrate
 
   return undefined
 }
 
 async function getVideoStreamDuration (path: string, existingProbe?: FfprobeData) {
   if (bitrate && !isNaN(bitrate)) return bitrate
 
   return undefined
 }
 
 async function getVideoStreamDuration (path: string, existingProbe?: FfprobeData) {

-  const metadata = await buildFileMetadata(path, existingProbe)
+  const metadata = existingProbe || await ffprobePromise(path)

 
   return Math.round(metadata.format.duration)
 }
 
 async function getVideoStream (path: string, existingProbe?: FfprobeData) {
 
   return Math.round(metadata.format.duration)
 }
 
 async function getVideoStream (path: string, existingProbe?: FfprobeData) {

-  const metadata = await buildFileMetadata(path, existingProbe)
+  const metadata = existingProbe || await ffprobePromise(path)

 
   return metadata.streams.find(s => s.codec_type === 'video')
 }
 
   return metadata.streams.find(s => s.codec_type === 'video')
 }


@@ -178,7 +172,6 @@ async function getVideoStream (path: string, existingProbe?: FfprobeData) {
 
 export {
   getVideoStreamDimensionsInfo,
 
 export {
   getVideoStreamDimensionsInfo,

-  buildFileMetadata,

   getMaxAudioBitrate,
   getVideoStream,
   getVideoStreamDuration,
   getMaxAudioBitrate,
   getVideoStream,
   getVideoStreamDuration,

diff --git a/shared/ffmpeg/index.ts b/shared/ffmpeg/index.ts
new file mode 100644 (file)
index 0000000..07a7d54
--- /dev/null
+++ b/shared/ffmpeg/index.ts
@@ -0,0 +1,8 @@
+export * from './ffmpeg-command-wrapper'
+export * from './ffmpeg-edition'
+export * from './ffmpeg-images'
+export * from './ffmpeg-live'
+export * from './ffmpeg-utils'
+export * from './ffmpeg-version'
+export * from './ffmpeg-vod'
+export * from './ffprobe'

diff --git a/shared/ffmpeg/shared/encoder-options.ts b/shared/ffmpeg/shared/encoder-options.ts
new file mode 100644 (file)
index 0000000..9692a6b
--- /dev/null
+++ b/shared/ffmpeg/shared/encoder-options.ts
@@ -0,0 +1,39 @@
+import { FfmpegCommand } from 'fluent-ffmpeg'
+import { EncoderOptions } from '@shared/models'
+import { buildStreamSuffix } from '../ffmpeg-utils'
+
+export function addDefaultEncoderGlobalParams (command: FfmpegCommand) {
+  // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375
+  command.outputOption('-max_muxing_queue_size 1024')
+         // strip all metadata
+         .outputOption('-map_metadata -1')
+         // allows import of source material with incompatible pixel formats (e.g. MJPEG video)
+         .outputOption('-pix_fmt yuv420p')
+}
+
+export function addDefaultEncoderParams (options: {
+  command: FfmpegCommand
+  encoder: 'libx264' | string
+  fps: number
+
+  streamNum?: number
+}) {
+  const { command, encoder, fps, streamNum } = options
+
+  if (encoder === 'libx264') {
+    // 3.1 is the minimal resource allocation for our highest supported resolution
+    command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1')
+
+    if (fps) {
+      // Keyframe interval of 2 seconds for faster seeking and resolution switching.
+      // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html
+      // https://superuser.com/a/908325
+      command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2))
+    }
+  }
+}
+
+export function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions) {
+  command.inputOptions(options.inputOptions ?? [])
+    .outputOptions(options.outputOptions ?? [])
+}

diff --git a/shared/ffmpeg/shared/index.ts b/shared/ffmpeg/shared/index.ts
new file mode 100644 (file)
index 0000000..51de031
--- /dev/null
+++ b/shared/ffmpeg/shared/index.ts
@@ -0,0 +1,2 @@
+export * from './encoder-options'
+export * from './presets'

diff --git a/shared/ffmpeg/shared/presets.ts b/shared/ffmpeg/shared/presets.ts
new file mode 100644 (file)
index 0000000..dcebdc1
--- /dev/null
+++ b/shared/ffmpeg/shared/presets.ts
@@ -0,0 +1,93 @@
+import { pick } from '@shared/core-utils'
+import { FFmpegCommandWrapper } from '../ffmpeg-command-wrapper'
+import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '../ffprobe'
+import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './encoder-options'
+import { getScaleFilter, StreamType } from '../ffmpeg-utils'
+
+export async function presetVOD (options: {
+  commandWrapper: FFmpegCommandWrapper
+
+  input: string
+
+  canCopyAudio: boolean
+  canCopyVideo: boolean
+
+  resolution: number
+  fps: number
+
+  scaleFilterValue?: string
+}) {
+  const { commandWrapper, input, resolution, fps, scaleFilterValue } = options
+  const command = commandWrapper.getCommand()
+
+  command.format('mp4')
+    .outputOption('-movflags faststart')
+
+  addDefaultEncoderGlobalParams(command)
+
+  const probe = await ffprobePromise(input)
+
+  // Audio encoder
+  const bitrate = await getVideoStreamBitrate(input, probe)
+  const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe)
+
+  let streamsToProcess: StreamType[] = [ 'audio', 'video' ]
+
+  if (!await hasAudioStream(input, probe)) {
+    command.noAudio()
+    streamsToProcess = [ 'video' ]
+  }
+
+  for (const streamType of streamsToProcess) {
+    const builderResult = await commandWrapper.getEncoderBuilderResult({
+      ...pick(options, [ 'canCopyAudio', 'canCopyVideo' ]),
+
+      input,
+      inputBitrate: bitrate,
+      inputRatio: videoStreamDimensions?.ratio || 0,
+
+      resolution,
+      fps,
+      streamType,
+
+      videoType: 'vod' as 'vod'
+    })
+
+    if (!builderResult) {
+      throw new Error('No available encoder found for stream ' + streamType)
+    }
+
+    commandWrapper.debugLog(
+      `Apply ffmpeg params from ${builderResult.encoder} for ${streamType} ` +
+      `stream of input ${input} using ${commandWrapper.getProfile()} profile.`,
+      { builderResult, resolution, fps }
+    )
+
+    if (streamType === 'video') {
+      command.videoCodec(builderResult.encoder)
+
+      if (scaleFilterValue) {
+        command.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`)
+      }
+    } else if (streamType === 'audio') {
+      command.audioCodec(builderResult.encoder)
+    }
+
+    applyEncoderOptions(command, builderResult.result)
+    addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps })
+  }
+}
+
+export function presetCopy (commandWrapper: FFmpegCommandWrapper) {
+  commandWrapper.getCommand()
+    .format('mp4')
+    .videoCodec('copy')
+    .audioCodec('copy')
+}
+
+export function presetOnlyAudio (commandWrapper: FFmpegCommandWrapper) {
+  commandWrapper.getCommand()
+    .format('mp4')
+    .audioCodec('copy')
+    .noVideo()
+}

diff --git a/shared/models/index.ts b/shared/models/index.ts
index 439e9c8e167d517e30581b6f2684ae0a79263f82..78f6e73e38d42c6fed8b42a145abb9f1a6ee5ca2 100644 (file)
--- a/shared/models/index.ts
+++ b/shared/models/index.ts
@@ -11,6 +11,7 @@ export * from './moderation'
 export * from './overviews'
 export * from './plugins'
 export * from './redundancy'
 export * from './overviews'
 export * from './plugins'
 export * from './redundancy'

+export * from './runners'

 export * from './search'
 export * from './server'
 export * from './tokens'
 export * from './search'
 export * from './server'
 export * from './tokens'

diff --git a/shared/models/runners/abort-runner-job-body.model.ts b/shared/models/runners/abort-runner-job-body.model.ts
new file mode 100644 (file)
index 0000000..0b9c46c
--- /dev/null
+++ b/shared/models/runners/abort-runner-job-body.model.ts
@@ -0,0 +1,6 @@
+export interface AbortRunnerJobBody {
+  runnerToken: string
+  jobToken: string
+
+  reason: string
+}

diff --git a/shared/models/runners/accept-runner-job-body.model.ts b/shared/models/runners/accept-runner-job-body.model.ts
new file mode 100644 (file)
index 0000000..cb266c4
--- /dev/null
+++ b/shared/models/runners/accept-runner-job-body.model.ts
@@ -0,0 +1,3 @@
+export interface AcceptRunnerJobBody {
+  runnerToken: string
+}

diff --git a/shared/models/runners/accept-runner-job-result.model.ts b/shared/models/runners/accept-runner-job-result.model.ts
new file mode 100644 (file)
index 0000000..f2094b9
--- /dev/null
+++ b/shared/models/runners/accept-runner-job-result.model.ts
@@ -0,0 +1,6 @@
+import { RunnerJobPayload } from './runner-job-payload.model'
+import { RunnerJob } from './runner-job.model'
+
+export interface AcceptRunnerJobResult <T extends RunnerJobPayload = RunnerJobPayload> {
+  job: RunnerJob<T> & { jobToken: string }
+}

diff --git a/shared/models/runners/error-runner-job-body.model.ts b/shared/models/runners/error-runner-job-body.model.ts
new file mode 100644 (file)
index 0000000..ac85684
--- /dev/null
+++ b/shared/models/runners/error-runner-job-body.model.ts
@@ -0,0 +1,6 @@
+export interface ErrorRunnerJobBody {
+  runnerToken: string
+  jobToken: string
+
+  message: string
+}

diff --git a/shared/models/runners/index.ts b/shared/models/runners/index.ts
new file mode 100644 (file)
index 0000000..a52b82d
--- /dev/null
+++ b/shared/models/runners/index.ts
@@ -0,0 +1,21 @@
+export * from './abort-runner-job-body.model'
+export * from './accept-runner-job-body.model'
+export * from './accept-runner-job-result.model'
+export * from './error-runner-job-body.model'
+export * from './list-runner-jobs-query.model'
+export * from './list-runner-registration-tokens.model'
+export * from './list-runners-query.model'
+export * from './register-runner-body.model'
+export * from './register-runner-result.model'
+export * from './request-runner-job-body.model'
+export * from './request-runner-job-result.model'
+export * from './runner-job-payload.model'
+export * from './runner-job-private-payload.model'
+export * from './runner-job-state.model'
+export * from './runner-job-success-body.model'
+export * from './runner-job-type.type'
+export * from './runner-job-update-body.model'
+export * from './runner-job.model'
+export * from './runner-registration-token'
+export * from './runner.model'
+export * from './unregister-runner-body.model'

diff --git a/shared/models/runners/list-runner-jobs-query.model.ts b/shared/models/runners/list-runner-jobs-query.model.ts
new file mode 100644 (file)
index 0000000..a5b62c5
--- /dev/null
+++ b/shared/models/runners/list-runner-jobs-query.model.ts
@@ -0,0 +1,6 @@
+export interface ListRunnerJobsQuery {
+  start?: number
+  count?: number
+  sort?: string
+  search?: string
+}

diff --git a/shared/models/runners/list-runner-registration-tokens.model.ts b/shared/models/runners/list-runner-registration-tokens.model.ts
new file mode 100644 (file)
index 0000000..872e059
--- /dev/null
+++ b/shared/models/runners/list-runner-registration-tokens.model.ts
@@ -0,0 +1,5 @@
+export interface ListRunnerRegistrationTokensQuery {
+  start?: number
+  count?: number
+  sort?: string
+}

diff --git a/shared/models/runners/list-runners-query.model.ts b/shared/models/runners/list-runners-query.model.ts
new file mode 100644 (file)
index 0000000..d4362e4
--- /dev/null
+++ b/shared/models/runners/list-runners-query.model.ts
@@ -0,0 +1,5 @@
+export interface ListRunnersQuery {
+  start?: number
+  count?: number
+  sort?: string
+}

diff --git a/shared/models/runners/register-runner-body.model.ts b/shared/models/runners/register-runner-body.model.ts
new file mode 100644 (file)
index 0000000..969bb35
--- /dev/null
+++ b/shared/models/runners/register-runner-body.model.ts
@@ -0,0 +1,6 @@
+export interface RegisterRunnerBody {
+  registrationToken: string
+
+  name: string
+  description?: string
+}

diff --git a/shared/models/runners/register-runner-result.model.ts b/shared/models/runners/register-runner-result.model.ts
new file mode 100644 (file)
index 0000000..e31776c
--- /dev/null
+++ b/shared/models/runners/register-runner-result.model.ts
@@ -0,0 +1,4 @@
+export interface RegisterRunnerResult {
+  id: number
+  runnerToken: string
+}

diff --git a/shared/models/runners/request-runner-job-body.model.ts b/shared/models/runners/request-runner-job-body.model.ts
new file mode 100644 (file)
index 0000000..0970d90
--- /dev/null
+++ b/shared/models/runners/request-runner-job-body.model.ts
@@ -0,0 +1,3 @@
+export interface RequestRunnerJobBody {
+  runnerToken: string
+}

diff --git a/shared/models/runners/request-runner-job-result.model.ts b/shared/models/runners/request-runner-job-result.model.ts
new file mode 100644 (file)
index 0000000..98601c4
--- /dev/null
+++ b/shared/models/runners/request-runner-job-result.model.ts
@@ -0,0 +1,10 @@
+import { RunnerJobPayload } from './runner-job-payload.model'
+import { RunnerJobType } from './runner-job-type.type'
+
+export interface RequestRunnerJobResult <P extends RunnerJobPayload = RunnerJobPayload> {
+  availableJobs: {
+    uuid: string
+    type: RunnerJobType
+    payload: P
+  }[]
+}

diff --git a/shared/models/runners/runner-job-payload.model.ts b/shared/models/runners/runner-job-payload.model.ts
new file mode 100644 (file)
index 0000000..8f0c171
--- /dev/null
+++ b/shared/models/runners/runner-job-payload.model.ts
@@ -0,0 +1,68 @@
+export type RunnerJobVODPayload =
+  RunnerJobVODWebVideoTranscodingPayload |
+  RunnerJobVODHLSTranscodingPayload |
+  RunnerJobVODAudioMergeTranscodingPayload
+
+export type RunnerJobPayload =
+  RunnerJobVODPayload |
+  RunnerJobLiveRTMPHLSTranscodingPayload
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobVODWebVideoTranscodingPayload {
+  input: {
+    videoFileUrl: string
+  }
+
+  output: {
+    resolution: number
+    fps: number
+  }
+}
+
+export interface RunnerJobVODHLSTranscodingPayload {
+  input: {
+    videoFileUrl: string
+  }
+
+  output: {
+    resolution: number
+    fps: number
+  }
+}
+
+export interface RunnerJobVODAudioMergeTranscodingPayload {
+  input: {
+    audioFileUrl: string
+    previewFileUrl: string
+  }
+
+  output: {
+    resolution: number
+    fps: number
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload {
+  return !!(payload as RunnerJobVODAudioMergeTranscodingPayload).input.audioFileUrl
+}
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobLiveRTMPHLSTranscodingPayload {
+  input: {
+    rtmpUrl: string
+  }
+
+  output: {
+    toTranscode: {
+      resolution: number
+      fps: number
+    }[]
+
+    segmentDuration: number
+    segmentListSize: number
+  }
+}

diff --git a/shared/models/runners/runner-job-private-payload.model.ts b/shared/models/runners/runner-job-private-payload.model.ts
new file mode 100644 (file)
index 0000000..c1d8d10
--- /dev/null
+++ b/shared/models/runners/runner-job-private-payload.model.ts
@@ -0,0 +1,34 @@
+export type RunnerJobVODPrivatePayload =
+  RunnerJobVODWebVideoTranscodingPrivatePayload |
+  RunnerJobVODAudioMergeTranscodingPrivatePayload |
+  RunnerJobVODHLSTranscodingPrivatePayload
+
+export type RunnerJobPrivatePayload =
+  RunnerJobVODPrivatePayload |
+  RunnerJobLiveRTMPHLSTranscodingPrivatePayload
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobVODWebVideoTranscodingPrivatePayload {
+  videoUUID: string
+  isNewVideo: boolean
+}
+
+export interface RunnerJobVODAudioMergeTranscodingPrivatePayload {
+  videoUUID: string
+  isNewVideo: boolean
+}
+
+export interface RunnerJobVODHLSTranscodingPrivatePayload {
+  videoUUID: string
+  isNewVideo: boolean
+  deleteWebVideoFiles: boolean
+}
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobLiveRTMPHLSTranscodingPrivatePayload {
+  videoUUID: string
+  masterPlaylistName: string
+  outputDirectory: string
+}

diff --git a/shared/models/runners/runner-job-state.model.ts b/shared/models/runners/runner-job-state.model.ts
new file mode 100644 (file)
index 0000000..738db38
--- /dev/null
+++ b/shared/models/runners/runner-job-state.model.ts
@@ -0,0 +1,10 @@
+export enum RunnerJobState {
+  PENDING = 1,
+  PROCESSING = 2,
+  COMPLETED = 3,
+  ERRORED = 4,
+  WAITING_FOR_PARENT_JOB = 5,
+  CANCELLED = 6,
+  PARENT_ERRORED = 7,
+  PARENT_CANCELLED = 8
+}

diff --git a/shared/models/runners/runner-job-success-body.model.ts b/shared/models/runners/runner-job-success-body.model.ts
new file mode 100644 (file)
index 0000000..223b755
--- /dev/null
+++ b/shared/models/runners/runner-job-success-body.model.ts
@@ -0,0 +1,41 @@
+export interface RunnerJobSuccessBody {
+  runnerToken: string
+  jobToken: string
+
+  payload: RunnerJobSuccessPayload
+}
+
+// ---------------------------------------------------------------------------
+
+export type RunnerJobSuccessPayload =
+  VODWebVideoTranscodingSuccess |
+  VODHLSTranscodingSuccess |
+  VODAudioMergeTranscodingSuccess |
+  LiveRTMPHLSTranscodingSuccess
+
+export interface VODWebVideoTranscodingSuccess {
+  videoFile: Blob | string
+}
+
+export interface VODHLSTranscodingSuccess {
+  videoFile: Blob | string
+  resolutionPlaylistFile: Blob | string
+}
+
+export interface VODAudioMergeTranscodingSuccess {
+  videoFile: Blob | string
+}
+
+export interface LiveRTMPHLSTranscodingSuccess {
+
+}
+
+export function isWebVideoOrAudioMergeTranscodingPayloadSuccess (
+  payload: RunnerJobSuccessPayload
+): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess {
+  return !!(payload as VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess)?.videoFile
+}
+
+export function isHLSTranscodingPayloadSuccess (payload: RunnerJobSuccessPayload): payload is VODHLSTranscodingSuccess {
+  return !!(payload as VODHLSTranscodingSuccess)?.resolutionPlaylistFile
+}

diff --git a/shared/models/runners/runner-job-type.type.ts b/shared/models/runners/runner-job-type.type.ts
new file mode 100644 (file)
index 0000000..36d3b9b
--- /dev/null
+++ b/shared/models/runners/runner-job-type.type.ts
@@ -0,0 +1,5 @@
+export type RunnerJobType =
+  'vod-web-video-transcoding' |
+  'vod-hls-transcoding' |
+  'vod-audio-merge-transcoding' |
+  'live-rtmp-hls-transcoding'

diff --git a/shared/models/runners/runner-job-update-body.model.ts b/shared/models/runners/runner-job-update-body.model.ts
new file mode 100644 (file)
index 0000000..ed94bbe
--- /dev/null
+++ b/shared/models/runners/runner-job-update-body.model.ts
@@ -0,0 +1,28 @@
+export interface RunnerJobUpdateBody {
+  runnerToken: string
+  jobToken: string
+
+  progress?: number
+  payload?: RunnerJobUpdatePayload
+}
+
+// ---------------------------------------------------------------------------
+
+export type RunnerJobUpdatePayload = LiveRTMPHLSTranscodingUpdatePayload
+
+export interface LiveRTMPHLSTranscodingUpdatePayload {
+  type: 'add-chunk' | 'remove-chunk'
+
+  masterPlaylistFile?: Blob | string
+
+  resolutionPlaylistFilename?: string
+  resolutionPlaylistFile?: Blob | string
+
+  videoChunkFilename: string
+  videoChunkFile?: Blob | string
+}
+
+export function isLiveRTMPHLSTranscodingUpdatePayload (value: RunnerJobUpdatePayload): value is LiveRTMPHLSTranscodingUpdatePayload {
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+  return !!(value as LiveRTMPHLSTranscodingUpdatePayload)?.videoChunkFilename
+}

diff --git a/shared/models/runners/runner-job.model.ts b/shared/models/runners/runner-job.model.ts
new file mode 100644 (file)
index 0000000..0800935
--- /dev/null
+++ b/shared/models/runners/runner-job.model.ts
@@ -0,0 +1,45 @@
+import { VideoConstant } from '../videos'
+import { RunnerJobPayload } from './runner-job-payload.model'
+import { RunnerJobPrivatePayload } from './runner-job-private-payload.model'
+import { RunnerJobState } from './runner-job-state.model'
+import { RunnerJobType } from './runner-job-type.type'
+
+export interface RunnerJob <T extends RunnerJobPayload = RunnerJobPayload> {
+  uuid: string
+
+  type: RunnerJobType
+
+  state: VideoConstant<RunnerJobState>
+
+  payload: T
+
+  failures: number
+  error: string | null
+
+  progress: number
+  priority: number
+
+  startedAt: Date | string
+  createdAt: Date | string
+  updatedAt: Date | string
+  finishedAt: Date | string
+
+  parent?: {
+    type: RunnerJobType
+    state: VideoConstant<RunnerJobState>
+    uuid: string
+  }
+
+  // If associated to a runner
+  runner?: {
+    id: number
+    name: string
+
+    description: string
+  }
+}
+
+// eslint-disable-next-line max-len
+export interface RunnerJobAdmin <T extends RunnerJobPayload = RunnerJobPayload, U extends RunnerJobPrivatePayload = RunnerJobPrivatePayload> extends RunnerJob<T> {
+  privatePayload: U
+}

diff --git a/shared/models/runners/runner-registration-token.ts b/shared/models/runners/runner-registration-token.ts
new file mode 100644 (file)
index 0000000..0a157aa
--- /dev/null
+++ b/shared/models/runners/runner-registration-token.ts
@@ -0,0 +1,10 @@
+export interface RunnerRegistrationToken {
+  id: number
+
+  registrationToken: string
+
+  createdAt: Date
+  updatedAt: Date
+
+  registeredRunnersCount: number
+}

diff --git a/shared/models/runners/runner.model.ts b/shared/models/runners/runner.model.ts
new file mode 100644 (file)
index 0000000..3284f29
--- /dev/null
+++ b/shared/models/runners/runner.model.ts
@@ -0,0 +1,12 @@
+export interface Runner {
+  id: number
+
+  name: string
+  description: string
+
+  ip: string
+  lastContact: Date | string
+
+  createdAt: Date | string
+  updatedAt: Date | string
+}

diff --git a/shared/models/runners/unregister-runner-body.model.ts b/shared/models/runners/unregister-runner-body.model.ts
new file mode 100644 (file)
index 0000000..d3465c5
--- /dev/null
+++ b/shared/models/runners/unregister-runner-body.model.ts
@@ -0,0 +1,3 @@
+export interface UnregisterRunnerBody {
+  runnerToken: string
+}

diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts
index 6ffe3a676a4a732d57544411fe14be25832fc07b..5d2c10278b453b7f95dcfc62a3b96f429fb64993 100644 (file)
--- a/shared/models/server/custom-config.model.ts
+++ b/shared/models/server/custom-config.model.ts
@@ -116,6 +116,10 @@ export interface CustomConfig {
     allowAdditionalExtensions: boolean
     allowAudioFiles: boolean
 
     allowAdditionalExtensions: boolean
     allowAudioFiles: boolean
 

+    remoteRunners: {
+      enabled: boolean
+    }
+

     threads: number
     concurrency: number
 
     threads: number
     concurrency: number
 


@@ -149,6 +153,9 @@ export interface CustomConfig {
 
     transcoding: {
       enabled: boolean
 
     transcoding: {
       enabled: boolean

+      remoteRunners: {
+        enabled: boolean
+      }

       threads: number
       profile: string
       resolutions: ConfigResolutions
       threads: number
       profile: string
       resolutions: ConfigResolutions

diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 9c0b5ea569256851d6587ab5900dcdcdf66b10bd..16187d133b1b9430863465f406ea725a532c841f 100644 (file)
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -18,6 +18,7 @@ export type JobType =
   | 'after-video-channel-import'
   | 'email'
   | 'federate-video'
   | 'after-video-channel-import'
   | 'email'
   | 'federate-video'

+  | 'transcoding-job-builder'

   | 'manage-video-torrent'
   | 'move-to-object-storage'
   | 'notify'
   | 'manage-video-torrent'
   | 'move-to-object-storage'
   | 'notify'


@@ -41,6 +42,10 @@ export interface Job {
   createdAt: Date | string
   finishedOn: Date | string
   processedOn: Date | string
   createdAt: Date | string
   finishedOn: Date | string
   processedOn: Date | string

+
+  parent?: {
+    id: string
+  }

 }
 
 export type ActivitypubHttpBroadcastPayload = {
 }
 
 export type ActivitypubHttpBroadcastPayload = {


@@ -139,30 +144,28 @@ interface BaseTranscodingPayload {
 export interface HLSTranscodingPayload extends BaseTranscodingPayload {
   type: 'new-resolution-to-hls'
   resolution: VideoResolution
 export interface HLSTranscodingPayload extends BaseTranscodingPayload {
   type: 'new-resolution-to-hls'
   resolution: VideoResolution

+  fps: number

   copyCodecs: boolean
 
   copyCodecs: boolean
 

-  hasAudio: boolean
-
-  autoDeleteWebTorrentIfNeeded: boolean
-  isMaxQuality: boolean
+  deleteWebTorrentFiles: boolean

 }
 
 export interface NewWebTorrentResolutionTranscodingPayload extends BaseTranscodingPayload {
   type: 'new-resolution-to-webtorrent'
   resolution: VideoResolution
 }
 
 export interface NewWebTorrentResolutionTranscodingPayload extends BaseTranscodingPayload {
   type: 'new-resolution-to-webtorrent'
   resolution: VideoResolution

-
-  hasAudio: boolean
-  createHLSIfNeeded: boolean
+  fps: number

 }
 
 export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload {
   type: 'merge-audio-to-webtorrent'
   resolution: VideoResolution
 }
 
 export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload {
   type: 'merge-audio-to-webtorrent'
   resolution: VideoResolution

-  createHLSIfNeeded: true
+  fps: number

 }
 
 export interface OptimizeTranscodingPayload extends BaseTranscodingPayload {
   type: 'optimize-to-webtorrent'
 }
 
 export interface OptimizeTranscodingPayload extends BaseTranscodingPayload {
   type: 'optimize-to-webtorrent'

+
+  quickTranscode: boolean

 }
 
 export type VideoTranscodingPayload =
 }
 
 export type VideoTranscodingPayload =


@@ -258,3 +261,27 @@ export interface FederateVideoPayload {
   videoUUID: string
   isNewVideo: boolean
 }
   videoUUID: string
   isNewVideo: boolean
 }

+
+// ---------------------------------------------------------------------------
+
+export interface TranscodingJobBuilderPayload {
+  videoUUID: string
+
+  optimizeJob?: {
+    isNewVideo: boolean
+  }
+
+  // Array of jobs to create
+  jobs?: {
+    type: 'video-transcoding'
+    payload: VideoTranscodingPayload
+    priority?: number
+  }[]
+
+  // Array of sequential jobs to create
+  sequentialJobs?: {
+    type: 'video-transcoding'
+    payload: VideoTranscodingPayload
+    priority?: number
+  }[][]
+}

diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts
index d0bd9a00feb19afd89015d6a04d108d3edc59593..38b9d0385ff9c61adfdb8ceeeb372135fa51170c 100644 (file)
--- a/shared/models/server/server-config.model.ts
+++ b/shared/models/server/server-config.model.ts
@@ -148,6 +148,10 @@ export interface ServerConfig {
 
     profile: string
     availableProfiles: string[]
 
     profile: string
     availableProfiles: string[]

+
+    remoteRunners: {
+      enabled: boolean
+    }

   }
 
   live: {
   }
 
   live: {


@@ -165,6 +169,10 @@ export interface ServerConfig {
     transcoding: {
       enabled: boolean
 
     transcoding: {
       enabled: boolean
 

+      remoteRunners: {
+        enabled: boolean
+      }
+

       enabledResolutions: number[]
 
       profile: string
       enabledResolutions: number[]
 
       profile: string

diff --git a/shared/models/server/server-error-code.enum.ts b/shared/models/server/server-error-code.enum.ts
index a39cde1b317ba98b548fbc6c6a31bb94df9ba862..24d3c6d21ae6bb6eb467710255652aacd789e271 100644 (file)
--- a/shared/models/server/server-error-code.enum.ts
+++ b/shared/models/server/server-error-code.enum.ts
@@ -45,7 +45,10 @@ export const enum ServerErrorCode {
   INVALID_TWO_FACTOR = 'invalid_two_factor',
 
   ACCOUNT_WAITING_FOR_APPROVAL = 'account_waiting_for_approval',
   INVALID_TWO_FACTOR = 'invalid_two_factor',
 
   ACCOUNT_WAITING_FOR_APPROVAL = 'account_waiting_for_approval',

-  ACCOUNT_APPROVAL_REJECTED = 'account_approval_rejected'
+  ACCOUNT_APPROVAL_REJECTED = 'account_approval_rejected',
+
+  RUNNER_JOB_NOT_IN_PROCESSING_STATE = 'runner_job_not_in_processing_state',
+  UNKNOWN_RUNNER_TOKEN = 'unknown_runner_token'

 }
 
 /**
 }
 
 /**

diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts
index 42e5c8cd637c8737984546d420d3d63bbc06c01a..a5a770b75d302df557640e4d61f43d666783c0e6 100644 (file)
--- a/shared/models/users/user-right.enum.ts
+++ b/shared/models/users/user-right.enum.ts
@@ -45,5 +45,7 @@ export const enum UserRight {
 
   MANAGE_VIDEO_IMPORTS = 27,
 
 
   MANAGE_VIDEO_IMPORTS = 27,
 

-  MANAGE_REGISTRATIONS = 28
+  MANAGE_REGISTRATIONS = 28,
+
+  MANAGE_RUNNERS = 29

 }
 }

diff --git a/shared/models/videos/live/live-video-error.enum.ts b/shared/models/videos/live/live-video-error.enum.ts
index 3a8e4afa0625b6feb0b0616954217f1e571c722f..a26453505343f0a0c0aec4bf8999961c817ca9ca 100644 (file)
--- a/shared/models/videos/live/live-video-error.enum.ts
+++ b/shared/models/videos/live/live-video-error.enum.ts
@@ -3,5 +3,7 @@ export const enum LiveVideoError {
   DURATION_EXCEEDED = 2,
   QUOTA_EXCEEDED = 3,
   FFMPEG_ERROR = 4,
   DURATION_EXCEEDED = 2,
   QUOTA_EXCEEDED = 3,
   FFMPEG_ERROR = 4,

-  BLACKLISTED = 5
+  BLACKLISTED = 5,
+  RUNNER_JOB_ERROR = 6,
+  RUNNER_JOB_CANCEL = 7

 }
 }