Implement remote runner jobs in server

Move ffmpeg functions to @shared

22 files changed, 337 insertions, 0 deletions

diff --git a/shared/models/runners/abort-runner-job-body.model.ts b/shared/models/runners/abort-runner-job-body.model.ts
new file mode 100644
index 000000000..0b9c46c91
--- /dev/null
+++ b/shared/models/runners/abort-runner-job-body.model.ts
@@ -0,0 +1,6 @@
+export interface AbortRunnerJobBody {
+  runnerToken: string
+  jobToken: string
+
+  reason: string
+}
diff --git a/shared/models/runners/accept-runner-job-body.model.ts b/shared/models/runners/accept-runner-job-body.model.ts
new file mode 100644
index 000000000..cb266c4e6
--- /dev/null
+++ b/shared/models/runners/accept-runner-job-body.model.ts
@@ -0,0 +1,3 @@
+export interface AcceptRunnerJobBody {
+  runnerToken: string
+}
diff --git a/shared/models/runners/accept-runner-job-result.model.ts b/shared/models/runners/accept-runner-job-result.model.ts
new file mode 100644
index 000000000..f2094b945
--- /dev/null
+++ b/shared/models/runners/accept-runner-job-result.model.ts
@@ -0,0 +1,6 @@
+import { RunnerJobPayload } from './runner-job-payload.model'
+import { RunnerJob } from './runner-job.model'
+
+export interface AcceptRunnerJobResult <T extends RunnerJobPayload = RunnerJobPayload> {
+  job: RunnerJob<T> & { jobToken: string }
+}
diff --git a/shared/models/runners/error-runner-job-body.model.ts b/shared/models/runners/error-runner-job-body.model.ts
new file mode 100644
index 000000000..ac8568409
--- /dev/null
+++ b/shared/models/runners/error-runner-job-body.model.ts
@@ -0,0 +1,6 @@
+export interface ErrorRunnerJobBody {
+  runnerToken: string
+  jobToken: string
+
+  message: string
+}
diff --git a/shared/models/runners/index.ts b/shared/models/runners/index.ts
new file mode 100644
index 000000000..a52b82d2e
--- /dev/null
+++ b/shared/models/runners/index.ts
@@ -0,0 +1,21 @@
+export * from './abort-runner-job-body.model'
+export * from './accept-runner-job-body.model'
+export * from './accept-runner-job-result.model'
+export * from './error-runner-job-body.model'
+export * from './list-runner-jobs-query.model'
+export * from './list-runner-registration-tokens.model'
+export * from './list-runners-query.model'
+export * from './register-runner-body.model'
+export * from './register-runner-result.model'
+export * from './request-runner-job-body.model'
+export * from './request-runner-job-result.model'
+export * from './runner-job-payload.model'
+export * from './runner-job-private-payload.model'
+export * from './runner-job-state.model'
+export * from './runner-job-success-body.model'
+export * from './runner-job-type.type'
+export * from './runner-job-update-body.model'
+export * from './runner-job.model'
+export * from './runner-registration-token'
+export * from './runner.model'
+export * from './unregister-runner-body.model'
diff --git a/shared/models/runners/list-runner-jobs-query.model.ts b/shared/models/runners/list-runner-jobs-query.model.ts
new file mode 100644
index 000000000..a5b62c55d
--- /dev/null
+++ b/shared/models/runners/list-runner-jobs-query.model.ts
@@ -0,0 +1,6 @@
+export interface ListRunnerJobsQuery {
+  start?: number
+  count?: number
+  sort?: string
+  search?: string
+}
diff --git a/shared/models/runners/list-runner-registration-tokens.model.ts b/shared/models/runners/list-runner-registration-tokens.model.ts
new file mode 100644
index 000000000..872e059cf
--- /dev/null
+++ b/shared/models/runners/list-runner-registration-tokens.model.ts
@@ -0,0 +1,5 @@
+export interface ListRunnerRegistrationTokensQuery {
+  start?: number
+  count?: number
+  sort?: string
+}
diff --git a/shared/models/runners/list-runners-query.model.ts b/shared/models/runners/list-runners-query.model.ts
new file mode 100644
index 000000000..d4362e4c5
--- /dev/null
+++ b/shared/models/runners/list-runners-query.model.ts
@@ -0,0 +1,5 @@
+export interface ListRunnersQuery {
+  start?: number
+  count?: number
+  sort?: string
+}
diff --git a/shared/models/runners/register-runner-body.model.ts b/shared/models/runners/register-runner-body.model.ts
new file mode 100644
index 000000000..969bb35e1
--- /dev/null
+++ b/shared/models/runners/register-runner-body.model.ts
@@ -0,0 +1,6 @@
+export interface RegisterRunnerBody {
+  registrationToken: string
+
+  name: string
+  description?: string
+}
diff --git a/shared/models/runners/register-runner-result.model.ts b/shared/models/runners/register-runner-result.model.ts
new file mode 100644
index 000000000..e31776c6a
--- /dev/null
+++ b/shared/models/runners/register-runner-result.model.ts
@@ -0,0 +1,4 @@
+export interface RegisterRunnerResult {
+  id: number
+  runnerToken: string
+}
diff --git a/shared/models/runners/request-runner-job-body.model.ts b/shared/models/runners/request-runner-job-body.model.ts
new file mode 100644
index 000000000..0970d9007
--- /dev/null
+++ b/shared/models/runners/request-runner-job-body.model.ts
@@ -0,0 +1,3 @@
+export interface RequestRunnerJobBody {
+  runnerToken: string
+}
diff --git a/shared/models/runners/request-runner-job-result.model.ts b/shared/models/runners/request-runner-job-result.model.ts
new file mode 100644
index 000000000..98601c42c
--- /dev/null
+++ b/shared/models/runners/request-runner-job-result.model.ts
@@ -0,0 +1,10 @@
+import { RunnerJobPayload } from './runner-job-payload.model'
+import { RunnerJobType } from './runner-job-type.type'
+
+export interface RequestRunnerJobResult <P extends RunnerJobPayload = RunnerJobPayload> {
+  availableJobs: {
+    uuid: string
+    type: RunnerJobType
+    payload: P
+  }[]
+}
diff --git a/shared/models/runners/runner-job-payload.model.ts b/shared/models/runners/runner-job-payload.model.ts
new file mode 100644
index 000000000..8f0c17135
--- /dev/null
+++ b/shared/models/runners/runner-job-payload.model.ts
@@ -0,0 +1,68 @@
+export type RunnerJobVODPayload =
+  RunnerJobVODWebVideoTranscodingPayload |
+  RunnerJobVODHLSTranscodingPayload |
+  RunnerJobVODAudioMergeTranscodingPayload
+
+export type RunnerJobPayload =
+  RunnerJobVODPayload |
+  RunnerJobLiveRTMPHLSTranscodingPayload
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobVODWebVideoTranscodingPayload {
+  input: {
+    videoFileUrl: string
+  }
+
+  output: {
+    resolution: number
+    fps: number
+  }
+}
+
+export interface RunnerJobVODHLSTranscodingPayload {
+  input: {
+    videoFileUrl: string
+  }
+
+  output: {
+    resolution: number
+    fps: number
+  }
+}
+
+export interface RunnerJobVODAudioMergeTranscodingPayload {
+  input: {
+    audioFileUrl: string
+    previewFileUrl: string
+  }
+
+  output: {
+    resolution: number
+    fps: number
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload {
+  return !!(payload as RunnerJobVODAudioMergeTranscodingPayload).input.audioFileUrl
+}
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobLiveRTMPHLSTranscodingPayload {
+  input: {
+    rtmpUrl: string
+  }
+
+  output: {
+    toTranscode: {
+      resolution: number
+      fps: number
+    }[]
+
+    segmentDuration: number
+    segmentListSize: number
+  }
+}
diff --git a/shared/models/runners/runner-job-private-payload.model.ts b/shared/models/runners/runner-job-private-payload.model.ts
new file mode 100644
index 000000000..c1d8d1045
--- /dev/null
+++ b/shared/models/runners/runner-job-private-payload.model.ts
@@ -0,0 +1,34 @@
+export type RunnerJobVODPrivatePayload =
+  RunnerJobVODWebVideoTranscodingPrivatePayload |
+  RunnerJobVODAudioMergeTranscodingPrivatePayload |
+  RunnerJobVODHLSTranscodingPrivatePayload
+
+export type RunnerJobPrivatePayload =
+  RunnerJobVODPrivatePayload |
+  RunnerJobLiveRTMPHLSTranscodingPrivatePayload
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobVODWebVideoTranscodingPrivatePayload {
+  videoUUID: string
+  isNewVideo: boolean
+}
+
+export interface RunnerJobVODAudioMergeTranscodingPrivatePayload {
+  videoUUID: string
+  isNewVideo: boolean
+}
+
+export interface RunnerJobVODHLSTranscodingPrivatePayload {
+  videoUUID: string
+  isNewVideo: boolean
+  deleteWebVideoFiles: boolean
+}
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobLiveRTMPHLSTranscodingPrivatePayload {
+  videoUUID: string
+  masterPlaylistName: string
+  outputDirectory: string
+}
diff --git a/shared/models/runners/runner-job-state.model.ts b/shared/models/runners/runner-job-state.model.ts
new file mode 100644
index 000000000..738db38b7
--- /dev/null
+++ b/shared/models/runners/runner-job-state.model.ts
@@ -0,0 +1,10 @@
+export enum RunnerJobState {
+  PENDING = 1,
+  PROCESSING = 2,
+  COMPLETED = 3,
+  ERRORED = 4,
+  WAITING_FOR_PARENT_JOB = 5,
+  CANCELLED = 6,
+  PARENT_ERRORED = 7,
+  PARENT_CANCELLED = 8
+}
diff --git a/shared/models/runners/runner-job-success-body.model.ts b/shared/models/runners/runner-job-success-body.model.ts
new file mode 100644
index 000000000..223b7552d
--- /dev/null
+++ b/shared/models/runners/runner-job-success-body.model.ts
@@ -0,0 +1,41 @@
+export interface RunnerJobSuccessBody {
+  runnerToken: string
+  jobToken: string
+
+  payload: RunnerJobSuccessPayload
+}
+
+// ---------------------------------------------------------------------------
+
+export type RunnerJobSuccessPayload =
+  VODWebVideoTranscodingSuccess |
+  VODHLSTranscodingSuccess |
+  VODAudioMergeTranscodingSuccess |
+  LiveRTMPHLSTranscodingSuccess
+
+export interface VODWebVideoTranscodingSuccess {
+  videoFile: Blob | string
+}
+
+export interface VODHLSTranscodingSuccess {
+  videoFile: Blob | string
+  resolutionPlaylistFile: Blob | string
+}
+
+export interface VODAudioMergeTranscodingSuccess {
+  videoFile: Blob | string
+}
+
+export interface LiveRTMPHLSTranscodingSuccess {
+
+}
+
+export function isWebVideoOrAudioMergeTranscodingPayloadSuccess (
+  payload: RunnerJobSuccessPayload
+): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess {
+  return !!(payload as VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess)?.videoFile
+}
+
+export function isHLSTranscodingPayloadSuccess (payload: RunnerJobSuccessPayload): payload is VODHLSTranscodingSuccess {
+  return !!(payload as VODHLSTranscodingSuccess)?.resolutionPlaylistFile
+}
diff --git a/shared/models/runners/runner-job-type.type.ts b/shared/models/runners/runner-job-type.type.ts
new file mode 100644
index 000000000..36d3b9b25
--- /dev/null
+++ b/shared/models/runners/runner-job-type.type.ts
@@ -0,0 +1,5 @@
+export type RunnerJobType =
+  'vod-web-video-transcoding' |
+  'vod-hls-transcoding' |
+  'vod-audio-merge-transcoding' |
+  'live-rtmp-hls-transcoding'
diff --git a/shared/models/runners/runner-job-update-body.model.ts b/shared/models/runners/runner-job-update-body.model.ts
new file mode 100644
index 000000000..ed94bbe63
--- /dev/null
+++ b/shared/models/runners/runner-job-update-body.model.ts
@@ -0,0 +1,28 @@
+export interface RunnerJobUpdateBody {
+  runnerToken: string
+  jobToken: string
+
+  progress?: number
+  payload?: RunnerJobUpdatePayload
+}
+
+// ---------------------------------------------------------------------------
+
+export type RunnerJobUpdatePayload = LiveRTMPHLSTranscodingUpdatePayload
+
+export interface LiveRTMPHLSTranscodingUpdatePayload {
+  type: 'add-chunk' | 'remove-chunk'
+
+  masterPlaylistFile?: Blob | string
+
+  resolutionPlaylistFilename?: string
+  resolutionPlaylistFile?: Blob | string
+
+  videoChunkFilename: string
+  videoChunkFile?: Blob | string
+}
+
+export function isLiveRTMPHLSTranscodingUpdatePayload (value: RunnerJobUpdatePayload): value is LiveRTMPHLSTranscodingUpdatePayload {
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+  return !!(value as LiveRTMPHLSTranscodingUpdatePayload)?.videoChunkFilename
+}
diff --git a/shared/models/runners/runner-job.model.ts b/shared/models/runners/runner-job.model.ts
new file mode 100644
index 000000000..080093563
--- /dev/null
+++ b/shared/models/runners/runner-job.model.ts
@@ -0,0 +1,45 @@
+import { VideoConstant } from '../videos'
+import { RunnerJobPayload } from './runner-job-payload.model'
+import { RunnerJobPrivatePayload } from './runner-job-private-payload.model'
+import { RunnerJobState } from './runner-job-state.model'
+import { RunnerJobType } from './runner-job-type.type'
+
+export interface RunnerJob <T extends RunnerJobPayload = RunnerJobPayload> {
+  uuid: string
+
+  type: RunnerJobType
+
+  state: VideoConstant<RunnerJobState>
+
+  payload: T
+
+  failures: number
+  error: string | null
+
+  progress: number
+  priority: number
+
+  startedAt: Date | string
+  createdAt: Date | string
+  updatedAt: Date | string
+  finishedAt: Date | string
+
+  parent?: {
+    type: RunnerJobType
+    state: VideoConstant<RunnerJobState>
+    uuid: string
+  }
+
+  // If associated to a runner
+  runner?: {
+    id: number
+    name: string
+
+    description: string
+  }
+}
+
+// eslint-disable-next-line max-len
+export interface RunnerJobAdmin <T extends RunnerJobPayload = RunnerJobPayload, U extends RunnerJobPrivatePayload = RunnerJobPrivatePayload> extends RunnerJob<T> {
+  privatePayload: U
+}
diff --git a/shared/models/runners/runner-registration-token.ts b/shared/models/runners/runner-registration-token.ts
new file mode 100644
index 000000000..0a157aa51
--- /dev/null
+++ b/shared/models/runners/runner-registration-token.ts
@@ -0,0 +1,10 @@
+export interface RunnerRegistrationToken {
+  id: number
+
+  registrationToken: string
+
+  createdAt: Date
+  updatedAt: Date
+
+  registeredRunnersCount: number
+}
diff --git a/shared/models/runners/runner.model.ts b/shared/models/runners/runner.model.ts
new file mode 100644
index 000000000..3284f2992
--- /dev/null
+++ b/shared/models/runners/runner.model.ts
@@ -0,0 +1,12 @@
+export interface Runner {
+  id: number
+
+  name: string
+  description: string
+
+  ip: string
+  lastContact: Date | string
+
+  createdAt: Date | string
+  updatedAt: Date | string
+}
diff --git a/shared/models/runners/unregister-runner-body.model.ts b/shared/models/runners/unregister-runner-body.model.ts
new file mode 100644
index 000000000..d3465c5d6
--- /dev/null
+++ b/shared/models/runners/unregister-runner-body.model.ts
@@ -0,0 +1,3 @@
+export interface UnregisterRunnerBody {
+  runnerToken: string
+}