Implement remote runner jobs in server

Move ffmpeg functions to @shared

5 files changed, 318 insertions, 0 deletions

diff --git a/server/middlewares/validators/runners/index.ts b/server/middlewares/validators/runners/index.ts
new file mode 100644
index 000000000..9a9629a80
--- /dev/null
+++ b/server/middlewares/validators/runners/index.ts
@@ -0,0 +1,3 @@
+export * from './jobs'
+export * from './registration-token'
+export * from './runners'
diff --git a/server/middlewares/validators/runners/job-files.ts b/server/middlewares/validators/runners/job-files.ts
new file mode 100644
index 000000000..56afa39aa
--- /dev/null
+++ b/server/middlewares/validators/runners/job-files.ts
@@ -0,0 +1,27 @@
+import express from 'express'
+import { HttpStatusCode } from '@shared/models'
+import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
+
+const tags = [ 'runner' ]
+
+export const runnerJobGetVideoTranscodingFileValidator = [
+  isValidVideoIdParam('videoId'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res)) return
+
+    if (!await doesVideoExist(req.params.videoId, res, 'all')) return
+
+    const runnerJob = res.locals.runnerJob
+
+    if (runnerJob.privatePayload.videoUUID !== res.locals.videoAll.uuid) {
+      return res.fail({
+        status: HttpStatusCode.FORBIDDEN_403,
+        message: 'Job is not associated to this video',
+        tags: [ ...tags, res.locals.videoAll.uuid ]
+      })
+    }
+
+    return next()
+  }
+]
diff --git a/server/middlewares/validators/runners/jobs.ts b/server/middlewares/validators/runners/jobs.ts
new file mode 100644
index 000000000..8cb87e946
--- /dev/null
+++ b/server/middlewares/validators/runners/jobs.ts
@@ -0,0 +1,156 @@
+import express from 'express'
+import { body, param } from 'express-validator'
+import { isUUIDValid } from '@server/helpers/custom-validators/misc'
+import {
+  isRunnerJobAbortReasonValid,
+  isRunnerJobErrorMessageValid,
+  isRunnerJobProgressValid,
+  isRunnerJobSuccessPayloadValid,
+  isRunnerJobTokenValid,
+  isRunnerJobUpdatePayloadValid
+} from '@server/helpers/custom-validators/runners/jobs'
+import { isRunnerTokenValid } from '@server/helpers/custom-validators/runners/runners'
+import { cleanUpReqFiles } from '@server/helpers/express-utils'
+import { RunnerJobModel } from '@server/models/runner/runner-job'
+import { HttpStatusCode, RunnerJobState, RunnerJobSuccessBody, RunnerJobUpdateBody, ServerErrorCode } from '@shared/models'
+import { areValidationErrors } from '../shared'
+
+const tags = [ 'runner' ]
+
+export const acceptRunnerJobValidator = [
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (res.locals.runnerJob.state !== RunnerJobState.PENDING) {
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        message: 'This runner job is not in pending state',
+        tags
+      })
+    }
+
+    return next()
+  }
+]
+
+export const abortRunnerJobValidator = [
+  body('reason').custom(isRunnerJobAbortReasonValid),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    return next()
+  }
+]
+
+export const updateRunnerJobValidator = [
+  body('progress').optional().custom(isRunnerJobProgressValid),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req)
+
+    const body = req.body as RunnerJobUpdateBody
+
+    if (isRunnerJobUpdatePayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        message: 'Payload is invalid',
+        tags
+      })
+    }
+
+    return next()
+  }
+]
+
+export const errorRunnerJobValidator = [
+  body('message').custom(isRunnerJobErrorMessageValid),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    return next()
+  }
+]
+
+export const successRunnerJobValidator = [
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    const body = req.body as RunnerJobSuccessBody
+
+    if (isRunnerJobSuccessPayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        message: 'Payload is invalid',
+        tags
+      })
+    }
+
+    return next()
+  }
+]
+
+export const runnerJobGetValidator = [
+  param('jobUUID').custom(isUUIDValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const runnerJob = await RunnerJobModel.loadWithRunner(req.params.jobUUID)
+
+    if (!runnerJob) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Unknown runner job',
+        tags
+      })
+    }
+
+    res.locals.runnerJob = runnerJob
+
+    return next()
+  }
+]
+
+export const jobOfRunnerGetValidator = [
+  param('jobUUID').custom(isUUIDValid),
+
+  body('runnerToken').custom(isRunnerTokenValid),
+  body('jobToken').custom(isRunnerJobTokenValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req)
+
+    const runnerJob = await RunnerJobModel.loadByRunnerAndJobTokensWithRunner({
+      uuid: req.params.jobUUID,
+      runnerToken: req.body.runnerToken,
+      jobToken: req.body.jobToken
+    })
+
+    if (!runnerJob) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Unknown runner job',
+        tags
+      })
+    }
+
+    if (runnerJob.state !== RunnerJobState.PROCESSING) {
+      cleanUpReqFiles(req)
+
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        type: ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE,
+        message: 'Job is not in "processing" state',
+        tags
+      })
+    }
+
+    res.locals.runnerJob = runnerJob
+
+    return next()
+  }
+]
diff --git a/server/middlewares/validators/runners/registration-token.ts b/server/middlewares/validators/runners/registration-token.ts
new file mode 100644
index 000000000..cc31d4a7e
--- /dev/null
+++ b/server/middlewares/validators/runners/registration-token.ts
@@ -0,0 +1,37 @@
+import express from 'express'
+import { param } from 'express-validator'
+import { isIdValid } from '@server/helpers/custom-validators/misc'
+import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
+import { forceNumber } from '@shared/core-utils'
+import { HttpStatusCode } from '@shared/models'
+import { areValidationErrors } from '../shared/utils'
+
+const tags = [ 'runner' ]
+
+const deleteRegistrationTokenValidator = [
+  param('id').custom(isIdValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const registrationToken = await RunnerRegistrationTokenModel.load(forceNumber(req.params.id))
+
+    if (!registrationToken) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Registration token not found',
+        tags
+      })
+    }
+
+    res.locals.runnerRegistrationToken = registrationToken
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  deleteRegistrationTokenValidator
+}
diff --git a/server/middlewares/validators/runners/runners.ts b/server/middlewares/validators/runners/runners.ts
new file mode 100644
index 000000000..71a1275d2
--- /dev/null
+++ b/server/middlewares/validators/runners/runners.ts
@@ -0,0 +1,95 @@
+import express from 'express'
+import { body, param } from 'express-validator'
+import { isIdValid } from '@server/helpers/custom-validators/misc'
+import {
+  isRunnerDescriptionValid,
+  isRunnerNameValid,
+  isRunnerRegistrationTokenValid,
+  isRunnerTokenValid
+} from '@server/helpers/custom-validators/runners/runners'
+import { RunnerModel } from '@server/models/runner/runner'
+import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token'
+import { forceNumber } from '@shared/core-utils'
+import { HttpStatusCode, RegisterRunnerBody, ServerErrorCode } from '@shared/models'
+import { areValidationErrors } from '../shared/utils'
+
+const tags = [ 'runner' ]
+
+const registerRunnerValidator = [
+  body('registrationToken').custom(isRunnerRegistrationTokenValid),
+  body('name').custom(isRunnerNameValid),
+  body('description').optional().custom(isRunnerDescriptionValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const body: RegisterRunnerBody = req.body
+
+    const runnerRegistrationToken = await RunnerRegistrationTokenModel.loadByRegistrationToken(body.registrationToken)
+
+    if (!runnerRegistrationToken) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Registration token is invalid',
+        tags
+      })
+    }
+
+    res.locals.runnerRegistrationToken = runnerRegistrationToken
+
+    return next()
+  }
+]
+
+const deleteRunnerValidator = [
+  param('runnerId').custom(isIdValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const runner = await RunnerModel.load(forceNumber(req.params.runnerId))
+
+    if (!runner) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Runner not found',
+        tags
+      })
+    }
+
+    res.locals.runner = runner
+
+    return next()
+  }
+]
+
+const getRunnerFromTokenValidator = [
+  body('runnerToken').custom(isRunnerTokenValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res, { tags })) return
+
+    const runner = await RunnerModel.loadByToken(req.body.runnerToken)
+
+    if (!runner) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Unknown runner token',
+        type: ServerErrorCode.UNKNOWN_RUNNER_TOKEN,
+        tags
+      })
+    }
+
+    res.locals.runner = runner
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  registerRunnerValidator,
+  deleteRunnerValidator,
+  getRunnerFromTokenValidator
+}