7 files changed, 192 insertions, 28 deletions

diff --git a/server/controllers/api/runners/jobs.ts b/server/controllers/api/runners/jobs.ts
index be5911b53..e9e2ddf49 100644
--- a/server/controllers/api/runners/jobs.ts
+++ b/server/controllers/api/runners/jobs.ts
@@ -5,7 +5,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { generateRunnerJobToken } from '@server/helpers/token-generator'
 import { MIMETYPES } from '@server/initializers/constants'
 import { sequelizeTypescript } from '@server/initializers/database'
-import { getRunnerJobHandlerClass, updateLastRunnerContact } from '@server/lib/runners'
+import { getRunnerJobHandlerClass, runnerJobCanBeCancelled, updateLastRunnerContact } from '@server/lib/runners'
 import {
   apiRateLimiter,
   asyncMiddleware,
@@ -23,6 +23,7 @@ import {
   errorRunnerJobValidator,
   getRunnerFromTokenValidator,
   jobOfRunnerGetValidatorFactory,
+  listRunnerJobsValidator,
   runnerJobGetValidator,
   successRunnerJobValidator,
   updateRunnerJobValidator
@@ -131,9 +132,17 @@ runnerJobsRouter.get('/jobs',
   runnerJobsSortValidator,
   setDefaultSort,
   setDefaultPagination,
+  listRunnerJobsValidator,
   asyncMiddleware(listRunnerJobs)
 )
 
+runnerJobsRouter.delete('/jobs/:jobUUID',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_RUNNERS),
+  asyncMiddleware(runnerJobGetValidator),
+  asyncMiddleware(deleteRunnerJob)
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -374,6 +383,21 @@ async function cancelRunnerJob (req: express.Request, res: express.Response) {
   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
+async function deleteRunnerJob (req: express.Request, res: express.Response) {
+  const runnerJob = res.locals.runnerJob
+
+  logger.info('Deleting job %s (%s)', runnerJob.uuid, runnerJob.type, lTags(runnerJob.uuid, runnerJob.type))
+
+  if (runnerJobCanBeCancelled(runnerJob)) {
+    const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob)
+    await new RunnerJobHandler().cancel({ runnerJob })
+  }
+
+  await runnerJob.destroy()
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
 async function listRunnerJobs (req: express.Request, res: express.Response) {
   const query: ListRunnerJobsQuery = req.query
 
@@ -381,7 +405,8 @@ async function listRunnerJobs (req: express.Request, res: express.Response) {
     start: query.start,
     count: query.count,
     sort: query.sort,
-    search: query.search
+    search: query.search,
+    stateOneOf: query.stateOneOf
   })
 
   return res.json({
diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts
index 725a7658f..6349e79ba 100644
--- a/server/helpers/custom-validators/runners/jobs.ts
+++ b/server/helpers/custom-validators/runners/jobs.ts
@@ -1,6 +1,6 @@
 import { UploadFilesForCheck } from 'express'
 import validator from 'validator'
-import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
+import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants'
 import {
   LiveRTMPHLSTranscodingSuccess,
   RunnerJobSuccessPayload,
@@ -11,7 +11,7 @@ import {
   VODHLSTranscodingSuccess,
   VODWebVideoTranscodingSuccess
 } from '@shared/models'
-import { exists, isFileValid, isSafeFilename } from '../misc'
+import { exists, isArray, isFileValid, isSafeFilename } from '../misc'
 
 const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS
 
@@ -56,6 +56,14 @@ function isRunnerJobErrorMessageValid (value: string) {
   return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE)
 }
 
+function isRunnerJobStateValid (value: any) {
+  return exists(value) && RUNNER_JOB_STATES[value] !== undefined
+}
+
+function isRunnerJobArrayOfStateValid (value: any) {
+  return isArray(value) && value.every(v => isRunnerJobStateValid(v))
+}
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -65,7 +73,9 @@ export {
   isRunnerJobTokenValid,
   isRunnerJobErrorMessageValid,
   isRunnerJobProgressValid,
-  isRunnerJobAbortReasonValid
+  isRunnerJobAbortReasonValid,
+  isRunnerJobArrayOfStateValid,
+  isRunnerJobStateValid
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/runners/runner.ts b/server/lib/runners/runner.ts
index 921cae6f2..947fdb3f0 100644
--- a/server/lib/runners/runner.ts
+++ b/server/lib/runners/runner.ts
@@ -2,8 +2,9 @@ import express from 'express'
 import { retryTransactionWrapper } from '@server/helpers/database-utils'
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { sequelizeTypescript } from '@server/initializers/database'
-import { MRunner } from '@server/types/models/runners'
+import { MRunner, MRunnerJob } from '@server/types/models/runners'
 import { RUNNER_JOBS } from '@server/initializers/constants'
+import { RunnerJobState } from '@shared/models'
 
 const lTags = loggerTagsFactory('runner')
 
@@ -32,6 +33,17 @@ function updateLastRunnerContact (req: express.Request, runner: MRunner) {
   .finally(() => updatingRunner.delete(runner.id))
 }
 
+function runnerJobCanBeCancelled (runnerJob: MRunnerJob) {
+  const allowedStates = new Set<RunnerJobState>([
+    RunnerJobState.PENDING,
+    RunnerJobState.PROCESSING,
+    RunnerJobState.WAITING_FOR_PARENT_JOB
+  ])
+
+  return allowedStates.has(runnerJob.state)
+}
+
 export {
-  updateLastRunnerContact
+  updateLastRunnerContact,
+  runnerJobCanBeCancelled
 }
diff --git a/server/middlewares/validators/runners/jobs.ts b/server/middlewares/validators/runners/jobs.ts
index 384b209ba..62f9340a5 100644
--- a/server/middlewares/validators/runners/jobs.ts
+++ b/server/middlewares/validators/runners/jobs.ts
@@ -1,8 +1,9 @@
 import express from 'express'
-import { body, param } from 'express-validator'
-import { isUUIDValid } from '@server/helpers/custom-validators/misc'
+import { body, param, query } from 'express-validator'
+import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc'
 import {
   isRunnerJobAbortReasonValid,
+  isRunnerJobArrayOfStateValid,
   isRunnerJobErrorMessageValid,
   isRunnerJobProgressValid,
   isRunnerJobSuccessPayloadValid,
@@ -12,7 +13,9 @@ import {
 import { isRunnerTokenValid } from '@server/helpers/custom-validators/runners/runners'
 import { cleanUpReqFiles } from '@server/helpers/express-utils'
 import { LiveManager } from '@server/lib/live'
+import { runnerJobCanBeCancelled } from '@server/lib/runners'
 import { RunnerJobModel } from '@server/models/runner/runner-job'
+import { arrayify } from '@shared/core-utils'
 import {
   HttpStatusCode,
   RunnerJobLiveRTMPHLSTranscodingPrivatePayload,
@@ -119,13 +122,7 @@ export const cancelRunnerJobValidator = [
   (req: express.Request, res: express.Response, next: express.NextFunction) => {
     const runnerJob = res.locals.runnerJob
 
-    const allowedStates = new Set<RunnerJobState>([
-      RunnerJobState.PENDING,
-      RunnerJobState.PROCESSING,
-      RunnerJobState.WAITING_FOR_PARENT_JOB
-    ])
-
-    if (allowedStates.has(runnerJob.state) !== true) {
+    if (runnerJobCanBeCancelled(runnerJob) !== true) {
       return res.fail({
         status: HttpStatusCode.BAD_REQUEST_400,
         message: 'Cannot cancel this job that is not in "pending", "processing" or "waiting for parent job" state',
@@ -137,6 +134,21 @@ export const cancelRunnerJobValidator = [
   }
 ]
 
+export const listRunnerJobsValidator = [
+  query('search')
+    .optional()
+    .custom(exists),
+
+  query('stateOneOf')
+    .optional()
+    .customSanitizer(arrayify)
+    .custom(isRunnerJobArrayOfStateValid),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    return next()
+  }
+]
+
 export const runnerJobGetValidator = [
   param('jobUUID').custom(isUUIDValid),
 
diff --git a/server/models/runner/runner-job.ts b/server/models/runner/runner-job.ts
index add6f9a43..f2ffd6a84 100644
--- a/server/models/runner/runner-job.ts
+++ b/server/models/runner/runner-job.ts
@@ -1,4 +1,4 @@
-import { FindOptions, Op, Transaction } from 'sequelize'
+import { Op, Transaction } from 'sequelize'
 import {
   AllowNull,
   BelongsTo,
@@ -13,7 +13,7 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { isUUIDValid } from '@server/helpers/custom-validators/misc'
+import { isArray, isUUIDValid } from '@server/helpers/custom-validators/misc'
 import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants'
 import { MRunnerJob, MRunnerJobRunner, MRunnerJobRunnerParent } from '@server/types/models/runners'
 import { RunnerJob, RunnerJobAdmin, RunnerJobPayload, RunnerJobPrivatePayload, RunnerJobState, RunnerJobType } from '@shared/models'
@@ -227,28 +227,38 @@ export class RunnerJobModel extends Model<Partial<AttributesOnly<RunnerJobModel>
     count: number
     sort: string
     search?: string
+    stateOneOf?: RunnerJobState[]
   }) {
-    const { start, count, sort, search } = options
+    const { start, count, sort, search, stateOneOf } = options
 
-    const query: FindOptions = {
+    const query = {
       offset: start,
       limit: count,
-      order: getSort(sort)
+      order: getSort(sort),
+      where: []
     }
 
     if (search) {
       if (isUUIDValid(search)) {
-        query.where = { uuid: search }
+        query.where.push({ uuid: search })
       } else {
-        query.where = {
+        query.where.push({
           [Op.or]: [
             searchAttribute(search, 'type'),
             searchAttribute(search, '$Runner.name$')
           ]
-        }
+        })
       }
     }
 
+    if (isArray(stateOneOf) && stateOneOf.length !== 0) {
+      query.where.push({
+        state: {
+          [Op.in]: stateOneOf
+        }
+      })
+    }
+
     return Promise.all([
       RunnerJobModel.scope([ ScopeNames.WITH_RUNNER ]).count(query),
       RunnerJobModel.scope([ ScopeNames.WITH_RUNNER, ScopeNames.WITH_PARENT ]).findAll<MRunnerJobRunnerParent>(query)
diff --git a/server/tests/api/check-params/runners.ts b/server/tests/api/check-params/runners.ts
index 9112ff716..7f9a0cd32 100644
--- a/server/tests/api/check-params/runners.ts
+++ b/server/tests/api/check-params/runners.ts
@@ -1,14 +1,14 @@
-import { basename } from 'path'
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+import { basename } from 'path'
 import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared'
 import {
   HttpStatusCode,
   isVideoStudioTaskIntro,
   RunnerJob,
   RunnerJobState,
+  RunnerJobStudioTranscodingPayload,
   RunnerJobSuccessPayload,
   RunnerJobUpdatePayload,
-  RunnerJobStudioTranscodingPayload,
   VideoPrivacy,
   VideoStudioTaskIntro
 } from '@shared/models'
@@ -236,6 +236,10 @@ describe('Test managing runners', function () {
         await checkBadSortPagination(server.url, path, server.accessToken)
       })
 
+      it('Should fail with an invalid state', async function () {
+        await server.runners.list({ start: 0, count: 5, sort: '-createdAt' })
+      })
+
       it('Should succeed to list with the correct params', async function () {
         await server.runners.list({ start: 0, count: 5, sort: '-createdAt' })
       })
@@ -307,8 +311,48 @@ describe('Test managing runners', function () {
         await checkBadSortPagination(server.url, path, server.accessToken)
       })
 
-      it('Should succeed to list with the correct params', async function () {
-        await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt' })
+      it('Should fail with an invalid state', async function () {
+        await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: 42 as any })
+        await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: [ 42 ] as any })
+      })
+
+      it('Should succeed with the correct params', async function () {
+        await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: [ RunnerJobState.COMPLETED ] })
+      })
+    })
+
+    describe('Delete', function () {
+      let jobUUID: string
+
+      before(async function () {
+        this.timeout(60000)
+
+        await server.videos.quickUpload({ name: 'video' })
+        await waitJobs([ server ])
+
+        const { availableJobs } = await server.runnerJobs.request({ runnerToken })
+        jobUUID = availableJobs[0].uuid
+      })
+
+      it('Should fail without oauth token', async function () {
+        await server.runnerJobs.deleteByAdmin({ token: null, jobUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
+      })
+
+      it('Should fail without admin rights', async function () {
+        await server.runnerJobs.deleteByAdmin({ token: userToken, jobUUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
+      })
+
+      it('Should fail with a bad job uuid', async function () {
+        await server.runnerJobs.deleteByAdmin({ jobUUID: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
+      })
+
+      it('Should fail with an unknown job uuid', async function () {
+        const jobUUID = badUUID
+        await server.runnerJobs.deleteByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+      })
+
+      it('Should succeed with the correct params', async function () {
+        await server.runnerJobs.deleteByAdmin({ jobUUID })
       })
     })
 
diff --git a/server/tests/api/runners/runner-common.ts b/server/tests/api/runners/runner-common.ts
index 34a51abe7..7fed75f40 100644
--- a/server/tests/api/runners/runner-common.ts
+++ b/server/tests/api/runners/runner-common.ts
@@ -339,6 +339,30 @@ describe('Test runner common actions', function () {
 
           expect(data).to.not.have.lengthOf(0)
           expect(total).to.not.equal(0)
+
+          for (const job of data) {
+            expect(job.type).to.include('hls')
+          }
+        }
+      })
+
+      it('Should filter jobs', async function () {
+        {
+          const { total, data } = await server.runnerJobs.list({ stateOneOf: [ RunnerJobState.WAITING_FOR_PARENT_JOB ] })
+
+          expect(data).to.not.have.lengthOf(0)
+          expect(total).to.not.equal(0)
+
+          for (const job of data) {
+            expect(job.state.label).to.equal('Waiting for parent job to finish')
+          }
+        }
+
+        {
+          const { total, data } = await server.runnerJobs.list({ stateOneOf: [ RunnerJobState.COMPLETED ] })
+
+          expect(data).to.have.lengthOf(0)
+          expect(total).to.equal(0)
         }
       })
     })
@@ -598,6 +622,33 @@ describe('Test runner common actions', function () {
       })
     })
 
+    describe('Remove', function () {
+
+      it('Should remove a pending job', async function () {
+        await server.videos.quickUpload({ name: 'video' })
+        await waitJobs([ server ])
+
+        {
+          const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
+
+          const pendingJob = data.find(j => j.state.id === RunnerJobState.PENDING)
+          jobUUID = pendingJob.uuid
+
+          await server.runnerJobs.deleteByAdmin({ jobUUID })
+        }
+
+        {
+          const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' })
+
+          const parent = data.find(j => j.uuid === jobUUID)
+          expect(parent).to.not.exist
+
+          const children = data.filter(j => j.parent?.uuid === jobUUID)
+          expect(children).to.have.lengthOf(0)
+        }
+      })
+    })
+
     describe('Stalled jobs', function () {
 
       it('Should abort stalled jobs', async function () {