9 files changed, 67 insertions, 39 deletions

diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index 0ff7b44a0..7d9dd61e9 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -5,11 +5,13 @@ import { doRequest } from '../../../helpers/requests'
 import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
 import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers/constants'
 import { ActorFollowScoreCache } from '../../files-cache'
+import { ContextType } from '@server/helpers/activitypub'
 
 export type ActivitypubHttpBroadcastPayload = {
   uris: string[]
   signatureActorId?: number
   body: any
+  contextType?: ContextType
 }
 
 async function processActivityPubHttpBroadcast (job: Bull.Job) {
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
index c70ce3be9..6b71e2891 100644
--- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
@@ -4,11 +4,13 @@ import { doRequest } from '../../../helpers/requests'
 import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
 import { JOB_REQUEST_TIMEOUT } from '../../../initializers/constants'
 import { ActorFollowScoreCache } from '../../files-cache'
+import { ContextType } from '@server/helpers/activitypub'
 
 export type ActivitypubHttpUnicastPayload = {
   uri: string
   signatureActorId?: number
   body: any
+  contextType?: ContextType
 }
 
 async function processActivityPubHttpUnicast (job: Bull.Job) {
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
index d3bde6e6a..54b35840d 100644
--- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
+++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
@@ -1,11 +1,11 @@
-import { buildSignedActivity } from '../../../../helpers/activitypub'
+import { buildSignedActivity, ContextType } from '../../../../helpers/activitypub'
 import { getServerActor } from '../../../../helpers/utils'
 import { ActorModel } from '../../../../models/activitypub/actor'
 import { sha256 } from '../../../../helpers/core-utils'
 import { HTTP_SIGNATURE } from '../../../../initializers/constants'
 import { MActor } from '../../../../typings/models'
 
-type Payload = { body: any, signatureActorId?: number }
+type Payload = { body: any, contextType?: ContextType, signatureActorId?: number }
 
 async function computeBody (payload: Payload) {
   let body = payload.body
@@ -13,7 +13,7 @@ async function computeBody (payload: Payload) {
   if (payload.signatureActorId) {
     const actorSignature = await ActorModel.load(payload.signatureActorId)
     if (!actorSignature) throw new Error('Unknown signature actor id.')
-    body = await buildSignedActivity(actorSignature, payload.body)
+    body = await buildSignedActivity(actorSignature, payload.body, payload.contextType)
   }
 
   return body
@@ -42,7 +42,7 @@ async function buildSignedRequestOptions (payload: Payload) {
 
 function buildGlobalHeaders (body: any) {
   return {
-    'Digest': buildDigest(body)
+    Digest: buildDigest(body)
   }
 }
 
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 99c991e72..be9e7d181 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -11,7 +11,7 @@ import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getVideoFilePath } from '@server/lib/video-paths'
 
 export type VideoFileImportPayload = {
-  videoUUID: string,
+  videoUUID: string
   filePath: string
 }
 
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 1fca17584..09f225cec 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -221,7 +221,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
         isNewVideo: true
       }
 
-      await JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
+      await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
     }
 
   } catch (err) {
diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts
new file mode 100644
index 000000000..319d7090e
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-redundancy.ts
@@ -0,0 +1,20 @@
+import * as Bull from 'bull'
+import { logger } from '../../../helpers/logger'
+import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler'
+
+export type VideoRedundancyPayload = {
+  videoId: number
+}
+
+async function processVideoRedundancy (job: Bull.Job) {
+  const payload = job.data as VideoRedundancyPayload
+  logger.info('Processing video redundancy in job %d.', job.id)
+
+  return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processVideoRedundancy
+}
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 39b9fac98..c020057c9 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -6,7 +6,6 @@ import { JobQueue } from '../job-queue'
 import { federateVideoIfNeeded } from '../../activitypub'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { sequelizeTypescript } from '../../../initializers'
-import * as Bluebird from 'bluebird'
 import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
 import { generateHlsPlaylist, mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewResolution } from '../../video-transcoding'
 import { Notifier } from '../../notifier'
@@ -40,8 +39,11 @@ interface OptimizeTranscodingPayload extends BaseTranscodingPayload {
   type: 'optimize'
 }
 
-export type VideoTranscodingPayload = HLSTranscodingPayload | NewResolutionTranscodingPayload
-  | OptimizeTranscodingPayload | MergeAudioTranscodingPayload
+export type VideoTranscodingPayload =
+  HLSTranscodingPayload
+  | NewResolutionTranscodingPayload
+  | OptimizeTranscodingPayload
+  | MergeAudioTranscodingPayload
 
 async function processVideoTranscoding (job: Bull.Job) {
   const payload = job.data as VideoTranscodingPayload
@@ -105,7 +107,7 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
 
   const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
     // Maybe the video changed in database, refresh it
-    let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t)
+    const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t)
     // Video does not exist anymore
     if (!videoDatabase) return undefined
 
@@ -122,8 +124,6 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
     await createHlsJobIfEnabled(hlsPayload)
 
     if (resolutionsEnabled.length !== 0) {
-      const tasks: (Bluebird<Bull.Job<any>> | Promise<Bull.Job<any>>)[] = []
-
       for (const resolution of resolutionsEnabled) {
         let dataInput: VideoTranscodingPayload
 
@@ -143,12 +143,9 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
           }
         }
 
-        const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
-        tasks.push(p)
+        JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
       }
 
-      await Promise.all(tasks)
-
       logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled })
     } else {
       // No transcoding to do, it's now published
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
index 73fa5ed04..2258cd029 100644
--- a/server/lib/job-queue/handlers/video-views.ts
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -23,6 +23,8 @@ async function processVideosViews () {
   for (const videoId of videoIds) {
     try {
       const views = await Redis.Instance.getVideoViews(videoId, hour)
+      await Redis.Instance.deleteVideoViews(videoId, hour)
+
       if (views) {
         logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
 
@@ -52,8 +54,6 @@ async function processVideosViews () {
           logger.error('Cannot create video views for video %d in hour %d.', videoId, hour, { err })
         }
       }
-
-      await Redis.Instance.deleteVideoViews(videoId, hour)
     } catch (err) {
       logger.error('Cannot update video views of video %d in hour %d.', videoId, hour, { err })
     }
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ec601e9ea..14acace7d 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import'
 import { processVideosViews } from './handlers/video-views'
 import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
 import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import'
+import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy'
 
 type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -24,20 +25,21 @@ type CreateJobArgument =
   { type: 'email', payload: EmailPayload } |
   { type: 'video-import', payload: VideoImportPayload } |
   { type: 'activitypub-refresher', payload: RefreshPayload } |
-  { type: 'videos-views', payload: {} }
+  { type: 'videos-views', payload: {} } |
+  { type: 'video-redundancy', payload: VideoRedundancyPayload }
 
-const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = {
+const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = {
   'activitypub-http-broadcast': processActivityPubHttpBroadcast,
   'activitypub-http-unicast': processActivityPubHttpUnicast,
   'activitypub-http-fetcher': processActivityPubHttpFetcher,
   'activitypub-follow': processActivityPubFollow,
   'video-file-import': processVideoFileImport,
   'video-transcoding': processVideoTranscoding,
-  'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3)
   'email': processEmail,
   'video-import': processVideoImport,
   'videos-views': processVideosViews,
-  'activitypub-refresher': refreshAPObject
+  'activitypub-refresher': refreshAPObject,
+  'video-redundancy': processVideoRedundancy
 }
 
 const jobTypes: JobType[] = [
@@ -50,20 +52,22 @@ const jobTypes: JobType[] = [
   'video-file-import',
   'video-import',
   'videos-views',
-  'activitypub-refresher'
+  'activitypub-refresher',
+  'video-redundancy'
 ]
 
 class JobQueue {
 
   private static instance: JobQueue
 
-  private queues: { [ id in JobType ]?: Bull.Queue } = {}
+  private queues: { [id in JobType]?: Bull.Queue } = {}
   private initialized = false
   private jobRedisPrefix: string
 
-  private constructor () {}
+  private constructor () {
+  }
 
-  async init () {
+  init () {
     // Already initialized
     if (this.initialized === true) return
     this.initialized = true
@@ -105,11 +109,16 @@ class JobQueue {
     }
   }
 
-  createJob (obj: CreateJobArgument) {
+  createJob (obj: CreateJobArgument): void {
+    this.createJobWithPromise(obj)
+         .catch(err => logger.error('Cannot create job.', { err, obj }))
+  }
+
+  createJobWithPromise (obj: CreateJobArgument) {
     const queue = this.queues[obj.type]
     if (queue === undefined) {
       logger.error('Unknown queue %s: cannot create job.', obj.type)
-      throw Error('Unknown queue, cannot create job')
+      return
     }
 
     const jobArgs: Bull.JobOptions = {
@@ -122,10 +131,10 @@ class JobQueue {
   }
 
   async listForApi (options: {
-    state: JobState,
-    start: number,
-    count: number,
-    asc?: boolean,
+    state: JobState
+    start: number
+    count: number
+    asc?: boolean
     jobType: JobType
   }): Promise<Bull.Job[]> {
     const { state, start, count, asc, jobType } = options
@@ -133,16 +142,14 @@ class JobQueue {
 
     const filteredJobTypes = this.filterJobTypes(jobType)
 
-    // TODO: optimize
     for (const jobType of filteredJobTypes) {
-      const queue = this.queues[ jobType ]
+      const queue = this.queues[jobType]
       if (queue === undefined) {
         logger.error('Unknown queue %s to list jobs.', jobType)
         continue
       }
 
-      // FIXME: Bull queue typings does not have getJobs method
-      const jobs = await (queue as any).getJobs(state, 0, start + count, asc)
+      const jobs = await queue.getJobs([ state ], 0, start + count, asc)
       results = results.concat(jobs)
     }
 
@@ -164,7 +171,7 @@ class JobQueue {
     const filteredJobTypes = this.filterJobTypes(jobType)
 
     for (const type of filteredJobTypes) {
-      const queue = this.queues[ type ]
+      const queue = this.queues[type]
       if (queue === undefined) {
         logger.error('Unknown queue %s to count jobs.', type)
         continue
@@ -172,7 +179,7 @@ class JobQueue {
 
       const counts = await queue.getJobCounts()
 
-      total += counts[ state ]
+      total += counts[state]
     }
 
     return total
@@ -188,7 +195,7 @@ class JobQueue {
   private addRepeatableJobs () {
     this.queues['videos-views'].add({}, {
       repeat: REPEAT_JOBS['videos-views']
-    })
+    }).catch(err => logger.error('Cannot add repeatable job.', { err }))
   }
 
   private filterJobTypes (jobType?: JobType) {