Don't stuck on active jobs

diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index d9d7a6e42770c25b0746a7ffa796a6c20e6ff6ff..fb3cc8f66e9f463bb41db283bcf0c1eb71b13ba1 100644 (file)
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -32,7 +32,7 @@ class JobQueue {
 
   private constructor () {}
 
-  init () {
+  async init () {
     // Already initialized
     if (this.initialized === true) return
     this.initialized = true
@@ -54,6 +54,8 @@ class JobQueue {
     })
     this.jobQueue.watchStuckJobs(5000)
 
+    await this.reactiveStuckJobs()
+
     for (const handlerName of Object.keys(handlers)) {
       this.jobQueue.process(handlerName, JOB_CONCURRENCY[handlerName], async (job, done) => {
         try {
@@ -117,6 +119,31 @@ class JobQueue {
     })
   }
 
+  private reactiveStuckJobs () {
+    const promises: Promise<any>[] = []
+
+    this.jobQueue.active((err, ids) => {
+      if (err) throw err
+
+      for (const id of ids) {
+        kue.Job.get(id, (err, job) => {
+          if (err) throw err
+
+          const p = new Promise((res, rej) => {
+            job.inactive(err => {
+              if (err) return rej(err)
+              return res()
+            })
+          })
+
+          promises.push(p)
+        })
+      }
+    })
+
+    return Promise.all(promises)
+  }
+
   static get Instance () {
     return this.instance || (this.instance = new this())
   }

diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts
index 7f67525ed5cba9a62f26154c125f784349047e40..f10ca856de7f5a85ea33ce70d56eb9f41f715672 100644 (file)
--- a/server/tests/real-world/real-world.ts
+++ b/server/tests/real-world/real-world.ts
@@ -347,7 +347,7 @@ function goodbye () {
 }
 
 async function isTherePendingRequests (servers: ServerInfo[]) {
-  const states: JobState[] = [ 'inactive', 'active' ]
+  const states: JobState[] = [ 'inactive', 'active', 'delayed' ]
   const tasks: Promise<any>[] = []
   let pendingRequests = false