Move job queue to redis

We'll use it as cache in the future.

/!\ You'll loose your old jobs (pending jobs too) so upgrade only when
you don't have pending job anymore.

4 files changed, 44 insertions, 30 deletions

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index cb043251a..329d0ffe8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -1,6 +1,6 @@
 import { IConfig } from 'config'
 import { dirname, join } from 'path'
-import { JobCategory, JobState, VideoRateType } from '../../shared/models'
+import { JobType, VideoRateType } from '../../shared/models'
 import { ActivityPubActorType } from '../../shared/models/activitypub'
 import { FollowState } from '../../shared/models/actors'
 import { VideoPrivacy } from '../../shared/models/videos'
@@ -12,7 +12,7 @@ let config: IConfig = require('config')
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 175
+const LAST_MIGRATION_VERSION = 180
 
 // ---------------------------------------------------------------------------
 
@@ -26,7 +26,7 @@ const PAGINATION_COUNT_DEFAULT = 15
 const SORTABLE_COLUMNS = {
   USERS: [ 'id', 'username', 'createdAt' ],
   ACCOUNTS: [ 'createdAt' ],
-  JOBS: [ 'id', 'createdAt' ],
+  JOBS: [ 'createdAt' ],
   VIDEO_ABUSES: [ 'id', 'createdAt' ],
   VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
   VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ],
@@ -61,23 +61,20 @@ const REMOTE_SCHEME = {
   WS: 'wss'
 }
 
-const JOB_STATES: { [ id: string ]: JobState } = {
-  PENDING: 'pending',
-  PROCESSING: 'processing',
-  ERROR: 'error',
-  SUCCESS: 'success'
-}
-const JOB_CATEGORIES: { [ id: string ]: JobCategory } = {
-  TRANSCODING: 'transcoding',
-  ACTIVITYPUB_HTTP: 'activitypub-http'
+const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
+  'activitypub-http-broadcast': 5,
+  'activitypub-http-unicast': 5,
+  'activitypub-http-fetcher': 5,
+  'video-file': 1
 }
-// How many maximum jobs we fetch from the database per cycle
-const JOBS_FETCH_LIMIT_PER_CYCLE = {
-  transcoding: 10,
-  httpRequest: 20
+const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
+  'activitypub-http-broadcast': 1,
+  'activitypub-http-unicast': 5,
+  'activitypub-http-fetcher': 1,
+  'video-file': 1
 }
-// 1 minutes
-let JOBS_FETCHING_INTERVAL = 60000
+// 2 days
+const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2
 
 // 1 hour
 let SCHEDULER_INTERVAL = 60000 * 60
@@ -96,6 +93,11 @@ const CONFIG = {
     USERNAME: config.get<string>('database.username'),
     PASSWORD: config.get<string>('database.password')
   },
+  REDIS: {
+    HOSTNAME: config.get<string>('redis.hostname'),
+    PORT: config.get<string>('redis.port'),
+    AUTH: config.get<string>('redis.auth')
+  },
   STORAGE: {
     AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
     LOG_DIR: buildPath(config.get<string>('storage.logs')),
@@ -284,7 +286,6 @@ const ACTIVITY_PUB = {
   PUBLIC: 'https://www.w3.org/ns/activitystreams#Public',
   COLLECTION_ITEMS_PER_PAGE: 10,
   FETCH_PAGE_LIMIT: 100,
-  MAX_HTTP_ATTEMPT: 5,
   URL_MIME_TYPES: {
     VIDEO: Object.keys(VIDEO_MIMETYPE_EXT),
     TORRENT: [ 'application/x-bittorrent' ],
@@ -358,7 +359,6 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '<!-- open graph and oembed tags -->'
 // Special constants for a test instance
 if (isTestInstance() === true) {
   ACTOR_FOLLOW_SCORE.BASE = 20
-  JOBS_FETCHING_INTERVAL = 1000
   REMOTE_SCHEME.HTTP = 'http'
   REMOTE_SCHEME.WS = 'ws'
   STATIC_MAX_AGE = '0'
@@ -381,10 +381,8 @@ export {
   CONFIG,
   CONSTRAINTS_FIELDS,
   EMBED_SIZE,
-  JOB_STATES,
-  JOBS_FETCH_LIMIT_PER_CYCLE,
-  JOBS_FETCHING_INTERVAL,
-  JOB_CATEGORIES,
+  JOB_CONCURRENCY,
+  JOB_ATTEMPTS,
   LAST_MIGRATION_VERSION,
   OAUTH_LIFETIME,
   OPENGRAPH_AND_OEMBED_COMMENT,
@@ -408,7 +406,8 @@ export {
   VIDEO_RATE_TYPES,
   VIDEO_MIMETYPE_EXT,
   AVATAR_MIMETYPE_EXT,
-  SCHEDULER_INTERVAL
+  SCHEDULER_INTERVAL,
+  JOB_COMPLETED_LIFETIME
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 852db68a0..b537ee59a 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -9,7 +9,6 @@ import { ActorModel } from '../models/activitypub/actor'
 import { ActorFollowModel } from '../models/activitypub/actor-follow'
 import { ApplicationModel } from '../models/application/application'
 import { AvatarModel } from '../models/avatar/avatar'
-import { JobModel } from '../models/job/job'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
 import { OAuthTokenModel } from '../models/oauth/oauth-token'
 import { ServerModel } from '../models/server/server'
@@ -61,7 +60,6 @@ async function initDatabaseModels (silent: boolean) {
     ActorFollowModel,
     AvatarModel,
     AccountModel,
-    JobModel,
     OAuthClientModel,
     OAuthTokenModel,
     ServerModel,
diff --git a/server/initializers/migrations/0100-activitypub.ts b/server/initializers/migrations/0100-activitypub.ts
index 8c5198f85..a7ebd804c 100644
--- a/server/initializers/migrations/0100-activitypub.ts
+++ b/server/initializers/migrations/0100-activitypub.ts
@@ -1,11 +1,10 @@
-import { values } from 'lodash'
 import * as Sequelize from 'sequelize'
 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
 import { shareVideoByServerAndChannel } from '../../lib/activitypub/share'
 import { getVideoActivityPubUrl, getVideoChannelActivityPubUrl } from '../../lib/activitypub/url'
 import { createLocalAccountWithoutKeys } from '../../lib/user'
 import { ApplicationModel } from '../../models/application/application'
-import { JOB_CATEGORIES, SERVER_ACTOR_NAME } from '../constants'
+import { SERVER_ACTOR_NAME } from '../constants'
 
 async function up (utils: {
   transaction: Sequelize.Transaction,
@@ -161,7 +160,7 @@ async function up (utils: {
 
   {
     const data = {
-      type: Sequelize.ENUM(values(JOB_CATEGORIES)),
+      type: Sequelize.ENUM('transcoding', 'activitypub-http'),
       defaultValue: 'transcoding',
       allowNull: false
     }
diff --git a/server/initializers/migrations/0180-job-table-delete.ts b/server/initializers/migrations/0180-job-table-delete.ts
new file mode 100644
index 000000000..df29145d0
--- /dev/null
+++ b/server/initializers/migrations/0180-job-table-delete.ts
@@ -0,0 +1,18 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+  await utils.queryInterface.dropTable('job')
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}