aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/initializers
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-01-25 15:05:18 +0100
committerChocobozzz <me@florianbigard.com>2018-01-25 18:41:17 +0100
commit94a5ff8a4a75d75bb9df542a39ce8769e7a7e6a4 (patch)
tree32a9148e0e4567f0c4ffae0412cbed20b84e8873 /server/initializers
parentd765fafc3faf0db9818eb1a07161df1cb1bc0efa (diff)
downloadPeerTube-94a5ff8a4a75d75bb9df542a39ce8769e7a7e6a4.tar.gz
PeerTube-94a5ff8a4a75d75bb9df542a39ce8769e7a7e6a4.tar.zst
PeerTube-94a5ff8a4a75d75bb9df542a39ce8769e7a7e6a4.zip
Move job queue to redis
We'll use it as cache in the future. /!\ You'll loose your old jobs (pending jobs too) so upgrade only when you don't have pending job anymore.
Diffstat (limited to 'server/initializers')
-rw-r--r--server/initializers/constants.ts49
-rw-r--r--server/initializers/database.ts2
-rw-r--r--server/initializers/migrations/0100-activitypub.ts5
-rw-r--r--server/initializers/migrations/0180-job-table-delete.ts18
4 files changed, 44 insertions, 30 deletions
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index cb043251a..329d0ffe8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -1,6 +1,6 @@
1import { IConfig } from 'config' 1import { IConfig } from 'config'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { JobCategory, JobState, VideoRateType } from '../../shared/models' 3import { JobType, VideoRateType } from '../../shared/models'
4import { ActivityPubActorType } from '../../shared/models/activitypub' 4import { ActivityPubActorType } from '../../shared/models/activitypub'
5import { FollowState } from '../../shared/models/actors' 5import { FollowState } from '../../shared/models/actors'
6import { VideoPrivacy } from '../../shared/models/videos' 6import { VideoPrivacy } from '../../shared/models/videos'
@@ -12,7 +12,7 @@ let config: IConfig = require('config')
12 12
13// --------------------------------------------------------------------------- 13// ---------------------------------------------------------------------------
14 14
15const LAST_MIGRATION_VERSION = 175 15const LAST_MIGRATION_VERSION = 180
16 16
17// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
18 18
@@ -26,7 +26,7 @@ const PAGINATION_COUNT_DEFAULT = 15
26const SORTABLE_COLUMNS = { 26const SORTABLE_COLUMNS = {
27 USERS: [ 'id', 'username', 'createdAt' ], 27 USERS: [ 'id', 'username', 'createdAt' ],
28 ACCOUNTS: [ 'createdAt' ], 28 ACCOUNTS: [ 'createdAt' ],
29 JOBS: [ 'id', 'createdAt' ], 29 JOBS: [ 'createdAt' ],
30 VIDEO_ABUSES: [ 'id', 'createdAt' ], 30 VIDEO_ABUSES: [ 'id', 'createdAt' ],
31 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], 31 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
32 VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ], 32 VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ],
@@ -61,23 +61,20 @@ const REMOTE_SCHEME = {
61 WS: 'wss' 61 WS: 'wss'
62} 62}
63 63
64const JOB_STATES: { [ id: string ]: JobState } = { 64const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
65 PENDING: 'pending', 65 'activitypub-http-broadcast': 5,
66 PROCESSING: 'processing', 66 'activitypub-http-unicast': 5,
67 ERROR: 'error', 67 'activitypub-http-fetcher': 5,
68 SUCCESS: 'success' 68 'video-file': 1
69}
70const JOB_CATEGORIES: { [ id: string ]: JobCategory } = {
71 TRANSCODING: 'transcoding',
72 ACTIVITYPUB_HTTP: 'activitypub-http'
73} 69}
74// How many maximum jobs we fetch from the database per cycle 70const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
75const JOBS_FETCH_LIMIT_PER_CYCLE = { 71 'activitypub-http-broadcast': 1,
76 transcoding: 10, 72 'activitypub-http-unicast': 5,
77 httpRequest: 20 73 'activitypub-http-fetcher': 1,
74 'video-file': 1
78} 75}
79// 1 minutes 76// 2 days
80let JOBS_FETCHING_INTERVAL = 60000 77const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2
81 78
82// 1 hour 79// 1 hour
83let SCHEDULER_INTERVAL = 60000 * 60 80let SCHEDULER_INTERVAL = 60000 * 60
@@ -96,6 +93,11 @@ const CONFIG = {
96 USERNAME: config.get<string>('database.username'), 93 USERNAME: config.get<string>('database.username'),
97 PASSWORD: config.get<string>('database.password') 94 PASSWORD: config.get<string>('database.password')
98 }, 95 },
96 REDIS: {
97 HOSTNAME: config.get<string>('redis.hostname'),
98 PORT: config.get<string>('redis.port'),
99 AUTH: config.get<string>('redis.auth')
100 },
99 STORAGE: { 101 STORAGE: {
100 AVATARS_DIR: buildPath(config.get<string>('storage.avatars')), 102 AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
101 LOG_DIR: buildPath(config.get<string>('storage.logs')), 103 LOG_DIR: buildPath(config.get<string>('storage.logs')),
@@ -284,7 +286,6 @@ const ACTIVITY_PUB = {
284 PUBLIC: 'https://www.w3.org/ns/activitystreams#Public', 286 PUBLIC: 'https://www.w3.org/ns/activitystreams#Public',
285 COLLECTION_ITEMS_PER_PAGE: 10, 287 COLLECTION_ITEMS_PER_PAGE: 10,
286 FETCH_PAGE_LIMIT: 100, 288 FETCH_PAGE_LIMIT: 100,
287 MAX_HTTP_ATTEMPT: 5,
288 URL_MIME_TYPES: { 289 URL_MIME_TYPES: {
289 VIDEO: Object.keys(VIDEO_MIMETYPE_EXT), 290 VIDEO: Object.keys(VIDEO_MIMETYPE_EXT),
290 TORRENT: [ 'application/x-bittorrent' ], 291 TORRENT: [ 'application/x-bittorrent' ],
@@ -358,7 +359,6 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '<!-- open graph and oembed tags -->'
358// Special constants for a test instance 359// Special constants for a test instance
359if (isTestInstance() === true) { 360if (isTestInstance() === true) {
360 ACTOR_FOLLOW_SCORE.BASE = 20 361 ACTOR_FOLLOW_SCORE.BASE = 20
361 JOBS_FETCHING_INTERVAL = 1000
362 REMOTE_SCHEME.HTTP = 'http' 362 REMOTE_SCHEME.HTTP = 'http'
363 REMOTE_SCHEME.WS = 'ws' 363 REMOTE_SCHEME.WS = 'ws'
364 STATIC_MAX_AGE = '0' 364 STATIC_MAX_AGE = '0'
@@ -381,10 +381,8 @@ export {
381 CONFIG, 381 CONFIG,
382 CONSTRAINTS_FIELDS, 382 CONSTRAINTS_FIELDS,
383 EMBED_SIZE, 383 EMBED_SIZE,
384 JOB_STATES, 384 JOB_CONCURRENCY,
385 JOBS_FETCH_LIMIT_PER_CYCLE, 385 JOB_ATTEMPTS,
386 JOBS_FETCHING_INTERVAL,
387 JOB_CATEGORIES,
388 LAST_MIGRATION_VERSION, 386 LAST_MIGRATION_VERSION,
389 OAUTH_LIFETIME, 387 OAUTH_LIFETIME,
390 OPENGRAPH_AND_OEMBED_COMMENT, 388 OPENGRAPH_AND_OEMBED_COMMENT,
@@ -408,7 +406,8 @@ export {
408 VIDEO_RATE_TYPES, 406 VIDEO_RATE_TYPES,
409 VIDEO_MIMETYPE_EXT, 407 VIDEO_MIMETYPE_EXT,
410 AVATAR_MIMETYPE_EXT, 408 AVATAR_MIMETYPE_EXT,
411 SCHEDULER_INTERVAL 409 SCHEDULER_INTERVAL,
410 JOB_COMPLETED_LIFETIME
412} 411}
413 412
414// --------------------------------------------------------------------------- 413// ---------------------------------------------------------------------------
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 852db68a0..b537ee59a 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -9,7 +9,6 @@ import { ActorModel } from '../models/activitypub/actor'
9import { ActorFollowModel } from '../models/activitypub/actor-follow' 9import { ActorFollowModel } from '../models/activitypub/actor-follow'
10import { ApplicationModel } from '../models/application/application' 10import { ApplicationModel } from '../models/application/application'
11import { AvatarModel } from '../models/avatar/avatar' 11import { AvatarModel } from '../models/avatar/avatar'
12import { JobModel } from '../models/job/job'
13import { OAuthClientModel } from '../models/oauth/oauth-client' 12import { OAuthClientModel } from '../models/oauth/oauth-client'
14import { OAuthTokenModel } from '../models/oauth/oauth-token' 13import { OAuthTokenModel } from '../models/oauth/oauth-token'
15import { ServerModel } from '../models/server/server' 14import { ServerModel } from '../models/server/server'
@@ -61,7 +60,6 @@ async function initDatabaseModels (silent: boolean) {
61 ActorFollowModel, 60 ActorFollowModel,
62 AvatarModel, 61 AvatarModel,
63 AccountModel, 62 AccountModel,
64 JobModel,
65 OAuthClientModel, 63 OAuthClientModel,
66 OAuthTokenModel, 64 OAuthTokenModel,
67 ServerModel, 65 ServerModel,
diff --git a/server/initializers/migrations/0100-activitypub.ts b/server/initializers/migrations/0100-activitypub.ts
index 8c5198f85..a7ebd804c 100644
--- a/server/initializers/migrations/0100-activitypub.ts
+++ b/server/initializers/migrations/0100-activitypub.ts
@@ -1,11 +1,10 @@
1import { values } from 'lodash'
2import * as Sequelize from 'sequelize' 1import * as Sequelize from 'sequelize'
3import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' 2import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
4import { shareVideoByServerAndChannel } from '../../lib/activitypub/share' 3import { shareVideoByServerAndChannel } from '../../lib/activitypub/share'
5import { getVideoActivityPubUrl, getVideoChannelActivityPubUrl } from '../../lib/activitypub/url' 4import { getVideoActivityPubUrl, getVideoChannelActivityPubUrl } from '../../lib/activitypub/url'
6import { createLocalAccountWithoutKeys } from '../../lib/user' 5import { createLocalAccountWithoutKeys } from '../../lib/user'
7import { ApplicationModel } from '../../models/application/application' 6import { ApplicationModel } from '../../models/application/application'
8import { JOB_CATEGORIES, SERVER_ACTOR_NAME } from '../constants' 7import { SERVER_ACTOR_NAME } from '../constants'
9 8
10async function up (utils: { 9async function up (utils: {
11 transaction: Sequelize.Transaction, 10 transaction: Sequelize.Transaction,
@@ -161,7 +160,7 @@ async function up (utils: {
161 160
162 { 161 {
163 const data = { 162 const data = {
164 type: Sequelize.ENUM(values(JOB_CATEGORIES)), 163 type: Sequelize.ENUM('transcoding', 'activitypub-http'),
165 defaultValue: 'transcoding', 164 defaultValue: 'transcoding',
166 allowNull: false 165 allowNull: false
167 } 166 }
diff --git a/server/initializers/migrations/0180-job-table-delete.ts b/server/initializers/migrations/0180-job-table-delete.ts
new file mode 100644
index 000000000..df29145d0
--- /dev/null
+++ b/server/initializers/migrations/0180-job-table-delete.ts
@@ -0,0 +1,18 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize
7}): Promise<void> {
8 await utils.queryInterface.dropTable('job')
9}
10
11function down (options) {
12 throw new Error('Not implemented.')
13}
14
15export {
16 up,
17 down
18}