diff options
author | Chocobozzz <me@florianbigard.com> | 2018-01-11 09:35:50 +0100 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-01-11 09:35:50 +0100 |
commit | 60650c77c8a2a98e92d869b237ae4900f369a8fc (patch) | |
tree | 7304a7591b5b23b99d219c4d06c6bd5c4c58c1c2 | |
parent | 7ae71355c40e9065f83d3fc77b6750d1929ac201 (diff) | |
download | PeerTube-60650c77c8a2a98e92d869b237ae4900f369a8fc.tar.gz PeerTube-60650c77c8a2a98e92d869b237ae4900f369a8fc.tar.zst PeerTube-60650c77c8a2a98e92d869b237ae4900f369a8fc.zip |
Add scores to follows and remove bad ones
21 files changed, 217 insertions, 133 deletions
diff --git a/client/src/app/+admin/follows/followers-list/followers-list.component.html b/client/src/app/+admin/follows/followers-list/followers-list.component.html index 9499a0433..d5b1b789d 100644 --- a/client/src/app/+admin/follows/followers-list/followers-list.component.html +++ b/client/src/app/+admin/follows/followers-list/followers-list.component.html | |||
@@ -3,8 +3,8 @@ | |||
3 | sortField="createdAt" (onLazyLoad)="loadLazy($event)" | 3 | sortField="createdAt" (onLazyLoad)="loadLazy($event)" |
4 | > | 4 | > |
5 | <p-column field="id" header="ID" [style]="{ width: '60px' }"></p-column> | 5 | <p-column field="id" header="ID" [style]="{ width: '60px' }"></p-column> |
6 | <p-column field="score" header="Score"></p-column> | ||
6 | <p-column field="follower.host" header="Host"></p-column> | 7 | <p-column field="follower.host" header="Host"></p-column> |
7 | <p-column field="follower.score" header="Score"></p-column> | ||
8 | <p-column field="state" header="State"></p-column> | 8 | <p-column field="state" header="State"></p-column> |
9 | <p-column field="createdAt" header="Created date" [sortable]="true"></p-column> | 9 | <p-column field="createdAt" header="Created date" [sortable]="true"></p-column> |
10 | </p-dataTable> | 10 | </p-dataTable> |
diff --git a/client/src/app/shared/misc/utils.ts b/client/src/app/shared/misc/utils.ts index 2739ff81a..23b46812b 100644 --- a/client/src/app/shared/misc/utils.ts +++ b/client/src/app/shared/misc/utils.ts | |||
@@ -31,7 +31,7 @@ function populateAsyncUserVideoChannels (authService: AuthService, channel: any[ | |||
31 | const videoChannels = user.videoChannels | 31 | const videoChannels = user.videoChannels |
32 | if (Array.isArray(videoChannels) === false) return | 32 | if (Array.isArray(videoChannels) === false) return |
33 | 33 | ||
34 | videoChannels.forEach(c => channel.push({ id: c.id, label: c.name })) | 34 | videoChannels.forEach(c => channel.push({ id: c.id, label: c.displayName })) |
35 | 35 | ||
36 | return res() | 36 | return res() |
37 | } | 37 | } |
diff --git a/client/src/app/videos/+video-edit/video-add.component.html b/client/src/app/videos/+video-edit/video-add.component.html index 2040ff9d4..34291c6c6 100644 --- a/client/src/app/videos/+video-edit/video-add.component.html +++ b/client/src/app/videos/+video-edit/video-add.component.html | |||
@@ -44,7 +44,6 @@ | |||
44 | [validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels" | 44 | [validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels" |
45 | ></my-video-edit> | 45 | ></my-video-edit> |
46 | 46 | ||
47 | |||
48 | <div class="submit-container"> | 47 | <div class="submit-container"> |
49 | <div *ngIf="videoUploaded === false" class="message-submit">Publish will be available when upload is finished</div> | 48 | <div *ngIf="videoUploaded === false" class="message-submit">Publish will be available when upload is finished</div> |
50 | 49 | ||
diff --git a/client/src/app/videos/+video-watch/video-watch.component.html b/client/src/app/videos/+video-watch/video-watch.component.html index a5c387638..5921b4b72 100644 --- a/client/src/app/videos/+video-watch/video-watch.component.html +++ b/client/src/app/videos/+video-watch/video-watch.component.html | |||
@@ -85,7 +85,7 @@ | |||
85 | </div> | 85 | </div> |
86 | 86 | ||
87 | <div class="video-info-channel"> | 87 | <div class="video-info-channel"> |
88 | {{ video.channel.name }} | 88 | {{ video.channel.displayName }} |
89 | <!-- Here will be the subscribe button --> | 89 | <!-- Here will be the subscribe button --> |
90 | </div> | 90 | </div> |
91 | 91 | ||
@@ -56,6 +56,7 @@ import { installApplication } from './server/initializers' | |||
56 | import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs' | 56 | import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs' |
57 | import { VideosPreviewCache } from './server/lib/cache' | 57 | import { VideosPreviewCache } from './server/lib/cache' |
58 | import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers' | 58 | import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers' |
59 | import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler' | ||
59 | 60 | ||
60 | // ----------- Command line ----------- | 61 | // ----------- Command line ----------- |
61 | 62 | ||
@@ -168,6 +169,8 @@ function onDatabaseInitDone () { | |||
168 | // ----------- Make the server listening ----------- | 169 | // ----------- Make the server listening ----------- |
169 | server.listen(port, () => { | 170 | server.listen(port, () => { |
170 | VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) | 171 | VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) |
172 | BadActorFollowScheduler.Instance.enable() | ||
173 | |||
171 | activitypubHttpJobScheduler.activate() | 174 | activitypubHttpJobScheduler.activate() |
172 | transcodingJobScheduler.activate() | 175 | transcodingJobScheduler.activate() |
173 | 176 | ||
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index c735e6daf..0c139912c 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -9,7 +9,7 @@ import { isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core | |||
9 | 9 | ||
10 | // --------------------------------------------------------------------------- | 10 | // --------------------------------------------------------------------------- |
11 | 11 | ||
12 | const LAST_MIGRATION_VERSION = 165 | 12 | const LAST_MIGRATION_VERSION = 170 |
13 | 13 | ||
14 | // --------------------------------------------------------------------------- | 14 | // --------------------------------------------------------------------------- |
15 | 15 | ||
@@ -40,12 +40,12 @@ const OAUTH_LIFETIME = { | |||
40 | 40 | ||
41 | // --------------------------------------------------------------------------- | 41 | // --------------------------------------------------------------------------- |
42 | 42 | ||
43 | // Number of points we add/remove from a friend after a successful/bad request | 43 | // Number of points we add/remove after a successful/bad request |
44 | const SERVERS_SCORE = { | 44 | const ACTOR_FOLLOW_SCORE = { |
45 | PENALTY: -10, | 45 | PENALTY: -10, |
46 | BONUS: 10, | 46 | BONUS: 10, |
47 | BASE: 100, | 47 | BASE: 1000, |
48 | MAX: 1000 | 48 | MAX: 10000 |
49 | } | 49 | } |
50 | 50 | ||
51 | const FOLLOW_STATES: { [ id: string ]: FollowState } = { | 51 | const FOLLOW_STATES: { [ id: string ]: FollowState } = { |
@@ -76,6 +76,9 @@ const JOBS_FETCH_LIMIT_PER_CYCLE = { | |||
76 | // 1 minutes | 76 | // 1 minutes |
77 | let JOBS_FETCHING_INTERVAL = 60000 | 77 | let JOBS_FETCHING_INTERVAL = 60000 |
78 | 78 | ||
79 | // 1 hour | ||
80 | let SCHEDULER_INTERVAL = 60000 * 60 | ||
81 | |||
79 | // --------------------------------------------------------------------------- | 82 | // --------------------------------------------------------------------------- |
80 | 83 | ||
81 | const CONFIG = { | 84 | const CONFIG = { |
@@ -346,7 +349,7 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '<!-- open graph and oembed tags -->' | |||
346 | 349 | ||
347 | // Special constants for a test instance | 350 | // Special constants for a test instance |
348 | if (isTestInstance() === true) { | 351 | if (isTestInstance() === true) { |
349 | SERVERS_SCORE.BASE = 20 | 352 | ACTOR_FOLLOW_SCORE.BASE = 20 |
350 | JOBS_FETCHING_INTERVAL = 1000 | 353 | JOBS_FETCHING_INTERVAL = 1000 |
351 | REMOTE_SCHEME.HTTP = 'http' | 354 | REMOTE_SCHEME.HTTP = 'http' |
352 | REMOTE_SCHEME.WS = 'ws' | 355 | REMOTE_SCHEME.WS = 'ws' |
@@ -354,6 +357,7 @@ if (isTestInstance() === true) { | |||
354 | ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 | 357 | ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 |
355 | ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 60 // 1 minute | 358 | ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 60 // 1 minute |
356 | CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB | 359 | CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB |
360 | SCHEDULER_INTERVAL = 10000 | ||
357 | } | 361 | } |
358 | 362 | ||
359 | CONFIG.WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT) | 363 | CONFIG.WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT) |
@@ -378,7 +382,7 @@ export { | |||
378 | OAUTH_LIFETIME, | 382 | OAUTH_LIFETIME, |
379 | OPENGRAPH_AND_OEMBED_COMMENT, | 383 | OPENGRAPH_AND_OEMBED_COMMENT, |
380 | PAGINATION_COUNT_DEFAULT, | 384 | PAGINATION_COUNT_DEFAULT, |
381 | SERVERS_SCORE, | 385 | ACTOR_FOLLOW_SCORE, |
382 | PREVIEWS_SIZE, | 386 | PREVIEWS_SIZE, |
383 | REMOTE_SCHEME, | 387 | REMOTE_SCHEME, |
384 | FOLLOW_STATES, | 388 | FOLLOW_STATES, |
@@ -396,5 +400,6 @@ export { | |||
396 | VIDEO_LICENCES, | 400 | VIDEO_LICENCES, |
397 | VIDEO_RATE_TYPES, | 401 | VIDEO_RATE_TYPES, |
398 | VIDEO_MIMETYPE_EXT, | 402 | VIDEO_MIMETYPE_EXT, |
399 | AVATAR_MIMETYPE_EXT | 403 | AVATAR_MIMETYPE_EXT, |
404 | SCHEDULER_INTERVAL | ||
400 | } | 405 | } |
diff --git a/server/initializers/migrations/0170-actor-follow-score.ts b/server/initializers/migrations/0170-actor-follow-score.ts new file mode 100644 index 000000000..2deabaf98 --- /dev/null +++ b/server/initializers/migrations/0170-actor-follow-score.ts | |||
@@ -0,0 +1,28 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | import { ACTOR_FOLLOW_SCORE } from '../index' | ||
3 | |||
4 | async function up (utils: { | ||
5 | transaction: Sequelize.Transaction, | ||
6 | queryInterface: Sequelize.QueryInterface, | ||
7 | sequelize: Sequelize.Sequelize | ||
8 | }): Promise<void> { | ||
9 | await utils.queryInterface.removeColumn('server', 'score') | ||
10 | |||
11 | const data = { | ||
12 | type: Sequelize.INTEGER, | ||
13 | allowNull: false, | ||
14 | defaultValue: ACTOR_FOLLOW_SCORE.BASE | ||
15 | } | ||
16 | |||
17 | await utils.queryInterface.addColumn('actorFollow', 'score', data) | ||
18 | |||
19 | } | ||
20 | |||
21 | function down (options) { | ||
22 | throw new Error('Not implemented.') | ||
23 | } | ||
24 | |||
25 | export { | ||
26 | up, | ||
27 | down | ||
28 | } | ||
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts index c20a48a4e..3f780e319 100644 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import { logger } from '../../../helpers/logger' | 1 | import { logger } from '../../../helpers/logger' |
2 | import { doRequest } from '../../../helpers/requests' | 2 | import { doRequest } from '../../../helpers/requests' |
3 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | ||
3 | import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' | 4 | import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' |
4 | 5 | ||
5 | async function process (payload: ActivityPubHttpPayload, jobId: number) { | 6 | async function process (payload: ActivityPubHttpPayload, jobId: number) { |
@@ -15,15 +16,22 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) { | |||
15 | httpSignature: httpSignatureOptions | 16 | httpSignature: httpSignatureOptions |
16 | } | 17 | } |
17 | 18 | ||
19 | const badUrls: string[] = [] | ||
20 | const goodUrls: string[] = [] | ||
21 | |||
18 | for (const uri of payload.uris) { | 22 | for (const uri of payload.uris) { |
19 | options.uri = uri | 23 | options.uri = uri |
20 | 24 | ||
21 | try { | 25 | try { |
22 | await doRequest(options) | 26 | await doRequest(options) |
27 | goodUrls.push(uri) | ||
23 | } catch (err) { | 28 | } catch (err) { |
24 | await maybeRetryRequestLater(err, payload, uri) | 29 | const isRetryingLater = await maybeRetryRequestLater(err, payload, uri) |
30 | if (isRetryingLater === false) badUrls.push(uri) | ||
25 | } | 31 | } |
26 | } | 32 | } |
33 | |||
34 | return ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes(goodUrls, badUrls, undefined) | ||
27 | } | 35 | } |
28 | 36 | ||
29 | function onError (err: Error, jobId: number) { | 37 | function onError (err: Error, jobId: number) { |
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts index d576cd42e..884ede5a3 100644 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts | |||
@@ -4,6 +4,7 @@ import { logger } from '../../../helpers/logger' | |||
4 | import { getServerActor } from '../../../helpers/utils' | 4 | import { getServerActor } from '../../../helpers/utils' |
5 | import { ACTIVITY_PUB } from '../../../initializers' | 5 | import { ACTIVITY_PUB } from '../../../initializers' |
6 | import { ActorModel } from '../../../models/activitypub/actor' | 6 | import { ActorModel } from '../../../models/activitypub/actor' |
7 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | ||
7 | import { JobHandler, JobScheduler } from '../job-scheduler' | 8 | import { JobHandler, JobScheduler } from '../job-scheduler' |
8 | 9 | ||
9 | import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler' | 10 | import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler' |
@@ -26,7 +27,7 @@ const jobCategory: JobCategory = 'activitypub-http' | |||
26 | 27 | ||
27 | const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers) | 28 | const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers) |
28 | 29 | ||
29 | function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) { | 30 | async function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) { |
30 | logger.warn('Cannot make request to %s.', uri, err) | 31 | logger.warn('Cannot make request to %s.', uri, err) |
31 | 32 | ||
32 | let attemptNumber = payload.attemptNumber || 1 | 33 | let attemptNumber = payload.attemptNumber || 1 |
@@ -39,8 +40,12 @@ function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, ur | |||
39 | uris: [ uri ], | 40 | uris: [ uri ], |
40 | attemptNumber | 41 | attemptNumber |
41 | }) | 42 | }) |
42 | return activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload) | 43 | await activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload) |
44 | |||
45 | return true | ||
43 | } | 46 | } |
47 | |||
48 | return false | ||
44 | } | 49 | } |
45 | 50 | ||
46 | async function computeBody (payload: ActivityPubHttpPayload) { | 51 | async function computeBody (payload: ActivityPubHttpPayload) { |
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts index 175ec6642..e02bd698e 100644 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import { logger } from '../../../helpers/logger' | 1 | import { logger } from '../../../helpers/logger' |
2 | import { doRequest } from '../../../helpers/requests' | 2 | import { doRequest } from '../../../helpers/requests' |
3 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | ||
3 | import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' | 4 | import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' |
4 | 5 | ||
5 | async function process (payload: ActivityPubHttpPayload, jobId: number) { | 6 | async function process (payload: ActivityPubHttpPayload, jobId: number) { |
@@ -18,8 +19,13 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) { | |||
18 | 19 | ||
19 | try { | 20 | try { |
20 | await doRequest(options) | 21 | await doRequest(options) |
22 | await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([ uri ], [], undefined) | ||
21 | } catch (err) { | 23 | } catch (err) { |
22 | await maybeRetryRequestLater(err, payload, uri) | 24 | const isRetryingLater = await maybeRetryRequestLater(err, payload, uri) |
25 | if (isRetryingLater === false) { | ||
26 | await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([], [ uri ], undefined) | ||
27 | } | ||
28 | |||
23 | throw err | 29 | throw err |
24 | } | 30 | } |
25 | } | 31 | } |
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts new file mode 100644 index 000000000..473544ddf --- /dev/null +++ b/server/lib/schedulers/abstract-scheduler.ts | |||
@@ -0,0 +1,16 @@ | |||
1 | import { SCHEDULER_INTERVAL } from '../../initializers' | ||
2 | |||
3 | export abstract class AbstractScheduler { | ||
4 | |||
5 | private interval: NodeJS.Timer | ||
6 | |||
7 | enable () { | ||
8 | this.interval = setInterval(() => this.execute(), SCHEDULER_INTERVAL) | ||
9 | } | ||
10 | |||
11 | disable () { | ||
12 | clearInterval(this.interval) | ||
13 | } | ||
14 | |||
15 | protected abstract execute () | ||
16 | } | ||
diff --git a/server/lib/schedulers/bad-actor-follow-scheduler.ts b/server/lib/schedulers/bad-actor-follow-scheduler.ts new file mode 100644 index 000000000..c6c285ece --- /dev/null +++ b/server/lib/schedulers/bad-actor-follow-scheduler.ts | |||
@@ -0,0 +1,24 @@ | |||
1 | import { logger } from '../../helpers/logger' | ||
2 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' | ||
3 | import { AbstractScheduler } from './abstract-scheduler' | ||
4 | |||
5 | export class BadActorFollowScheduler extends AbstractScheduler { | ||
6 | |||
7 | private static instance: AbstractScheduler | ||
8 | |||
9 | private constructor () { | ||
10 | super() | ||
11 | } | ||
12 | |||
13 | async execute () { | ||
14 | try { | ||
15 | await ActorFollowModel.removeBadActorFollows() | ||
16 | } catch (err) { | ||
17 | logger.error('Error in bad actor follows scheduler.', err) | ||
18 | } | ||
19 | } | ||
20 | |||
21 | static get Instance () { | ||
22 | return this.instance || (this.instance = new this()) | ||
23 | } | ||
24 | } | ||
diff --git a/server/models/account/account.ts b/server/models/account/account.ts index 47336d1e0..f81c50180 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts | |||
@@ -179,7 +179,6 @@ export class AccountModel extends Model<AccountModel> { | |||
179 | const actor = this.Actor.toFormattedJSON() | 179 | const actor = this.Actor.toFormattedJSON() |
180 | const account = { | 180 | const account = { |
181 | id: this.id, | 181 | id: this.id, |
182 | name: this.Actor.preferredUsername, | ||
183 | displayName: this.name, | 182 | displayName: this.name, |
184 | createdAt: this.createdAt, | 183 | createdAt: this.createdAt, |
185 | updatedAt: this.updatedAt | 184 | updatedAt: this.updatedAt |
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index 5fcc3449d..78a65a0ff 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts | |||
@@ -1,8 +1,14 @@ | |||
1 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
2 | import { values } from 'lodash' | 2 | import { values } from 'lodash' |
3 | import * as Sequelize from 'sequelize' | 3 | import * as Sequelize from 'sequelize' |
4 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' | 4 | import { |
5 | AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, IsInt, Max, Model, Table, | ||
6 | UpdatedAt | ||
7 | } from 'sequelize-typescript' | ||
5 | import { FollowState } from '../../../shared/models/actors' | 8 | import { FollowState } from '../../../shared/models/actors' |
9 | import { AccountFollow } from '../../../shared/models/actors/follow.model' | ||
10 | import { logger } from '../../helpers/logger' | ||
11 | import { ACTOR_FOLLOW_SCORE } from '../../initializers' | ||
6 | import { FOLLOW_STATES } from '../../initializers/constants' | 12 | import { FOLLOW_STATES } from '../../initializers/constants' |
7 | import { ServerModel } from '../server/server' | 13 | import { ServerModel } from '../server/server' |
8 | import { getSort } from '../utils' | 14 | import { getSort } from '../utils' |
@@ -20,6 +26,9 @@ import { ActorModel } from './actor' | |||
20 | { | 26 | { |
21 | fields: [ 'actorId', 'targetActorId' ], | 27 | fields: [ 'actorId', 'targetActorId' ], |
22 | unique: true | 28 | unique: true |
29 | }, | ||
30 | { | ||
31 | fields: [ 'score' ] | ||
23 | } | 32 | } |
24 | ] | 33 | ] |
25 | }) | 34 | }) |
@@ -29,6 +38,13 @@ export class ActorFollowModel extends Model<ActorFollowModel> { | |||
29 | @Column(DataType.ENUM(values(FOLLOW_STATES))) | 38 | @Column(DataType.ENUM(values(FOLLOW_STATES))) |
30 | state: FollowState | 39 | state: FollowState |
31 | 40 | ||
41 | @AllowNull(false) | ||
42 | @Default(ACTOR_FOLLOW_SCORE.BASE) | ||
43 | @IsInt | ||
44 | @Max(ACTOR_FOLLOW_SCORE.MAX) | ||
45 | @Column | ||
46 | score: number | ||
47 | |||
32 | @CreatedAt | 48 | @CreatedAt |
33 | createdAt: Date | 49 | createdAt: Date |
34 | 50 | ||
@@ -63,6 +79,34 @@ export class ActorFollowModel extends Model<ActorFollowModel> { | |||
63 | }) | 79 | }) |
64 | ActorFollowing: ActorModel | 80 | ActorFollowing: ActorModel |
65 | 81 | ||
82 | // Remove actor follows with a score of 0 (too many requests where they were unreachable) | ||
83 | static async removeBadActorFollows () { | ||
84 | const actorFollows = await ActorFollowModel.listBadActorFollows() | ||
85 | |||
86 | const actorFollowsRemovePromises = actorFollows.map(actorFollow => actorFollow.destroy()) | ||
87 | await Promise.all(actorFollowsRemovePromises) | ||
88 | |||
89 | const numberOfActorFollowsRemoved = actorFollows.length | ||
90 | |||
91 | if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved) | ||
92 | } | ||
93 | |||
94 | static updateActorFollowsScoreAndRemoveBadOnes (goodInboxes: string[], badInboxes: string[], t: Sequelize.Transaction) { | ||
95 | if (goodInboxes.length === 0 && badInboxes.length === 0) return | ||
96 | |||
97 | logger.info('Updating %d good actor follows and %d bad actor follows scores.', goodInboxes.length, badInboxes.length) | ||
98 | |||
99 | if (goodInboxes.length !== 0) { | ||
100 | ActorFollowModel.incrementScores(goodInboxes, ACTOR_FOLLOW_SCORE.BONUS, t) | ||
101 | .catch(err => logger.error('Cannot increment scores of good actor follows.', err)) | ||
102 | } | ||
103 | |||
104 | if (badInboxes.length !== 0) { | ||
105 | ActorFollowModel.incrementScores(badInboxes, ACTOR_FOLLOW_SCORE.PENALTY, t) | ||
106 | .catch(err => logger.error('Cannot decrement scores of bad actor follows.', err)) | ||
107 | } | ||
108 | } | ||
109 | |||
66 | static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Sequelize.Transaction) { | 110 | static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Sequelize.Transaction) { |
67 | const query = { | 111 | const query = { |
68 | where: { | 112 | where: { |
@@ -260,7 +304,37 @@ export class ActorFollowModel extends Model<ActorFollowModel> { | |||
260 | } | 304 | } |
261 | } | 305 | } |
262 | 306 | ||
263 | toFormattedJSON () { | 307 | private static incrementScores (inboxUrls: string[], value: number, t: Sequelize.Transaction) { |
308 | const inboxUrlsString = inboxUrls.map(url => `'${url}'`).join(',') | ||
309 | |||
310 | const query = 'UPDATE "actorFollow" SET "score" = "score" +' + value + ' ' + | ||
311 | 'WHERE id IN (' + | ||
312 | 'SELECT "actorFollow"."id" FROM "actorFollow" ' + | ||
313 | 'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."actorId" ' + | ||
314 | 'WHERE "actor"."inboxUrl" IN (' + inboxUrlsString + ') OR "actor"."sharedInboxUrl" IN (' + inboxUrlsString + ')' + | ||
315 | ')' | ||
316 | |||
317 | const options = { | ||
318 | type: Sequelize.QueryTypes.BULKUPDATE, | ||
319 | transaction: t | ||
320 | } | ||
321 | |||
322 | return ActorFollowModel.sequelize.query(query, options) | ||
323 | } | ||
324 | |||
325 | private static listBadActorFollows () { | ||
326 | const query = { | ||
327 | where: { | ||
328 | score: { | ||
329 | [Sequelize.Op.lte]: 0 | ||
330 | } | ||
331 | } | ||
332 | } | ||
333 | |||
334 | return ActorFollowModel.findAll(query) | ||
335 | } | ||
336 | |||
337 | toFormattedJSON (): AccountFollow { | ||
264 | const follower = this.ActorFollower.toFormattedJSON() | 338 | const follower = this.ActorFollower.toFormattedJSON() |
265 | const following = this.ActorFollowing.toFormattedJSON() | 339 | const following = this.ActorFollowing.toFormattedJSON() |
266 | 340 | ||
@@ -268,6 +342,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> { | |||
268 | id: this.id, | 342 | id: this.id, |
269 | follower, | 343 | follower, |
270 | following, | 344 | following, |
345 | score: this.score, | ||
271 | state: this.state, | 346 | state: this.state, |
272 | createdAt: this.createdAt, | 347 | createdAt: this.createdAt, |
273 | updatedAt: this.updatedAt | 348 | updatedAt: this.updatedAt |
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index b88e06b41..912d8d748 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts | |||
@@ -204,7 +204,7 @@ export class ActorModel extends Model<ActorModel> { | |||
204 | VideoChannel: VideoChannelModel | 204 | VideoChannel: VideoChannelModel |
205 | 205 | ||
206 | static load (id: number) { | 206 | static load (id: number) { |
207 | return ActorModel.scope(ScopeNames.FULL).findById(id) | 207 | return ActorModel.unscoped().findById(id) |
208 | } | 208 | } |
209 | 209 | ||
210 | static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { | 210 | static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { |
@@ -267,20 +267,17 @@ export class ActorModel extends Model<ActorModel> { | |||
267 | avatar = this.Avatar.toFormattedJSON() | 267 | avatar = this.Avatar.toFormattedJSON() |
268 | } | 268 | } |
269 | 269 | ||
270 | let score: number | ||
271 | if (this.Server) { | ||
272 | score = this.Server.score | ||
273 | } | ||
274 | |||
275 | return { | 270 | return { |
276 | id: this.id, | 271 | id: this.id, |
277 | url: this.url, | 272 | url: this.url, |
278 | uuid: this.uuid, | 273 | uuid: this.uuid, |
274 | name: this.preferredUsername, | ||
279 | host: this.getHost(), | 275 | host: this.getHost(), |
280 | score, | ||
281 | followingCount: this.followingCount, | 276 | followingCount: this.followingCount, |
282 | followersCount: this.followersCount, | 277 | followersCount: this.followersCount, |
283 | avatar | 278 | avatar, |
279 | createdAt: this.createdAt, | ||
280 | updatedAt: this.updatedAt | ||
284 | } | 281 | } |
285 | } | 282 | } |
286 | 283 | ||
diff --git a/server/models/server/server.ts b/server/models/server/server.ts index d35aa0ca4..c43146156 100644 --- a/server/models/server/server.ts +++ b/server/models/server/server.ts | |||
@@ -1,8 +1,5 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import { AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' |
2 | import { AllowNull, Column, CreatedAt, Default, Is, IsInt, Max, Model, Table, UpdatedAt } from 'sequelize-typescript' | ||
3 | import { isHostValid } from '../../helpers/custom-validators/servers' | 2 | import { isHostValid } from '../../helpers/custom-validators/servers' |
4 | import { logger } from '../../helpers/logger' | ||
5 | import { SERVERS_SCORE } from '../../initializers' | ||
6 | import { throwIfNotValid } from '../utils' | 3 | import { throwIfNotValid } from '../utils' |
7 | 4 | ||
8 | @Table({ | 5 | @Table({ |
@@ -11,9 +8,6 @@ import { throwIfNotValid } from '../utils' | |||
11 | { | 8 | { |
12 | fields: [ 'host' ], | 9 | fields: [ 'host' ], |
13 | unique: true | 10 | unique: true |
14 | }, | ||
15 | { | ||
16 | fields: [ 'score' ] | ||
17 | } | 11 | } |
18 | ] | 12 | ] |
19 | }) | 13 | }) |
@@ -24,86 +18,9 @@ export class ServerModel extends Model<ServerModel> { | |||
24 | @Column | 18 | @Column |
25 | host: string | 19 | host: string |
26 | 20 | ||
27 | @AllowNull(false) | ||
28 | @Default(SERVERS_SCORE.BASE) | ||
29 | @IsInt | ||
30 | @Max(SERVERS_SCORE.MAX) | ||
31 | @Column | ||
32 | score: number | ||
33 | |||
34 | @CreatedAt | 21 | @CreatedAt |
35 | createdAt: Date | 22 | createdAt: Date |
36 | 23 | ||
37 | @UpdatedAt | 24 | @UpdatedAt |
38 | updatedAt: Date | 25 | updatedAt: Date |
39 | |||
40 | static updateServersScoreAndRemoveBadOnes (goodServers: number[], badServers: number[]) { | ||
41 | logger.info('Updating %d good servers and %d bad servers scores.', goodServers.length, badServers.length) | ||
42 | |||
43 | if (goodServers.length !== 0) { | ||
44 | ServerModel.incrementScores(goodServers, SERVERS_SCORE.BONUS) | ||
45 | .catch(err => { | ||
46 | logger.error('Cannot increment scores of good servers.', err) | ||
47 | }) | ||
48 | } | ||
49 | |||
50 | if (badServers.length !== 0) { | ||
51 | ServerModel.incrementScores(badServers, SERVERS_SCORE.PENALTY) | ||
52 | .then(() => ServerModel.removeBadServers()) | ||
53 | .catch(err => { | ||
54 | if (err) logger.error('Cannot decrement scores of bad servers.', err) | ||
55 | }) | ||
56 | |||
57 | } | ||
58 | } | ||
59 | |||
60 | // Remove servers with a score of 0 (too many requests where they were unreachable) | ||
61 | private static async removeBadServers () { | ||
62 | try { | ||
63 | const servers = await ServerModel.listBadServers() | ||
64 | |||
65 | const serversRemovePromises = servers.map(server => server.destroy()) | ||
66 | await Promise.all(serversRemovePromises) | ||
67 | |||
68 | const numberOfServersRemoved = servers.length | ||
69 | |||
70 | if (numberOfServersRemoved) { | ||
71 | logger.info('Removed %d servers.', numberOfServersRemoved) | ||
72 | } else { | ||
73 | logger.info('No need to remove bad servers.') | ||
74 | } | ||
75 | } catch (err) { | ||
76 | logger.error('Cannot remove bad servers.', err) | ||
77 | } | ||
78 | } | ||
79 | |||
80 | private static incrementScores (ids: number[], value: number) { | ||
81 | const update = { | ||
82 | score: Sequelize.literal('score +' + value) | ||
83 | } | ||
84 | |||
85 | const options = { | ||
86 | where: { | ||
87 | id: { | ||
88 | [Sequelize.Op.in]: ids | ||
89 | } | ||
90 | }, | ||
91 | // In this case score is a literal and not an integer so we do not validate it | ||
92 | validate: false | ||
93 | } | ||
94 | |||
95 | return ServerModel.update(update, options) | ||
96 | } | ||
97 | |||
98 | private static listBadServers () { | ||
99 | const query = { | ||
100 | where: { | ||
101 | score: { | ||
102 | [Sequelize.Op.lte]: 0 | ||
103 | } | ||
104 | } | ||
105 | } | ||
106 | |||
107 | return ServerModel.findAll(query) | ||
108 | } | ||
109 | } | 26 | } |
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index acc2486b3..e2cbf0422 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts | |||
@@ -228,7 +228,7 @@ export class VideoChannelModel extends Model<VideoChannelModel> { | |||
228 | const actor = this.Actor.toFormattedJSON() | 228 | const actor = this.Actor.toFormattedJSON() |
229 | const account = { | 229 | const account = { |
230 | id: this.id, | 230 | id: this.id, |
231 | name: this.name, | 231 | displayName: this.name, |
232 | description: this.description, | 232 | description: this.description, |
233 | isLocal: this.Actor.isOwned(), | 233 | isLocal: this.Actor.isOwned(), |
234 | createdAt: this.createdAt, | 234 | createdAt: this.createdAt, |
diff --git a/shared/models/actors/account.model.ts b/shared/models/actors/account.model.ts index e4dbc81e5..5cc12c18f 100644 --- a/shared/models/actors/account.model.ts +++ b/shared/models/actors/account.model.ts | |||
@@ -1,15 +1,5 @@ | |||
1 | import { Avatar } from '../avatars/avatar.model' | 1 | import { Actor } from './actor.model' |
2 | 2 | ||
3 | export interface Account { | 3 | export interface Account extends Actor { |
4 | id: number | ||
5 | uuid: string | ||
6 | url: string | ||
7 | name: string | ||
8 | displayName: string | 4 | displayName: string |
9 | host: string | ||
10 | followingCount: number | ||
11 | followersCount: number | ||
12 | createdAt: Date | ||
13 | updatedAt: Date | ||
14 | avatar: Avatar | ||
15 | } | 5 | } |
diff --git a/shared/models/actors/actor.model.ts b/shared/models/actors/actor.model.ts new file mode 100644 index 000000000..f91616519 --- /dev/null +++ b/shared/models/actors/actor.model.ts | |||
@@ -0,0 +1,14 @@ | |||
1 | import { Avatar } from '../avatars/avatar.model' | ||
2 | |||
3 | export interface Actor { | ||
4 | id: number | ||
5 | uuid: string | ||
6 | url: string | ||
7 | name: string | ||
8 | host: string | ||
9 | followingCount: number | ||
10 | followersCount: number | ||
11 | createdAt: Date | ||
12 | updatedAt: Date | ||
13 | avatar: Avatar | ||
14 | } | ||
diff --git a/shared/models/actors/follow.model.ts b/shared/models/actors/follow.model.ts index cdc3da560..70562bfc7 100644 --- a/shared/models/actors/follow.model.ts +++ b/shared/models/actors/follow.model.ts | |||
@@ -1,11 +1,12 @@ | |||
1 | import { Account } from './account.model' | 1 | import { Actor } from './actor.model' |
2 | 2 | ||
3 | export type FollowState = 'pending' | 'accepted' | 3 | export type FollowState = 'pending' | 'accepted' |
4 | 4 | ||
5 | export interface AccountFollow { | 5 | export interface AccountFollow { |
6 | id: number | 6 | id: number |
7 | follower: Account | 7 | follower: Actor |
8 | following: Account | 8 | following: Actor |
9 | score: number | ||
9 | state: FollowState | 10 | state: FollowState |
10 | createdAt: Date | 11 | createdAt: Date |
11 | updatedAt: Date | 12 | updatedAt: Date |
diff --git a/shared/models/videos/video-channel.model.ts b/shared/models/videos/video-channel.model.ts index d1a952826..b164fb555 100644 --- a/shared/models/videos/video-channel.model.ts +++ b/shared/models/videos/video-channel.model.ts | |||
@@ -1,13 +1,10 @@ | |||
1 | import { Actor } from '../actors/actor.model' | ||
1 | import { Video } from './video.model' | 2 | import { Video } from './video.model' |
2 | 3 | ||
3 | export interface VideoChannel { | 4 | export interface VideoChannel extends Actor { |
4 | id: number | 5 | displayName: string |
5 | name: string | ||
6 | url: string | ||
7 | description: string | 6 | description: string |
8 | isLocal: boolean | 7 | isLocal: boolean |
9 | createdAt: Date | string | ||
10 | updatedAt: Date | string | ||
11 | owner?: { | 8 | owner?: { |
12 | name: string | 9 | name: string |
13 | uuid: string | 10 | uuid: string |