aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-01-11 09:35:50 +0100
committerChocobozzz <me@florianbigard.com>2018-01-11 09:35:50 +0100
commit60650c77c8a2a98e92d869b237ae4900f369a8fc (patch)
tree7304a7591b5b23b99d219c4d06c6bd5c4c58c1c2
parent7ae71355c40e9065f83d3fc77b6750d1929ac201 (diff)
downloadPeerTube-60650c77c8a2a98e92d869b237ae4900f369a8fc.tar.gz
PeerTube-60650c77c8a2a98e92d869b237ae4900f369a8fc.tar.zst
PeerTube-60650c77c8a2a98e92d869b237ae4900f369a8fc.zip
Add scores to follows and remove bad ones
-rw-r--r--client/src/app/+admin/follows/followers-list/followers-list.component.html2
-rw-r--r--client/src/app/shared/misc/utils.ts2
-rw-r--r--client/src/app/videos/+video-edit/video-add.component.html1
-rw-r--r--client/src/app/videos/+video-watch/video-watch.component.html2
-rw-r--r--server.ts3
-rw-r--r--server/initializers/constants.ts21
-rw-r--r--server/initializers/migrations/0170-actor-follow-score.ts28
-rw-r--r--server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts10
-rw-r--r--server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts9
-rw-r--r--server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts8
-rw-r--r--server/lib/schedulers/abstract-scheduler.ts16
-rw-r--r--server/lib/schedulers/bad-actor-follow-scheduler.ts24
-rw-r--r--server/models/account/account.ts1
-rw-r--r--server/models/activitypub/actor-follow.ts79
-rw-r--r--server/models/activitypub/actor.ts13
-rw-r--r--server/models/server/server.ts85
-rw-r--r--server/models/video/video-channel.ts2
-rw-r--r--shared/models/actors/account.model.ts14
-rw-r--r--shared/models/actors/actor.model.ts14
-rw-r--r--shared/models/actors/follow.model.ts7
-rw-r--r--shared/models/videos/video-channel.model.ts9
21 files changed, 217 insertions, 133 deletions
diff --git a/client/src/app/+admin/follows/followers-list/followers-list.component.html b/client/src/app/+admin/follows/followers-list/followers-list.component.html
index 9499a0433..d5b1b789d 100644
--- a/client/src/app/+admin/follows/followers-list/followers-list.component.html
+++ b/client/src/app/+admin/follows/followers-list/followers-list.component.html
@@ -3,8 +3,8 @@
3 sortField="createdAt" (onLazyLoad)="loadLazy($event)" 3 sortField="createdAt" (onLazyLoad)="loadLazy($event)"
4> 4>
5 <p-column field="id" header="ID" [style]="{ width: '60px' }"></p-column> 5 <p-column field="id" header="ID" [style]="{ width: '60px' }"></p-column>
6 <p-column field="score" header="Score"></p-column>
6 <p-column field="follower.host" header="Host"></p-column> 7 <p-column field="follower.host" header="Host"></p-column>
7 <p-column field="follower.score" header="Score"></p-column>
8 <p-column field="state" header="State"></p-column> 8 <p-column field="state" header="State"></p-column>
9 <p-column field="createdAt" header="Created date" [sortable]="true"></p-column> 9 <p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
10</p-dataTable> 10</p-dataTable>
diff --git a/client/src/app/shared/misc/utils.ts b/client/src/app/shared/misc/utils.ts
index 2739ff81a..23b46812b 100644
--- a/client/src/app/shared/misc/utils.ts
+++ b/client/src/app/shared/misc/utils.ts
@@ -31,7 +31,7 @@ function populateAsyncUserVideoChannels (authService: AuthService, channel: any[
31 const videoChannels = user.videoChannels 31 const videoChannels = user.videoChannels
32 if (Array.isArray(videoChannels) === false) return 32 if (Array.isArray(videoChannels) === false) return
33 33
34 videoChannels.forEach(c => channel.push({ id: c.id, label: c.name })) 34 videoChannels.forEach(c => channel.push({ id: c.id, label: c.displayName }))
35 35
36 return res() 36 return res()
37 } 37 }
diff --git a/client/src/app/videos/+video-edit/video-add.component.html b/client/src/app/videos/+video-edit/video-add.component.html
index 2040ff9d4..34291c6c6 100644
--- a/client/src/app/videos/+video-edit/video-add.component.html
+++ b/client/src/app/videos/+video-edit/video-add.component.html
@@ -44,7 +44,6 @@
44 [validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels" 44 [validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
45 ></my-video-edit> 45 ></my-video-edit>
46 46
47
48 <div class="submit-container"> 47 <div class="submit-container">
49 <div *ngIf="videoUploaded === false" class="message-submit">Publish will be available when upload is finished</div> 48 <div *ngIf="videoUploaded === false" class="message-submit">Publish will be available when upload is finished</div>
50 49
diff --git a/client/src/app/videos/+video-watch/video-watch.component.html b/client/src/app/videos/+video-watch/video-watch.component.html
index a5c387638..5921b4b72 100644
--- a/client/src/app/videos/+video-watch/video-watch.component.html
+++ b/client/src/app/videos/+video-watch/video-watch.component.html
@@ -85,7 +85,7 @@
85 </div> 85 </div>
86 86
87 <div class="video-info-channel"> 87 <div class="video-info-channel">
88 {{ video.channel.name }} 88 {{ video.channel.displayName }}
89 <!-- Here will be the subscribe button --> 89 <!-- Here will be the subscribe button -->
90 </div> 90 </div>
91 91
diff --git a/server.ts b/server.ts
index a52c47083..99077a173 100644
--- a/server.ts
+++ b/server.ts
@@ -56,6 +56,7 @@ import { installApplication } from './server/initializers'
56import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs' 56import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs'
57import { VideosPreviewCache } from './server/lib/cache' 57import { VideosPreviewCache } from './server/lib/cache'
58import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers' 58import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers'
59import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler'
59 60
60// ----------- Command line ----------- 61// ----------- Command line -----------
61 62
@@ -168,6 +169,8 @@ function onDatabaseInitDone () {
168 // ----------- Make the server listening ----------- 169 // ----------- Make the server listening -----------
169 server.listen(port, () => { 170 server.listen(port, () => {
170 VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) 171 VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE)
172 BadActorFollowScheduler.Instance.enable()
173
171 activitypubHttpJobScheduler.activate() 174 activitypubHttpJobScheduler.activate()
172 transcodingJobScheduler.activate() 175 transcodingJobScheduler.activate()
173 176
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index c735e6daf..0c139912c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -9,7 +9,7 @@ import { isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core
9 9
10// --------------------------------------------------------------------------- 10// ---------------------------------------------------------------------------
11 11
12const LAST_MIGRATION_VERSION = 165 12const LAST_MIGRATION_VERSION = 170
13 13
14// --------------------------------------------------------------------------- 14// ---------------------------------------------------------------------------
15 15
@@ -40,12 +40,12 @@ const OAUTH_LIFETIME = {
40 40
41// --------------------------------------------------------------------------- 41// ---------------------------------------------------------------------------
42 42
43// Number of points we add/remove from a friend after a successful/bad request 43// Number of points we add/remove after a successful/bad request
44const SERVERS_SCORE = { 44const ACTOR_FOLLOW_SCORE = {
45 PENALTY: -10, 45 PENALTY: -10,
46 BONUS: 10, 46 BONUS: 10,
47 BASE: 100, 47 BASE: 1000,
48 MAX: 1000 48 MAX: 10000
49} 49}
50 50
51const FOLLOW_STATES: { [ id: string ]: FollowState } = { 51const FOLLOW_STATES: { [ id: string ]: FollowState } = {
@@ -76,6 +76,9 @@ const JOBS_FETCH_LIMIT_PER_CYCLE = {
76// 1 minutes 76// 1 minutes
77let JOBS_FETCHING_INTERVAL = 60000 77let JOBS_FETCHING_INTERVAL = 60000
78 78
79// 1 hour
80let SCHEDULER_INTERVAL = 60000 * 60
81
79// --------------------------------------------------------------------------- 82// ---------------------------------------------------------------------------
80 83
81const CONFIG = { 84const CONFIG = {
@@ -346,7 +349,7 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '<!-- open graph and oembed tags -->'
346 349
347// Special constants for a test instance 350// Special constants for a test instance
348if (isTestInstance() === true) { 351if (isTestInstance() === true) {
349 SERVERS_SCORE.BASE = 20 352 ACTOR_FOLLOW_SCORE.BASE = 20
350 JOBS_FETCHING_INTERVAL = 1000 353 JOBS_FETCHING_INTERVAL = 1000
351 REMOTE_SCHEME.HTTP = 'http' 354 REMOTE_SCHEME.HTTP = 'http'
352 REMOTE_SCHEME.WS = 'ws' 355 REMOTE_SCHEME.WS = 'ws'
@@ -354,6 +357,7 @@ if (isTestInstance() === true) {
354 ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 357 ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
355 ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 60 // 1 minute 358 ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 60 // 1 minute
356 CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB 359 CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
360 SCHEDULER_INTERVAL = 10000
357} 361}
358 362
359CONFIG.WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT) 363CONFIG.WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT)
@@ -378,7 +382,7 @@ export {
378 OAUTH_LIFETIME, 382 OAUTH_LIFETIME,
379 OPENGRAPH_AND_OEMBED_COMMENT, 383 OPENGRAPH_AND_OEMBED_COMMENT,
380 PAGINATION_COUNT_DEFAULT, 384 PAGINATION_COUNT_DEFAULT,
381 SERVERS_SCORE, 385 ACTOR_FOLLOW_SCORE,
382 PREVIEWS_SIZE, 386 PREVIEWS_SIZE,
383 REMOTE_SCHEME, 387 REMOTE_SCHEME,
384 FOLLOW_STATES, 388 FOLLOW_STATES,
@@ -396,5 +400,6 @@ export {
396 VIDEO_LICENCES, 400 VIDEO_LICENCES,
397 VIDEO_RATE_TYPES, 401 VIDEO_RATE_TYPES,
398 VIDEO_MIMETYPE_EXT, 402 VIDEO_MIMETYPE_EXT,
399 AVATAR_MIMETYPE_EXT 403 AVATAR_MIMETYPE_EXT,
404 SCHEDULER_INTERVAL
400} 405}
diff --git a/server/initializers/migrations/0170-actor-follow-score.ts b/server/initializers/migrations/0170-actor-follow-score.ts
new file mode 100644
index 000000000..2deabaf98
--- /dev/null
+++ b/server/initializers/migrations/0170-actor-follow-score.ts
@@ -0,0 +1,28 @@
1import * as Sequelize from 'sequelize'
2import { ACTOR_FOLLOW_SCORE } from '../index'
3
4async function up (utils: {
5 transaction: Sequelize.Transaction,
6 queryInterface: Sequelize.QueryInterface,
7 sequelize: Sequelize.Sequelize
8}): Promise<void> {
9 await utils.queryInterface.removeColumn('server', 'score')
10
11 const data = {
12 type: Sequelize.INTEGER,
13 allowNull: false,
14 defaultValue: ACTOR_FOLLOW_SCORE.BASE
15 }
16
17 await utils.queryInterface.addColumn('actorFollow', 'score', data)
18
19}
20
21function down (options) {
22 throw new Error('Not implemented.')
23}
24
25export {
26 up,
27 down
28}
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts
index c20a48a4e..3f780e319 100644
--- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts
@@ -1,5 +1,6 @@
1import { logger } from '../../../helpers/logger' 1import { logger } from '../../../helpers/logger'
2import { doRequest } from '../../../helpers/requests' 2import { doRequest } from '../../../helpers/requests'
3import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
3import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' 4import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler'
4 5
5async function process (payload: ActivityPubHttpPayload, jobId: number) { 6async function process (payload: ActivityPubHttpPayload, jobId: number) {
@@ -15,15 +16,22 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) {
15 httpSignature: httpSignatureOptions 16 httpSignature: httpSignatureOptions
16 } 17 }
17 18
19 const badUrls: string[] = []
20 const goodUrls: string[] = []
21
18 for (const uri of payload.uris) { 22 for (const uri of payload.uris) {
19 options.uri = uri 23 options.uri = uri
20 24
21 try { 25 try {
22 await doRequest(options) 26 await doRequest(options)
27 goodUrls.push(uri)
23 } catch (err) { 28 } catch (err) {
24 await maybeRetryRequestLater(err, payload, uri) 29 const isRetryingLater = await maybeRetryRequestLater(err, payload, uri)
30 if (isRetryingLater === false) badUrls.push(uri)
25 } 31 }
26 } 32 }
33
34 return ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes(goodUrls, badUrls, undefined)
27} 35}
28 36
29function onError (err: Error, jobId: number) { 37function onError (err: Error, jobId: number) {
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
index d576cd42e..884ede5a3 100644
--- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
@@ -4,6 +4,7 @@ import { logger } from '../../../helpers/logger'
4import { getServerActor } from '../../../helpers/utils' 4import { getServerActor } from '../../../helpers/utils'
5import { ACTIVITY_PUB } from '../../../initializers' 5import { ACTIVITY_PUB } from '../../../initializers'
6import { ActorModel } from '../../../models/activitypub/actor' 6import { ActorModel } from '../../../models/activitypub/actor'
7import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
7import { JobHandler, JobScheduler } from '../job-scheduler' 8import { JobHandler, JobScheduler } from '../job-scheduler'
8 9
9import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler' 10import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler'
@@ -26,7 +27,7 @@ const jobCategory: JobCategory = 'activitypub-http'
26 27
27const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers) 28const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers)
28 29
29function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) { 30async function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) {
30 logger.warn('Cannot make request to %s.', uri, err) 31 logger.warn('Cannot make request to %s.', uri, err)
31 32
32 let attemptNumber = payload.attemptNumber || 1 33 let attemptNumber = payload.attemptNumber || 1
@@ -39,8 +40,12 @@ function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, ur
39 uris: [ uri ], 40 uris: [ uri ],
40 attemptNumber 41 attemptNumber
41 }) 42 })
42 return activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload) 43 await activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload)
44
45 return true
43 } 46 }
47
48 return false
44} 49}
45 50
46async function computeBody (payload: ActivityPubHttpPayload) { 51async function computeBody (payload: ActivityPubHttpPayload) {
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts
index 175ec6642..e02bd698e 100644
--- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts
@@ -1,5 +1,6 @@
1import { logger } from '../../../helpers/logger' 1import { logger } from '../../../helpers/logger'
2import { doRequest } from '../../../helpers/requests' 2import { doRequest } from '../../../helpers/requests'
3import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
3import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' 4import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler'
4 5
5async function process (payload: ActivityPubHttpPayload, jobId: number) { 6async function process (payload: ActivityPubHttpPayload, jobId: number) {
@@ -18,8 +19,13 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) {
18 19
19 try { 20 try {
20 await doRequest(options) 21 await doRequest(options)
22 await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([ uri ], [], undefined)
21 } catch (err) { 23 } catch (err) {
22 await maybeRetryRequestLater(err, payload, uri) 24 const isRetryingLater = await maybeRetryRequestLater(err, payload, uri)
25 if (isRetryingLater === false) {
26 await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([], [ uri ], undefined)
27 }
28
23 throw err 29 throw err
24 } 30 }
25} 31}
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts
new file mode 100644
index 000000000..473544ddf
--- /dev/null
+++ b/server/lib/schedulers/abstract-scheduler.ts
@@ -0,0 +1,16 @@
1import { SCHEDULER_INTERVAL } from '../../initializers'
2
3export abstract class AbstractScheduler {
4
5 private interval: NodeJS.Timer
6
7 enable () {
8 this.interval = setInterval(() => this.execute(), SCHEDULER_INTERVAL)
9 }
10
11 disable () {
12 clearInterval(this.interval)
13 }
14
15 protected abstract execute ()
16}
diff --git a/server/lib/schedulers/bad-actor-follow-scheduler.ts b/server/lib/schedulers/bad-actor-follow-scheduler.ts
new file mode 100644
index 000000000..c6c285ece
--- /dev/null
+++ b/server/lib/schedulers/bad-actor-follow-scheduler.ts
@@ -0,0 +1,24 @@
1import { logger } from '../../helpers/logger'
2import { ActorFollowModel } from '../../models/activitypub/actor-follow'
3import { AbstractScheduler } from './abstract-scheduler'
4
5export class BadActorFollowScheduler extends AbstractScheduler {
6
7 private static instance: AbstractScheduler
8
9 private constructor () {
10 super()
11 }
12
13 async execute () {
14 try {
15 await ActorFollowModel.removeBadActorFollows()
16 } catch (err) {
17 logger.error('Error in bad actor follows scheduler.', err)
18 }
19 }
20
21 static get Instance () {
22 return this.instance || (this.instance = new this())
23 }
24}
diff --git a/server/models/account/account.ts b/server/models/account/account.ts
index 47336d1e0..f81c50180 100644
--- a/server/models/account/account.ts
+++ b/server/models/account/account.ts
@@ -179,7 +179,6 @@ export class AccountModel extends Model<AccountModel> {
179 const actor = this.Actor.toFormattedJSON() 179 const actor = this.Actor.toFormattedJSON()
180 const account = { 180 const account = {
181 id: this.id, 181 id: this.id,
182 name: this.Actor.preferredUsername,
183 displayName: this.name, 182 displayName: this.name,
184 createdAt: this.createdAt, 183 createdAt: this.createdAt,
185 updatedAt: this.updatedAt 184 updatedAt: this.updatedAt
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts
index 5fcc3449d..78a65a0ff 100644
--- a/server/models/activitypub/actor-follow.ts
+++ b/server/models/activitypub/actor-follow.ts
@@ -1,8 +1,14 @@
1import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import { values } from 'lodash' 2import { values } from 'lodash'
3import * as Sequelize from 'sequelize' 3import * as Sequelize from 'sequelize'
4import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' 4import {
5 AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, IsInt, Max, Model, Table,
6 UpdatedAt
7} from 'sequelize-typescript'
5import { FollowState } from '../../../shared/models/actors' 8import { FollowState } from '../../../shared/models/actors'
9import { AccountFollow } from '../../../shared/models/actors/follow.model'
10import { logger } from '../../helpers/logger'
11import { ACTOR_FOLLOW_SCORE } from '../../initializers'
6import { FOLLOW_STATES } from '../../initializers/constants' 12import { FOLLOW_STATES } from '../../initializers/constants'
7import { ServerModel } from '../server/server' 13import { ServerModel } from '../server/server'
8import { getSort } from '../utils' 14import { getSort } from '../utils'
@@ -20,6 +26,9 @@ import { ActorModel } from './actor'
20 { 26 {
21 fields: [ 'actorId', 'targetActorId' ], 27 fields: [ 'actorId', 'targetActorId' ],
22 unique: true 28 unique: true
29 },
30 {
31 fields: [ 'score' ]
23 } 32 }
24 ] 33 ]
25}) 34})
@@ -29,6 +38,13 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
29 @Column(DataType.ENUM(values(FOLLOW_STATES))) 38 @Column(DataType.ENUM(values(FOLLOW_STATES)))
30 state: FollowState 39 state: FollowState
31 40
41 @AllowNull(false)
42 @Default(ACTOR_FOLLOW_SCORE.BASE)
43 @IsInt
44 @Max(ACTOR_FOLLOW_SCORE.MAX)
45 @Column
46 score: number
47
32 @CreatedAt 48 @CreatedAt
33 createdAt: Date 49 createdAt: Date
34 50
@@ -63,6 +79,34 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
63 }) 79 })
64 ActorFollowing: ActorModel 80 ActorFollowing: ActorModel
65 81
82 // Remove actor follows with a score of 0 (too many requests where they were unreachable)
83 static async removeBadActorFollows () {
84 const actorFollows = await ActorFollowModel.listBadActorFollows()
85
86 const actorFollowsRemovePromises = actorFollows.map(actorFollow => actorFollow.destroy())
87 await Promise.all(actorFollowsRemovePromises)
88
89 const numberOfActorFollowsRemoved = actorFollows.length
90
91 if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved)
92 }
93
94 static updateActorFollowsScoreAndRemoveBadOnes (goodInboxes: string[], badInboxes: string[], t: Sequelize.Transaction) {
95 if (goodInboxes.length === 0 && badInboxes.length === 0) return
96
97 logger.info('Updating %d good actor follows and %d bad actor follows scores.', goodInboxes.length, badInboxes.length)
98
99 if (goodInboxes.length !== 0) {
100 ActorFollowModel.incrementScores(goodInboxes, ACTOR_FOLLOW_SCORE.BONUS, t)
101 .catch(err => logger.error('Cannot increment scores of good actor follows.', err))
102 }
103
104 if (badInboxes.length !== 0) {
105 ActorFollowModel.incrementScores(badInboxes, ACTOR_FOLLOW_SCORE.PENALTY, t)
106 .catch(err => logger.error('Cannot decrement scores of bad actor follows.', err))
107 }
108 }
109
66 static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Sequelize.Transaction) { 110 static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Sequelize.Transaction) {
67 const query = { 111 const query = {
68 where: { 112 where: {
@@ -260,7 +304,37 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
260 } 304 }
261 } 305 }
262 306
263 toFormattedJSON () { 307 private static incrementScores (inboxUrls: string[], value: number, t: Sequelize.Transaction) {
308 const inboxUrlsString = inboxUrls.map(url => `'${url}'`).join(',')
309
310 const query = 'UPDATE "actorFollow" SET "score" = "score" +' + value + ' ' +
311 'WHERE id IN (' +
312 'SELECT "actorFollow"."id" FROM "actorFollow" ' +
313 'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."actorId" ' +
314 'WHERE "actor"."inboxUrl" IN (' + inboxUrlsString + ') OR "actor"."sharedInboxUrl" IN (' + inboxUrlsString + ')' +
315 ')'
316
317 const options = {
318 type: Sequelize.QueryTypes.BULKUPDATE,
319 transaction: t
320 }
321
322 return ActorFollowModel.sequelize.query(query, options)
323 }
324
325 private static listBadActorFollows () {
326 const query = {
327 where: {
328 score: {
329 [Sequelize.Op.lte]: 0
330 }
331 }
332 }
333
334 return ActorFollowModel.findAll(query)
335 }
336
337 toFormattedJSON (): AccountFollow {
264 const follower = this.ActorFollower.toFormattedJSON() 338 const follower = this.ActorFollower.toFormattedJSON()
265 const following = this.ActorFollowing.toFormattedJSON() 339 const following = this.ActorFollowing.toFormattedJSON()
266 340
@@ -268,6 +342,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
268 id: this.id, 342 id: this.id,
269 follower, 343 follower,
270 following, 344 following,
345 score: this.score,
271 state: this.state, 346 state: this.state,
272 createdAt: this.createdAt, 347 createdAt: this.createdAt,
273 updatedAt: this.updatedAt 348 updatedAt: this.updatedAt
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index b88e06b41..912d8d748 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -204,7 +204,7 @@ export class ActorModel extends Model<ActorModel> {
204 VideoChannel: VideoChannelModel 204 VideoChannel: VideoChannelModel
205 205
206 static load (id: number) { 206 static load (id: number) {
207 return ActorModel.scope(ScopeNames.FULL).findById(id) 207 return ActorModel.unscoped().findById(id)
208 } 208 }
209 209
210 static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { 210 static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) {
@@ -267,20 +267,17 @@ export class ActorModel extends Model<ActorModel> {
267 avatar = this.Avatar.toFormattedJSON() 267 avatar = this.Avatar.toFormattedJSON()
268 } 268 }
269 269
270 let score: number
271 if (this.Server) {
272 score = this.Server.score
273 }
274
275 return { 270 return {
276 id: this.id, 271 id: this.id,
277 url: this.url, 272 url: this.url,
278 uuid: this.uuid, 273 uuid: this.uuid,
274 name: this.preferredUsername,
279 host: this.getHost(), 275 host: this.getHost(),
280 score,
281 followingCount: this.followingCount, 276 followingCount: this.followingCount,
282 followersCount: this.followersCount, 277 followersCount: this.followersCount,
283 avatar 278 avatar,
279 createdAt: this.createdAt,
280 updatedAt: this.updatedAt
284 } 281 }
285 } 282 }
286 283
diff --git a/server/models/server/server.ts b/server/models/server/server.ts
index d35aa0ca4..c43146156 100644
--- a/server/models/server/server.ts
+++ b/server/models/server/server.ts
@@ -1,8 +1,5 @@
1import * as Sequelize from 'sequelize' 1import { AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
2import { AllowNull, Column, CreatedAt, Default, Is, IsInt, Max, Model, Table, UpdatedAt } from 'sequelize-typescript'
3import { isHostValid } from '../../helpers/custom-validators/servers' 2import { isHostValid } from '../../helpers/custom-validators/servers'
4import { logger } from '../../helpers/logger'
5import { SERVERS_SCORE } from '../../initializers'
6import { throwIfNotValid } from '../utils' 3import { throwIfNotValid } from '../utils'
7 4
8@Table({ 5@Table({
@@ -11,9 +8,6 @@ import { throwIfNotValid } from '../utils'
11 { 8 {
12 fields: [ 'host' ], 9 fields: [ 'host' ],
13 unique: true 10 unique: true
14 },
15 {
16 fields: [ 'score' ]
17 } 11 }
18 ] 12 ]
19}) 13})
@@ -24,86 +18,9 @@ export class ServerModel extends Model<ServerModel> {
24 @Column 18 @Column
25 host: string 19 host: string
26 20
27 @AllowNull(false)
28 @Default(SERVERS_SCORE.BASE)
29 @IsInt
30 @Max(SERVERS_SCORE.MAX)
31 @Column
32 score: number
33
34 @CreatedAt 21 @CreatedAt
35 createdAt: Date 22 createdAt: Date
36 23
37 @UpdatedAt 24 @UpdatedAt
38 updatedAt: Date 25 updatedAt: Date
39
40 static updateServersScoreAndRemoveBadOnes (goodServers: number[], badServers: number[]) {
41 logger.info('Updating %d good servers and %d bad servers scores.', goodServers.length, badServers.length)
42
43 if (goodServers.length !== 0) {
44 ServerModel.incrementScores(goodServers, SERVERS_SCORE.BONUS)
45 .catch(err => {
46 logger.error('Cannot increment scores of good servers.', err)
47 })
48 }
49
50 if (badServers.length !== 0) {
51 ServerModel.incrementScores(badServers, SERVERS_SCORE.PENALTY)
52 .then(() => ServerModel.removeBadServers())
53 .catch(err => {
54 if (err) logger.error('Cannot decrement scores of bad servers.', err)
55 })
56
57 }
58 }
59
60 // Remove servers with a score of 0 (too many requests where they were unreachable)
61 private static async removeBadServers () {
62 try {
63 const servers = await ServerModel.listBadServers()
64
65 const serversRemovePromises = servers.map(server => server.destroy())
66 await Promise.all(serversRemovePromises)
67
68 const numberOfServersRemoved = servers.length
69
70 if (numberOfServersRemoved) {
71 logger.info('Removed %d servers.', numberOfServersRemoved)
72 } else {
73 logger.info('No need to remove bad servers.')
74 }
75 } catch (err) {
76 logger.error('Cannot remove bad servers.', err)
77 }
78 }
79
80 private static incrementScores (ids: number[], value: number) {
81 const update = {
82 score: Sequelize.literal('score +' + value)
83 }
84
85 const options = {
86 where: {
87 id: {
88 [Sequelize.Op.in]: ids
89 }
90 },
91 // In this case score is a literal and not an integer so we do not validate it
92 validate: false
93 }
94
95 return ServerModel.update(update, options)
96 }
97
98 private static listBadServers () {
99 const query = {
100 where: {
101 score: {
102 [Sequelize.Op.lte]: 0
103 }
104 }
105 }
106
107 return ServerModel.findAll(query)
108 }
109} 26}
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts
index acc2486b3..e2cbf0422 100644
--- a/server/models/video/video-channel.ts
+++ b/server/models/video/video-channel.ts
@@ -228,7 +228,7 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
228 const actor = this.Actor.toFormattedJSON() 228 const actor = this.Actor.toFormattedJSON()
229 const account = { 229 const account = {
230 id: this.id, 230 id: this.id,
231 name: this.name, 231 displayName: this.name,
232 description: this.description, 232 description: this.description,
233 isLocal: this.Actor.isOwned(), 233 isLocal: this.Actor.isOwned(),
234 createdAt: this.createdAt, 234 createdAt: this.createdAt,
diff --git a/shared/models/actors/account.model.ts b/shared/models/actors/account.model.ts
index e4dbc81e5..5cc12c18f 100644
--- a/shared/models/actors/account.model.ts
+++ b/shared/models/actors/account.model.ts
@@ -1,15 +1,5 @@
1import { Avatar } from '../avatars/avatar.model' 1import { Actor } from './actor.model'
2 2
3export interface Account { 3export interface Account extends Actor {
4 id: number
5 uuid: string
6 url: string
7 name: string
8 displayName: string 4 displayName: string
9 host: string
10 followingCount: number
11 followersCount: number
12 createdAt: Date
13 updatedAt: Date
14 avatar: Avatar
15} 5}
diff --git a/shared/models/actors/actor.model.ts b/shared/models/actors/actor.model.ts
new file mode 100644
index 000000000..f91616519
--- /dev/null
+++ b/shared/models/actors/actor.model.ts
@@ -0,0 +1,14 @@
1import { Avatar } from '../avatars/avatar.model'
2
3export interface Actor {
4 id: number
5 uuid: string
6 url: string
7 name: string
8 host: string
9 followingCount: number
10 followersCount: number
11 createdAt: Date
12 updatedAt: Date
13 avatar: Avatar
14}
diff --git a/shared/models/actors/follow.model.ts b/shared/models/actors/follow.model.ts
index cdc3da560..70562bfc7 100644
--- a/shared/models/actors/follow.model.ts
+++ b/shared/models/actors/follow.model.ts
@@ -1,11 +1,12 @@
1import { Account } from './account.model' 1import { Actor } from './actor.model'
2 2
3export type FollowState = 'pending' | 'accepted' 3export type FollowState = 'pending' | 'accepted'
4 4
5export interface AccountFollow { 5export interface AccountFollow {
6 id: number 6 id: number
7 follower: Account 7 follower: Actor
8 following: Account 8 following: Actor
9 score: number
9 state: FollowState 10 state: FollowState
10 createdAt: Date 11 createdAt: Date
11 updatedAt: Date 12 updatedAt: Date
diff --git a/shared/models/videos/video-channel.model.ts b/shared/models/videos/video-channel.model.ts
index d1a952826..b164fb555 100644
--- a/shared/models/videos/video-channel.model.ts
+++ b/shared/models/videos/video-channel.model.ts
@@ -1,13 +1,10 @@
1import { Actor } from '../actors/actor.model'
1import { Video } from './video.model' 2import { Video } from './video.model'
2 3
3export interface VideoChannel { 4export interface VideoChannel extends Actor {
4 id: number 5 displayName: string
5 name: string
6 url: string
7 description: string 6 description: string
8 isLocal: boolean 7 isLocal: boolean
9 createdAt: Date | string
10 updatedAt: Date | string
11 owner?: { 8 owner?: {
12 name: string 9 name: string
13 uuid: string 10 uuid: string