aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/request
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib/request')
-rw-r--r--server/lib/request/abstract-request-scheduler.ts168
-rw-r--r--server/lib/request/index.ts4
-rw-r--r--server/lib/request/request-scheduler.ts96
-rw-r--r--server/lib/request/request-video-event-scheduler.ts129
-rw-r--r--server/lib/request/request-video-qadu-scheduler.ts148
5 files changed, 0 insertions, 545 deletions
diff --git a/server/lib/request/abstract-request-scheduler.ts b/server/lib/request/abstract-request-scheduler.ts
deleted file mode 100644
index f838c47f2..000000000
--- a/server/lib/request/abstract-request-scheduler.ts
+++ /dev/null
@@ -1,168 +0,0 @@
1import { isEmpty } from 'lodash'
2import * as Bluebird from 'bluebird'
3
4import { database as db } from '../../initializers/database'
5import { logger, makeSecureRequest } from '../../helpers'
6import { AbstractRequestClass, AbstractRequestToPodClass, PodInstance } from '../../models'
7import {
8 API_VERSION,
9 REQUESTS_IN_PARALLEL,
10 REQUESTS_INTERVAL
11} from '../../initializers'
12
13interface RequestsObjects<U> {
14 [ id: string ]: {
15 toPod: PodInstance
16 endpoint: string
17 ids: number[] // ids
18 datas: U[]
19 }
20}
21
22abstract class AbstractRequestScheduler <T> {
23 requestInterval: number
24 limitPods: number
25 limitPerPod: number
26
27 protected lastRequestTimestamp: number
28 protected timer: NodeJS.Timer
29 protected description: string
30
31 constructor () {
32 this.lastRequestTimestamp = 0
33 this.timer = null
34 this.requestInterval = REQUESTS_INTERVAL
35 }
36
37 abstract getRequestModel (): AbstractRequestClass<T>
38 abstract getRequestToPodModel (): AbstractRequestToPodClass
39 abstract buildRequestsObjects (requestsGrouped: T): RequestsObjects<any>
40
41 activate () {
42 logger.info('Requests scheduler activated.')
43 this.lastRequestTimestamp = Date.now()
44
45 this.timer = setInterval(() => {
46 this.lastRequestTimestamp = Date.now()
47 this.makeRequests()
48 }, this.requestInterval)
49 }
50
51 deactivate () {
52 logger.info('Requests scheduler deactivated.')
53 clearInterval(this.timer)
54 this.timer = null
55 }
56
57 forceSend () {
58 logger.info('Force requests scheduler sending.')
59 this.makeRequests()
60 }
61
62 remainingMilliSeconds () {
63 if (this.timer === null) return -1
64
65 return REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp)
66 }
67
68 remainingRequestsCount () {
69 return this.getRequestModel().countTotalRequests()
70 }
71
72 flush () {
73 return this.getRequestModel().removeAll()
74 }
75
76 // ---------------------------------------------------------------------------
77
78 // Make a requests to friends of a certain type
79 protected async makeRequest (toPod: PodInstance, requestEndpoint: string, requestsToMake: any) {
80 const params = {
81 toPod: toPod,
82 method: 'POST' as 'POST',
83 path: '/api/' + API_VERSION + '/remote/' + requestEndpoint,
84 data: requestsToMake // Requests we need to make
85 }
86
87 // Make multiple retry requests to all of pods
88 // The function fire some useful callbacks
89 try {
90 const { response } = await makeSecureRequest(params)
91
92 // 400 because if the other pod is not up to date, it may not understand our request
93 if ([ 200, 201, 204, 400 ].indexOf(response.statusCode) === -1) {
94 throw new Error('Status code not 20x or 400 : ' + response.statusCode)
95 }
96 } catch (err) {
97 logger.error('Error sending secure request to %s pod.', toPod.host, err)
98
99 throw err
100 }
101 }
102
103 // Make all the requests of the scheduler
104 protected async makeRequests () {
105 let requestsGrouped: T
106
107 try {
108 requestsGrouped = await this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod)
109 } catch (err) {
110 logger.error('Cannot get the list of "%s".', this.description, { error: err.stack })
111 throw err
112 }
113
114 // We want to group requests by destinations pod and endpoint
115 const requestsToMake = this.buildRequestsObjects(requestsGrouped)
116
117 // If there are no requests, abort
118 if (isEmpty(requestsToMake) === true) {
119 logger.info('No "%s" to make.', this.description)
120 return { goodPods: [], badPods: [] }
121 }
122
123 logger.info('Making "%s" to friends.', this.description)
124
125 const goodPods: number[] = []
126 const badPods: number[] = []
127
128 await Bluebird.map(Object.keys(requestsToMake), async hashKey => {
129 const requestToMake = requestsToMake[hashKey]
130 const toPod: PodInstance = requestToMake.toPod
131
132 try {
133 await this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas)
134 logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids })
135 goodPods.push(requestToMake.toPod.id)
136
137 this.afterRequestHook()
138
139 // Remove the pod id of these request ids
140 await this.getRequestToPodModel()
141 .removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id)
142 } catch (err) {
143 badPods.push(requestToMake.toPod.id)
144 logger.info('Cannot make request to %s.', toPod.host, err)
145 }
146 }, { concurrency: REQUESTS_IN_PARALLEL })
147
148 this.afterRequestsHook()
149
150 // All the requests were made, we update the pods score
151 db.Pod.updatePodsScore(goodPods, badPods)
152 }
153
154 protected afterRequestHook () {
155 // Nothing to do, let children re-implement it
156 }
157
158 protected afterRequestsHook () {
159 // Nothing to do, let children re-implement it
160 }
161}
162
163// ---------------------------------------------------------------------------
164
165export {
166 AbstractRequestScheduler,
167 RequestsObjects
168}
diff --git a/server/lib/request/index.ts b/server/lib/request/index.ts
deleted file mode 100644
index 47d60e5b4..000000000
--- a/server/lib/request/index.ts
+++ /dev/null
@@ -1,4 +0,0 @@
1export * from './abstract-request-scheduler'
2export * from './request-scheduler'
3export * from './request-video-event-scheduler'
4export * from './request-video-qadu-scheduler'
diff --git a/server/lib/request/request-scheduler.ts b/server/lib/request/request-scheduler.ts
deleted file mode 100644
index c3f7f6429..000000000
--- a/server/lib/request/request-scheduler.ts
+++ /dev/null
@@ -1,96 +0,0 @@
1import * as Sequelize from 'sequelize'
2
3import { database as db } from '../../initializers/database'
4import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler'
5import { logger } from '../../helpers'
6import { REQUESTS_LIMIT_PODS, REQUESTS_LIMIT_PER_POD } from '../../initializers'
7import { RequestsGrouped } from '../../models'
8import { RequestEndpoint, RemoteVideoRequest } from '../../../shared'
9
10export type RequestSchedulerOptions = {
11 type: string
12 endpoint: RequestEndpoint
13 data: Object
14 toIds: number[]
15 transaction: Sequelize.Transaction
16}
17
18class RequestScheduler extends AbstractRequestScheduler<RequestsGrouped> {
19 constructor () {
20 super()
21
22 // We limit the size of the requests
23 this.limitPods = REQUESTS_LIMIT_PODS
24 this.limitPerPod = REQUESTS_LIMIT_PER_POD
25
26 this.description = 'requests'
27 }
28
29 getRequestModel () {
30 return db.Request
31 }
32
33 getRequestToPodModel () {
34 return db.RequestToPod
35 }
36
37 buildRequestsObjects (requestsGrouped: RequestsGrouped) {
38 const requestsToMakeGrouped: RequestsObjects<RemoteVideoRequest> = {}
39
40 for (const toPodId of Object.keys(requestsGrouped)) {
41 for (const data of requestsGrouped[toPodId]) {
42 const request = data.request
43 const pod = data.pod
44 const hashKey = toPodId + request.endpoint
45
46 if (!requestsToMakeGrouped[hashKey]) {
47 requestsToMakeGrouped[hashKey] = {
48 toPod: pod,
49 endpoint: request.endpoint,
50 ids: [], // request ids, to delete them from the DB in the future
51 datas: [] // requests data,
52 }
53 }
54
55 requestsToMakeGrouped[hashKey].ids.push(request.id)
56 requestsToMakeGrouped[hashKey].datas.push(request.request)
57 }
58 }
59
60 return requestsToMakeGrouped
61 }
62
63 async createRequest ({ type, endpoint, data, toIds, transaction }: RequestSchedulerOptions) {
64 // If there are no destination pods abort
65 if (toIds.length === 0) return undefined
66
67 const createQuery = {
68 endpoint,
69 request: {
70 type: type,
71 data: data
72 }
73 }
74
75 const dbRequestOptions: Sequelize.CreateOptions = {
76 transaction
77 }
78
79 const request = await db.Request.create(createQuery, dbRequestOptions)
80 await request.setPods(toIds, dbRequestOptions)
81 }
82
83 // ---------------------------------------------------------------------------
84
85 afterRequestsHook () {
86 // Flush requests with no pod
87 this.getRequestModel().removeWithEmptyTo()
88 .catch(err => logger.error('Error when removing requests with no pods.', err))
89 }
90}
91
92// ---------------------------------------------------------------------------
93
94export {
95 RequestScheduler
96}
diff --git a/server/lib/request/request-video-event-scheduler.ts b/server/lib/request/request-video-event-scheduler.ts
deleted file mode 100644
index 5f21287f0..000000000
--- a/server/lib/request/request-video-event-scheduler.ts
+++ /dev/null
@@ -1,129 +0,0 @@
1import * as Sequelize from 'sequelize'
2
3import { database as db } from '../../initializers/database'
4import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler'
5import {
6 REQUESTS_VIDEO_EVENT_LIMIT_PODS,
7 REQUESTS_VIDEO_EVENT_LIMIT_PER_POD,
8 REQUEST_VIDEO_EVENT_ENDPOINT
9} from '../../initializers'
10import { RequestsVideoEventGrouped } from '../../models'
11import { RequestVideoEventType, RemoteVideoEventRequest, RemoteVideoEventType } from '../../../shared'
12
13export type RequestVideoEventSchedulerOptions = {
14 type: RequestVideoEventType
15 videoId: number
16 count?: number
17 transaction?: Sequelize.Transaction
18}
19
20class RequestVideoEventScheduler extends AbstractRequestScheduler<RequestsVideoEventGrouped> {
21 constructor () {
22 super()
23
24 // We limit the size of the requests
25 this.limitPods = REQUESTS_VIDEO_EVENT_LIMIT_PODS
26 this.limitPerPod = REQUESTS_VIDEO_EVENT_LIMIT_PER_POD
27
28 this.description = 'video event requests'
29 }
30
31 getRequestModel () {
32 return db.RequestVideoEvent
33 }
34
35 getRequestToPodModel () {
36 return db.RequestVideoEvent
37 }
38
39 buildRequestsObjects (eventRequests: RequestsVideoEventGrouped) {
40 const requestsToMakeGrouped: RequestsObjects<RemoteVideoEventRequest> = {}
41
42 /* Example:
43 {
44 pod1: {
45 video1: { views: 4, likes: 5 },
46 video2: { likes: 5 }
47 }
48 }
49 */
50 const eventsPerVideoPerPod: {
51 [ podId: string ]: {
52 [ videoUUID: string ]: {
53 views?: number
54 likes?: number
55 dislikes?: number
56 }
57 }
58 } = {}
59
60 // We group video events per video and per pod
61 // We add the counts of the same event types
62 for (const toPodId of Object.keys(eventRequests)) {
63 for (const eventToProcess of eventRequests[toPodId]) {
64 if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {}
65
66 if (!requestsToMakeGrouped[toPodId]) {
67 requestsToMakeGrouped[toPodId] = {
68 toPod: eventToProcess.pod,
69 endpoint: REQUEST_VIDEO_EVENT_ENDPOINT,
70 ids: [], // request ids, to delete them from the DB in the future
71 datas: [] // requests data
72 }
73 }
74 requestsToMakeGrouped[toPodId].ids.push(eventToProcess.id)
75
76 const eventsPerVideo = eventsPerVideoPerPod[toPodId]
77 const uuid = eventToProcess.video.uuid
78 if (!eventsPerVideo[uuid]) eventsPerVideo[uuid] = {}
79
80 const events = eventsPerVideo[uuid]
81 if (!events[eventToProcess.type]) events[eventToProcess.type] = 0
82
83 events[eventToProcess.type] += eventToProcess.count
84 }
85 }
86
87 // Now we build our requests array per pod
88 for (const toPodId of Object.keys(eventsPerVideoPerPod)) {
89 const eventsForPod = eventsPerVideoPerPod[toPodId]
90
91 for (const uuid of Object.keys(eventsForPod)) {
92 const eventsForVideo = eventsForPod[uuid]
93
94 for (const eventType of Object.keys(eventsForVideo)) {
95 requestsToMakeGrouped[toPodId].datas.push({
96 data: {
97 uuid,
98 eventType: eventType as RemoteVideoEventType,
99 count: +eventsForVideo[eventType]
100 }
101 })
102 }
103 }
104 }
105
106 return requestsToMakeGrouped
107 }
108
109 createRequest ({ type, videoId, count, transaction }: RequestVideoEventSchedulerOptions) {
110 if (count === undefined) count = 1
111
112 const dbRequestOptions: Sequelize.CreateOptions = {}
113 if (transaction) dbRequestOptions.transaction = transaction
114
115 const createQuery = {
116 type,
117 count,
118 videoId
119 }
120
121 return db.RequestVideoEvent.create(createQuery, dbRequestOptions)
122 }
123}
124
125// ---------------------------------------------------------------------------
126
127export {
128 RequestVideoEventScheduler
129}
diff --git a/server/lib/request/request-video-qadu-scheduler.ts b/server/lib/request/request-video-qadu-scheduler.ts
deleted file mode 100644
index 24ee59d29..000000000
--- a/server/lib/request/request-video-qadu-scheduler.ts
+++ /dev/null
@@ -1,148 +0,0 @@
1import * as Sequelize from 'sequelize'
2
3import { database as db } from '../../initializers/database'
4import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler'
5import { logger } from '../../helpers'
6import {
7 REQUESTS_VIDEO_QADU_LIMIT_PODS,
8 REQUESTS_VIDEO_QADU_LIMIT_PER_POD,
9 REQUEST_VIDEO_QADU_ENDPOINT,
10 REQUEST_VIDEO_QADU_TYPES
11} from '../../initializers'
12import { RequestsVideoQaduGrouped, PodInstance } from '../../models'
13import { RemoteQaduVideoRequest, RequestVideoQaduType } from '../../../shared'
14
15// We create a custom interface because we need "videos" attribute for our computations
16interface RequestsObjectsCustom<U> extends RequestsObjects<U> {
17 [ id: string ]: {
18 toPod: PodInstance
19 endpoint: string
20 ids: number[] // ids
21 datas: U[]
22
23 videos: {
24 [ uuid: string ]: {
25 uuid: string
26 likes?: number
27 dislikes?: number
28 views?: number
29 }
30 }
31 }
32}
33
34export type RequestVideoQaduSchedulerOptions = {
35 type: RequestVideoQaduType
36 videoId: number
37 transaction?: Sequelize.Transaction
38}
39
40class RequestVideoQaduScheduler extends AbstractRequestScheduler<RequestsVideoQaduGrouped> {
41 constructor () {
42 super()
43
44 // We limit the size of the requests
45 this.limitPods = REQUESTS_VIDEO_QADU_LIMIT_PODS
46 this.limitPerPod = REQUESTS_VIDEO_QADU_LIMIT_PER_POD
47
48 this.description = 'video QADU requests'
49 }
50
51 getRequestModel () {
52 return db.RequestVideoQadu
53 }
54
55 getRequestToPodModel () {
56 return db.RequestVideoQadu
57 }
58
59 buildRequestsObjects (requests: RequestsVideoQaduGrouped) {
60 const requestsToMakeGrouped: RequestsObjectsCustom<RemoteQaduVideoRequest> = {}
61
62 for (const toPodId of Object.keys(requests)) {
63 for (const data of requests[toPodId]) {
64 const request = data.request
65 const video = data.video
66 const pod = data.pod
67 const hashKey = toPodId
68
69 if (!requestsToMakeGrouped[hashKey]) {
70 requestsToMakeGrouped[hashKey] = {
71 toPod: pod,
72 endpoint: REQUEST_VIDEO_QADU_ENDPOINT,
73 ids: [], // request ids, to delete them from the DB in the future
74 datas: [], // requests data
75 videos: {}
76 }
77 }
78
79 // Maybe another attribute was filled for this video
80 let videoData = requestsToMakeGrouped[hashKey].videos[video.id]
81 if (!videoData) videoData = { uuid: null }
82
83 switch (request.type) {
84 case REQUEST_VIDEO_QADU_TYPES.LIKES:
85 videoData.likes = video.likes
86 break
87
88 case REQUEST_VIDEO_QADU_TYPES.DISLIKES:
89 videoData.dislikes = video.dislikes
90 break
91
92 case REQUEST_VIDEO_QADU_TYPES.VIEWS:
93 videoData.views = video.views
94 break
95
96 default:
97 logger.error('Unknown request video QADU type %s.', request.type)
98 return undefined
99 }
100
101 // Do not forget the uuid so the remote pod can identify the video
102 videoData.uuid = video.uuid
103 requestsToMakeGrouped[hashKey].ids.push(request.id)
104
105 // Maybe there are multiple quick and dirty update for the same video
106 // We use this hash map to dedupe them
107 requestsToMakeGrouped[hashKey].videos[video.id] = videoData
108 }
109 }
110
111 // Now we deduped similar quick and dirty updates, we can build our requests data
112 for (const hashKey of Object.keys(requestsToMakeGrouped)) {
113 for (const videoUUID of Object.keys(requestsToMakeGrouped[hashKey].videos)) {
114 const videoData = requestsToMakeGrouped[hashKey].videos[videoUUID]
115
116 requestsToMakeGrouped[hashKey].datas.push({
117 data: videoData
118 })
119 }
120
121 // We don't need it anymore, it was just to build our data array
122 delete requestsToMakeGrouped[hashKey].videos
123 }
124
125 return requestsToMakeGrouped
126 }
127
128 async createRequest ({ type, videoId, transaction }: RequestVideoQaduSchedulerOptions) {
129 const dbRequestOptions: Sequelize.BulkCreateOptions = {}
130 if (transaction) dbRequestOptions.transaction = transaction
131
132 // Send the update to all our friends
133 const podIds = await db.Pod.listAllIds(transaction)
134 const queries = []
135 for (const podId of podIds) {
136 queries.push({ type, videoId, podId })
137 }
138
139 await db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions)
140 return undefined
141 }
142}
143
144// ---------------------------------------------------------------------------
145
146export {
147 RequestVideoQaduScheduler
148}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-27 23:29:21 +0000