]>
Commit | Line | Data |
---|---|---|
4d4e5cd4 | 1 | import * as request from 'request' |
69818c93 | 2 | import * as Sequelize from 'sequelize' |
6fcd19ba | 3 | import * as Promise from 'bluebird' |
9f10b292 | 4 | |
e02643f3 | 5 | import { database as db } from '../initializers/database' |
65fcc311 C |
6 | import { |
7 | API_VERSION, | |
8 | CONFIG, | |
9 | REQUESTS_IN_PARALLEL, | |
10 | REQUEST_ENDPOINTS, | |
11 | REQUEST_ENDPOINT_ACTIONS, | |
12 | REMOTE_SCHEME | |
13 | } from '../initializers' | |
14 | import { | |
15 | logger, | |
16 | getMyPublicCert, | |
17 | makeSecureRequest, | |
6fcd19ba | 18 | makeRetryRequest |
65fcc311 C |
19 | } from '../helpers' |
20 | import { | |
21 | RequestScheduler, | |
69818c93 C |
22 | RequestSchedulerOptions, |
23 | ||
65fcc311 | 24 | RequestVideoQaduScheduler, |
69818c93 C |
25 | RequestVideoQaduSchedulerOptions, |
26 | ||
27 | RequestVideoEventScheduler, | |
28 | RequestVideoEventSchedulerOptions | |
65fcc311 | 29 | } from './request' |
ee9e7b61 C |
30 | import { |
31 | PodInstance, | |
32 | VideoInstance | |
33 | } from '../models' | |
34 | import { | |
35 | RequestEndpoint, | |
36 | RequestVideoEventType, | |
37 | RequestVideoQaduType | |
38 | } from '../../shared' | |
69818c93 | 39 | |
ee9e7b61 C |
40 | type QaduParam = { videoId: string, type: RequestVideoQaduType } |
41 | type EventParam = { videoId: string, type: RequestVideoEventType } | |
65fcc311 C |
42 | |
43 | const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] | |
9e167724 | 44 | |
fe783f6b | 45 | const requestScheduler = new RequestScheduler() |
99fdec46 C |
46 | const requestVideoQaduScheduler = new RequestVideoQaduScheduler() |
47 | const requestVideoEventScheduler = new RequestVideoEventScheduler() | |
da691c46 | 48 | |
65fcc311 | 49 | function activateSchedulers () { |
c1a7ab7f | 50 | requestScheduler.activate() |
99fdec46 C |
51 | requestVideoQaduScheduler.activate() |
52 | requestVideoEventScheduler.activate() | |
c1a7ab7f C |
53 | } |
54 | ||
6fcd19ba | 55 | function addVideoToFriends (videoData: Object, transaction: Sequelize.Transaction) { |
ed04d94f | 56 | const options = { |
da691c46 | 57 | type: ENDPOINT_ACTIONS.ADD, |
65fcc311 | 58 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
ed04d94f C |
59 | data: videoData, |
60 | transaction | |
61 | } | |
6fcd19ba | 62 | return createRequest(options) |
9f10b292 C |
63 | } |
64 | ||
6fcd19ba | 65 | function updateVideoToFriends (videoData: Object, transaction: Sequelize.Transaction) { |
ed04d94f | 66 | const options = { |
da691c46 | 67 | type: ENDPOINT_ACTIONS.UPDATE, |
65fcc311 | 68 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
ed04d94f C |
69 | data: videoData, |
70 | transaction | |
71 | } | |
6fcd19ba | 72 | return createRequest(options) |
55fa55a9 C |
73 | } |
74 | ||
69818c93 | 75 | function removeVideoToFriends (videoParams: Object) { |
ed04d94f | 76 | const options = { |
da691c46 | 77 | type: ENDPOINT_ACTIONS.REMOVE, |
65fcc311 | 78 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
69818c93 C |
79 | data: videoParams, |
80 | transaction: null | |
ed04d94f | 81 | } |
6fcd19ba | 82 | return createRequest(options) |
55fa55a9 C |
83 | } |
84 | ||
6fcd19ba | 85 | function reportAbuseVideoToFriend (reportData: Object, video: VideoInstance, transaction: Sequelize.Transaction) { |
bd14d16a | 86 | const options = { |
da691c46 | 87 | type: ENDPOINT_ACTIONS.REPORT_ABUSE, |
65fcc311 | 88 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
bd14d16a | 89 | data: reportData, |
69818c93 | 90 | toIds: [ video.Author.podId ], |
6fcd19ba | 91 | transaction |
bd14d16a | 92 | } |
6fcd19ba | 93 | return createRequest(options) |
7b1f49de C |
94 | } |
95 | ||
6fcd19ba | 96 | function quickAndDirtyUpdateVideoToFriends (qaduParam: QaduParam, transaction?: Sequelize.Transaction) { |
9e167724 | 97 | const options = { |
69818c93 C |
98 | videoId: qaduParam.videoId, |
99 | type: qaduParam.type, | |
9e167724 C |
100 | transaction |
101 | } | |
6fcd19ba | 102 | return createVideoQaduRequest(options) |
9e167724 C |
103 | } |
104 | ||
6fcd19ba | 105 | function quickAndDirtyUpdatesVideoToFriends (qadusParams: QaduParam[], transaction: Sequelize.Transaction) { |
d38b8281 C |
106 | const tasks = [] |
107 | ||
108 | qadusParams.forEach(function (qaduParams) { | |
6fcd19ba | 109 | tasks.push(quickAndDirtyUpdateVideoToFriends(qaduParams, transaction)) |
d38b8281 C |
110 | }) |
111 | ||
6fcd19ba | 112 | return Promise.all(tasks) |
d38b8281 C |
113 | } |
114 | ||
6fcd19ba | 115 | function addEventToRemoteVideo (eventParam: EventParam, transaction?: Sequelize.Transaction) { |
e4c87ec2 | 116 | const options = { |
69818c93 C |
117 | videoId: eventParam.videoId, |
118 | type: eventParam.type, | |
e4c87ec2 C |
119 | transaction |
120 | } | |
6fcd19ba | 121 | return createVideoEventRequest(options) |
e4c87ec2 C |
122 | } |
123 | ||
6fcd19ba | 124 | function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) { |
d38b8281 C |
125 | const tasks = [] |
126 | ||
127 | eventsParams.forEach(function (eventParams) { | |
6fcd19ba | 128 | tasks.push(addEventToRemoteVideo(eventParams, transaction)) |
d38b8281 C |
129 | }) |
130 | ||
6fcd19ba | 131 | return Promise.all(tasks) |
d38b8281 C |
132 | } |
133 | ||
6fcd19ba C |
134 | function hasFriends () { |
135 | return db.Pod.countAll().then(count => count !== 0) | |
9f10b292 C |
136 | } |
137 | ||
6fcd19ba | 138 | function makeFriends (hosts: string[]) { |
bc503c2a | 139 | const podsScore = {} |
9f10b292 C |
140 | |
141 | logger.info('Make friends!') | |
6fcd19ba C |
142 | return getMyPublicCert() |
143 | .then(cert => { | |
709756b8 | 144 | return Promise.each(hosts, host => computeForeignPodsList(host, podsScore)).then(() => cert) |
6fcd19ba C |
145 | }) |
146 | .then(cert => { | |
bc503c2a | 147 | logger.debug('Pods scores computed.', { podsScore: podsScore }) |
49abbbbe | 148 | const podsList = computeWinningPods(hosts, podsScore) |
bc503c2a | 149 | logger.debug('Pods that we keep.', { podsToKeep: podsList }) |
9f10b292 | 150 | |
6fcd19ba | 151 | return makeRequestsToWinningPods(cert, podsList) |
c173e565 | 152 | }) |
9f10b292 C |
153 | } |
154 | ||
6fcd19ba | 155 | function quitFriends () { |
9f10b292 | 156 | // Stop pool requests |
c1a7ab7f | 157 | requestScheduler.deactivate() |
9f10b292 | 158 | |
6fcd19ba C |
159 | return requestScheduler.flush() |
160 | .then(() => { | |
161 | return requestVideoQaduScheduler.flush() | |
162 | }) | |
163 | .then(() => { | |
164 | return db.Pod.list() | |
165 | }) | |
166 | .then(pods => { | |
528a9efa | 167 | const requestParams = { |
69818c93 | 168 | method: 'POST' as 'POST', |
65fcc311 | 169 | path: '/api/' + API_VERSION + '/remote/pods/remove', |
65fcc311 | 170 | toPod: null |
c173e565 C |
171 | } |
172 | ||
e7ea2817 | 173 | // Announce we quit them |
528a9efa C |
174 | // We don't care if the request fails |
175 | // The other pod will exclude us automatically after a while | |
6fcd19ba | 176 | return Promise.map(pods, pod => { |
528a9efa | 177 | requestParams.toPod = pod |
709756b8 | 178 | |
6fcd19ba C |
179 | return makeSecureRequest(requestParams) |
180 | }, { concurrency: REQUESTS_IN_PARALLEL }) | |
181 | .then(() => pods) | |
182 | .catch(err => { | |
183 | logger.error('Some errors while quitting friends.', { err: err }) | |
184 | // Don't stop the process | |
e7ea2817 | 185 | }) |
6fcd19ba C |
186 | }) |
187 | .then(pods => { | |
188 | const tasks = [] | |
189 | pods.forEach(pod => tasks.push(pod.destroy())) | |
e7ea2817 | 190 | |
6fcd19ba C |
191 | return Promise.all(pods) |
192 | }) | |
193 | .then(() => { | |
194 | logger.info('Removed all remote videos.') | |
195 | // Don't forget to re activate the scheduler, even if there was an error | |
196 | return requestScheduler.activate() | |
197 | }) | |
198 | .finally(() => requestScheduler.activate()) | |
9f10b292 | 199 | } |
c173e565 | 200 | |
69818c93 | 201 | function sendOwnedVideosToPod (podId: number) { |
6fcd19ba C |
202 | db.Video.listOwnedAndPopulateAuthorAndTags() |
203 | .then(videosList => { | |
204 | const tasks = [] | |
205 | videosList.forEach(video => { | |
206 | const promise = video.toAddRemoteJSON() | |
207 | .then(remoteVideo => { | |
208 | const options = { | |
209 | type: 'add', | |
210 | endpoint: REQUEST_ENDPOINTS.VIDEOS, | |
211 | data: remoteVideo, | |
212 | toIds: [ podId ], | |
213 | transaction: null | |
214 | } | |
215 | return createRequest(options) | |
216 | }) | |
217 | .catch(err => { | |
218 | logger.error('Cannot convert video to remote.', { error: err }) | |
219 | // Don't break the process | |
220 | return undefined | |
221 | }) | |
222 | ||
223 | tasks.push(promise) | |
528a9efa | 224 | }) |
6fcd19ba C |
225 | |
226 | return Promise.all(tasks) | |
528a9efa | 227 | }) |
9f10b292 | 228 | } |
c173e565 | 229 | |
99fdec46 C |
230 | function getRequestScheduler () { |
231 | return requestScheduler | |
232 | } | |
233 | ||
234 | function getRequestVideoQaduScheduler () { | |
235 | return requestVideoQaduScheduler | |
236 | } | |
237 | ||
238 | function getRequestVideoEventScheduler () { | |
239 | return requestVideoEventScheduler | |
240 | } | |
241 | ||
9f10b292 | 242 | // --------------------------------------------------------------------------- |
c173e565 | 243 | |
65fcc311 C |
244 | export { |
245 | activateSchedulers, | |
246 | addVideoToFriends, | |
247 | updateVideoToFriends, | |
248 | reportAbuseVideoToFriend, | |
249 | quickAndDirtyUpdateVideoToFriends, | |
250 | quickAndDirtyUpdatesVideoToFriends, | |
251 | addEventToRemoteVideo, | |
252 | addEventsToRemoteVideo, | |
253 | hasFriends, | |
254 | makeFriends, | |
255 | quitFriends, | |
256 | removeVideoToFriends, | |
257 | sendOwnedVideosToPod, | |
258 | getRequestScheduler, | |
259 | getRequestVideoQaduScheduler, | |
260 | getRequestVideoEventScheduler | |
261 | } | |
c173e565 | 262 | |
9f10b292 | 263 | // --------------------------------------------------------------------------- |
c173e565 | 264 | |
6fcd19ba C |
265 | function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) { |
266 | // TODO: type res | |
267 | return getForeignPodsList(host).then((res: any) => { | |
55fa55a9 | 268 | const foreignPodsList = res.data |
528a9efa C |
269 | |
270 | // Let's give 1 point to the pod we ask the friends list | |
49abbbbe | 271 | foreignPodsList.push({ host }) |
89d1d8ba | 272 | |
6fcd19ba | 273 | foreignPodsList.forEach(foreignPod => { |
49abbbbe | 274 | const foreignPodHost = foreignPod.host |
89d1d8ba | 275 | |
49abbbbe C |
276 | if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++ |
277 | else podsScore[foreignPodHost] = 1 | |
89d1d8ba | 278 | }) |
cbe2f7c3 | 279 | |
6fcd19ba | 280 | return undefined |
89d1d8ba C |
281 | }) |
282 | } | |
283 | ||
69818c93 | 284 | function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) { |
89d1d8ba C |
285 | // Build the list of pods to add |
286 | // Only add a pod if it exists in more than a half base pods | |
bc503c2a | 287 | const podsList = [] |
49abbbbe | 288 | const baseScore = hosts.length / 2 |
1e4b0080 | 289 | |
6fcd19ba | 290 | Object.keys(podsScore).forEach(podHost => { |
2c49ca42 | 291 | // If the pod is not me and with a good score we add it |
49abbbbe C |
292 | if (isMe(podHost) === false && podsScore[podHost] > baseScore) { |
293 | podsList.push({ host: podHost }) | |
2c49ca42 | 294 | } |
89d1d8ba | 295 | }) |
e7ea2817 | 296 | |
bc503c2a | 297 | return podsList |
89d1d8ba C |
298 | } |
299 | ||
6fcd19ba C |
300 | function getForeignPodsList (host: string) { |
301 | return new Promise((res, rej) => { | |
302 | const path = '/api/' + API_VERSION + '/pods' | |
c173e565 | 303 | |
6fcd19ba C |
304 | request.get(REMOTE_SCHEME.HTTP + '://' + host + path, function (err, response, body) { |
305 | if (err) return rej(err) | |
8425cb89 | 306 | |
6fcd19ba C |
307 | try { |
308 | const json = JSON.parse(body) | |
309 | return res(json) | |
310 | } catch (err) { | |
311 | return rej(err) | |
312 | } | |
313 | }) | |
9f10b292 C |
314 | }) |
315 | } | |
89d1d8ba | 316 | |
6fcd19ba | 317 | function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) { |
89d1d8ba | 318 | // Stop pool requests |
c1a7ab7f | 319 | requestScheduler.deactivate() |
89d1d8ba | 320 | // Flush pool requests |
c1a7ab7f | 321 | requestScheduler.forceSend() |
89d1d8ba | 322 | |
6fcd19ba | 323 | return Promise.map(podsList, pod => { |
528a9efa | 324 | const params = { |
65fcc311 | 325 | url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/pods/', |
69818c93 | 326 | method: 'POST' as 'POST', |
528a9efa | 327 | json: { |
65fcc311 C |
328 | host: CONFIG.WEBSERVER.HOST, |
329 | email: CONFIG.ADMIN.EMAIL, | |
528a9efa C |
330 | publicKey: cert |
331 | } | |
89d1d8ba C |
332 | } |
333 | ||
6fcd19ba C |
334 | return makeRetryRequest(params) |
335 | .then(({ response, body }) => { | |
336 | body = body as { cert: string, email: string } | |
337 | ||
338 | if (response.statusCode === 200) { | |
339 | const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email }) | |
340 | return podObj.save() | |
341 | .then(podCreated => { | |
342 | ||
343 | // Add our videos to the request scheduler | |
344 | sendOwnedVideosToPod(podCreated.id) | |
345 | }) | |
346 | .catch(err => { | |
347 | logger.error('Cannot add friend %s pod.', pod.host, { error: err }) | |
348 | }) | |
349 | } else { | |
350 | logger.error('Status not 200 for %s pod.', pod.host) | |
351 | } | |
352 | }) | |
353 | .catch(err => { | |
354 | logger.error('Error with adding %s pod.', pod.host, { error: err.stack }) | |
528a9efa | 355 | // Don't break the process |
6fcd19ba C |
356 | }) |
357 | }, { concurrency: REQUESTS_IN_PARALLEL }) | |
358 | .then(() => logger.debug('makeRequestsToWinningPods finished.')) | |
359 | .finally(() => { | |
528a9efa C |
360 | // Final callback, we've ended all the requests |
361 | // Now we made new friends, we can re activate the pool of requests | |
c1a7ab7f | 362 | requestScheduler.activate() |
89d1d8ba C |
363 | }) |
364 | } | |
00057e85 | 365 | |
55fa55a9 | 366 | // Wrapper that populate "toIds" argument with all our friends if it is not specified |
69818c93 C |
367 | type CreateRequestOptions = { |
368 | type: string | |
ee9e7b61 | 369 | endpoint: RequestEndpoint |
69818c93 C |
370 | data: Object |
371 | toIds?: number[] | |
372 | transaction: Sequelize.Transaction | |
373 | } | |
6fcd19ba C |
374 | function createRequest (options: CreateRequestOptions) { |
375 | if (options.toIds !== undefined) return requestScheduler.createRequest(options as RequestSchedulerOptions) | |
feb4bdfd | 376 | |
55fa55a9 | 377 | // If the "toIds" pods is not specified, we send the request to all our friends |
6fcd19ba | 378 | return db.Pod.listAllIds(options.transaction).then(podIds => { |
ed04d94f | 379 | const newOptions = Object.assign(options, { toIds: podIds }) |
6fcd19ba | 380 | return requestScheduler.createRequest(newOptions) |
00057e85 C |
381 | }) |
382 | } | |
2c49ca42 | 383 | |
6fcd19ba C |
384 | function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) { |
385 | return requestVideoQaduScheduler.createRequest(options) | |
9e167724 C |
386 | } |
387 | ||
6fcd19ba C |
388 | function createVideoEventRequest (options: RequestVideoEventSchedulerOptions) { |
389 | return requestVideoEventScheduler.createRequest(options) | |
e4c87ec2 C |
390 | } |
391 | ||
69818c93 | 392 | function isMe (host: string) { |
65fcc311 | 393 | return host === CONFIG.WEBSERVER.HOST |
2c49ca42 | 394 | } |