]>
Commit | Line | Data |
---|---|---|
65fcc311 C |
1 | import { each, eachLimit, eachSeries, series, waterfall } from 'async' |
2 | import request = require('request') | |
9f10b292 | 3 | |
feb4bdfd | 4 | const db = require('../initializers/database') |
65fcc311 C |
5 | import { |
6 | API_VERSION, | |
7 | CONFIG, | |
8 | REQUESTS_IN_PARALLEL, | |
9 | REQUEST_ENDPOINTS, | |
10 | REQUEST_ENDPOINT_ACTIONS, | |
11 | REMOTE_SCHEME | |
12 | } from '../initializers' | |
13 | import { | |
14 | logger, | |
15 | getMyPublicCert, | |
16 | makeSecureRequest, | |
17 | makeRetryRequest, | |
18 | createEmptyCallback | |
19 | } from '../helpers' | |
20 | import { | |
21 | RequestScheduler, | |
22 | RequestVideoQaduScheduler, | |
23 | RequestVideoEventScheduler | |
24 | } from './request' | |
25 | ||
26 | const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] | |
9e167724 | 27 | |
fe783f6b | 28 | const requestScheduler = new RequestScheduler() |
99fdec46 C |
29 | const requestVideoQaduScheduler = new RequestVideoQaduScheduler() |
30 | const requestVideoEventScheduler = new RequestVideoEventScheduler() | |
da691c46 | 31 | |
65fcc311 | 32 | function activateSchedulers () { |
c1a7ab7f | 33 | requestScheduler.activate() |
99fdec46 C |
34 | requestVideoQaduScheduler.activate() |
35 | requestVideoEventScheduler.activate() | |
c1a7ab7f C |
36 | } |
37 | ||
ed04d94f C |
38 | function addVideoToFriends (videoData, transaction, callback) { |
39 | const options = { | |
da691c46 | 40 | type: ENDPOINT_ACTIONS.ADD, |
65fcc311 | 41 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
ed04d94f C |
42 | data: videoData, |
43 | transaction | |
44 | } | |
45 | createRequest(options, callback) | |
9f10b292 C |
46 | } |
47 | ||
ed04d94f C |
48 | function updateVideoToFriends (videoData, transaction, callback) { |
49 | const options = { | |
da691c46 | 50 | type: ENDPOINT_ACTIONS.UPDATE, |
65fcc311 | 51 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
ed04d94f C |
52 | data: videoData, |
53 | transaction | |
54 | } | |
55 | createRequest(options, callback) | |
55fa55a9 C |
56 | } |
57 | ||
58 | function removeVideoToFriends (videoParams) { | |
ed04d94f | 59 | const options = { |
da691c46 | 60 | type: ENDPOINT_ACTIONS.REMOVE, |
65fcc311 | 61 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
ed04d94f C |
62 | data: videoParams |
63 | } | |
64 | createRequest(options) | |
55fa55a9 C |
65 | } |
66 | ||
67 | function reportAbuseVideoToFriend (reportData, video) { | |
bd14d16a | 68 | const options = { |
da691c46 | 69 | type: ENDPOINT_ACTIONS.REPORT_ABUSE, |
65fcc311 | 70 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
bd14d16a C |
71 | data: reportData, |
72 | toIds: [ video.Author.podId ] | |
73 | } | |
74 | createRequest(options) | |
7b1f49de C |
75 | } |
76 | ||
65fcc311 | 77 | function quickAndDirtyUpdateVideoToFriends (qaduParams, transaction?, callback?) { |
9e167724 | 78 | const options = { |
d38b8281 C |
79 | videoId: qaduParams.videoId, |
80 | type: qaduParams.type, | |
9e167724 C |
81 | transaction |
82 | } | |
83 | return createVideoQaduRequest(options, callback) | |
84 | } | |
85 | ||
d38b8281 C |
86 | function quickAndDirtyUpdatesVideoToFriends (qadusParams, transaction, finalCallback) { |
87 | const tasks = [] | |
88 | ||
89 | qadusParams.forEach(function (qaduParams) { | |
90 | const fun = function (callback) { | |
91 | quickAndDirtyUpdateVideoToFriends(qaduParams, transaction, callback) | |
92 | } | |
93 | ||
94 | tasks.push(fun) | |
95 | }) | |
96 | ||
97 | series(tasks, finalCallback) | |
98 | } | |
99 | ||
65fcc311 | 100 | function addEventToRemoteVideo (eventParams, transaction?, callback?) { |
e4c87ec2 | 101 | const options = { |
d38b8281 C |
102 | videoId: eventParams.videoId, |
103 | type: eventParams.type, | |
e4c87ec2 C |
104 | transaction |
105 | } | |
106 | createVideoEventRequest(options, callback) | |
107 | } | |
108 | ||
d38b8281 C |
109 | function addEventsToRemoteVideo (eventsParams, transaction, finalCallback) { |
110 | const tasks = [] | |
111 | ||
112 | eventsParams.forEach(function (eventParams) { | |
113 | const fun = function (callback) { | |
114 | addEventToRemoteVideo(eventParams, transaction, callback) | |
115 | } | |
116 | ||
117 | tasks.push(fun) | |
118 | }) | |
119 | ||
120 | series(tasks, finalCallback) | |
121 | } | |
122 | ||
9f10b292 | 123 | function hasFriends (callback) { |
feb4bdfd | 124 | db.Pod.countAll(function (err, count) { |
9f10b292 C |
125 | if (err) return callback(err) |
126 | ||
bc503c2a C |
127 | const hasFriends = (count !== 0) |
128 | callback(null, hasFriends) | |
9f10b292 C |
129 | }) |
130 | } | |
131 | ||
49abbbbe | 132 | function makeFriends (hosts, callback) { |
bc503c2a | 133 | const podsScore = {} |
9f10b292 C |
134 | |
135 | logger.info('Make friends!') | |
65fcc311 | 136 | getMyPublicCert(function (err, cert) { |
9f10b292 C |
137 | if (err) { |
138 | logger.error('Cannot read public cert.') | |
139 | return callback(err) | |
140 | } | |
c173e565 | 141 | |
49abbbbe C |
142 | eachSeries(hosts, function (host, callbackEach) { |
143 | computeForeignPodsList(host, podsScore, callbackEach) | |
89d1d8ba | 144 | }, function (err) { |
c173e565 C |
145 | if (err) return callback(err) |
146 | ||
bc503c2a | 147 | logger.debug('Pods scores computed.', { podsScore: podsScore }) |
49abbbbe | 148 | const podsList = computeWinningPods(hosts, podsScore) |
bc503c2a | 149 | logger.debug('Pods that we keep.', { podsToKeep: podsList }) |
9f10b292 | 150 | |
bc503c2a | 151 | makeRequestsToWinningPods(cert, podsList, callback) |
c173e565 | 152 | }) |
9f10b292 | 153 | }) |
9f10b292 C |
154 | } |
155 | ||
156 | function quitFriends (callback) { | |
157 | // Stop pool requests | |
c1a7ab7f | 158 | requestScheduler.deactivate() |
9f10b292 | 159 | |
1a42c9e2 | 160 | waterfall([ |
7920c273 | 161 | function flushRequests (callbackAsync) { |
9e167724 C |
162 | requestScheduler.flush(err => callbackAsync(err)) |
163 | }, | |
164 | ||
165 | function flushVideoQaduRequests (callbackAsync) { | |
99fdec46 | 166 | requestVideoQaduScheduler.flush(err => callbackAsync(err)) |
7920c273 C |
167 | }, |
168 | ||
e7ea2817 | 169 | function getPodsList (callbackAsync) { |
feb4bdfd | 170 | return db.Pod.list(callbackAsync) |
e7ea2817 C |
171 | }, |
172 | ||
173 | function announceIQuitMyFriends (pods, callbackAsync) { | |
528a9efa | 174 | const requestParams = { |
e7ea2817 | 175 | method: 'POST', |
65fcc311 C |
176 | path: '/api/' + API_VERSION + '/remote/pods/remove', |
177 | sign: true, | |
178 | toPod: null | |
c173e565 C |
179 | } |
180 | ||
e7ea2817 | 181 | // Announce we quit them |
528a9efa C |
182 | // We don't care if the request fails |
183 | // The other pod will exclude us automatically after a while | |
65fcc311 | 184 | eachLimit(pods, REQUESTS_IN_PARALLEL, function (pod, callbackEach) { |
528a9efa | 185 | requestParams.toPod = pod |
65fcc311 | 186 | makeSecureRequest(requestParams, callbackEach) |
528a9efa C |
187 | }, function (err) { |
188 | if (err) { | |
189 | logger.error('Some errors while quitting friends.', { err: err }) | |
190 | // Don't stop the process | |
191 | } | |
192 | ||
80a6c9e7 | 193 | return callbackAsync(null, pods) |
e7ea2817 C |
194 | }) |
195 | }, | |
c173e565 | 196 | |
80a6c9e7 | 197 | function removePodsFromDB (pods, callbackAsync) { |
65fcc311 | 198 | each(pods, function (pod: any, callbackEach) { |
feb4bdfd | 199 | pod.destroy().asCallback(callbackEach) |
aaf61f38 | 200 | }, callbackAsync) |
e7ea2817 C |
201 | } |
202 | ], function (err) { | |
203 | // Don't forget to re activate the scheduler, even if there was an error | |
c1a7ab7f | 204 | requestScheduler.activate() |
e7ea2817 C |
205 | |
206 | if (err) return callback(err) | |
207 | ||
208 | logger.info('Removed all remote videos.') | |
209 | return callback(null) | |
9f10b292 C |
210 | }) |
211 | } | |
c173e565 | 212 | |
528a9efa | 213 | function sendOwnedVideosToPod (podId) { |
7920c273 | 214 | db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { |
528a9efa C |
215 | if (err) { |
216 | logger.error('Cannot get the list of videos we own.') | |
217 | return | |
218 | } | |
219 | ||
220 | videosList.forEach(function (video) { | |
7b1f49de | 221 | video.toAddRemoteJSON(function (err, remoteVideo) { |
528a9efa C |
222 | if (err) { |
223 | logger.error('Cannot convert video to remote.', { error: err }) | |
224 | // Don't break the process | |
225 | return | |
226 | } | |
227 | ||
bd14d16a C |
228 | const options = { |
229 | type: 'add', | |
65fcc311 | 230 | endpoint: REQUEST_ENDPOINTS.VIDEOS, |
bd14d16a C |
231 | data: remoteVideo, |
232 | toIds: [ podId ] | |
233 | } | |
234 | createRequest(options) | |
528a9efa C |
235 | }) |
236 | }) | |
237 | }) | |
9f10b292 | 238 | } |
c173e565 | 239 | |
99fdec46 C |
240 | function getRequestScheduler () { |
241 | return requestScheduler | |
242 | } | |
243 | ||
244 | function getRequestVideoQaduScheduler () { | |
245 | return requestVideoQaduScheduler | |
246 | } | |
247 | ||
248 | function getRequestVideoEventScheduler () { | |
249 | return requestVideoEventScheduler | |
250 | } | |
251 | ||
9f10b292 | 252 | // --------------------------------------------------------------------------- |
c173e565 | 253 | |
65fcc311 C |
254 | export { |
255 | activateSchedulers, | |
256 | addVideoToFriends, | |
257 | updateVideoToFriends, | |
258 | reportAbuseVideoToFriend, | |
259 | quickAndDirtyUpdateVideoToFriends, | |
260 | quickAndDirtyUpdatesVideoToFriends, | |
261 | addEventToRemoteVideo, | |
262 | addEventsToRemoteVideo, | |
263 | hasFriends, | |
264 | makeFriends, | |
265 | quitFriends, | |
266 | removeVideoToFriends, | |
267 | sendOwnedVideosToPod, | |
268 | getRequestScheduler, | |
269 | getRequestVideoQaduScheduler, | |
270 | getRequestVideoEventScheduler | |
271 | } | |
c173e565 | 272 | |
9f10b292 | 273 | // --------------------------------------------------------------------------- |
c173e565 | 274 | |
49abbbbe | 275 | function computeForeignPodsList (host, podsScore, callback) { |
55fa55a9 | 276 | getForeignPodsList(host, function (err, res) { |
89d1d8ba | 277 | if (err) return callback(err) |
528a9efa | 278 | |
55fa55a9 | 279 | const foreignPodsList = res.data |
528a9efa C |
280 | |
281 | // Let's give 1 point to the pod we ask the friends list | |
49abbbbe | 282 | foreignPodsList.push({ host }) |
89d1d8ba | 283 | |
bc503c2a | 284 | foreignPodsList.forEach(function (foreignPod) { |
49abbbbe | 285 | const foreignPodHost = foreignPod.host |
89d1d8ba | 286 | |
49abbbbe C |
287 | if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++ |
288 | else podsScore[foreignPodHost] = 1 | |
89d1d8ba | 289 | }) |
cbe2f7c3 | 290 | |
1e4b0080 | 291 | return callback() |
89d1d8ba C |
292 | }) |
293 | } | |
294 | ||
49abbbbe | 295 | function computeWinningPods (hosts, podsScore) { |
89d1d8ba C |
296 | // Build the list of pods to add |
297 | // Only add a pod if it exists in more than a half base pods | |
bc503c2a | 298 | const podsList = [] |
49abbbbe | 299 | const baseScore = hosts.length / 2 |
1e4b0080 | 300 | |
49abbbbe | 301 | Object.keys(podsScore).forEach(function (podHost) { |
2c49ca42 | 302 | // If the pod is not me and with a good score we add it |
49abbbbe C |
303 | if (isMe(podHost) === false && podsScore[podHost] > baseScore) { |
304 | podsList.push({ host: podHost }) | |
2c49ca42 | 305 | } |
89d1d8ba | 306 | }) |
e7ea2817 | 307 | |
bc503c2a | 308 | return podsList |
89d1d8ba C |
309 | } |
310 | ||
49abbbbe | 311 | function getForeignPodsList (host, callback) { |
65fcc311 | 312 | const path = '/api/' + API_VERSION + '/pods' |
c173e565 | 313 | |
65fcc311 | 314 | request.get(REMOTE_SCHEME.HTTP + '://' + host + path, function (err, response, body) { |
9f10b292 | 315 | if (err) return callback(err) |
8425cb89 | 316 | |
39f87cb2 C |
317 | try { |
318 | const json = JSON.parse(body) | |
319 | return callback(null, json) | |
320 | } catch (err) { | |
321 | return callback(err) | |
322 | } | |
9f10b292 C |
323 | }) |
324 | } | |
89d1d8ba | 325 | |
bc503c2a | 326 | function makeRequestsToWinningPods (cert, podsList, callback) { |
89d1d8ba | 327 | // Stop pool requests |
c1a7ab7f | 328 | requestScheduler.deactivate() |
89d1d8ba | 329 | // Flush pool requests |
c1a7ab7f | 330 | requestScheduler.forceSend() |
89d1d8ba | 331 | |
65fcc311 | 332 | eachLimit(podsList, REQUESTS_IN_PARALLEL, function (pod: any, callbackEach) { |
528a9efa | 333 | const params = { |
65fcc311 | 334 | url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/pods/', |
528a9efa C |
335 | method: 'POST', |
336 | json: { | |
65fcc311 C |
337 | host: CONFIG.WEBSERVER.HOST, |
338 | email: CONFIG.ADMIN.EMAIL, | |
528a9efa C |
339 | publicKey: cert |
340 | } | |
89d1d8ba C |
341 | } |
342 | ||
65fcc311 | 343 | makeRetryRequest(params, function (err, res, body) { |
528a9efa | 344 | if (err) { |
49abbbbe | 345 | logger.error('Error with adding %s pod.', pod.host, { error: err }) |
528a9efa C |
346 | // Don't break the process |
347 | return callbackEach() | |
348 | } | |
89d1d8ba | 349 | |
528a9efa | 350 | if (res.statusCode === 200) { |
4793c343 | 351 | const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email }) |
feb4bdfd | 352 | podObj.save().asCallback(function (err, podCreated) { |
a3ee6fa2 | 353 | if (err) { |
49abbbbe | 354 | logger.error('Cannot add friend %s pod.', pod.host, { error: err }) |
a3ee6fa2 C |
355 | return callbackEach() |
356 | } | |
89d1d8ba | 357 | |
528a9efa | 358 | // Add our videos to the request scheduler |
7920c273 | 359 | sendOwnedVideosToPod(podCreated.id) |
89d1d8ba | 360 | |
528a9efa C |
361 | return callbackEach() |
362 | }) | |
363 | } else { | |
49abbbbe | 364 | logger.error('Status not 200 for %s pod.', pod.host) |
528a9efa | 365 | return callbackEach() |
89d1d8ba | 366 | } |
528a9efa C |
367 | }) |
368 | }, function endRequests () { | |
369 | // Final callback, we've ended all the requests | |
370 | // Now we made new friends, we can re activate the pool of requests | |
c1a7ab7f | 371 | requestScheduler.activate() |
528a9efa C |
372 | |
373 | logger.debug('makeRequestsToWinningPods finished.') | |
374 | return callback() | |
89d1d8ba C |
375 | }) |
376 | } | |
00057e85 | 377 | |
55fa55a9 | 378 | // Wrapper that populate "toIds" argument with all our friends if it is not specified |
ed04d94f | 379 | // { type, endpoint, data, toIds, transaction } |
65fcc311 C |
380 | function createRequest (options, callback?) { |
381 | if (!callback) callback = function () { /* empty */ } | |
c1a7ab7f | 382 | if (options.toIds) return requestScheduler.createRequest(options, callback) |
feb4bdfd | 383 | |
55fa55a9 | 384 | // If the "toIds" pods is not specified, we send the request to all our friends |
ed04d94f | 385 | db.Pod.listAllIds(options.transaction, function (err, podIds) { |
feb4bdfd C |
386 | if (err) { |
387 | logger.error('Cannot get pod ids', { error: err }) | |
388 | return | |
389 | } | |
390 | ||
ed04d94f | 391 | const newOptions = Object.assign(options, { toIds: podIds }) |
c1a7ab7f | 392 | return requestScheduler.createRequest(newOptions, callback) |
00057e85 C |
393 | }) |
394 | } | |
2c49ca42 | 395 | |
9e167724 | 396 | function createVideoQaduRequest (options, callback) { |
65fcc311 | 397 | if (!callback) callback = createEmptyCallback() |
9e167724 | 398 | |
99fdec46 | 399 | requestVideoQaduScheduler.createRequest(options, callback) |
9e167724 C |
400 | } |
401 | ||
e4c87ec2 | 402 | function createVideoEventRequest (options, callback) { |
65fcc311 | 403 | if (!callback) callback = createEmptyCallback() |
e4c87ec2 | 404 | |
99fdec46 | 405 | requestVideoEventScheduler.createRequest(options, callback) |
e4c87ec2 C |
406 | } |
407 | ||
49abbbbe | 408 | function isMe (host) { |
65fcc311 | 409 | return host === CONFIG.WEBSERVER.HOST |
2c49ca42 | 410 | } |