1 import * as request from 'request'
2 import * as Sequelize from 'sequelize'
3 import * as Bluebird from 'bluebird'
4 import { join } from 'path'
6 import { database as db } from '../initializers/database'
12 REQUEST_ENDPOINT_ACTIONS,
15 } from '../initializers'
24 RequestSchedulerOptions,
26 RequestVideoQaduScheduler,
27 RequestVideoQaduSchedulerOptions,
29 RequestVideoEventScheduler,
30 RequestVideoEventSchedulerOptions
38 RequestVideoEventType,
40 RemoteVideoCreateData,
41 RemoteVideoUpdateData,
42 RemoteVideoRemoveData,
43 RemoteVideoReportAbuseData,
45 RemoteVideoRequestType,
47 RemoteVideoChannelCreateData,
48 RemoteVideoChannelUpdateData,
49 RemoteVideoChannelRemoveData,
50 RemoteVideoAuthorCreateData,
51 RemoteVideoAuthorRemoveData
54 type QaduParam = { videoId: number, type: RequestVideoQaduType }
55 type EventParam = { videoId: number, type: RequestVideoEventType }
57 const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
59 const requestScheduler = new RequestScheduler()
60 const requestVideoQaduScheduler = new RequestVideoQaduScheduler()
61 const requestVideoEventScheduler = new RequestVideoEventScheduler()
63 function activateSchedulers () {
64 requestScheduler.activate()
65 requestVideoQaduScheduler.activate()
66 requestVideoEventScheduler.activate()
69 function addVideoToFriends (videoData: RemoteVideoCreateData, transaction: Sequelize.Transaction) {
71 type: ENDPOINT_ACTIONS.ADD_VIDEO,
72 endpoint: REQUEST_ENDPOINTS.VIDEOS,
76 return createRequest(options)
79 function updateVideoToFriends (videoData: RemoteVideoUpdateData, transaction: Sequelize.Transaction) {
81 type: ENDPOINT_ACTIONS.UPDATE_VIDEO,
82 endpoint: REQUEST_ENDPOINTS.VIDEOS,
86 return createRequest(options)
89 function removeVideoToFriends (videoParams: RemoteVideoRemoveData, transaction?: Sequelize.Transaction) {
91 type: ENDPOINT_ACTIONS.REMOVE_VIDEO,
92 endpoint: REQUEST_ENDPOINTS.VIDEOS,
96 return createRequest(options)
99 function addVideoAuthorToFriends (authorData: RemoteVideoAuthorCreateData, transaction: Sequelize.Transaction) {
101 type: ENDPOINT_ACTIONS.ADD_AUTHOR,
102 endpoint: REQUEST_ENDPOINTS.VIDEOS,
106 return createRequest(options)
109 function removeVideoAuthorToFriends (authorData: RemoteVideoAuthorRemoveData, transaction?: Sequelize.Transaction) {
111 type: ENDPOINT_ACTIONS.REMOVE_AUTHOR,
112 endpoint: REQUEST_ENDPOINTS.VIDEOS,
116 return createRequest(options)
119 function addVideoChannelToFriends (videoChannelData: RemoteVideoChannelCreateData, transaction: Sequelize.Transaction) {
121 type: ENDPOINT_ACTIONS.ADD_CHANNEL,
122 endpoint: REQUEST_ENDPOINTS.VIDEOS,
123 data: videoChannelData,
126 return createRequest(options)
129 function updateVideoChannelToFriends (videoChannelData: RemoteVideoChannelUpdateData, transaction: Sequelize.Transaction) {
131 type: ENDPOINT_ACTIONS.UPDATE_CHANNEL,
132 endpoint: REQUEST_ENDPOINTS.VIDEOS,
133 data: videoChannelData,
136 return createRequest(options)
139 function removeVideoChannelToFriends (videoChannelParams: RemoteVideoChannelRemoveData, transaction?: Sequelize.Transaction) {
141 type: ENDPOINT_ACTIONS.REMOVE_CHANNEL,
142 endpoint: REQUEST_ENDPOINTS.VIDEOS,
143 data: videoChannelParams,
146 return createRequest(options)
149 function reportAbuseVideoToFriend (reportData: RemoteVideoReportAbuseData, video: VideoInstance, transaction: Sequelize.Transaction) {
151 type: ENDPOINT_ACTIONS.REPORT_ABUSE,
152 endpoint: REQUEST_ENDPOINTS.VIDEOS,
154 toIds: [ video.VideoChannel.Author.podId ],
157 return createRequest(options)
160 function quickAndDirtyUpdateVideoToFriends (qaduParam: QaduParam, transaction?: Sequelize.Transaction) {
162 videoId: qaduParam.videoId,
163 type: qaduParam.type,
166 return createVideoQaduRequest(options)
169 function quickAndDirtyUpdatesVideoToFriends (qadusParams: QaduParam[], transaction: Sequelize.Transaction) {
172 qadusParams.forEach(qaduParams => {
173 tasks.push(quickAndDirtyUpdateVideoToFriends(qaduParams, transaction))
176 return Promise.all(tasks)
179 function addEventToRemoteVideo (eventParam: EventParam, transaction?: Sequelize.Transaction) {
181 videoId: eventParam.videoId,
182 type: eventParam.type,
185 return createVideoEventRequest(options)
188 function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) {
191 for (const eventParams of eventsParams) {
192 tasks.push(addEventToRemoteVideo(eventParams, transaction))
195 return Promise.all(tasks)
198 async function hasFriends () {
199 const count = await db.Pod.countAll()
204 async function makeFriends (hosts: string[]) {
207 logger.info('Make friends!')
208 const cert = await getMyPublicCert()
210 for (const host of hosts) {
211 await computeForeignPodsList(host, podsScore)
214 logger.debug('Pods scores computed.', { podsScore: podsScore })
216 const podsList = computeWinningPods(hosts, podsScore)
217 logger.debug('Pods that we keep.', { podsToKeep: podsList })
219 return makeRequestsToWinningPods(cert, podsList)
222 async function quitFriends () {
223 // Stop pool requests
224 requestScheduler.deactivate()
227 await requestScheduler.flush()
229 await requestVideoQaduScheduler.flush()
231 const pods = await db.Pod.list()
232 const requestParams = {
233 method: 'POST' as 'POST',
234 path: '/api/' + API_VERSION + '/remote/pods/remove',
238 // Announce we quit them
239 // We don't care if the request fails
240 // The other pod will exclude us automatically after a while
242 await Bluebird.map(pods, pod => {
243 requestParams.toPod = pod
245 return makeSecureRequest(requestParams)
246 }, { concurrency: REQUESTS_IN_PARALLEL })
247 } catch (err) { // Don't stop the process
248 logger.error('Some errors while quitting friends.', err)
252 for (const pod of pods) {
253 tasks.push(pod.destroy())
255 await Promise.all(pods)
257 logger.info('Removed all remote videos.')
259 requestScheduler.activate()
261 // Don't forget to re activate the scheduler, even if there was an error
262 requestScheduler.activate()
268 async function sendOwnedDataToPod (podId: number) {
269 // First send authors
270 await sendOwnedAuthorsToPod(podId)
271 await sendOwnedChannelsToPod(podId)
272 await sendOwnedVideosToPod(podId)
275 async function sendOwnedChannelsToPod (podId: number) {
276 const videoChannels = await db.VideoChannel.listOwned()
278 const tasks: Promise<any>[] = []
279 for (const videoChannel of videoChannels) {
280 const remoteVideoChannel = videoChannel.toAddRemoteJSON()
282 type: 'add-channel' as 'add-channel',
283 endpoint: REQUEST_ENDPOINTS.VIDEOS,
284 data: remoteVideoChannel,
289 const p = createRequest(options)
293 await Promise.all(tasks)
296 async function sendOwnedAuthorsToPod (podId: number) {
297 const authors = await db.Author.listOwned()
298 const tasks: Promise<any>[] = []
300 for (const author of authors) {
301 const remoteAuthor = author.toAddRemoteJSON()
303 type: 'add-author' as 'add-author',
304 endpoint: REQUEST_ENDPOINTS.VIDEOS,
310 const p = createRequest(options)
314 await Promise.all(tasks)
317 async function sendOwnedVideosToPod (podId: number) {
318 const videosList = await db.Video.listOwnedAndPopulateAuthorAndTags()
319 const tasks: Bluebird<any>[] = []
321 for (const video of videosList) {
322 const promise = video.toAddRemoteJSON()
323 .then(remoteVideo => {
325 type: 'add-video' as 'add-video',
326 endpoint: REQUEST_ENDPOINTS.VIDEOS,
331 return createRequest(options)
334 logger.error('Cannot convert video to remote.', err)
335 // Don't break the process
342 await Promise.all(tasks)
345 function fetchRemotePreview (video: VideoInstance) {
346 const host = video.VideoChannel.Author.Pod.host
347 const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
349 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
352 async function removeFriend (pod: PodInstance) {
353 const requestParams = {
354 method: 'POST' as 'POST',
355 path: '/api/' + API_VERSION + '/remote/pods/remove',
360 await makeSecureRequest(requestParams)
362 logger.warn('Cannot notify friends %s we are quitting him.', pod.host, err)
368 logger.info('Removed friend %s.', pod.host)
370 logger.error('Cannot destroy friend %s.', pod.host, err)
374 function getRequestScheduler () {
375 return requestScheduler
378 function getRequestVideoQaduScheduler () {
379 return requestVideoQaduScheduler
382 function getRequestVideoEventScheduler () {
383 return requestVideoEventScheduler
386 // ---------------------------------------------------------------------------
391 removeVideoAuthorToFriends,
392 updateVideoToFriends,
393 addVideoAuthorToFriends,
394 reportAbuseVideoToFriend,
395 quickAndDirtyUpdateVideoToFriends,
396 quickAndDirtyUpdatesVideoToFriends,
397 addEventToRemoteVideo,
398 addEventsToRemoteVideo,
403 removeVideoToFriends,
406 getRequestVideoQaduScheduler,
407 getRequestVideoEventScheduler,
409 addVideoChannelToFriends,
410 updateVideoChannelToFriends,
411 removeVideoChannelToFriends
414 // ---------------------------------------------------------------------------
416 async function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) {
417 const result = await getForeignPodsList(host)
418 const foreignPodsList: { host: string }[] = result.data
420 // Let's give 1 point to the pod we ask the friends list
421 foreignPodsList.push({ host })
423 for (const foreignPod of foreignPodsList) {
424 const foreignPodHost = foreignPod.host
426 if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++
427 else podsScore[foreignPodHost] = 1
433 function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) {
434 // Build the list of pods to add
435 // Only add a pod if it exists in more than a half base pods
437 const baseScore = hosts.length / 2
439 for (const podHost of Object.keys(podsScore)) {
440 // If the pod is not me and with a good score we add it
441 if (isMe(podHost) === false && podsScore[podHost] > baseScore) {
442 podsList.push({ host: podHost })
449 function getForeignPodsList (host: string) {
450 return new Promise< ResultList<FormattedPod> >((res, rej) => {
451 const path = '/api/' + API_VERSION + '/remote/pods/list'
453 request.post(REMOTE_SCHEME.HTTP + '://' + host + path, (err, response, body) => {
454 if (err) return rej(err)
457 const json: ResultList<FormattedPod> = JSON.parse(body)
466 async function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) {
467 // Stop pool requests
468 requestScheduler.deactivate()
469 // Flush pool requests
470 requestScheduler.forceSend()
473 await Bluebird.map(podsList, async pod => {
475 url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/remote/pods/add',
476 method: 'POST' as 'POST',
478 host: CONFIG.WEBSERVER.HOST,
479 email: CONFIG.ADMIN.EMAIL,
484 const { response, body } = await makeRetryRequest(params)
485 const typedBody = body as { cert: string, email: string }
487 if (response.statusCode === 200) {
488 const podObj = db.Pod.build({ host: pod.host, publicKey: typedBody.cert, email: typedBody.email })
490 let podCreated: PodInstance
492 podCreated = await podObj.save()
494 logger.error('Cannot add friend %s pod.', pod.host, err)
497 // Add our videos to the request scheduler
498 sendOwnedDataToPod(podCreated.id)
499 .catch(err => logger.warn('Cannot send owned data to pod %d.', podCreated.id, err))
501 logger.error('Status not 200 for %s pod.', pod.host)
503 }, { concurrency: REQUESTS_IN_PARALLEL })
505 logger.debug('makeRequestsToWinningPods finished.')
507 requestScheduler.activate()
509 // Final callback, we've ended all the requests
510 // Now we made new friends, we can re activate the pool of requests
511 requestScheduler.activate()
515 // Wrapper that populate "toIds" argument with all our friends if it is not specified
516 type CreateRequestOptions = {
517 type: RemoteVideoRequestType
518 endpoint: RequestEndpoint
521 transaction: Sequelize.Transaction
523 async function createRequest (options: CreateRequestOptions) {
524 if (options.toIds !== undefined) {
525 await requestScheduler.createRequest(options as RequestSchedulerOptions)
529 // If the "toIds" pods is not specified, we send the request to all our friends
530 const podIds = await db.Pod.listAllIds(options.transaction)
532 const newOptions = Object.assign(options, { toIds: podIds })
533 await requestScheduler.createRequest(newOptions)
538 function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) {
539 return requestVideoQaduScheduler.createRequest(options)
542 function createVideoEventRequest (options: RequestVideoEventSchedulerOptions) {
543 return requestVideoEventScheduler.createRequest(options)
546 function isMe (host: string) {
547 return host === CONFIG.WEBSERVER.HOST