1 import * as request from 'request'
2 import * as Sequelize from 'sequelize'
3 import * as Promise from 'bluebird'
4 import { join } from 'path'
6 import { database as db } from '../initializers/database'
12 REQUEST_ENDPOINT_ACTIONS,
15 } from '../initializers'
24 RequestSchedulerOptions,
26 RequestVideoQaduScheduler,
27 RequestVideoQaduSchedulerOptions,
29 RequestVideoEventScheduler,
30 RequestVideoEventSchedulerOptions
38 RequestVideoEventType,
40 RemoteVideoCreateData,
41 RemoteVideoUpdateData,
42 RemoteVideoRemoveData,
43 RemoteVideoReportAbuseData,
48 type QaduParam = { videoId: number, type: RequestVideoQaduType }
49 type EventParam = { videoId: number, type: RequestVideoEventType }
51 const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
53 const requestScheduler = new RequestScheduler()
54 const requestVideoQaduScheduler = new RequestVideoQaduScheduler()
55 const requestVideoEventScheduler = new RequestVideoEventScheduler()
57 function activateSchedulers () {
58 requestScheduler.activate()
59 requestVideoQaduScheduler.activate()
60 requestVideoEventScheduler.activate()
63 function addVideoToFriends (videoData: RemoteVideoCreateData, transaction: Sequelize.Transaction) {
65 type: ENDPOINT_ACTIONS.ADD,
66 endpoint: REQUEST_ENDPOINTS.VIDEOS,
70 return createRequest(options)
73 function updateVideoToFriends (videoData: RemoteVideoUpdateData, transaction: Sequelize.Transaction) {
75 type: ENDPOINT_ACTIONS.UPDATE,
76 endpoint: REQUEST_ENDPOINTS.VIDEOS,
80 return createRequest(options)
83 function removeVideoToFriends (videoParams: RemoteVideoRemoveData, transaction: Sequelize.Transaction) {
85 type: ENDPOINT_ACTIONS.REMOVE,
86 endpoint: REQUEST_ENDPOINTS.VIDEOS,
90 return createRequest(options)
93 function reportAbuseVideoToFriend (reportData: RemoteVideoReportAbuseData, video: VideoInstance, transaction: Sequelize.Transaction) {
95 type: ENDPOINT_ACTIONS.REPORT_ABUSE,
96 endpoint: REQUEST_ENDPOINTS.VIDEOS,
98 toIds: [ video.Author.podId ],
101 return createRequest(options)
104 function quickAndDirtyUpdateVideoToFriends (qaduParam: QaduParam, transaction?: Sequelize.Transaction) {
106 videoId: qaduParam.videoId,
107 type: qaduParam.type,
110 return createVideoQaduRequest(options)
113 function quickAndDirtyUpdatesVideoToFriends (qadusParams: QaduParam[], transaction: Sequelize.Transaction) {
116 qadusParams.forEach(qaduParams => {
117 tasks.push(quickAndDirtyUpdateVideoToFriends(qaduParams, transaction))
120 return Promise.all(tasks)
123 function addEventToRemoteVideo (eventParam: EventParam, transaction?: Sequelize.Transaction) {
125 videoId: eventParam.videoId,
126 type: eventParam.type,
129 return createVideoEventRequest(options)
132 function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) {
135 eventsParams.forEach(eventParams => {
136 tasks.push(addEventToRemoteVideo(eventParams, transaction))
139 return Promise.all(tasks)
142 function hasFriends () {
143 return db.Pod.countAll().then(count => count !== 0)
146 function makeFriends (hosts: string[]) {
149 logger.info('Make friends!')
150 return getMyPublicCert()
152 return Promise.each(hosts, host => computeForeignPodsList(host, podsScore)).then(() => cert)
155 logger.debug('Pods scores computed.', { podsScore: podsScore })
156 const podsList = computeWinningPods(hosts, podsScore)
157 logger.debug('Pods that we keep.', { podsToKeep: podsList })
159 return makeRequestsToWinningPods(cert, podsList)
163 function quitFriends () {
164 // Stop pool requests
165 requestScheduler.deactivate()
167 return requestScheduler.flush()
169 return requestVideoQaduScheduler.flush()
175 const requestParams = {
176 method: 'POST' as 'POST',
177 path: '/api/' + API_VERSION + '/remote/pods/remove',
181 // Announce we quit them
182 // We don't care if the request fails
183 // The other pod will exclude us automatically after a while
184 return Promise.map(pods, pod => {
185 requestParams.toPod = pod
187 return makeSecureRequest(requestParams)
188 }, { concurrency: REQUESTS_IN_PARALLEL })
191 logger.error('Some errors while quitting friends.', err)
192 // Don't stop the process
198 pods.forEach(pod => tasks.push(pod.destroy()))
200 return Promise.all(pods)
203 logger.info('Removed all remote videos.')
204 // Don't forget to re activate the scheduler, even if there was an error
205 return requestScheduler.activate()
207 .finally(() => requestScheduler.activate())
210 function sendOwnedVideosToPod (podId: number) {
211 db.Video.listOwnedAndPopulateAuthorAndTags()
212 .then(videosList => {
214 videosList.forEach(video => {
215 const promise = video.toAddRemoteJSON()
216 .then(remoteVideo => {
219 endpoint: REQUEST_ENDPOINTS.VIDEOS,
224 return createRequest(options)
227 logger.error('Cannot convert video to remote.', err)
228 // Don't break the process
235 return Promise.all(tasks)
239 function fetchRemotePreview (pod: PodInstance, video: VideoInstance) {
240 const host = video.Author.Pod.host
241 const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
243 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
246 function removeFriend (pod: PodInstance) {
247 const requestParams = {
248 method: 'POST' as 'POST',
249 path: '/api/' + API_VERSION + '/remote/pods/remove',
253 return makeSecureRequest(requestParams)
254 .catch(err => logger.warn('Cannot notify friends %s we are quitting him.', pod.host, err))
255 .then(() => pod.destroy())
256 .then(() => logger.info('Removed friend %s.', pod.host))
257 .catch(err => logger.error('Cannot destroy friend %s.', pod.host, err))
260 function getRequestScheduler () {
261 return requestScheduler
264 function getRequestVideoQaduScheduler () {
265 return requestVideoQaduScheduler
268 function getRequestVideoEventScheduler () {
269 return requestVideoEventScheduler
272 // ---------------------------------------------------------------------------
277 updateVideoToFriends,
278 reportAbuseVideoToFriend,
279 quickAndDirtyUpdateVideoToFriends,
280 quickAndDirtyUpdatesVideoToFriends,
281 addEventToRemoteVideo,
282 addEventsToRemoteVideo,
287 removeVideoToFriends,
288 sendOwnedVideosToPod,
290 getRequestVideoQaduScheduler,
291 getRequestVideoEventScheduler,
295 // ---------------------------------------------------------------------------
297 function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) {
299 return getForeignPodsList(host).then(res => {
300 const foreignPodsList: { host: string }[] = res.data
302 // Let's give 1 point to the pod we ask the friends list
303 foreignPodsList.push({ host })
305 foreignPodsList.forEach(foreignPod => {
306 const foreignPodHost = foreignPod.host
308 if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++
309 else podsScore[foreignPodHost] = 1
316 function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) {
317 // Build the list of pods to add
318 // Only add a pod if it exists in more than a half base pods
320 const baseScore = hosts.length / 2
322 Object.keys(podsScore).forEach(podHost => {
323 // If the pod is not me and with a good score we add it
324 if (isMe(podHost) === false && podsScore[podHost] > baseScore) {
325 podsList.push({ host: podHost })
332 function getForeignPodsList (host: string) {
333 return new Promise< ResultList<FormattedPod> >((res, rej) => {
334 const path = '/api/' + API_VERSION + '/remote/pods/list'
336 request.post(REMOTE_SCHEME.HTTP + '://' + host + path, (err, response, body) => {
337 if (err) return rej(err)
340 const json = JSON.parse(body)
349 function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) {
350 // Stop pool requests
351 requestScheduler.deactivate()
352 // Flush pool requests
353 requestScheduler.forceSend()
355 return Promise.map(podsList, pod => {
357 url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/remote/pods/add',
358 method: 'POST' as 'POST',
360 host: CONFIG.WEBSERVER.HOST,
361 email: CONFIG.ADMIN.EMAIL,
366 return makeRetryRequest(params)
367 .then(({ response, body }) => {
368 body = body as { cert: string, email: string }
370 if (response.statusCode === 200) {
371 const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email })
373 .then(podCreated => {
375 // Add our videos to the request scheduler
376 sendOwnedVideosToPod(podCreated.id)
379 logger.error('Cannot add friend %s pod.', pod.host, err)
382 logger.error('Status not 200 for %s pod.', pod.host)
386 logger.error('Error with adding %s pod.', pod.host, { error: err.stack })
387 // Don't break the process
389 }, { concurrency: REQUESTS_IN_PARALLEL })
390 .then(() => logger.debug('makeRequestsToWinningPods finished.'))
392 // Final callback, we've ended all the requests
393 // Now we made new friends, we can re activate the pool of requests
394 requestScheduler.activate()
398 // Wrapper that populate "toIds" argument with all our friends if it is not specified
399 type CreateRequestOptions = {
401 endpoint: RequestEndpoint
404 transaction: Sequelize.Transaction
406 function createRequest (options: CreateRequestOptions) {
407 if (options.toIds !== undefined) return requestScheduler.createRequest(options as RequestSchedulerOptions)
409 // If the "toIds" pods is not specified, we send the request to all our friends
410 return db.Pod.listAllIds(options.transaction).then(podIds => {
411 const newOptions = Object.assign(options, { toIds: podIds })
412 return requestScheduler.createRequest(newOptions)
416 function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) {
417 return requestVideoQaduScheduler.createRequest(options)
420 function createVideoEventRequest (options: RequestVideoEventSchedulerOptions) {
421 return requestVideoEventScheduler.createRequest(options)
424 function isMe (host: string) {
425 return host === CONFIG.WEBSERVER.HOST