1 import * as request from 'request'
2 import * as Sequelize from 'sequelize'
3 import * as Promise from 'bluebird'
5 import { database as db } from '../initializers/database'
11 REQUEST_ENDPOINT_ACTIONS,
13 } from '../initializers'
22 RequestSchedulerOptions,
24 RequestVideoQaduScheduler,
25 RequestVideoQaduSchedulerOptions,
27 RequestVideoEventScheduler,
28 RequestVideoEventSchedulerOptions
36 RequestVideoEventType,
38 RemoteVideoCreateData,
39 RemoteVideoUpdateData,
40 RemoteVideoRemoveData,
41 RemoteVideoReportAbuseData,
46 type QaduParam = { videoId: number, type: RequestVideoQaduType }
47 type EventParam = { videoId: number, type: RequestVideoEventType }
49 const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
51 const requestScheduler = new RequestScheduler()
52 const requestVideoQaduScheduler = new RequestVideoQaduScheduler()
53 const requestVideoEventScheduler = new RequestVideoEventScheduler()
55 function activateSchedulers () {
56 requestScheduler.activate()
57 requestVideoQaduScheduler.activate()
58 requestVideoEventScheduler.activate()
61 function addVideoToFriends (videoData: RemoteVideoCreateData, transaction: Sequelize.Transaction) {
63 type: ENDPOINT_ACTIONS.ADD,
64 endpoint: REQUEST_ENDPOINTS.VIDEOS,
68 return createRequest(options)
71 function updateVideoToFriends (videoData: RemoteVideoUpdateData, transaction: Sequelize.Transaction) {
73 type: ENDPOINT_ACTIONS.UPDATE,
74 endpoint: REQUEST_ENDPOINTS.VIDEOS,
78 return createRequest(options)
81 function removeVideoToFriends (videoParams: RemoteVideoRemoveData) {
83 type: ENDPOINT_ACTIONS.REMOVE,
84 endpoint: REQUEST_ENDPOINTS.VIDEOS,
88 return createRequest(options)
91 function reportAbuseVideoToFriend (reportData: RemoteVideoReportAbuseData, video: VideoInstance, transaction: Sequelize.Transaction) {
93 type: ENDPOINT_ACTIONS.REPORT_ABUSE,
94 endpoint: REQUEST_ENDPOINTS.VIDEOS,
96 toIds: [ video.Author.podId ],
99 return createRequest(options)
102 function quickAndDirtyUpdateVideoToFriends (qaduParam: QaduParam, transaction?: Sequelize.Transaction) {
104 videoId: qaduParam.videoId,
105 type: qaduParam.type,
108 return createVideoQaduRequest(options)
111 function quickAndDirtyUpdatesVideoToFriends (qadusParams: QaduParam[], transaction: Sequelize.Transaction) {
114 qadusParams.forEach(qaduParams => {
115 tasks.push(quickAndDirtyUpdateVideoToFriends(qaduParams, transaction))
118 return Promise.all(tasks)
121 function addEventToRemoteVideo (eventParam: EventParam, transaction?: Sequelize.Transaction) {
123 videoId: eventParam.videoId,
124 type: eventParam.type,
127 return createVideoEventRequest(options)
130 function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) {
133 eventsParams.forEach(eventParams => {
134 tasks.push(addEventToRemoteVideo(eventParams, transaction))
137 return Promise.all(tasks)
140 function hasFriends () {
141 return db.Pod.countAll().then(count => count !== 0)
144 function makeFriends (hosts: string[]) {
147 logger.info('Make friends!')
148 return getMyPublicCert()
150 return Promise.each(hosts, host => computeForeignPodsList(host, podsScore)).then(() => cert)
153 logger.debug('Pods scores computed.', { podsScore: podsScore })
154 const podsList = computeWinningPods(hosts, podsScore)
155 logger.debug('Pods that we keep.', { podsToKeep: podsList })
157 return makeRequestsToWinningPods(cert, podsList)
161 function quitFriends () {
162 // Stop pool requests
163 requestScheduler.deactivate()
165 return requestScheduler.flush()
167 return requestVideoQaduScheduler.flush()
173 const requestParams = {
174 method: 'POST' as 'POST',
175 path: '/api/' + API_VERSION + '/remote/pods/remove',
179 // Announce we quit them
180 // We don't care if the request fails
181 // The other pod will exclude us automatically after a while
182 return Promise.map(pods, pod => {
183 requestParams.toPod = pod
185 return makeSecureRequest(requestParams)
186 }, { concurrency: REQUESTS_IN_PARALLEL })
189 logger.error('Some errors while quitting friends.', err)
190 // Don't stop the process
195 pods.forEach(pod => tasks.push(pod.destroy()))
197 return Promise.all(pods)
200 logger.info('Removed all remote videos.')
201 // Don't forget to re activate the scheduler, even if there was an error
202 return requestScheduler.activate()
204 .finally(() => requestScheduler.activate())
207 function sendOwnedVideosToPod (podId: number) {
208 db.Video.listOwnedAndPopulateAuthorAndTags()
209 .then(videosList => {
211 videosList.forEach(video => {
212 const promise = video.toAddRemoteJSON()
213 .then(remoteVideo => {
216 endpoint: REQUEST_ENDPOINTS.VIDEOS,
221 return createRequest(options)
224 logger.error('Cannot convert video to remote.', err)
225 // Don't break the process
232 return Promise.all(tasks)
236 function getRequestScheduler () {
237 return requestScheduler
240 function getRequestVideoQaduScheduler () {
241 return requestVideoQaduScheduler
244 function getRequestVideoEventScheduler () {
245 return requestVideoEventScheduler
248 // ---------------------------------------------------------------------------
253 updateVideoToFriends,
254 reportAbuseVideoToFriend,
255 quickAndDirtyUpdateVideoToFriends,
256 quickAndDirtyUpdatesVideoToFriends,
257 addEventToRemoteVideo,
258 addEventsToRemoteVideo,
262 removeVideoToFriends,
263 sendOwnedVideosToPod,
265 getRequestVideoQaduScheduler,
266 getRequestVideoEventScheduler
269 // ---------------------------------------------------------------------------
271 function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) {
273 return getForeignPodsList(host).then(res => {
274 const foreignPodsList: { host: string }[] = res.data
276 // Let's give 1 point to the pod we ask the friends list
277 foreignPodsList.push({ host })
279 foreignPodsList.forEach(foreignPod => {
280 const foreignPodHost = foreignPod.host
282 if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++
283 else podsScore[foreignPodHost] = 1
290 function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) {
291 // Build the list of pods to add
292 // Only add a pod if it exists in more than a half base pods
294 const baseScore = hosts.length / 2
296 Object.keys(podsScore).forEach(podHost => {
297 // If the pod is not me and with a good score we add it
298 if (isMe(podHost) === false && podsScore[podHost] > baseScore) {
299 podsList.push({ host: podHost })
306 function getForeignPodsList (host: string) {
307 return new Promise< ResultList<FormatedPod> >((res, rej) => {
308 const path = '/api/' + API_VERSION + '/pods'
310 request.get(REMOTE_SCHEME.HTTP + '://' + host + path, (err, response, body) => {
311 if (err) return rej(err)
314 const json = JSON.parse(body)
323 function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) {
324 // Stop pool requests
325 requestScheduler.deactivate()
326 // Flush pool requests
327 requestScheduler.forceSend()
329 return Promise.map(podsList, pod => {
331 url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/pods/',
332 method: 'POST' as 'POST',
334 host: CONFIG.WEBSERVER.HOST,
335 email: CONFIG.ADMIN.EMAIL,
340 return makeRetryRequest(params)
341 .then(({ response, body }) => {
342 body = body as { cert: string, email: string }
344 if (response.statusCode === 200) {
345 const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email })
347 .then(podCreated => {
349 // Add our videos to the request scheduler
350 sendOwnedVideosToPod(podCreated.id)
353 logger.error('Cannot add friend %s pod.', pod.host, err)
356 logger.error('Status not 200 for %s pod.', pod.host)
360 logger.error('Error with adding %s pod.', pod.host, { error: err.stack })
361 // Don't break the process
363 }, { concurrency: REQUESTS_IN_PARALLEL })
364 .then(() => logger.debug('makeRequestsToWinningPods finished.'))
366 // Final callback, we've ended all the requests
367 // Now we made new friends, we can re activate the pool of requests
368 requestScheduler.activate()
372 // Wrapper that populate "toIds" argument with all our friends if it is not specified
373 type CreateRequestOptions = {
375 endpoint: RequestEndpoint
378 transaction: Sequelize.Transaction
380 function createRequest (options: CreateRequestOptions) {
381 if (options.toIds !== undefined) return requestScheduler.createRequest(options as RequestSchedulerOptions)
383 // If the "toIds" pods is not specified, we send the request to all our friends
384 return db.Pod.listAllIds(options.transaction).then(podIds => {
385 const newOptions = Object.assign(options, { toIds: podIds })
386 return requestScheduler.createRequest(newOptions)
390 function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) {
391 return requestVideoQaduScheduler.createRequest(options)
394 function createVideoEventRequest (options: RequestVideoEventSchedulerOptions) {
395 return requestVideoEventScheduler.createRequest(options)
398 function isMe (host: string) {
399 return host === CONFIG.WEBSERVER.HOST