aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/cache/videos-preview-cache.ts2
-rw-r--r--server/lib/friends.ts567
-rw-r--r--server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts4
-rw-r--r--server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts4
-rw-r--r--server/lib/video-channel.ts2
5 files changed, 6 insertions, 573 deletions
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts
index 0fe4d2f78..791ad1cbf 100644
--- a/server/lib/cache/videos-preview-cache.ts
+++ b/server/lib/cache/videos-preview-cache.ts
@@ -43,7 +43,7 @@ class VideosPreviewCache {
43 } 43 }
44 44
45 private async loadPreviews (key: string) { 45 private async loadPreviews (key: string) {
46 const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key) 46 const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(key)
47 if (!video) return undefined 47 if (!video) return undefined
48 48
49 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) 49 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
diff --git a/server/lib/friends.ts b/server/lib/friends.ts
deleted file mode 100644
index 5c9baef47..000000000
--- a/server/lib/friends.ts
+++ /dev/null
@@ -1,567 +0,0 @@
1import * as request from 'request'
2import * as Sequelize from 'sequelize'
3import * as Bluebird from 'bluebird'
4import { join } from 'path'
5
6import { database as db } from '../initializers/database'
7import {
8 API_VERSION,
9 CONFIG,
10 REQUESTS_IN_PARALLEL,
11 REQUEST_ENDPOINTS,
12 REQUEST_ENDPOINT_ACTIONS,
13 REMOTE_SCHEME,
14 STATIC_PATHS
15} from '../initializers'
16import {
17 logger,
18 getMyPublicCert,
19 makeSecureRequest,
20 makeRetryRequest
21} from '../helpers'
22import {
23 RequestScheduler,
24 RequestSchedulerOptions,
25
26 RequestVideoQaduScheduler,
27 RequestVideoQaduSchedulerOptions,
28
29 RequestVideoEventScheduler,
30 RequestVideoEventSchedulerOptions
31} from './request'
32import {
33 PodInstance,
34 VideoInstance
35} from '../models'
36import {
37 RequestEndpoint,
38 RequestVideoEventType,
39 RequestVideoQaduType,
40 RemoteVideoCreateData,
41 RemoteVideoUpdateData,
42 RemoteVideoRemoveData,
43 RemoteVideoReportAbuseData,
44 ResultList,
45 RemoteVideoRequestType,
46 Pod as FormattedPod,
47 RemoteVideoChannelCreateData,
48 RemoteVideoChannelUpdateData,
49 RemoteVideoChannelRemoveData,
50 RemoteVideoAuthorCreateData,
51 RemoteVideoAuthorRemoveData
52} from '../../shared'
53
54type QaduParam = { videoId: number, type: RequestVideoQaduType }
55type EventParam = { videoId: number, type: RequestVideoEventType }
56
57const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
58
59const requestScheduler = new RequestScheduler()
60const requestVideoQaduScheduler = new RequestVideoQaduScheduler()
61const requestVideoEventScheduler = new RequestVideoEventScheduler()
62
63function activateSchedulers () {
64 requestScheduler.activate()
65 requestVideoQaduScheduler.activate()
66 requestVideoEventScheduler.activate()
67}
68
69function addVideoToFriends (videoData: RemoteVideoCreateData, transaction: Sequelize.Transaction) {
70 const options = {
71 type: ENDPOINT_ACTIONS.ADD_VIDEO,
72 endpoint: REQUEST_ENDPOINTS.VIDEOS,
73 data: videoData,
74 transaction
75 }
76 return createRequest(options)
77}
78
79function updateVideoToFriends (videoData: RemoteVideoUpdateData, transaction: Sequelize.Transaction) {
80 const options = {
81 type: ENDPOINT_ACTIONS.UPDATE_VIDEO,
82 endpoint: REQUEST_ENDPOINTS.VIDEOS,
83 data: videoData,
84 transaction
85 }
86 return createRequest(options)
87}
88
89function removeVideoToFriends (videoParams: RemoteVideoRemoveData, transaction?: Sequelize.Transaction) {
90 const options = {
91 type: ENDPOINT_ACTIONS.REMOVE_VIDEO,
92 endpoint: REQUEST_ENDPOINTS.VIDEOS,
93 data: videoParams,
94 transaction
95 }
96 return createRequest(options)
97}
98
99function addVideoAuthorToFriends (authorData: RemoteVideoAuthorCreateData, transaction: Sequelize.Transaction) {
100 const options = {
101 type: ENDPOINT_ACTIONS.ADD_AUTHOR,
102 endpoint: REQUEST_ENDPOINTS.VIDEOS,
103 data: authorData,
104 transaction
105 }
106 return createRequest(options)
107}
108
109function removeVideoAuthorToFriends (authorData: RemoteVideoAuthorRemoveData, transaction?: Sequelize.Transaction) {
110 const options = {
111 type: ENDPOINT_ACTIONS.REMOVE_AUTHOR,
112 endpoint: REQUEST_ENDPOINTS.VIDEOS,
113 data: authorData,
114 transaction
115 }
116 return createRequest(options)
117}
118
119function addVideoChannelToFriends (videoChannelData: RemoteVideoChannelCreateData, transaction: Sequelize.Transaction) {
120 const options = {
121 type: ENDPOINT_ACTIONS.ADD_CHANNEL,
122 endpoint: REQUEST_ENDPOINTS.VIDEOS,
123 data: videoChannelData,
124 transaction
125 }
126 return createRequest(options)
127}
128
129function updateVideoChannelToFriends (videoChannelData: RemoteVideoChannelUpdateData, transaction: Sequelize.Transaction) {
130 const options = {
131 type: ENDPOINT_ACTIONS.UPDATE_CHANNEL,
132 endpoint: REQUEST_ENDPOINTS.VIDEOS,
133 data: videoChannelData,
134 transaction
135 }
136 return createRequest(options)
137}
138
139function removeVideoChannelToFriends (videoChannelParams: RemoteVideoChannelRemoveData, transaction?: Sequelize.Transaction) {
140 const options = {
141 type: ENDPOINT_ACTIONS.REMOVE_CHANNEL,
142 endpoint: REQUEST_ENDPOINTS.VIDEOS,
143 data: videoChannelParams,
144 transaction
145 }
146 return createRequest(options)
147}
148
149function reportAbuseVideoToFriend (reportData: RemoteVideoReportAbuseData, video: VideoInstance, transaction: Sequelize.Transaction) {
150 const options = {
151 type: ENDPOINT_ACTIONS.REPORT_ABUSE,
152 endpoint: REQUEST_ENDPOINTS.VIDEOS,
153 data: reportData,
154 toIds: [ video.VideoChannel.Author.podId ],
155 transaction
156 }
157 return createRequest(options)
158}
159
160function quickAndDirtyUpdateVideoToFriends (qaduParam: QaduParam, transaction?: Sequelize.Transaction) {
161 const options = {
162 videoId: qaduParam.videoId,
163 type: qaduParam.type,
164 transaction
165 }
166 return createVideoQaduRequest(options)
167}
168
169function quickAndDirtyUpdatesVideoToFriends (qadusParams: QaduParam[], transaction: Sequelize.Transaction) {
170 const tasks = []
171
172 qadusParams.forEach(qaduParams => {
173 tasks.push(quickAndDirtyUpdateVideoToFriends(qaduParams, transaction))
174 })
175
176 return Promise.all(tasks)
177}
178
179function addEventToRemoteVideo (eventParam: EventParam, transaction?: Sequelize.Transaction) {
180 const options = {
181 videoId: eventParam.videoId,
182 type: eventParam.type,
183 transaction
184 }
185 return createVideoEventRequest(options)
186}
187
188function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) {
189 const tasks = []
190
191 for (const eventParams of eventsParams) {
192 tasks.push(addEventToRemoteVideo(eventParams, transaction))
193 }
194
195 return Promise.all(tasks)
196}
197
198async function hasFriends () {
199 const count = await db.Pod.countAll()
200
201 return count !== 0
202}
203
204async function makeFriends (hosts: string[]) {
205 const podsScore = {}
206
207 logger.info('Make friends!')
208 const cert = await getMyPublicCert()
209
210 for (const host of hosts) {
211 await computeForeignPodsList(host, podsScore)
212 }
213
214 logger.debug('Pods scores computed.', { podsScore: podsScore })
215
216 const podsList = computeWinningPods(hosts, podsScore)
217 logger.debug('Pods that we keep.', { podsToKeep: podsList })
218
219 return makeRequestsToWinningPods(cert, podsList)
220}
221
222async function quitFriends () {
223 // Stop pool requests
224 requestScheduler.deactivate()
225
226 try {
227 await requestScheduler.flush()
228
229 await requestVideoQaduScheduler.flush()
230
231 const pods = await db.Pod.list()
232 const requestParams = {
233 method: 'POST' as 'POST',
234 path: '/api/' + API_VERSION + '/remote/pods/remove',
235 toPod: null
236 }
237
238 // Announce we quit them
239 // We don't care if the request fails
240 // The other pod will exclude us automatically after a while
241 try {
242 await Bluebird.map(pods, pod => {
243 requestParams.toPod = pod
244
245 return makeSecureRequest(requestParams)
246 }, { concurrency: REQUESTS_IN_PARALLEL })
247 } catch (err) { // Don't stop the process
248 logger.error('Some errors while quitting friends.', err)
249 }
250
251 const tasks = []
252 for (const pod of pods) {
253 tasks.push(pod.destroy())
254 }
255 await Promise.all(pods)
256
257 logger.info('Removed all remote videos.')
258
259 requestScheduler.activate()
260 } catch (err) {
261 // Don't forget to re activate the scheduler, even if there was an error
262 requestScheduler.activate()
263
264 throw err
265 }
266}
267
268async function sendOwnedDataToPod (podId: number) {
269 // First send authors
270 await sendOwnedAuthorsToPod(podId)
271 await sendOwnedChannelsToPod(podId)
272 await sendOwnedVideosToPod(podId)
273}
274
275async function sendOwnedChannelsToPod (podId: number) {
276 const videoChannels = await db.VideoChannel.listOwned()
277
278 const tasks: Promise<any>[] = []
279 for (const videoChannel of videoChannels) {
280 const remoteVideoChannel = videoChannel.toAddRemoteJSON()
281 const options = {
282 type: 'add-channel' as 'add-channel',
283 endpoint: REQUEST_ENDPOINTS.VIDEOS,
284 data: remoteVideoChannel,
285 toIds: [ podId ],
286 transaction: null
287 }
288
289 const p = createRequest(options)
290 tasks.push(p)
291 }
292
293 await Promise.all(tasks)
294}
295
296async function sendOwnedAuthorsToPod (podId: number) {
297 const authors = await db.Author.listOwned()
298 const tasks: Promise<any>[] = []
299
300 for (const author of authors) {
301 const remoteAuthor = author.toAddRemoteJSON()
302 const options = {
303 type: 'add-author' as 'add-author',
304 endpoint: REQUEST_ENDPOINTS.VIDEOS,
305 data: remoteAuthor,
306 toIds: [ podId ],
307 transaction: null
308 }
309
310 const p = createRequest(options)
311 tasks.push(p)
312 }
313
314 await Promise.all(tasks)
315}
316
317async function sendOwnedVideosToPod (podId: number) {
318 const videosList = await db.Video.listOwnedAndPopulateAuthorAndTags()
319 const tasks: Bluebird<any>[] = []
320
321 for (const video of videosList) {
322 const promise = video.toAddRemoteJSON()
323 .then(remoteVideo => {
324 const options = {
325 type: 'add-video' as 'add-video',
326 endpoint: REQUEST_ENDPOINTS.VIDEOS,
327 data: remoteVideo,
328 toIds: [ podId ],
329 transaction: null
330 }
331 return createRequest(options)
332 })
333 .catch(err => {
334 logger.error('Cannot convert video to remote.', err)
335 // Don't break the process
336 return undefined
337 })
338
339 tasks.push(promise)
340 }
341
342 await Promise.all(tasks)
343}
344
345function fetchRemotePreview (video: VideoInstance) {
346 const host = video.VideoChannel.Author.Pod.host
347 const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
348
349 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
350}
351
352function fetchRemoteDescription (video: VideoInstance) {
353 const host = video.VideoChannel.Author.Pod.host
354 const path = video.getDescriptionPath()
355
356 const requestOptions = {
357 url: REMOTE_SCHEME.HTTP + '://' + host + path,
358 json: true
359 }
360
361 return new Promise<string>((res, rej) => {
362 request.get(requestOptions, (err, response, body) => {
363 if (err) return rej(err)
364
365 return res(body.description ? body.description : '')
366 })
367 })
368}
369
370async function removeFriend (pod: PodInstance) {
371 const requestParams = {
372 method: 'POST' as 'POST',
373 path: '/api/' + API_VERSION + '/remote/pods/remove',
374 toPod: pod
375 }
376
377 try {
378 await makeSecureRequest(requestParams)
379 } catch (err) {
380 logger.warn('Cannot notify friends %s we are quitting him.', pod.host, err)
381 }
382
383 try {
384 await pod.destroy()
385
386 logger.info('Removed friend %s.', pod.host)
387 } catch (err) {
388 logger.error('Cannot destroy friend %s.', pod.host, err)
389 }
390}
391
392function getRequestScheduler () {
393 return requestScheduler
394}
395
396function getRequestVideoQaduScheduler () {
397 return requestVideoQaduScheduler
398}
399
400function getRequestVideoEventScheduler () {
401 return requestVideoEventScheduler
402}
403
404// ---------------------------------------------------------------------------
405
406export {
407 activateSchedulers,
408 addVideoToFriends,
409 removeVideoAuthorToFriends,
410 updateVideoToFriends,
411 addVideoAuthorToFriends,
412 reportAbuseVideoToFriend,
413 quickAndDirtyUpdateVideoToFriends,
414 quickAndDirtyUpdatesVideoToFriends,
415 addEventToRemoteVideo,
416 addEventsToRemoteVideo,
417 hasFriends,
418 makeFriends,
419 quitFriends,
420 removeFriend,
421 removeVideoToFriends,
422 sendOwnedDataToPod,
423 getRequestScheduler,
424 getRequestVideoQaduScheduler,
425 getRequestVideoEventScheduler,
426 fetchRemotePreview,
427 addVideoChannelToFriends,
428 fetchRemoteDescription,
429 updateVideoChannelToFriends,
430 removeVideoChannelToFriends
431}
432
433// ---------------------------------------------------------------------------
434
435async function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) {
436 const result = await getForeignPodsList(host)
437 const foreignPodsList: { host: string }[] = result.data
438
439 // Let's give 1 point to the pod we ask the friends list
440 foreignPodsList.push({ host })
441
442 for (const foreignPod of foreignPodsList) {
443 const foreignPodHost = foreignPod.host
444
445 if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++
446 else podsScore[foreignPodHost] = 1
447 }
448
449 return undefined
450}
451
452function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) {
453 // Build the list of pods to add
454 // Only add a pod if it exists in more than a half base pods
455 const podsList = []
456 const baseScore = hosts.length / 2
457
458 for (const podHost of Object.keys(podsScore)) {
459 // If the pod is not me and with a good score we add it
460 if (isMe(podHost) === false && podsScore[podHost] > baseScore) {
461 podsList.push({ host: podHost })
462 }
463 }
464
465 return podsList
466}
467
468function getForeignPodsList (host: string) {
469 return new Promise< ResultList<FormattedPod> >((res, rej) => {
470 const path = '/api/' + API_VERSION + '/remote/pods/list'
471
472 request.post(REMOTE_SCHEME.HTTP + '://' + host + path, (err, response, body) => {
473 if (err) return rej(err)
474
475 try {
476 const json: ResultList<FormattedPod> = JSON.parse(body)
477 return res(json)
478 } catch (err) {
479 return rej(err)
480 }
481 })
482 })
483}
484
485async function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) {
486 // Stop pool requests
487 requestScheduler.deactivate()
488 // Flush pool requests
489 requestScheduler.forceSend()
490
491 try {
492 await Bluebird.map(podsList, async pod => {
493 const params = {
494 url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/remote/pods/add',
495 method: 'POST' as 'POST',
496 json: {
497 host: CONFIG.WEBSERVER.HOST,
498 email: CONFIG.ADMIN.EMAIL,
499 publicKey: cert
500 }
501 }
502
503 const { response, body } = await makeRetryRequest(params)
504 const typedBody = body as { cert: string, email: string }
505
506 if (response.statusCode === 200) {
507 const podObj = db.Pod.build({ host: pod.host, publicKey: typedBody.cert, email: typedBody.email })
508
509 let podCreated: PodInstance
510 try {
511 podCreated = await podObj.save()
512 } catch (err) {
513 logger.error('Cannot add friend %s pod.', pod.host, err)
514 }
515
516 // Add our videos to the request scheduler
517 sendOwnedDataToPod(podCreated.id)
518 .catch(err => logger.warn('Cannot send owned data to pod %d.', podCreated.id, err))
519 } else {
520 logger.error('Status not 200 for %s pod.', pod.host)
521 }
522 }, { concurrency: REQUESTS_IN_PARALLEL })
523
524 logger.debug('makeRequestsToWinningPods finished.')
525
526 requestScheduler.activate()
527 } catch (err) {
528 // Final callback, we've ended all the requests
529 // Now we made new friends, we can re activate the pool of requests
530 requestScheduler.activate()
531 }
532}
533
534// Wrapper that populate "toIds" argument with all our friends if it is not specified
535type CreateRequestOptions = {
536 type: RemoteVideoRequestType
537 endpoint: RequestEndpoint
538 data: Object
539 toIds?: number[]
540 transaction: Sequelize.Transaction
541}
542async function createRequest (options: CreateRequestOptions) {
543 if (options.toIds !== undefined) {
544 await requestScheduler.createRequest(options as RequestSchedulerOptions)
545 return undefined
546 }
547
548 // If the "toIds" pods is not specified, we send the request to all our friends
549 const podIds = await db.Pod.listAllIds(options.transaction)
550
551 const newOptions = Object.assign(options, { toIds: podIds })
552 await requestScheduler.createRequest(newOptions)
553
554 return undefined
555}
556
557function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) {
558 return requestVideoQaduScheduler.createRequest(options)
559}
560
561function createVideoEventRequest (options: RequestVideoEventSchedulerOptions) {
562 return requestVideoEventScheduler.createRequest(options)
563}
564
565function isMe (host: string) {
566 return host === CONFIG.WEBSERVER.HOST
567}
diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts
index ccded4721..f019c28bc 100644
--- a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts
+++ b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts
@@ -7,7 +7,7 @@ import { addVideoToFriends } from '../../friends'
7import { JobScheduler } from '../job-scheduler' 7import { JobScheduler } from '../job-scheduler'
8 8
9async function process (data: { videoUUID: string }, jobId: number) { 9async function process (data: { videoUUID: string }, jobId: number) {
10 const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID) 10 const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID)
11 // No video, maybe deleted? 11 // No video, maybe deleted?
12 if (!video) { 12 if (!video) {
13 logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) 13 logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid })
@@ -30,7 +30,7 @@ async function onSuccess (jobId: number, video: VideoInstance) {
30 logger.info('Job %d is a success.', jobId) 30 logger.info('Job %d is a success.', jobId)
31 31
32 // Maybe the video changed in database, refresh it 32 // Maybe the video changed in database, refresh it
33 const videoDatabase = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(video.uuid) 33 const videoDatabase = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(video.uuid)
34 // Video does not exist anymore 34 // Video does not exist anymore
35 if (!videoDatabase) return undefined 35 if (!videoDatabase) return undefined
36 36
diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts
index 853645510..397b95795 100644
--- a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts
+++ b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts
@@ -5,7 +5,7 @@ import { VideoInstance } from '../../../models'
5import { VideoResolution } from '../../../../shared' 5import { VideoResolution } from '../../../../shared'
6 6
7async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { 7async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) {
8 const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID) 8 const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID)
9 // No video, maybe deleted? 9 // No video, maybe deleted?
10 if (!video) { 10 if (!video) {
11 logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) 11 logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid })
@@ -28,7 +28,7 @@ async function onSuccess (jobId: number, video: VideoInstance) {
28 logger.info('Job %d is a success.', jobId) 28 logger.info('Job %d is a success.', jobId)
29 29
30 // Maybe the video changed in database, refresh it 30 // Maybe the video changed in database, refresh it
31 const videoDatabase = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(video.uuid) 31 const videoDatabase = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(video.uuid)
32 // Video does not exist anymore 32 // Video does not exist anymore
33 if (!videoDatabase) return undefined 33 if (!videoDatabase) return undefined
34 34
diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts
index a6dd4d061..f81383ce8 100644
--- a/server/lib/video-channel.ts
+++ b/server/lib/video-channel.ts
@@ -11,7 +11,7 @@ async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account
11 name: videoChannelInfo.name, 11 name: videoChannelInfo.name,
12 description: videoChannelInfo.description, 12 description: videoChannelInfo.description,
13 remote: false, 13 remote: false,
14 authorId: account.id 14 accountId: account.id
15 } 15 }
16 16
17 const videoChannel = db.VideoChannel.build(videoChannelData) 17 const videoChannel = db.VideoChannel.build(videoChannelData)
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-15 01:26:33 +0000