diff options
Diffstat (limited to 'server/lib')
109 files changed, 4120 insertions, 3150 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts deleted file mode 100644 index 5fe7381c9..000000000 --- a/server/lib/activitypub/actor.ts +++ /dev/null | |||
@@ -1,594 +0,0 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { extname } from 'path' | ||
3 | import { Op, Transaction } from 'sequelize' | ||
4 | import { URL } from 'url' | ||
5 | import { v4 as uuidv4 } from 'uuid' | ||
6 | import { getServerActor } from '@server/models/application/application' | ||
7 | import { ActorImageType } from '@shared/models' | ||
8 | import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' | ||
9 | import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | ||
10 | import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' | ||
11 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | ||
12 | import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' | ||
13 | import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor' | ||
14 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | ||
15 | import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' | ||
16 | import { logger } from '../../helpers/logger' | ||
17 | import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' | ||
18 | import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests' | ||
19 | import { getUrlFromWebfinger } from '../../helpers/webfinger' | ||
20 | import { MIMETYPES, WEBSERVER } from '../../initializers/constants' | ||
21 | import { sequelizeTypescript } from '../../initializers/database' | ||
22 | import { AccountModel } from '../../models/account/account' | ||
23 | import { ActorImageModel } from '../../models/account/actor-image' | ||
24 | import { ActorModel } from '../../models/activitypub/actor' | ||
25 | import { ServerModel } from '../../models/server/server' | ||
26 | import { VideoChannelModel } from '../../models/video/video-channel' | ||
27 | import { | ||
28 | MAccount, | ||
29 | MAccountDefault, | ||
30 | MActor, | ||
31 | MActorAccountChannelId, | ||
32 | MActorAccountChannelIdActor, | ||
33 | MActorAccountId, | ||
34 | MActorFull, | ||
35 | MActorFullActor, | ||
36 | MActorId, | ||
37 | MActorImage, | ||
38 | MActorImages, | ||
39 | MChannel | ||
40 | } from '../../types/models' | ||
41 | import { JobQueue } from '../job-queue' | ||
42 | |||
43 | // Set account keys, this could be long so process after the account creation and do not block the client | ||
44 | async function generateAndSaveActorKeys <T extends MActor> (actor: T) { | ||
45 | const { publicKey, privateKey } = await createPrivateAndPublicKeys() | ||
46 | |||
47 | actor.publicKey = publicKey | ||
48 | actor.privateKey = privateKey | ||
49 | |||
50 | return actor.save() | ||
51 | } | ||
52 | |||
53 | function getOrCreateActorAndServerAndModel ( | ||
54 | activityActor: string | ActivityPubActor, | ||
55 | fetchType: 'all', | ||
56 | recurseIfNeeded?: boolean, | ||
57 | updateCollections?: boolean | ||
58 | ): Promise<MActorFullActor> | ||
59 | |||
60 | function getOrCreateActorAndServerAndModel ( | ||
61 | activityActor: string | ActivityPubActor, | ||
62 | fetchType?: 'association-ids', | ||
63 | recurseIfNeeded?: boolean, | ||
64 | updateCollections?: boolean | ||
65 | ): Promise<MActorAccountChannelId> | ||
66 | |||
67 | async function getOrCreateActorAndServerAndModel ( | ||
68 | activityActor: string | ActivityPubActor, | ||
69 | fetchType: ActorFetchByUrlType = 'association-ids', | ||
70 | recurseIfNeeded = true, | ||
71 | updateCollections = false | ||
72 | ): Promise<MActorFullActor | MActorAccountChannelId> { | ||
73 | const actorUrl = getAPId(activityActor) | ||
74 | let created = false | ||
75 | let accountPlaylistsUrl: string | ||
76 | |||
77 | let actor = await fetchActorByUrl(actorUrl, fetchType) | ||
78 | // Orphan actor (not associated to an account of channel) so recreate it | ||
79 | if (actor && (!actor.Account && !actor.VideoChannel)) { | ||
80 | await actor.destroy() | ||
81 | actor = null | ||
82 | } | ||
83 | |||
84 | // We don't have this actor in our database, fetch it on remote | ||
85 | if (!actor) { | ||
86 | const { result } = await fetchRemoteActor(actorUrl) | ||
87 | if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) | ||
88 | |||
89 | // Create the attributed to actor | ||
90 | // In PeerTube a video channel is owned by an account | ||
91 | let ownerActor: MActorFullActor | ||
92 | if (recurseIfNeeded === true && result.actor.type === 'Group') { | ||
93 | const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person') | ||
94 | if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url) | ||
95 | |||
96 | if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) { | ||
97 | throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`) | ||
98 | } | ||
99 | |||
100 | try { | ||
101 | // Don't recurse another time | ||
102 | const recurseIfNeeded = false | ||
103 | ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded) | ||
104 | } catch (err) { | ||
105 | logger.error('Cannot get or create account attributed to video channel ' + actorUrl) | ||
106 | throw new Error(err) | ||
107 | } | ||
108 | } | ||
109 | |||
110 | actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor) | ||
111 | created = true | ||
112 | accountPlaylistsUrl = result.playlists | ||
113 | } | ||
114 | |||
115 | if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor | ||
116 | if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor | ||
117 | |||
118 | const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) | ||
119 | if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') | ||
120 | |||
121 | if ((created === true || refreshed === true) && updateCollections === true) { | ||
122 | const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } | ||
123 | await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) | ||
124 | } | ||
125 | |||
126 | // We created a new account: fetch the playlists | ||
127 | if (created === true && actor.Account && accountPlaylistsUrl) { | ||
128 | const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' } | ||
129 | await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) | ||
130 | } | ||
131 | |||
132 | return actorRefreshed | ||
133 | } | ||
134 | |||
135 | function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) { | ||
136 | return new ActorModel({ | ||
137 | type, | ||
138 | url, | ||
139 | preferredUsername, | ||
140 | uuid, | ||
141 | publicKey: null, | ||
142 | privateKey: null, | ||
143 | followersCount: 0, | ||
144 | followingCount: 0, | ||
145 | inboxUrl: url + '/inbox', | ||
146 | outboxUrl: url + '/outbox', | ||
147 | sharedInboxUrl: WEBSERVER.URL + '/inbox', | ||
148 | followersUrl: url + '/followers', | ||
149 | followingUrl: url + '/following' | ||
150 | }) as MActor | ||
151 | } | ||
152 | |||
153 | async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) { | ||
154 | const followersCount = await fetchActorTotalItems(attributes.followers) | ||
155 | const followingCount = await fetchActorTotalItems(attributes.following) | ||
156 | |||
157 | actorInstance.type = attributes.type | ||
158 | actorInstance.preferredUsername = attributes.preferredUsername | ||
159 | actorInstance.url = attributes.id | ||
160 | actorInstance.publicKey = attributes.publicKey.publicKeyPem | ||
161 | actorInstance.followersCount = followersCount | ||
162 | actorInstance.followingCount = followingCount | ||
163 | actorInstance.inboxUrl = attributes.inbox | ||
164 | actorInstance.outboxUrl = attributes.outbox | ||
165 | actorInstance.followersUrl = attributes.followers | ||
166 | actorInstance.followingUrl = attributes.following | ||
167 | |||
168 | if (attributes.published) actorInstance.remoteCreatedAt = new Date(attributes.published) | ||
169 | |||
170 | if (attributes.endpoints?.sharedInbox) { | ||
171 | actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox | ||
172 | } | ||
173 | } | ||
174 | |||
175 | type ImageInfo = { | ||
176 | name: string | ||
177 | fileUrl: string | ||
178 | height: number | ||
179 | width: number | ||
180 | onDisk?: boolean | ||
181 | } | ||
182 | async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) { | ||
183 | const oldImageModel = type === ActorImageType.AVATAR | ||
184 | ? actor.Avatar | ||
185 | : actor.Banner | ||
186 | |||
187 | if (oldImageModel) { | ||
188 | // Don't update the avatar if the file URL did not change | ||
189 | if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor | ||
190 | |||
191 | try { | ||
192 | await oldImageModel.destroy({ transaction: t }) | ||
193 | |||
194 | setActorImage(actor, type, null) | ||
195 | } catch (err) { | ||
196 | logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) | ||
197 | } | ||
198 | } | ||
199 | |||
200 | if (imageInfo) { | ||
201 | const imageModel = await ActorImageModel.create({ | ||
202 | filename: imageInfo.name, | ||
203 | onDisk: imageInfo.onDisk ?? false, | ||
204 | fileUrl: imageInfo.fileUrl, | ||
205 | height: imageInfo.height, | ||
206 | width: imageInfo.width, | ||
207 | type | ||
208 | }, { transaction: t }) | ||
209 | |||
210 | setActorImage(actor, type, imageModel) | ||
211 | } | ||
212 | |||
213 | return actor | ||
214 | } | ||
215 | |||
216 | async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) { | ||
217 | try { | ||
218 | if (type === ActorImageType.AVATAR) { | ||
219 | await actor.Avatar.destroy({ transaction: t }) | ||
220 | |||
221 | actor.avatarId = null | ||
222 | actor.Avatar = null | ||
223 | } else { | ||
224 | await actor.Banner.destroy({ transaction: t }) | ||
225 | |||
226 | actor.bannerId = null | ||
227 | actor.Banner = null | ||
228 | } | ||
229 | } catch (err) { | ||
230 | logger.error('Cannot remove old image of actor %s.', actor.url, { err }) | ||
231 | } | ||
232 | |||
233 | return actor | ||
234 | } | ||
235 | |||
236 | async function fetchActorTotalItems (url: string) { | ||
237 | try { | ||
238 | const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true }) | ||
239 | |||
240 | return body.totalItems || 0 | ||
241 | } catch (err) { | ||
242 | logger.warn('Cannot fetch remote actor count %s.', url, { err }) | ||
243 | return 0 | ||
244 | } | ||
245 | } | ||
246 | |||
247 | function getImageInfoIfExists (actorJSON: ActivityPubActor, type: ActorImageType) { | ||
248 | const mimetypes = MIMETYPES.IMAGE | ||
249 | const icon = type === ActorImageType.AVATAR | ||
250 | ? actorJSON.icon | ||
251 | : actorJSON.image | ||
252 | |||
253 | if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined | ||
254 | |||
255 | let extension: string | ||
256 | |||
257 | if (icon.mediaType) { | ||
258 | extension = mimetypes.MIMETYPE_EXT[icon.mediaType] | ||
259 | } else { | ||
260 | const tmp = extname(icon.url) | ||
261 | |||
262 | if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp | ||
263 | } | ||
264 | |||
265 | if (!extension) return undefined | ||
266 | |||
267 | return { | ||
268 | name: uuidv4() + extension, | ||
269 | fileUrl: icon.url, | ||
270 | height: icon.height, | ||
271 | width: icon.width, | ||
272 | type | ||
273 | } | ||
274 | } | ||
275 | |||
276 | async function addFetchOutboxJob (actor: Pick<ActorModel, 'id' | 'outboxUrl'>) { | ||
277 | // Don't fetch ourselves | ||
278 | const serverActor = await getServerActor() | ||
279 | if (serverActor.id === actor.id) { | ||
280 | logger.error('Cannot fetch our own outbox!') | ||
281 | return undefined | ||
282 | } | ||
283 | |||
284 | const payload = { | ||
285 | uri: actor.outboxUrl, | ||
286 | type: 'activity' as 'activity' | ||
287 | } | ||
288 | |||
289 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | ||
290 | } | ||
291 | |||
292 | async function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> ( | ||
293 | actorArg: T, | ||
294 | fetchedType: ActorFetchByUrlType | ||
295 | ): Promise<{ actor: T | MActorFull, refreshed: boolean }> { | ||
296 | if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } | ||
297 | |||
298 | // We need more attributes | ||
299 | const actor = fetchedType === 'all' | ||
300 | ? actorArg as MActorFull | ||
301 | : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
302 | |||
303 | try { | ||
304 | let actorUrl: string | ||
305 | try { | ||
306 | actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
307 | } catch (err) { | ||
308 | logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err) | ||
309 | actorUrl = actor.url | ||
310 | } | ||
311 | |||
312 | const { result } = await fetchRemoteActor(actorUrl) | ||
313 | |||
314 | if (result === undefined) { | ||
315 | logger.warn('Cannot fetch remote actor in refresh actor.') | ||
316 | return { actor, refreshed: false } | ||
317 | } | ||
318 | |||
319 | return sequelizeTypescript.transaction(async t => { | ||
320 | updateInstanceWithAnother(actor, result.actor) | ||
321 | |||
322 | await updateActorImageInstance(actor, ActorImageType.AVATAR, result.avatar, t) | ||
323 | await updateActorImageInstance(actor, ActorImageType.BANNER, result.banner, t) | ||
324 | |||
325 | // Force update | ||
326 | actor.setDataValue('updatedAt', new Date()) | ||
327 | await actor.save({ transaction: t }) | ||
328 | |||
329 | if (actor.Account) { | ||
330 | actor.Account.name = result.name | ||
331 | actor.Account.description = result.summary | ||
332 | |||
333 | await actor.Account.save({ transaction: t }) | ||
334 | } else if (actor.VideoChannel) { | ||
335 | actor.VideoChannel.name = result.name | ||
336 | actor.VideoChannel.description = result.summary | ||
337 | actor.VideoChannel.support = result.support | ||
338 | |||
339 | await actor.VideoChannel.save({ transaction: t }) | ||
340 | } | ||
341 | |||
342 | return { refreshed: true, actor } | ||
343 | }) | ||
344 | } catch (err) { | ||
345 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
346 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) | ||
347 | actor.Account | ||
348 | ? await actor.Account.destroy() | ||
349 | : await actor.VideoChannel.destroy() | ||
350 | |||
351 | return { actor: undefined, refreshed: false } | ||
352 | } | ||
353 | |||
354 | logger.warn('Cannot refresh actor %s.', actor.url, { err }) | ||
355 | return { actor, refreshed: false } | ||
356 | } | ||
357 | } | ||
358 | |||
359 | export { | ||
360 | getOrCreateActorAndServerAndModel, | ||
361 | buildActorInstance, | ||
362 | generateAndSaveActorKeys, | ||
363 | fetchActorTotalItems, | ||
364 | getImageInfoIfExists, | ||
365 | updateActorInstance, | ||
366 | deleteActorImageInstance, | ||
367 | refreshActorIfNeeded, | ||
368 | updateActorImageInstance, | ||
369 | addFetchOutboxJob | ||
370 | } | ||
371 | |||
372 | // --------------------------------------------------------------------------- | ||
373 | |||
374 | function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) { | ||
375 | const id = imageModel | ||
376 | ? imageModel.id | ||
377 | : null | ||
378 | |||
379 | if (type === ActorImageType.AVATAR) { | ||
380 | actorModel.avatarId = id | ||
381 | actorModel.Avatar = imageModel | ||
382 | } else { | ||
383 | actorModel.bannerId = id | ||
384 | actorModel.Banner = imageModel | ||
385 | } | ||
386 | |||
387 | return actorModel | ||
388 | } | ||
389 | |||
390 | function saveActorAndServerAndModelIfNotExist ( | ||
391 | result: FetchRemoteActorResult, | ||
392 | ownerActor?: MActorFullActor, | ||
393 | t?: Transaction | ||
394 | ): Bluebird<MActorFullActor> | Promise<MActorFullActor> { | ||
395 | const actor = result.actor | ||
396 | |||
397 | if (t !== undefined) return save(t) | ||
398 | |||
399 | return sequelizeTypescript.transaction(t => save(t)) | ||
400 | |||
401 | async function save (t: Transaction) { | ||
402 | const actorHost = new URL(actor.url).host | ||
403 | |||
404 | const serverOptions = { | ||
405 | where: { | ||
406 | host: actorHost | ||
407 | }, | ||
408 | defaults: { | ||
409 | host: actorHost | ||
410 | }, | ||
411 | transaction: t | ||
412 | } | ||
413 | const [ server ] = await ServerModel.findOrCreate(serverOptions) | ||
414 | |||
415 | // Save our new account in database | ||
416 | actor.serverId = server.id | ||
417 | |||
418 | // Avatar? | ||
419 | if (result.avatar) { | ||
420 | const avatar = await ActorImageModel.create({ | ||
421 | filename: result.avatar.name, | ||
422 | fileUrl: result.avatar.fileUrl, | ||
423 | width: result.avatar.width, | ||
424 | height: result.avatar.height, | ||
425 | onDisk: false, | ||
426 | type: ActorImageType.AVATAR | ||
427 | }, { transaction: t }) | ||
428 | |||
429 | actor.avatarId = avatar.id | ||
430 | } | ||
431 | |||
432 | // Banner? | ||
433 | if (result.banner) { | ||
434 | const banner = await ActorImageModel.create({ | ||
435 | filename: result.banner.name, | ||
436 | fileUrl: result.banner.fileUrl, | ||
437 | width: result.banner.width, | ||
438 | height: result.banner.height, | ||
439 | onDisk: false, | ||
440 | type: ActorImageType.BANNER | ||
441 | }, { transaction: t }) | ||
442 | |||
443 | actor.bannerId = banner.id | ||
444 | } | ||
445 | |||
446 | // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists | ||
447 | // (which could be false in a retried query) | ||
448 | const [ actorCreated, created ] = await ActorModel.findOrCreate<MActorFullActor>({ | ||
449 | defaults: actor.toJSON(), | ||
450 | where: { | ||
451 | [Op.or]: [ | ||
452 | { | ||
453 | url: actor.url | ||
454 | }, | ||
455 | { | ||
456 | serverId: actor.serverId, | ||
457 | preferredUsername: actor.preferredUsername | ||
458 | } | ||
459 | ] | ||
460 | }, | ||
461 | transaction: t | ||
462 | }) | ||
463 | |||
464 | // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards | ||
465 | if (created !== true && actorCreated.url !== actor.url) { | ||
466 | // Only fix http://example.com/account/djidane to https://example.com/account/djidane | ||
467 | if (actorCreated.url.replace(/^http:\/\//, '') !== actor.url.replace(/^https:\/\//, '')) { | ||
468 | throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${actor.url}`) | ||
469 | } | ||
470 | |||
471 | actorCreated.url = actor.url | ||
472 | await actorCreated.save({ transaction: t }) | ||
473 | } | ||
474 | |||
475 | if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { | ||
476 | actorCreated.Account = await saveAccount(actorCreated, result, t) as MAccountDefault | ||
477 | actorCreated.Account.Actor = actorCreated | ||
478 | } else if (actorCreated.type === 'Group') { // Video channel | ||
479 | const channel = await saveVideoChannel(actorCreated, result, ownerActor, t) | ||
480 | actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: ownerActor.Account }) | ||
481 | } | ||
482 | |||
483 | actorCreated.Server = server | ||
484 | |||
485 | return actorCreated | ||
486 | } | ||
487 | } | ||
488 | |||
489 | type ImageResult = { | ||
490 | name: string | ||
491 | fileUrl: string | ||
492 | height: number | ||
493 | width: number | ||
494 | } | ||
495 | |||
496 | type FetchRemoteActorResult = { | ||
497 | actor: MActor | ||
498 | name: string | ||
499 | summary: string | ||
500 | support?: string | ||
501 | playlists?: string | ||
502 | avatar?: ImageResult | ||
503 | banner?: ImageResult | ||
504 | attributedTo: ActivityPubAttributedTo[] | ||
505 | } | ||
506 | async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { | ||
507 | logger.info('Fetching remote actor %s.', actorUrl) | ||
508 | |||
509 | const requestResult = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true }) | ||
510 | const actorJSON = requestResult.body | ||
511 | |||
512 | if (sanitizeAndCheckActorObject(actorJSON) === false) { | ||
513 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) | ||
514 | return { result: undefined, statusCode: requestResult.statusCode } | ||
515 | } | ||
516 | |||
517 | if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { | ||
518 | logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) | ||
519 | return { result: undefined, statusCode: requestResult.statusCode } | ||
520 | } | ||
521 | |||
522 | const followersCount = await fetchActorTotalItems(actorJSON.followers) | ||
523 | const followingCount = await fetchActorTotalItems(actorJSON.following) | ||
524 | |||
525 | const actor = new ActorModel({ | ||
526 | type: actorJSON.type, | ||
527 | preferredUsername: actorJSON.preferredUsername, | ||
528 | url: actorJSON.id, | ||
529 | publicKey: actorJSON.publicKey.publicKeyPem, | ||
530 | privateKey: null, | ||
531 | followersCount: followersCount, | ||
532 | followingCount: followingCount, | ||
533 | inboxUrl: actorJSON.inbox, | ||
534 | outboxUrl: actorJSON.outbox, | ||
535 | followersUrl: actorJSON.followers, | ||
536 | followingUrl: actorJSON.following, | ||
537 | |||
538 | sharedInboxUrl: actorJSON.endpoints?.sharedInbox | ||
539 | ? actorJSON.endpoints.sharedInbox | ||
540 | : null | ||
541 | }) | ||
542 | |||
543 | const avatarInfo = getImageInfoIfExists(actorJSON, ActorImageType.AVATAR) | ||
544 | const bannerInfo = getImageInfoIfExists(actorJSON, ActorImageType.BANNER) | ||
545 | |||
546 | const name = actorJSON.name || actorJSON.preferredUsername | ||
547 | return { | ||
548 | statusCode: requestResult.statusCode, | ||
549 | result: { | ||
550 | actor, | ||
551 | name, | ||
552 | avatar: avatarInfo, | ||
553 | banner: bannerInfo, | ||
554 | summary: actorJSON.summary, | ||
555 | support: actorJSON.support, | ||
556 | playlists: actorJSON.playlists, | ||
557 | attributedTo: actorJSON.attributedTo | ||
558 | } | ||
559 | } | ||
560 | } | ||
561 | |||
562 | async function saveAccount (actor: MActorId, result: FetchRemoteActorResult, t: Transaction) { | ||
563 | const [ accountCreated ] = await AccountModel.findOrCreate({ | ||
564 | defaults: { | ||
565 | name: result.name, | ||
566 | description: result.summary, | ||
567 | actorId: actor.id | ||
568 | }, | ||
569 | where: { | ||
570 | actorId: actor.id | ||
571 | }, | ||
572 | transaction: t | ||
573 | }) | ||
574 | |||
575 | return accountCreated as MAccount | ||
576 | } | ||
577 | |||
578 | async function saveVideoChannel (actor: MActorId, result: FetchRemoteActorResult, ownerActor: MActorAccountId, t: Transaction) { | ||
579 | const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ | ||
580 | defaults: { | ||
581 | name: result.name, | ||
582 | description: result.summary, | ||
583 | support: result.support, | ||
584 | actorId: actor.id, | ||
585 | accountId: ownerActor.Account.id | ||
586 | }, | ||
587 | where: { | ||
588 | actorId: actor.id | ||
589 | }, | ||
590 | transaction: t | ||
591 | }) | ||
592 | |||
593 | return videoChannelCreated as MChannel | ||
594 | } | ||
diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts new file mode 100644 index 000000000..8681ea02a --- /dev/null +++ b/server/lib/activitypub/actors/get.ts | |||
@@ -0,0 +1,122 @@ | |||
1 | |||
2 | import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub' | ||
3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { JobQueue } from '@server/lib/job-queue' | ||
6 | import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders' | ||
7 | import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models' | ||
8 | import { ActivityPubActor } from '@shared/models' | ||
9 | import { refreshActorIfNeeded } from './refresh' | ||
10 | import { APActorCreator, fetchRemoteActor } from './shared' | ||
11 | |||
12 | function getOrCreateAPActor ( | ||
13 | activityActor: string | ActivityPubActor, | ||
14 | fetchType: 'all', | ||
15 | recurseIfNeeded?: boolean, | ||
16 | updateCollections?: boolean | ||
17 | ): Promise<MActorFullActor> | ||
18 | |||
19 | function getOrCreateAPActor ( | ||
20 | activityActor: string | ActivityPubActor, | ||
21 | fetchType?: 'association-ids', | ||
22 | recurseIfNeeded?: boolean, | ||
23 | updateCollections?: boolean | ||
24 | ): Promise<MActorAccountChannelId> | ||
25 | |||
26 | async function getOrCreateAPActor ( | ||
27 | activityActor: string | ActivityPubActor, | ||
28 | fetchType: ActorLoadByUrlType = 'association-ids', | ||
29 | recurseIfNeeded = true, | ||
30 | updateCollections = false | ||
31 | ): Promise<MActorFullActor | MActorAccountChannelId> { | ||
32 | const actorUrl = getAPId(activityActor) | ||
33 | let actor = await loadActorFromDB(actorUrl, fetchType) | ||
34 | |||
35 | let created = false | ||
36 | let accountPlaylistsUrl: string | ||
37 | |||
38 | // We don't have this actor in our database, fetch it on remote | ||
39 | if (!actor) { | ||
40 | const { actorObject } = await fetchRemoteActor(actorUrl) | ||
41 | if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) | ||
42 | |||
43 | // actorUrl is just an alias/rediraction, so process object id instead | ||
44 | if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections) | ||
45 | |||
46 | // Create the attributed to actor | ||
47 | // In PeerTube a video channel is owned by an account | ||
48 | let ownerActor: MActorFullActor | ||
49 | if (recurseIfNeeded === true && actorObject.type === 'Group') { | ||
50 | ownerActor = await getOrCreateAPOwner(actorObject, actorUrl) | ||
51 | } | ||
52 | |||
53 | const creator = new APActorCreator(actorObject, ownerActor) | ||
54 | actor = await retryTransactionWrapper(creator.create.bind(creator)) | ||
55 | created = true | ||
56 | accountPlaylistsUrl = actorObject.playlists | ||
57 | } | ||
58 | |||
59 | if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor | ||
60 | if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor | ||
61 | |||
62 | const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType }) | ||
63 | if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') | ||
64 | |||
65 | await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections) | ||
66 | await schedulePlaylistFetchIfNeeded(actor, created, accountPlaylistsUrl) | ||
67 | |||
68 | return actorRefreshed | ||
69 | } | ||
70 | |||
71 | // --------------------------------------------------------------------------- | ||
72 | |||
73 | export { | ||
74 | getOrCreateAPActor | ||
75 | } | ||
76 | |||
77 | // --------------------------------------------------------------------------- | ||
78 | |||
79 | async function loadActorFromDB (actorUrl: string, fetchType: ActorLoadByUrlType) { | ||
80 | let actor = await loadActorByUrl(actorUrl, fetchType) | ||
81 | |||
82 | // Orphan actor (not associated to an account of channel) so recreate it | ||
83 | if (actor && (!actor.Account && !actor.VideoChannel)) { | ||
84 | await actor.destroy() | ||
85 | actor = null | ||
86 | } | ||
87 | |||
88 | return actor | ||
89 | } | ||
90 | |||
91 | function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) { | ||
92 | const accountAttributedTo = actorObject.attributedTo.find(a => a.type === 'Person') | ||
93 | if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actorUrl) | ||
94 | |||
95 | if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) { | ||
96 | throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`) | ||
97 | } | ||
98 | |||
99 | try { | ||
100 | // Don't recurse another time | ||
101 | const recurseIfNeeded = false | ||
102 | return getOrCreateAPActor(accountAttributedTo.id, 'all', recurseIfNeeded) | ||
103 | } catch (err) { | ||
104 | logger.error('Cannot get or create account attributed to video channel ' + actorUrl) | ||
105 | throw new Error(err) | ||
106 | } | ||
107 | } | ||
108 | |||
109 | async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) { | ||
110 | if ((created === true || refreshed === true) && updateCollections === true) { | ||
111 | const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } | ||
112 | await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) | ||
113 | } | ||
114 | } | ||
115 | |||
116 | async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) { | ||
117 | // We created a new account: fetch the playlists | ||
118 | if (created === true && actor.Account && accountPlaylistsUrl) { | ||
119 | const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' } | ||
120 | await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) | ||
121 | } | ||
122 | } | ||
diff --git a/server/lib/activitypub/actors/image.ts b/server/lib/activitypub/actors/image.ts new file mode 100644 index 000000000..443ad0a63 --- /dev/null +++ b/server/lib/activitypub/actors/image.ts | |||
@@ -0,0 +1,94 @@ | |||
1 | import { Transaction } from 'sequelize/types' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { ActorImageModel } from '@server/models/actor/actor-image' | ||
4 | import { MActorImage, MActorImages } from '@server/types/models' | ||
5 | import { ActorImageType } from '@shared/models' | ||
6 | |||
7 | type ImageInfo = { | ||
8 | name: string | ||
9 | fileUrl: string | ||
10 | height: number | ||
11 | width: number | ||
12 | onDisk?: boolean | ||
13 | } | ||
14 | |||
15 | async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) { | ||
16 | const oldImageModel = type === ActorImageType.AVATAR | ||
17 | ? actor.Avatar | ||
18 | : actor.Banner | ||
19 | |||
20 | if (oldImageModel) { | ||
21 | // Don't update the avatar if the file URL did not change | ||
22 | if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor | ||
23 | |||
24 | try { | ||
25 | await oldImageModel.destroy({ transaction: t }) | ||
26 | |||
27 | setActorImage(actor, type, null) | ||
28 | } catch (err) { | ||
29 | logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) | ||
30 | } | ||
31 | } | ||
32 | |||
33 | if (imageInfo) { | ||
34 | const imageModel = await ActorImageModel.create({ | ||
35 | filename: imageInfo.name, | ||
36 | onDisk: imageInfo.onDisk ?? false, | ||
37 | fileUrl: imageInfo.fileUrl, | ||
38 | height: imageInfo.height, | ||
39 | width: imageInfo.width, | ||
40 | type | ||
41 | }, { transaction: t }) | ||
42 | |||
43 | setActorImage(actor, type, imageModel) | ||
44 | } | ||
45 | |||
46 | return actor | ||
47 | } | ||
48 | |||
49 | async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) { | ||
50 | try { | ||
51 | if (type === ActorImageType.AVATAR) { | ||
52 | await actor.Avatar.destroy({ transaction: t }) | ||
53 | |||
54 | actor.avatarId = null | ||
55 | actor.Avatar = null | ||
56 | } else { | ||
57 | await actor.Banner.destroy({ transaction: t }) | ||
58 | |||
59 | actor.bannerId = null | ||
60 | actor.Banner = null | ||
61 | } | ||
62 | } catch (err) { | ||
63 | logger.error('Cannot remove old image of actor %s.', actor.url, { err }) | ||
64 | } | ||
65 | |||
66 | return actor | ||
67 | } | ||
68 | |||
69 | // --------------------------------------------------------------------------- | ||
70 | |||
71 | export { | ||
72 | ImageInfo, | ||
73 | |||
74 | updateActorImageInstance, | ||
75 | deleteActorImageInstance | ||
76 | } | ||
77 | |||
78 | // --------------------------------------------------------------------------- | ||
79 | |||
80 | function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) { | ||
81 | const id = imageModel | ||
82 | ? imageModel.id | ||
83 | : null | ||
84 | |||
85 | if (type === ActorImageType.AVATAR) { | ||
86 | actorModel.avatarId = id | ||
87 | actorModel.Avatar = imageModel | ||
88 | } else { | ||
89 | actorModel.bannerId = id | ||
90 | actorModel.Banner = imageModel | ||
91 | } | ||
92 | |||
93 | return actorModel | ||
94 | } | ||
diff --git a/server/lib/activitypub/actors/index.ts b/server/lib/activitypub/actors/index.ts new file mode 100644 index 000000000..5ee2a6f1a --- /dev/null +++ b/server/lib/activitypub/actors/index.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | export * from './get' | ||
2 | export * from './image' | ||
3 | export * from './keys' | ||
4 | export * from './refresh' | ||
5 | export * from './updater' | ||
6 | export * from './webfinger' | ||
diff --git a/server/lib/activitypub/actors/keys.ts b/server/lib/activitypub/actors/keys.ts new file mode 100644 index 000000000..c3d18abd8 --- /dev/null +++ b/server/lib/activitypub/actors/keys.ts | |||
@@ -0,0 +1,16 @@ | |||
1 | import { createPrivateAndPublicKeys } from '@server/helpers/peertube-crypto' | ||
2 | import { MActor } from '@server/types/models' | ||
3 | |||
4 | // Set account keys, this could be long so process after the account creation and do not block the client | ||
5 | async function generateAndSaveActorKeys <T extends MActor> (actor: T) { | ||
6 | const { publicKey, privateKey } = await createPrivateAndPublicKeys() | ||
7 | |||
8 | actor.publicKey = publicKey | ||
9 | actor.privateKey = privateKey | ||
10 | |||
11 | return actor.save() | ||
12 | } | ||
13 | |||
14 | export { | ||
15 | generateAndSaveActorKeys | ||
16 | } | ||
diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts new file mode 100644 index 000000000..b2fe3932f --- /dev/null +++ b/server/lib/activitypub/actors/refresh.ts | |||
@@ -0,0 +1,81 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
2 | import { PromiseCache } from '@server/helpers/promise-cache' | ||
3 | import { PeerTubeRequestError } from '@server/helpers/requests' | ||
4 | import { ActorLoadByUrlType } from '@server/lib/model-loaders' | ||
5 | import { ActorModel } from '@server/models/actor/actor' | ||
6 | import { MActorAccountChannelId, MActorFull } from '@server/types/models' | ||
7 | import { HttpStatusCode } from '@shared/core-utils' | ||
8 | import { fetchRemoteActor } from './shared' | ||
9 | import { APActorUpdater } from './updater' | ||
10 | import { getUrlFromWebfinger } from './webfinger' | ||
11 | |||
12 | type RefreshResult <T> = Promise<{ actor: T | MActorFull, refreshed: boolean }> | ||
13 | |||
14 | type RefreshOptions <T> = { | ||
15 | actor: T | ||
16 | fetchedType: ActorLoadByUrlType | ||
17 | } | ||
18 | |||
19 | const promiseCache = new PromiseCache(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url) | ||
20 | |||
21 | function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <T> { | ||
22 | const actorArg = options.actor | ||
23 | if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false }) | ||
24 | |||
25 | return promiseCache.run(options) | ||
26 | } | ||
27 | |||
28 | export { | ||
29 | refreshActorIfNeeded | ||
30 | } | ||
31 | |||
32 | // --------------------------------------------------------------------------- | ||
33 | |||
34 | async function doRefresh <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <MActorFull> { | ||
35 | const { actor: actorArg, fetchedType } = options | ||
36 | |||
37 | // We need more attributes | ||
38 | const actor = fetchedType === 'all' | ||
39 | ? actorArg as MActorFull | ||
40 | : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
41 | |||
42 | const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url) | ||
43 | |||
44 | logger.info('Refreshing actor %s.', actor.url, lTags()) | ||
45 | |||
46 | try { | ||
47 | const actorUrl = await getActorUrl(actor) | ||
48 | const { actorObject } = await fetchRemoteActor(actorUrl) | ||
49 | |||
50 | if (actorObject === undefined) { | ||
51 | logger.warn('Cannot fetch remote actor in refresh actor.') | ||
52 | return { actor, refreshed: false } | ||
53 | } | ||
54 | |||
55 | const updater = new APActorUpdater(actorObject, actor) | ||
56 | await updater.update() | ||
57 | |||
58 | return { refreshed: true, actor } | ||
59 | } catch (err) { | ||
60 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
61 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url, lTags()) | ||
62 | |||
63 | actor.Account | ||
64 | ? await actor.Account.destroy() | ||
65 | : await actor.VideoChannel.destroy() | ||
66 | |||
67 | return { actor: undefined, refreshed: false } | ||
68 | } | ||
69 | |||
70 | logger.warn('Cannot refresh actor %s.', actor.url, { err, ...lTags() }) | ||
71 | return { actor, refreshed: false } | ||
72 | } | ||
73 | } | ||
74 | |||
75 | function getActorUrl (actor: MActorFull) { | ||
76 | return getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
77 | .catch(err => { | ||
78 | logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err) | ||
79 | return actor.url | ||
80 | }) | ||
81 | } | ||
diff --git a/server/lib/activitypub/actors/shared/creator.ts b/server/lib/activitypub/actors/shared/creator.ts new file mode 100644 index 000000000..999aed97d --- /dev/null +++ b/server/lib/activitypub/actors/shared/creator.ts | |||
@@ -0,0 +1,149 @@ | |||
1 | import { Op, Transaction } from 'sequelize' | ||
2 | import { sequelizeTypescript } from '@server/initializers/database' | ||
3 | import { AccountModel } from '@server/models/account/account' | ||
4 | import { ActorModel } from '@server/models/actor/actor' | ||
5 | import { ServerModel } from '@server/models/server/server' | ||
6 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
7 | import { MAccount, MAccountDefault, MActor, MActorFullActor, MActorId, MActorImages, MChannel, MServer } from '@server/types/models' | ||
8 | import { ActivityPubActor, ActorImageType } from '@shared/models' | ||
9 | import { updateActorImageInstance } from '../image' | ||
10 | import { getActorAttributesFromObject, getActorDisplayNameFromObject, getImageInfoFromObject } from './object-to-model-attributes' | ||
11 | import { fetchActorFollowsCount } from './url-to-object' | ||
12 | |||
13 | export class APActorCreator { | ||
14 | |||
15 | constructor ( | ||
16 | private readonly actorObject: ActivityPubActor, | ||
17 | private readonly ownerActor?: MActorFullActor | ||
18 | ) { | ||
19 | |||
20 | } | ||
21 | |||
22 | async create (): Promise<MActorFullActor> { | ||
23 | const { followersCount, followingCount } = await fetchActorFollowsCount(this.actorObject) | ||
24 | |||
25 | const actorInstance = new ActorModel(getActorAttributesFromObject(this.actorObject, followersCount, followingCount)) | ||
26 | |||
27 | return sequelizeTypescript.transaction(async t => { | ||
28 | const server = await this.setServer(actorInstance, t) | ||
29 | |||
30 | await this.setImageIfNeeded(actorInstance, ActorImageType.AVATAR, t) | ||
31 | await this.setImageIfNeeded(actorInstance, ActorImageType.BANNER, t) | ||
32 | |||
33 | const { actorCreated, created } = await this.saveActor(actorInstance, t) | ||
34 | |||
35 | await this.tryToFixActorUrlIfNeeded(actorCreated, actorInstance, created, t) | ||
36 | |||
37 | if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { // Account or PeerTube instance | ||
38 | actorCreated.Account = await this.saveAccount(actorCreated, t) as MAccountDefault | ||
39 | actorCreated.Account.Actor = actorCreated | ||
40 | } | ||
41 | |||
42 | if (actorCreated.type === 'Group') { // Video channel | ||
43 | const channel = await this.saveVideoChannel(actorCreated, t) | ||
44 | actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: this.ownerActor.Account }) | ||
45 | } | ||
46 | |||
47 | actorCreated.Server = server | ||
48 | |||
49 | return actorCreated | ||
50 | }) | ||
51 | } | ||
52 | |||
53 | private async setServer (actor: MActor, t: Transaction) { | ||
54 | const actorHost = new URL(actor.url).host | ||
55 | |||
56 | const serverOptions = { | ||
57 | where: { | ||
58 | host: actorHost | ||
59 | }, | ||
60 | defaults: { | ||
61 | host: actorHost | ||
62 | }, | ||
63 | transaction: t | ||
64 | } | ||
65 | const [ server ] = await ServerModel.findOrCreate(serverOptions) | ||
66 | |||
67 | // Save our new account in database | ||
68 | actor.serverId = server.id | ||
69 | |||
70 | return server as MServer | ||
71 | } | ||
72 | |||
73 | private async setImageIfNeeded (actor: MActor, type: ActorImageType, t: Transaction) { | ||
74 | const imageInfo = getImageInfoFromObject(this.actorObject, type) | ||
75 | if (!imageInfo) return | ||
76 | |||
77 | return updateActorImageInstance(actor as MActorImages, type, imageInfo, t) | ||
78 | } | ||
79 | |||
80 | private async saveActor (actor: MActor, t: Transaction) { | ||
81 | // Force the actor creation using findOrCreate() instead of save() | ||
82 | // Sometimes Sequelize skips the save() when it thinks the instance already exists | ||
83 | // (which could be false in a retried query) | ||
84 | const [ actorCreated, created ] = await ActorModel.findOrCreate<MActorFullActor>({ | ||
85 | defaults: actor.toJSON(), | ||
86 | where: { | ||
87 | [Op.or]: [ | ||
88 | { | ||
89 | url: actor.url | ||
90 | }, | ||
91 | { | ||
92 | serverId: actor.serverId, | ||
93 | preferredUsername: actor.preferredUsername | ||
94 | } | ||
95 | ] | ||
96 | }, | ||
97 | transaction: t | ||
98 | }) | ||
99 | |||
100 | return { actorCreated, created } | ||
101 | } | ||
102 | |||
103 | private async tryToFixActorUrlIfNeeded (actorCreated: MActor, newActor: MActor, created: boolean, t: Transaction) { | ||
104 | // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards | ||
105 | if (created !== true && actorCreated.url !== newActor.url) { | ||
106 | // Only fix http://example.com/account/djidane to https://example.com/account/djidane | ||
107 | if (actorCreated.url.replace(/^http:\/\//, '') !== newActor.url.replace(/^https:\/\//, '')) { | ||
108 | throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${newActor.url}`) | ||
109 | } | ||
110 | |||
111 | actorCreated.url = newActor.url | ||
112 | await actorCreated.save({ transaction: t }) | ||
113 | } | ||
114 | } | ||
115 | |||
116 | private async saveAccount (actor: MActorId, t: Transaction) { | ||
117 | const [ accountCreated ] = await AccountModel.findOrCreate({ | ||
118 | defaults: { | ||
119 | name: getActorDisplayNameFromObject(this.actorObject), | ||
120 | description: this.actorObject.summary, | ||
121 | actorId: actor.id | ||
122 | }, | ||
123 | where: { | ||
124 | actorId: actor.id | ||
125 | }, | ||
126 | transaction: t | ||
127 | }) | ||
128 | |||
129 | return accountCreated as MAccount | ||
130 | } | ||
131 | |||
132 | private async saveVideoChannel (actor: MActorId, t: Transaction) { | ||
133 | const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ | ||
134 | defaults: { | ||
135 | name: getActorDisplayNameFromObject(this.actorObject), | ||
136 | description: this.actorObject.summary, | ||
137 | support: this.actorObject.support, | ||
138 | actorId: actor.id, | ||
139 | accountId: this.ownerActor.Account.id | ||
140 | }, | ||
141 | where: { | ||
142 | actorId: actor.id | ||
143 | }, | ||
144 | transaction: t | ||
145 | }) | ||
146 | |||
147 | return videoChannelCreated as MChannel | ||
148 | } | ||
149 | } | ||
diff --git a/server/lib/activitypub/actors/shared/index.ts b/server/lib/activitypub/actors/shared/index.ts new file mode 100644 index 000000000..52af1a8e1 --- /dev/null +++ b/server/lib/activitypub/actors/shared/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './creator' | ||
2 | export * from './object-to-model-attributes' | ||
3 | export * from './url-to-object' | ||
diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts new file mode 100644 index 000000000..1612b3ad0 --- /dev/null +++ b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts | |||
@@ -0,0 +1,70 @@ | |||
1 | import { getLowercaseExtension } from '@server/helpers/core-utils' | ||
2 | import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' | ||
3 | import { buildUUID } from '@server/helpers/uuid' | ||
4 | import { MIMETYPES } from '@server/initializers/constants' | ||
5 | import { ActorModel } from '@server/models/actor/actor' | ||
6 | import { FilteredModelAttributes } from '@server/types' | ||
7 | import { ActivityPubActor, ActorImageType } from '@shared/models' | ||
8 | |||
9 | function getActorAttributesFromObject ( | ||
10 | actorObject: ActivityPubActor, | ||
11 | followersCount: number, | ||
12 | followingCount: number | ||
13 | ): FilteredModelAttributes<ActorModel> { | ||
14 | return { | ||
15 | type: actorObject.type, | ||
16 | preferredUsername: actorObject.preferredUsername, | ||
17 | url: actorObject.id, | ||
18 | publicKey: actorObject.publicKey.publicKeyPem, | ||
19 | privateKey: null, | ||
20 | followersCount, | ||
21 | followingCount, | ||
22 | inboxUrl: actorObject.inbox, | ||
23 | outboxUrl: actorObject.outbox, | ||
24 | followersUrl: actorObject.followers, | ||
25 | followingUrl: actorObject.following, | ||
26 | |||
27 | sharedInboxUrl: actorObject.endpoints?.sharedInbox | ||
28 | ? actorObject.endpoints.sharedInbox | ||
29 | : null | ||
30 | } | ||
31 | } | ||
32 | |||
33 | function getImageInfoFromObject (actorObject: ActivityPubActor, type: ActorImageType) { | ||
34 | const mimetypes = MIMETYPES.IMAGE | ||
35 | const icon = type === ActorImageType.AVATAR | ||
36 | ? actorObject.icon | ||
37 | : actorObject.image | ||
38 | |||
39 | if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined | ||
40 | |||
41 | let extension: string | ||
42 | |||
43 | if (icon.mediaType) { | ||
44 | extension = mimetypes.MIMETYPE_EXT[icon.mediaType] | ||
45 | } else { | ||
46 | const tmp = getLowercaseExtension(icon.url) | ||
47 | |||
48 | if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp | ||
49 | } | ||
50 | |||
51 | if (!extension) return undefined | ||
52 | |||
53 | return { | ||
54 | name: buildUUID() + extension, | ||
55 | fileUrl: icon.url, | ||
56 | height: icon.height, | ||
57 | width: icon.width, | ||
58 | type | ||
59 | } | ||
60 | } | ||
61 | |||
62 | function getActorDisplayNameFromObject (actorObject: ActivityPubActor) { | ||
63 | return actorObject.name || actorObject.preferredUsername | ||
64 | } | ||
65 | |||
66 | export { | ||
67 | getActorAttributesFromObject, | ||
68 | getImageInfoFromObject, | ||
69 | getActorDisplayNameFromObject | ||
70 | } | ||
diff --git a/server/lib/activitypub/actors/shared/url-to-object.ts b/server/lib/activitypub/actors/shared/url-to-object.ts new file mode 100644 index 000000000..f4f16b044 --- /dev/null +++ b/server/lib/activitypub/actors/shared/url-to-object.ts | |||
@@ -0,0 +1,54 @@ | |||
1 | |||
2 | import { checkUrlsSameHost } from '@server/helpers/activitypub' | ||
3 | import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { doJSONRequest } from '@server/helpers/requests' | ||
6 | import { ActivityPubActor, ActivityPubOrderedCollection } from '@shared/models' | ||
7 | |||
8 | async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> { | ||
9 | logger.info('Fetching remote actor %s.', actorUrl) | ||
10 | |||
11 | const { body, statusCode } = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true }) | ||
12 | |||
13 | if (sanitizeAndCheckActorObject(body) === false) { | ||
14 | logger.debug('Remote actor JSON is not valid.', { actorJSON: body }) | ||
15 | return { actorObject: undefined, statusCode: statusCode } | ||
16 | } | ||
17 | |||
18 | if (checkUrlsSameHost(body.id, actorUrl) !== true) { | ||
19 | logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, body.id) | ||
20 | return { actorObject: undefined, statusCode: statusCode } | ||
21 | } | ||
22 | |||
23 | return { | ||
24 | statusCode, | ||
25 | |||
26 | actorObject: body | ||
27 | } | ||
28 | } | ||
29 | |||
30 | async function fetchActorFollowsCount (actorObject: ActivityPubActor) { | ||
31 | const followersCount = await fetchActorTotalItems(actorObject.followers) | ||
32 | const followingCount = await fetchActorTotalItems(actorObject.following) | ||
33 | |||
34 | return { followersCount, followingCount } | ||
35 | } | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | export { | ||
39 | fetchActorFollowsCount, | ||
40 | fetchRemoteActor | ||
41 | } | ||
42 | |||
43 | // --------------------------------------------------------------------------- | ||
44 | |||
45 | async function fetchActorTotalItems (url: string) { | ||
46 | try { | ||
47 | const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true }) | ||
48 | |||
49 | return body.totalItems || 0 | ||
50 | } catch (err) { | ||
51 | logger.warn('Cannot fetch remote actor count %s.', url, { err }) | ||
52 | return 0 | ||
53 | } | ||
54 | } | ||
diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts new file mode 100644 index 000000000..de5e03eee --- /dev/null +++ b/server/lib/activitypub/actors/updater.ts | |||
@@ -0,0 +1,90 @@ | |||
1 | import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
4 | import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models' | ||
5 | import { ActivityPubActor, ActorImageType } from '@shared/models' | ||
6 | import { updateActorImageInstance } from './image' | ||
7 | import { fetchActorFollowsCount } from './shared' | ||
8 | import { getImageInfoFromObject } from './shared/object-to-model-attributes' | ||
9 | |||
10 | export class APActorUpdater { | ||
11 | |||
12 | private accountOrChannel: MAccount | MChannel | ||
13 | |||
14 | private readonly actorFieldsSave: object | ||
15 | private readonly accountOrChannelFieldsSave: object | ||
16 | |||
17 | constructor ( | ||
18 | private readonly actorObject: ActivityPubActor, | ||
19 | private readonly actor: MActorFull | ||
20 | ) { | ||
21 | this.actorFieldsSave = this.actor.toJSON() | ||
22 | |||
23 | if (this.actorObject.type === 'Group') this.accountOrChannel = this.actor.VideoChannel | ||
24 | else this.accountOrChannel = this.actor.Account | ||
25 | |||
26 | this.accountOrChannelFieldsSave = this.accountOrChannel.toJSON() | ||
27 | } | ||
28 | |||
29 | async update () { | ||
30 | const avatarInfo = getImageInfoFromObject(this.actorObject, ActorImageType.AVATAR) | ||
31 | const bannerInfo = getImageInfoFromObject(this.actorObject, ActorImageType.BANNER) | ||
32 | |||
33 | try { | ||
34 | await this.updateActorInstance(this.actor, this.actorObject) | ||
35 | |||
36 | this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername | ||
37 | this.accountOrChannel.description = this.actorObject.summary | ||
38 | |||
39 | if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support | ||
40 | |||
41 | await runInReadCommittedTransaction(async t => { | ||
42 | await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t) | ||
43 | await updateActorImageInstance(this.actor, ActorImageType.BANNER, bannerInfo, t) | ||
44 | }) | ||
45 | |||
46 | await runInReadCommittedTransaction(async t => { | ||
47 | await this.actor.save({ transaction: t }) | ||
48 | await this.accountOrChannel.save({ transaction: t }) | ||
49 | }) | ||
50 | |||
51 | logger.info('Remote account %s updated', this.actorObject.url) | ||
52 | } catch (err) { | ||
53 | if (this.actor !== undefined && this.actorFieldsSave !== undefined) { | ||
54 | resetSequelizeInstance(this.actor, this.actorFieldsSave) | ||
55 | } | ||
56 | |||
57 | if (this.accountOrChannel !== undefined && this.accountOrChannelFieldsSave !== undefined) { | ||
58 | resetSequelizeInstance(this.accountOrChannel, this.accountOrChannelFieldsSave) | ||
59 | } | ||
60 | |||
61 | // This is just a debug because we will retry the insert | ||
62 | logger.debug('Cannot update the remote account.', { err }) | ||
63 | throw err | ||
64 | } | ||
65 | } | ||
66 | |||
67 | private async updateActorInstance (actorInstance: MActor, actorObject: ActivityPubActor) { | ||
68 | const { followersCount, followingCount } = await fetchActorFollowsCount(actorObject) | ||
69 | |||
70 | actorInstance.type = actorObject.type | ||
71 | actorInstance.preferredUsername = actorObject.preferredUsername | ||
72 | actorInstance.url = actorObject.id | ||
73 | actorInstance.publicKey = actorObject.publicKey.publicKeyPem | ||
74 | actorInstance.followersCount = followersCount | ||
75 | actorInstance.followingCount = followingCount | ||
76 | actorInstance.inboxUrl = actorObject.inbox | ||
77 | actorInstance.outboxUrl = actorObject.outbox | ||
78 | actorInstance.followersUrl = actorObject.followers | ||
79 | actorInstance.followingUrl = actorObject.following | ||
80 | |||
81 | if (actorObject.published) actorInstance.remoteCreatedAt = new Date(actorObject.published) | ||
82 | |||
83 | if (actorObject.endpoints?.sharedInbox) { | ||
84 | actorInstance.sharedInboxUrl = actorObject.endpoints.sharedInbox | ||
85 | } | ||
86 | |||
87 | // Force actor update | ||
88 | actorInstance.changed('updatedAt', true) | ||
89 | } | ||
90 | } | ||
diff --git a/server/lib/activitypub/actors/webfinger.ts b/server/lib/activitypub/actors/webfinger.ts new file mode 100644 index 000000000..1c7ec4717 --- /dev/null +++ b/server/lib/activitypub/actors/webfinger.ts | |||
@@ -0,0 +1,67 @@ | |||
1 | import * as WebFinger from 'webfinger.js' | ||
2 | import { isProdInstance } from '@server/helpers/core-utils' | ||
3 | import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' | ||
4 | import { REQUEST_TIMEOUT, WEBSERVER } from '@server/initializers/constants' | ||
5 | import { ActorModel } from '@server/models/actor/actor' | ||
6 | import { MActorFull } from '@server/types/models' | ||
7 | import { WebFingerData } from '@shared/models' | ||
8 | |||
9 | const webfinger = new WebFinger({ | ||
10 | webfist_fallback: false, | ||
11 | tls_only: isProdInstance(), | ||
12 | uri_fallback: false, | ||
13 | request_timeout: REQUEST_TIMEOUT | ||
14 | }) | ||
15 | |||
16 | async function loadActorUrlOrGetFromWebfinger (uriArg: string) { | ||
17 | // Handle strings like @toto@example.com | ||
18 | const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg | ||
19 | |||
20 | const [ name, host ] = uri.split('@') | ||
21 | let actor: MActorFull | ||
22 | |||
23 | if (!host || host === WEBSERVER.HOST) { | ||
24 | actor = await ActorModel.loadLocalByName(name) | ||
25 | } else { | ||
26 | actor = await ActorModel.loadByNameAndHost(name, host) | ||
27 | } | ||
28 | |||
29 | if (actor) return actor.url | ||
30 | |||
31 | return getUrlFromWebfinger(uri) | ||
32 | } | ||
33 | |||
34 | async function getUrlFromWebfinger (uri: string) { | ||
35 | const webfingerData: WebFingerData = await webfingerLookup(uri) | ||
36 | return getLinkOrThrow(webfingerData) | ||
37 | } | ||
38 | |||
39 | // --------------------------------------------------------------------------- | ||
40 | |||
41 | export { | ||
42 | getUrlFromWebfinger, | ||
43 | loadActorUrlOrGetFromWebfinger | ||
44 | } | ||
45 | |||
46 | // --------------------------------------------------------------------------- | ||
47 | |||
48 | function getLinkOrThrow (webfingerData: WebFingerData) { | ||
49 | if (Array.isArray(webfingerData.links) === false) throw new Error('WebFinger links is not an array.') | ||
50 | |||
51 | const selfLink = webfingerData.links.find(l => l.rel === 'self') | ||
52 | if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) { | ||
53 | throw new Error('Cannot find self link or href is not a valid URL.') | ||
54 | } | ||
55 | |||
56 | return selfLink.href | ||
57 | } | ||
58 | |||
59 | function webfingerLookup (nameWithHost: string) { | ||
60 | return new Promise<WebFingerData>((res, rej) => { | ||
61 | webfinger.lookup(nameWithHost, (err, p) => { | ||
62 | if (err) return rej(err) | ||
63 | |||
64 | return res(p.object) | ||
65 | }) | ||
66 | }) | ||
67 | } | ||
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts index 2986714d3..d0558f191 100644 --- a/server/lib/activitypub/audience.ts +++ b/server/lib/activitypub/audience.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { ActivityAudience } from '../../../shared/models/activitypub' | 2 | import { ActivityAudience } from '../../../shared/models/activitypub' |
3 | import { ACTIVITY_PUB } from '../../initializers/constants' | 3 | import { ACTIVITY_PUB } from '../../initializers/constants' |
4 | import { ActorModel } from '../../models/activitypub/actor' | 4 | import { ActorModel } from '../../models/actor/actor' |
5 | import { VideoModel } from '../../models/video/video' | 5 | import { VideoModel } from '../../models/video/video' |
6 | import { VideoShareModel } from '../../models/video/video-share' | 6 | import { VideoShareModel } from '../../models/video/video-share' |
7 | import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '../../types/models' | 7 | import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '../../types/models' |
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 2e6dd34e0..a16d2cd93 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -1,54 +1,27 @@ | |||
1 | import { CacheFileObject } from '../../../shared/index' | ||
2 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | ||
3 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
4 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
5 | import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' | 2 | import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' |
3 | import { CacheFileObject } from '../../../shared/index' | ||
4 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | ||
6 | 6 | ||
7 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { | 7 | async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { |
8 | 8 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) | |
9 | if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { | ||
10 | const url = cacheFileObject.url | ||
11 | |||
12 | const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) | ||
13 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | ||
14 | 9 | ||
15 | return { | 10 | if (redundancyModel) { |
16 | expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, | 11 | return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) |
17 | url: cacheFileObject.id, | ||
18 | fileUrl: url.href, | ||
19 | strategy: null, | ||
20 | videoStreamingPlaylistId: playlist.id, | ||
21 | actorId: byActor.id | ||
22 | } | ||
23 | } | 12 | } |
24 | 13 | ||
25 | const url = cacheFileObject.url | 14 | return createCacheFile(cacheFileObject, video, byActor, t) |
26 | const videoFile = video.VideoFiles.find(f => { | ||
27 | return f.resolution === url.height && f.fps === url.fps | ||
28 | }) | ||
29 | |||
30 | if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) | ||
31 | |||
32 | return { | ||
33 | expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, | ||
34 | url: cacheFileObject.id, | ||
35 | fileUrl: url.href, | ||
36 | strategy: null, | ||
37 | videoFileId: videoFile.id, | ||
38 | actorId: byActor.id | ||
39 | } | ||
40 | } | 15 | } |
41 | 16 | ||
42 | async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { | 17 | // --------------------------------------------------------------------------- |
43 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) | ||
44 | 18 | ||
45 | if (!redundancyModel) { | 19 | export { |
46 | await createCacheFile(cacheFileObject, video, byActor, t) | 20 | createOrUpdateCacheFile |
47 | } else { | ||
48 | await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) | ||
49 | } | ||
50 | } | 21 | } |
51 | 22 | ||
23 | // --------------------------------------------------------------------------- | ||
24 | |||
52 | function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { | 25 | function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { |
53 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) | 26 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) |
54 | 27 | ||
@@ -74,9 +47,37 @@ function updateCacheFile ( | |||
74 | return redundancyModel.save({ transaction: t }) | 47 | return redundancyModel.save({ transaction: t }) |
75 | } | 48 | } |
76 | 49 | ||
77 | export { | 50 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { |
78 | createOrUpdateCacheFile, | 51 | |
79 | createCacheFile, | 52 | if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { |
80 | updateCacheFile, | 53 | const url = cacheFileObject.url |
81 | cacheFileActivityObjectToDBAttributes | 54 | |
55 | const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) | ||
56 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | ||
57 | |||
58 | return { | ||
59 | expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, | ||
60 | url: cacheFileObject.id, | ||
61 | fileUrl: url.href, | ||
62 | strategy: null, | ||
63 | videoStreamingPlaylistId: playlist.id, | ||
64 | actorId: byActor.id | ||
65 | } | ||
66 | } | ||
67 | |||
68 | const url = cacheFileObject.url | ||
69 | const videoFile = video.VideoFiles.find(f => { | ||
70 | return f.resolution === url.height && f.fps === url.fps | ||
71 | }) | ||
72 | |||
73 | if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) | ||
74 | |||
75 | return { | ||
76 | expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, | ||
77 | url: cacheFileObject.id, | ||
78 | fileUrl: url.href, | ||
79 | strategy: null, | ||
80 | videoFileId: videoFile.id, | ||
81 | actorId: byActor.id | ||
82 | } | ||
82 | } | 83 | } |
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 278abf7de..cd117f571 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts | |||
@@ -3,7 +3,7 @@ import { URL } from 'url' | |||
3 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | 3 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' |
4 | import { logger } from '../../helpers/logger' | 4 | import { logger } from '../../helpers/logger' |
5 | import { doJSONRequest } from '../../helpers/requests' | 5 | import { doJSONRequest } from '../../helpers/requests' |
6 | import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' | 6 | import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants' |
7 | 7 | ||
8 | type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) | 8 | type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) |
9 | type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) | 9 | type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) |
@@ -13,10 +13,7 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction | |||
13 | 13 | ||
14 | logger.info('Crawling ActivityPub data on %s.', url) | 14 | logger.info('Crawling ActivityPub data on %s.', url) |
15 | 15 | ||
16 | const options = { | 16 | const options = { activityPub: true } |
17 | activityPub: true, | ||
18 | timeout: REQUEST_TIMEOUT | ||
19 | } | ||
20 | 17 | ||
21 | const startDate = new Date() | 18 | const startDate = new Date() |
22 | 19 | ||
diff --git a/server/lib/activitypub/follow.ts b/server/lib/activitypub/follow.ts index 351499bd1..c1bd667e0 100644 --- a/server/lib/activitypub/follow.ts +++ b/server/lib/activitypub/follow.ts | |||
@@ -1,12 +1,13 @@ | |||
1 | import { MActorFollowActors } from '../../types/models' | 1 | import { Transaction } from 'sequelize' |
2 | import { getServerActor } from '@server/models/application/application' | ||
3 | import { logger } from '../../helpers/logger' | ||
2 | import { CONFIG } from '../../initializers/config' | 4 | import { CONFIG } from '../../initializers/config' |
3 | import { SERVER_ACTOR_NAME } from '../../initializers/constants' | 5 | import { SERVER_ACTOR_NAME } from '../../initializers/constants' |
4 | import { JobQueue } from '../job-queue' | ||
5 | import { logger } from '../../helpers/logger' | ||
6 | import { ServerModel } from '../../models/server/server' | 6 | import { ServerModel } from '../../models/server/server' |
7 | import { getServerActor } from '@server/models/application/application' | 7 | import { MActorFollowActors } from '../../types/models' |
8 | import { JobQueue } from '../job-queue' | ||
8 | 9 | ||
9 | async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors) { | 10 | async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors, transaction?: Transaction) { |
10 | if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return | 11 | if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return |
11 | 12 | ||
12 | const follower = actorFollow.ActorFollower | 13 | const follower = actorFollow.ActorFollower |
@@ -16,7 +17,7 @@ async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors) { | |||
16 | 17 | ||
17 | const me = await getServerActor() | 18 | const me = await getServerActor() |
18 | 19 | ||
19 | const server = await ServerModel.load(follower.serverId) | 20 | const server = await ServerModel.load(follower.serverId, transaction) |
20 | const host = server.host | 21 | const host = server.host |
21 | 22 | ||
22 | const payload = { | 23 | const payload = { |
diff --git a/server/lib/activitypub/outbox.ts b/server/lib/activitypub/outbox.ts new file mode 100644 index 000000000..ecdc33a77 --- /dev/null +++ b/server/lib/activitypub/outbox.ts | |||
@@ -0,0 +1,24 @@ | |||
1 | import { logger } from '@server/helpers/logger' | ||
2 | import { ActorModel } from '@server/models/actor/actor' | ||
3 | import { getServerActor } from '@server/models/application/application' | ||
4 | import { JobQueue } from '../job-queue' | ||
5 | |||
6 | async function addFetchOutboxJob (actor: Pick<ActorModel, 'id' | 'outboxUrl'>) { | ||
7 | // Don't fetch ourselves | ||
8 | const serverActor = await getServerActor() | ||
9 | if (serverActor.id === actor.id) { | ||
10 | logger.error('Cannot fetch our own outbox!') | ||
11 | return undefined | ||
12 | } | ||
13 | |||
14 | const payload = { | ||
15 | uri: actor.outboxUrl, | ||
16 | type: 'activity' as 'activity' | ||
17 | } | ||
18 | |||
19 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | ||
20 | } | ||
21 | |||
22 | export { | ||
23 | addFetchOutboxJob | ||
24 | } | ||
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts deleted file mode 100644 index 7166c68a6..000000000 --- a/server/lib/activitypub/playlist.ts +++ /dev/null | |||
@@ -1,204 +0,0 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' | ||
3 | import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' | ||
4 | import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' | ||
5 | import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' | ||
6 | import { checkUrlsSameHost } from '../../helpers/activitypub' | ||
7 | import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' | ||
8 | import { isArray } from '../../helpers/custom-validators/misc' | ||
9 | import { logger } from '../../helpers/logger' | ||
10 | import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests' | ||
11 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | ||
12 | import { sequelizeTypescript } from '../../initializers/database' | ||
13 | import { VideoPlaylistModel } from '../../models/video/video-playlist' | ||
14 | import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' | ||
15 | import { MAccountDefault, MAccountId, MVideoId } from '../../types/models' | ||
16 | import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist' | ||
17 | import { FilteredModelAttributes } from '../../types/sequelize' | ||
18 | import { createPlaylistMiniatureFromUrl } from '../thumbnail' | ||
19 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
20 | import { crawlCollectionPage } from './crawl' | ||
21 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | ||
22 | |||
23 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { | ||
24 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | ||
25 | ? VideoPlaylistPrivacy.PUBLIC | ||
26 | : VideoPlaylistPrivacy.UNLISTED | ||
27 | |||
28 | return { | ||
29 | name: playlistObject.name, | ||
30 | description: playlistObject.content, | ||
31 | privacy, | ||
32 | url: playlistObject.id, | ||
33 | uuid: playlistObject.uuid, | ||
34 | ownerAccountId: byAccount.id, | ||
35 | videoChannelId: null, | ||
36 | createdAt: new Date(playlistObject.published), | ||
37 | updatedAt: new Date(playlistObject.updated) | ||
38 | } | ||
39 | } | ||
40 | |||
41 | function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) { | ||
42 | return { | ||
43 | position: elementObject.position, | ||
44 | url: elementObject.id, | ||
45 | startTimestamp: elementObject.startTimestamp || null, | ||
46 | stopTimestamp: elementObject.stopTimestamp || null, | ||
47 | videoPlaylistId: videoPlaylist.id, | ||
48 | videoId: video.id | ||
49 | } | ||
50 | } | ||
51 | |||
52 | async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) { | ||
53 | await Bluebird.map(playlistUrls, async playlistUrl => { | ||
54 | try { | ||
55 | const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) | ||
56 | if (exists === true) return | ||
57 | |||
58 | // Fetch url | ||
59 | const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true }) | ||
60 | |||
61 | if (!isPlaylistObjectValid(body)) { | ||
62 | throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`) | ||
63 | } | ||
64 | |||
65 | if (!isArray(body.to)) { | ||
66 | throw new Error('Playlist does not have an audience.') | ||
67 | } | ||
68 | |||
69 | return createOrUpdateVideoPlaylist(body, account, body.to) | ||
70 | } catch (err) { | ||
71 | logger.warn('Cannot add playlist element %s.', playlistUrl, { err }) | ||
72 | } | ||
73 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
74 | } | ||
75 | |||
76 | async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { | ||
77 | const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to) | ||
78 | |||
79 | if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) { | ||
80 | const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0]) | ||
81 | |||
82 | if (actor.VideoChannel) { | ||
83 | playlistAttributes.videoChannelId = actor.VideoChannel.id | ||
84 | } else { | ||
85 | logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject }) | ||
86 | } | ||
87 | } | ||
88 | |||
89 | const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true }) | ||
90 | |||
91 | let accItems: string[] = [] | ||
92 | await crawlCollectionPage<string>(playlistObject.id, items => { | ||
93 | accItems = accItems.concat(items) | ||
94 | |||
95 | return Promise.resolve() | ||
96 | }) | ||
97 | |||
98 | const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null) | ||
99 | |||
100 | if (playlistObject.icon) { | ||
101 | try { | ||
102 | const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist }) | ||
103 | await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined) | ||
104 | } catch (err) { | ||
105 | logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err }) | ||
106 | } | ||
107 | } else if (refreshedPlaylist.hasThumbnail()) { | ||
108 | await refreshedPlaylist.Thumbnail.destroy() | ||
109 | refreshedPlaylist.Thumbnail = null | ||
110 | } | ||
111 | |||
112 | return resetVideoPlaylistElements(accItems, refreshedPlaylist) | ||
113 | } | ||
114 | |||
115 | async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> { | ||
116 | if (!videoPlaylist.isOutdated()) return videoPlaylist | ||
117 | |||
118 | try { | ||
119 | const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) | ||
120 | |||
121 | if (playlistObject === undefined) { | ||
122 | logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url) | ||
123 | |||
124 | await videoPlaylist.setAsRefreshed() | ||
125 | return videoPlaylist | ||
126 | } | ||
127 | |||
128 | const byAccount = videoPlaylist.OwnerAccount | ||
129 | await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to) | ||
130 | |||
131 | return videoPlaylist | ||
132 | } catch (err) { | ||
133 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
134 | logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url) | ||
135 | |||
136 | await videoPlaylist.destroy() | ||
137 | return undefined | ||
138 | } | ||
139 | |||
140 | logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err }) | ||
141 | |||
142 | await videoPlaylist.setAsRefreshed() | ||
143 | return videoPlaylist | ||
144 | } | ||
145 | } | ||
146 | |||
147 | // --------------------------------------------------------------------------- | ||
148 | |||
149 | export { | ||
150 | createAccountPlaylists, | ||
151 | playlistObjectToDBAttributes, | ||
152 | playlistElementObjectToDBAttributes, | ||
153 | createOrUpdateVideoPlaylist, | ||
154 | refreshVideoPlaylistIfNeeded | ||
155 | } | ||
156 | |||
157 | // --------------------------------------------------------------------------- | ||
158 | |||
159 | async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { | ||
160 | const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = [] | ||
161 | |||
162 | await Bluebird.map(elementUrls, async elementUrl => { | ||
163 | try { | ||
164 | const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true }) | ||
165 | |||
166 | if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`) | ||
167 | |||
168 | if (checkUrlsSameHost(body.id, elementUrl) !== true) { | ||
169 | throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) | ||
170 | } | ||
171 | |||
172 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' }) | ||
173 | |||
174 | elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video)) | ||
175 | } catch (err) { | ||
176 | logger.warn('Cannot add playlist element %s.', elementUrl, { err }) | ||
177 | } | ||
178 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
179 | |||
180 | await sequelizeTypescript.transaction(async t => { | ||
181 | await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) | ||
182 | |||
183 | for (const element of elementsToCreate) { | ||
184 | await VideoPlaylistElementModel.create(element, { transaction: t }) | ||
185 | } | ||
186 | }) | ||
187 | |||
188 | logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length) | ||
189 | |||
190 | return undefined | ||
191 | } | ||
192 | |||
193 | async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { | ||
194 | logger.info('Fetching remote playlist %s.', playlistUrl) | ||
195 | |||
196 | const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true }) | ||
197 | |||
198 | if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { | ||
199 | logger.debug('Remote video playlist JSON is not valid.', { body }) | ||
200 | return { statusCode, playlistObject: undefined } | ||
201 | } | ||
202 | |||
203 | return { statusCode, playlistObject: body } | ||
204 | } | ||
diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts new file mode 100644 index 000000000..ea3e61ac5 --- /dev/null +++ b/server/lib/activitypub/playlists/create-update.ts | |||
@@ -0,0 +1,156 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { getAPId } from '@server/helpers/activitypub' | ||
3 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
5 | import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' | ||
6 | import { sequelizeTypescript } from '@server/initializers/database' | ||
7 | import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail' | ||
8 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | ||
9 | import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' | ||
10 | import { FilteredModelAttributes } from '@server/types' | ||
11 | import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models' | ||
12 | import { AttributesOnly } from '@shared/core-utils' | ||
13 | import { PlaylistObject } from '@shared/models' | ||
14 | import { getOrCreateAPActor } from '../actors' | ||
15 | import { crawlCollectionPage } from '../crawl' | ||
16 | import { getOrCreateAPVideo } from '../videos' | ||
17 | import { | ||
18 | fetchRemotePlaylistElement, | ||
19 | fetchRemoteVideoPlaylist, | ||
20 | playlistElementObjectToDBAttributes, | ||
21 | playlistObjectToDBAttributes | ||
22 | } from './shared' | ||
23 | |||
24 | const lTags = loggerTagsFactory('ap', 'video-playlist') | ||
25 | |||
26 | async function createAccountPlaylists (playlistUrls: string[]) { | ||
27 | await Bluebird.map(playlistUrls, async playlistUrl => { | ||
28 | try { | ||
29 | const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) | ||
30 | if (exists === true) return | ||
31 | |||
32 | const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) | ||
33 | |||
34 | if (playlistObject === undefined) { | ||
35 | throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`) | ||
36 | } | ||
37 | |||
38 | return createOrUpdateVideoPlaylist(playlistObject) | ||
39 | } catch (err) { | ||
40 | logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) }) | ||
41 | } | ||
42 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
43 | } | ||
44 | |||
45 | async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) { | ||
46 | const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to) | ||
47 | |||
48 | await setVideoChannel(playlistObject, playlistAttributes) | ||
49 | |||
50 | const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true }) | ||
51 | |||
52 | const playlistElementUrls = await fetchElementUrls(playlistObject) | ||
53 | |||
54 | // Refetch playlist from DB since elements fetching could be long in time | ||
55 | const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null) | ||
56 | |||
57 | await updatePlaylistThumbnail(playlistObject, playlist) | ||
58 | |||
59 | const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist) | ||
60 | playlist.setVideosLength(elementsLength) | ||
61 | |||
62 | return playlist | ||
63 | } | ||
64 | |||
65 | // --------------------------------------------------------------------------- | ||
66 | |||
67 | export { | ||
68 | createAccountPlaylists, | ||
69 | createOrUpdateVideoPlaylist | ||
70 | } | ||
71 | |||
72 | // --------------------------------------------------------------------------- | ||
73 | |||
74 | async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) { | ||
75 | if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) { | ||
76 | throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) | ||
77 | } | ||
78 | |||
79 | const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all') | ||
80 | |||
81 | if (!actor.VideoChannel) { | ||
82 | logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) | ||
83 | return | ||
84 | } | ||
85 | |||
86 | playlistAttributes.videoChannelId = actor.VideoChannel.id | ||
87 | playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id | ||
88 | } | ||
89 | |||
90 | async function fetchElementUrls (playlistObject: PlaylistObject) { | ||
91 | let accItems: string[] = [] | ||
92 | await crawlCollectionPage<string>(playlistObject.id, items => { | ||
93 | accItems = accItems.concat(items) | ||
94 | |||
95 | return Promise.resolve() | ||
96 | }) | ||
97 | |||
98 | return accItems | ||
99 | } | ||
100 | |||
101 | async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) { | ||
102 | if (playlistObject.icon) { | ||
103 | let thumbnailModel: MThumbnail | ||
104 | |||
105 | try { | ||
106 | thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) | ||
107 | await playlist.setAndSaveThumbnail(thumbnailModel, undefined) | ||
108 | } catch (err) { | ||
109 | logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) | ||
110 | |||
111 | if (thumbnailModel) await thumbnailModel.removeThumbnail() | ||
112 | } | ||
113 | |||
114 | return | ||
115 | } | ||
116 | |||
117 | // Playlist does not have an icon, destroy existing one | ||
118 | if (playlist.hasThumbnail()) { | ||
119 | await playlist.Thumbnail.destroy() | ||
120 | playlist.Thumbnail = null | ||
121 | } | ||
122 | } | ||
123 | |||
124 | async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { | ||
125 | const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist) | ||
126 | |||
127 | await sequelizeTypescript.transaction(async t => { | ||
128 | await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) | ||
129 | |||
130 | for (const element of elementsToCreate) { | ||
131 | await VideoPlaylistElementModel.create(element, { transaction: t }) | ||
132 | } | ||
133 | }) | ||
134 | |||
135 | logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url)) | ||
136 | |||
137 | return elementsToCreate.length | ||
138 | } | ||
139 | |||
140 | async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) { | ||
141 | const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = [] | ||
142 | |||
143 | await Bluebird.map(elementUrls, async elementUrl => { | ||
144 | try { | ||
145 | const { elementObject } = await fetchRemotePlaylistElement(elementUrl) | ||
146 | |||
147 | const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' }) | ||
148 | |||
149 | elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video)) | ||
150 | } catch (err) { | ||
151 | logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) }) | ||
152 | } | ||
153 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
154 | |||
155 | return elementsToCreate | ||
156 | } | ||
diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts new file mode 100644 index 000000000..2c19c503a --- /dev/null +++ b/server/lib/activitypub/playlists/get.ts | |||
@@ -0,0 +1,35 @@ | |||
1 | import { getAPId } from '@server/helpers/activitypub' | ||
2 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | ||
3 | import { MVideoPlaylistFullSummary } from '@server/types/models' | ||
4 | import { APObject } from '@shared/models' | ||
5 | import { createOrUpdateVideoPlaylist } from './create-update' | ||
6 | import { scheduleRefreshIfNeeded } from './refresh' | ||
7 | import { fetchRemoteVideoPlaylist } from './shared' | ||
8 | |||
9 | async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObject): Promise<MVideoPlaylistFullSummary> { | ||
10 | const playlistUrl = getAPId(playlistObjectArg) | ||
11 | |||
12 | const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl) | ||
13 | |||
14 | if (playlistFromDatabase) { | ||
15 | scheduleRefreshIfNeeded(playlistFromDatabase) | ||
16 | |||
17 | return playlistFromDatabase | ||
18 | } | ||
19 | |||
20 | const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) | ||
21 | if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl) | ||
22 | |||
23 | // playlistUrl is just an alias/rediraction, so process object id instead | ||
24 | if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject) | ||
25 | |||
26 | const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject) | ||
27 | |||
28 | return playlistCreated | ||
29 | } | ||
30 | |||
31 | // --------------------------------------------------------------------------- | ||
32 | |||
33 | export { | ||
34 | getOrCreateAPVideoPlaylist | ||
35 | } | ||
diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts new file mode 100644 index 000000000..e2470a674 --- /dev/null +++ b/server/lib/activitypub/playlists/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './get' | ||
2 | export * from './create-update' | ||
3 | export * from './refresh' | ||
diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts new file mode 100644 index 000000000..ef3cb3fe4 --- /dev/null +++ b/server/lib/activitypub/playlists/refresh.ts | |||
@@ -0,0 +1,53 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
2 | import { PeerTubeRequestError } from '@server/helpers/requests' | ||
3 | import { JobQueue } from '@server/lib/job-queue' | ||
4 | import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models' | ||
5 | import { HttpStatusCode } from '@shared/core-utils' | ||
6 | import { createOrUpdateVideoPlaylist } from './create-update' | ||
7 | import { fetchRemoteVideoPlaylist } from './shared' | ||
8 | |||
9 | function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) { | ||
10 | if (!playlist.isOutdated()) return | ||
11 | |||
12 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } }) | ||
13 | } | ||
14 | |||
15 | async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> { | ||
16 | if (!videoPlaylist.isOutdated()) return videoPlaylist | ||
17 | |||
18 | const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url) | ||
19 | |||
20 | logger.info('Refreshing playlist %s.', videoPlaylist.url, lTags()) | ||
21 | |||
22 | try { | ||
23 | const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) | ||
24 | |||
25 | if (playlistObject === undefined) { | ||
26 | logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags()) | ||
27 | |||
28 | await videoPlaylist.setAsRefreshed() | ||
29 | return videoPlaylist | ||
30 | } | ||
31 | |||
32 | await createOrUpdateVideoPlaylist(playlistObject) | ||
33 | |||
34 | return videoPlaylist | ||
35 | } catch (err) { | ||
36 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
37 | logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags()) | ||
38 | |||
39 | await videoPlaylist.destroy() | ||
40 | return undefined | ||
41 | } | ||
42 | |||
43 | logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() }) | ||
44 | |||
45 | await videoPlaylist.setAsRefreshed() | ||
46 | return videoPlaylist | ||
47 | } | ||
48 | } | ||
49 | |||
50 | export { | ||
51 | scheduleRefreshIfNeeded, | ||
52 | refreshVideoPlaylistIfNeeded | ||
53 | } | ||
diff --git a/server/lib/activitypub/playlists/shared/index.ts b/server/lib/activitypub/playlists/shared/index.ts new file mode 100644 index 000000000..a217f2291 --- /dev/null +++ b/server/lib/activitypub/playlists/shared/index.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | export * from './object-to-model-attributes' | ||
2 | export * from './url-to-object' | ||
diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts new file mode 100644 index 000000000..70fd335bc --- /dev/null +++ b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { ACTIVITY_PUB } from '@server/initializers/constants' | ||
2 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | ||
3 | import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' | ||
4 | import { MVideoId, MVideoPlaylistId } from '@server/types/models' | ||
5 | import { AttributesOnly } from '@shared/core-utils' | ||
6 | import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models' | ||
7 | |||
8 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) { | ||
9 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | ||
10 | ? VideoPlaylistPrivacy.PUBLIC | ||
11 | : VideoPlaylistPrivacy.UNLISTED | ||
12 | |||
13 | return { | ||
14 | name: playlistObject.name, | ||
15 | description: playlistObject.content, | ||
16 | privacy, | ||
17 | url: playlistObject.id, | ||
18 | uuid: playlistObject.uuid, | ||
19 | ownerAccountId: null, | ||
20 | videoChannelId: null, | ||
21 | createdAt: new Date(playlistObject.published), | ||
22 | updatedAt: new Date(playlistObject.updated) | ||
23 | } as AttributesOnly<VideoPlaylistModel> | ||
24 | } | ||
25 | |||
26 | function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) { | ||
27 | return { | ||
28 | position: elementObject.position, | ||
29 | url: elementObject.id, | ||
30 | startTimestamp: elementObject.startTimestamp || null, | ||
31 | stopTimestamp: elementObject.stopTimestamp || null, | ||
32 | videoPlaylistId: videoPlaylist.id, | ||
33 | videoId: video.id | ||
34 | } as AttributesOnly<VideoPlaylistElementModel> | ||
35 | } | ||
36 | |||
37 | export { | ||
38 | playlistObjectToDBAttributes, | ||
39 | playlistElementObjectToDBAttributes | ||
40 | } | ||
diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts new file mode 100644 index 000000000..ec8c01255 --- /dev/null +++ b/server/lib/activitypub/playlists/shared/url-to-object.ts | |||
@@ -0,0 +1,47 @@ | |||
1 | import { isArray } from 'lodash' | ||
2 | import { checkUrlsSameHost } from '@server/helpers/activitypub' | ||
3 | import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist' | ||
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
5 | import { doJSONRequest } from '@server/helpers/requests' | ||
6 | import { PlaylistElementObject, PlaylistObject } from '@shared/models' | ||
7 | |||
8 | async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { | ||
9 | const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl) | ||
10 | |||
11 | logger.info('Fetching remote playlist %s.', playlistUrl, lTags()) | ||
12 | |||
13 | const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true }) | ||
14 | |||
15 | if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { | ||
16 | logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() }) | ||
17 | return { statusCode, playlistObject: undefined } | ||
18 | } | ||
19 | |||
20 | if (!isArray(body.to)) { | ||
21 | logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() }) | ||
22 | return { statusCode, playlistObject: undefined } | ||
23 | } | ||
24 | |||
25 | return { statusCode, playlistObject: body } | ||
26 | } | ||
27 | |||
28 | async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> { | ||
29 | const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl) | ||
30 | |||
31 | logger.debug('Fetching remote playlist element %s.', elementUrl, lTags()) | ||
32 | |||
33 | const { body, statusCode } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true }) | ||
34 | |||
35 | if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`) | ||
36 | |||
37 | if (checkUrlsSameHost(body.id, elementUrl) !== true) { | ||
38 | throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) | ||
39 | } | ||
40 | |||
41 | return { statusCode, elementObject: body } | ||
42 | } | ||
43 | |||
44 | export { | ||
45 | fetchRemoteVideoPlaylist, | ||
46 | fetchRemotePlaylistElement | ||
47 | } | ||
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 1799829f8..077b01eda 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts | |||
@@ -1,8 +1,8 @@ | |||
1 | import { ActivityAccept } from '../../../../shared/models/activitypub' | 1 | import { ActivityAccept } from '../../../../shared/models/activitypub' |
2 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 2 | import { ActorFollowModel } from '../../../models/actor/actor-follow' |
3 | import { addFetchOutboxJob } from '../actor' | ||
4 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 3 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
5 | import { MActorDefault, MActorSignature } from '../../../types/models' | 4 | import { MActorDefault, MActorSignature } from '../../../types/models' |
5 | import { addFetchOutboxJob } from '../outbox' | ||
6 | 6 | ||
7 | async function processAcceptActivity (options: APProcessorOptions<ActivityAccept>) { | 7 | async function processAcceptActivity (options: APProcessorOptions<ActivityAccept>) { |
8 | const { byActor: targetActor, inboxActor } = options | 8 | const { byActor: targetActor, inboxActor } = options |
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 63082466e..ec23c705e 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts | |||
@@ -3,7 +3,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' | |||
3 | import { sequelizeTypescript } from '../../../initializers/database' | 3 | import { sequelizeTypescript } from '../../../initializers/database' |
4 | import { VideoShareModel } from '../../../models/video/video-share' | 4 | import { VideoShareModel } from '../../../models/video/video-share' |
5 | import { forwardVideoRelatedActivity } from '../send/utils' | 5 | import { forwardVideoRelatedActivity } from '../send/utils' |
6 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 6 | import { getOrCreateAPVideo } from '../videos' |
7 | import { Notifier } from '../../notifier' | 7 | import { Notifier } from '../../notifier' |
8 | import { logger } from '../../../helpers/logger' | 8 | import { logger } from '../../../helpers/logger' |
9 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 9 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
@@ -32,7 +32,7 @@ async function processVideoShare (actorAnnouncer: MActorSignature, activity: Act | |||
32 | let videoCreated: boolean | 32 | let videoCreated: boolean |
33 | 33 | ||
34 | try { | 34 | try { |
35 | const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) | 35 | const result = await getOrCreateAPVideo({ videoObject: objectUri }) |
36 | video = result.video | 36 | video = result.video |
37 | videoCreated = result.created | 37 | videoCreated = result.created |
38 | } catch (err) { | 38 | } catch (err) { |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 9cded4dec..70e048d6e 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | ||
1 | import { isRedundancyAccepted } from '@server/lib/redundancy' | 2 | import { isRedundancyAccepted } from '@server/lib/redundancy' |
2 | import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared' | 3 | import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared' |
3 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' | 4 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' |
@@ -9,11 +10,10 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model' | |||
9 | import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' | 10 | import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' |
10 | import { Notifier } from '../../notifier' | 11 | import { Notifier } from '../../notifier' |
11 | import { createOrUpdateCacheFile } from '../cache-file' | 12 | import { createOrUpdateCacheFile } from '../cache-file' |
12 | import { createOrUpdateVideoPlaylist } from '../playlist' | 13 | import { createOrUpdateVideoPlaylist } from '../playlists' |
13 | import { forwardVideoRelatedActivity } from '../send/utils' | 14 | import { forwardVideoRelatedActivity } from '../send/utils' |
14 | import { resolveThread } from '../video-comments' | 15 | import { resolveThread } from '../video-comments' |
15 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 16 | import { getOrCreateAPVideo } from '../videos' |
16 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | ||
17 | 17 | ||
18 | async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { | 18 | async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { |
19 | const { activity, byActor } = options | 19 | const { activity, byActor } = options |
@@ -55,7 +55,7 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) { | |||
55 | const videoToCreateData = activity.object as VideoObject | 55 | const videoToCreateData = activity.object as VideoObject |
56 | 56 | ||
57 | const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } | 57 | const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } |
58 | const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData, syncParam }) | 58 | const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) |
59 | 59 | ||
60 | if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) | 60 | if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) |
61 | 61 | ||
@@ -67,7 +67,7 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor | |||
67 | 67 | ||
68 | const cacheFile = activity.object as CacheFileObject | 68 | const cacheFile = activity.object as CacheFileObject |
69 | 69 | ||
70 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) | 70 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) |
71 | 71 | ||
72 | await sequelizeTypescript.transaction(async t => { | 72 | await sequelizeTypescript.transaction(async t => { |
73 | return createOrUpdateCacheFile(cacheFile, video, byActor, t) | 73 | return createOrUpdateCacheFile(cacheFile, video, byActor, t) |
@@ -128,5 +128,5 @@ async function processCreatePlaylist (activity: ActivityCreate, byActor: MActorS | |||
128 | 128 | ||
129 | if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) | 129 | if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) |
130 | 130 | ||
131 | await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) | 131 | await createOrUpdateVideoPlaylist(playlistObject, activity.to) |
132 | } | 132 | } |
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts index 88a968318..1d2279df5 100644 --- a/server/lib/activitypub/process/process-delete.ts +++ b/server/lib/activitypub/process/process-delete.ts | |||
@@ -2,7 +2,7 @@ import { ActivityDelete } from '../../../../shared/models/activitypub' | |||
2 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 2 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
3 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
4 | import { sequelizeTypescript } from '../../../initializers/database' | 4 | import { sequelizeTypescript } from '../../../initializers/database' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/actor/actor' |
6 | import { VideoModel } from '../../../models/video/video' | 6 | import { VideoModel } from '../../../models/video/video' |
7 | import { VideoCommentModel } from '../../../models/video/video-comment' | 7 | import { VideoCommentModel } from '../../../models/video/video-comment' |
8 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | 8 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' |
@@ -16,7 +16,6 @@ import { | |||
16 | MChannelActor, | 16 | MChannelActor, |
17 | MCommentOwnerVideo | 17 | MCommentOwnerVideo |
18 | } from '../../../types/models' | 18 | } from '../../../types/models' |
19 | import { markCommentAsDeleted } from '../../video-comment' | ||
20 | import { forwardVideoRelatedActivity } from '../send/utils' | 19 | import { forwardVideoRelatedActivity } from '../send/utils' |
21 | 20 | ||
22 | async function processDeleteActivity (options: APProcessorOptions<ActivityDelete>) { | 21 | async function processDeleteActivity (options: APProcessorOptions<ActivityDelete>) { |
@@ -130,7 +129,7 @@ async function processDeleteVideoChannel (videoChannelToRemove: MChannelActor) { | |||
130 | 129 | ||
131 | function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) { | 130 | function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) { |
132 | // Already deleted | 131 | // Already deleted |
133 | if (videoComment.isDeleted()) return | 132 | if (videoComment.isDeleted()) return Promise.resolve() |
134 | 133 | ||
135 | logger.debug('Removing remote video comment "%s".', videoComment.url) | 134 | logger.debug('Removing remote video comment "%s".', videoComment.url) |
136 | 135 | ||
@@ -139,11 +138,9 @@ function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCom | |||
139 | throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`) | 138 | throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`) |
140 | } | 139 | } |
141 | 140 | ||
142 | await sequelizeTypescript.transaction(async t => { | 141 | videoComment.markAsDeleted() |
143 | markCommentAsDeleted(videoComment) | ||
144 | 142 | ||
145 | await videoComment.save() | 143 | await videoComment.save({ transaction: t }) |
146 | }) | ||
147 | 144 | ||
148 | if (videoComment.Video.isOwned()) { | 145 | if (videoComment.Video.isOwned()) { |
149 | // Don't resend the activity to the sender | 146 | // Don't resend the activity to the sender |
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts index 089c7b881..ecc57cd10 100644 --- a/server/lib/activitypub/process/process-dislike.ts +++ b/server/lib/activitypub/process/process-dislike.ts | |||
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat | |||
6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
7 | import { MActorSignature } from '../../../types/models' | 7 | import { MActorSignature } from '../../../types/models' |
8 | import { forwardVideoRelatedActivity } from '../send/utils' | 8 | import { forwardVideoRelatedActivity } from '../send/utils' |
9 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 9 | import { getOrCreateAPVideo } from '../videos' |
10 | 10 | ||
11 | async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) { | 11 | async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) { |
12 | const { activity, byActor } = options | 12 | const { activity, byActor } = options |
@@ -30,7 +30,7 @@ async function processDislike (activity: ActivityCreate | ActivityDislike, byAct | |||
30 | 30 | ||
31 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | 31 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) |
32 | 32 | ||
33 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) | 33 | const { video } = await getOrCreateAPVideo({ videoObject: dislikeObject }) |
34 | 34 | ||
35 | return sequelizeTypescript.transaction(async t => { | 35 | return sequelizeTypescript.transaction(async t => { |
36 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) | 36 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) |
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index 38d684512..f85238f8e 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts | |||
@@ -1,17 +1,17 @@ | |||
1 | import { getServerActor } from '@server/models/application/application' | ||
1 | import { ActivityFollow } from '../../../../shared/models/activitypub' | 2 | import { ActivityFollow } from '../../../../shared/models/activitypub' |
3 | import { getAPId } from '../../../helpers/activitypub' | ||
2 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
3 | import { logger } from '../../../helpers/logger' | 5 | import { logger } from '../../../helpers/logger' |
4 | import { sequelizeTypescript } from '../../../initializers/database' | ||
5 | import { ActorModel } from '../../../models/activitypub/actor' | ||
6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | ||
7 | import { sendAccept, sendReject } from '../send' | ||
8 | import { Notifier } from '../../notifier' | ||
9 | import { getAPId } from '../../../helpers/activitypub' | ||
10 | import { CONFIG } from '../../../initializers/config' | 6 | import { CONFIG } from '../../../initializers/config' |
7 | import { sequelizeTypescript } from '../../../initializers/database' | ||
8 | import { ActorModel } from '../../../models/actor/actor' | ||
9 | import { ActorFollowModel } from '../../../models/actor/actor-follow' | ||
11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 10 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
12 | import { MActorFollowActors, MActorSignature } from '../../../types/models' | 11 | import { MActorFollowActors, MActorSignature } from '../../../types/models' |
12 | import { Notifier } from '../../notifier' | ||
13 | import { autoFollowBackIfNeeded } from '../follow' | 13 | import { autoFollowBackIfNeeded } from '../follow' |
14 | import { getServerActor } from '@server/models/application/application' | 14 | import { sendAccept, sendReject } from '../send' |
15 | 15 | ||
16 | async function processFollowActivity (options: APProcessorOptions<ActivityFollow>) { | 16 | async function processFollowActivity (options: APProcessorOptions<ActivityFollow>) { |
17 | const { activity, byActor } = options | 17 | const { activity, byActor } = options |
@@ -43,7 +43,7 @@ async function processFollow (byActor: MActorSignature, activityId: string, targ | |||
43 | if (isFollowingInstance && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { | 43 | if (isFollowingInstance && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { |
44 | logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) | 44 | logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) |
45 | 45 | ||
46 | await sendReject(activityId, byActor, targetActor) | 46 | sendReject(activityId, byActor, targetActor) |
47 | 47 | ||
48 | return { actorFollow: undefined as MActorFollowActors } | 48 | return { actorFollow: undefined as MActorFollowActors } |
49 | } | 49 | } |
@@ -84,8 +84,9 @@ async function processFollow (byActor: MActorSignature, activityId: string, targ | |||
84 | 84 | ||
85 | // Target sends to actor he accepted the follow request | 85 | // Target sends to actor he accepted the follow request |
86 | if (actorFollow.state === 'accepted') { | 86 | if (actorFollow.state === 'accepted') { |
87 | await sendAccept(actorFollow) | 87 | sendAccept(actorFollow) |
88 | await autoFollowBackIfNeeded(actorFollow) | 88 | |
89 | await autoFollowBackIfNeeded(actorFollow, t) | ||
89 | } | 90 | } |
90 | 91 | ||
91 | return { actorFollow, created, isFollowingInstance, targetActor } | 92 | return { actorFollow, created, isFollowingInstance, targetActor } |
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 8688b3b47..cd4e86cbb 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts | |||
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat | |||
6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
7 | import { MActorSignature } from '../../../types/models' | 7 | import { MActorSignature } from '../../../types/models' |
8 | import { forwardVideoRelatedActivity } from '../send/utils' | 8 | import { forwardVideoRelatedActivity } from '../send/utils' |
9 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 9 | import { getOrCreateAPVideo } from '../videos' |
10 | 10 | ||
11 | async function processLikeActivity (options: APProcessorOptions<ActivityLike>) { | 11 | async function processLikeActivity (options: APProcessorOptions<ActivityLike>) { |
12 | const { activity, byActor } = options | 12 | const { activity, byActor } = options |
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: MActorSignature, activity: ActivityLik | |||
27 | const byAccount = byActor.Account | 27 | const byAccount = byActor.Account |
28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) | 28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) |
29 | 29 | ||
30 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) | 30 | const { video } = await getOrCreateAPVideo({ videoObject: videoUrl }) |
31 | 31 | ||
32 | return sequelizeTypescript.transaction(async t => { | 32 | return sequelizeTypescript.transaction(async t => { |
33 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) | 33 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) |
diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts index 03b669fd9..7f7ab305f 100644 --- a/server/lib/activitypub/process/process-reject.ts +++ b/server/lib/activitypub/process/process-reject.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { ActivityReject } from '../../../../shared/models/activitypub/activity' | 1 | import { ActivityReject } from '../../../../shared/models/activitypub/activity' |
2 | import { sequelizeTypescript } from '../../../initializers/database' | 2 | import { sequelizeTypescript } from '../../../initializers/database' |
3 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 3 | import { ActorFollowModel } from '../../../models/actor/actor-follow' |
4 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 4 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
5 | import { MActor } from '../../../types/models' | 5 | import { MActor } from '../../../types/models' |
6 | 6 | ||
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index e520c2f0d..d4b2a795f 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -4,14 +4,14 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' | |||
4 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
5 | import { sequelizeTypescript } from '../../../initializers/database' | 5 | import { sequelizeTypescript } from '../../../initializers/database' |
6 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 6 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
7 | import { ActorModel } from '../../../models/activitypub/actor' | 7 | import { ActorModel } from '../../../models/actor/actor' |
8 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 8 | import { ActorFollowModel } from '../../../models/actor/actor-follow' |
9 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 9 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
10 | import { VideoShareModel } from '../../../models/video/video-share' | 10 | import { VideoShareModel } from '../../../models/video/video-share' |
11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
12 | import { MActorSignature } from '../../../types/models' | 12 | import { MActorSignature } from '../../../types/models' |
13 | import { forwardVideoRelatedActivity } from '../send/utils' | 13 | import { forwardVideoRelatedActivity } from '../send/utils' |
14 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 14 | import { getOrCreateAPVideo } from '../videos' |
15 | 15 | ||
16 | async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { | 16 | async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { |
17 | const { activity, byActor } = options | 17 | const { activity, byActor } = options |
@@ -55,7 +55,7 @@ export { | |||
55 | async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { | 55 | async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { |
56 | const likeActivity = activity.object as ActivityLike | 56 | const likeActivity = activity.object as ActivityLike |
57 | 57 | ||
58 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) | 58 | const { video } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) |
59 | 59 | ||
60 | return sequelizeTypescript.transaction(async t => { | 60 | return sequelizeTypescript.transaction(async t => { |
61 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | 61 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
@@ -80,7 +80,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU | |||
80 | ? activity.object | 80 | ? activity.object |
81 | : activity.object.object as DislikeObject | 81 | : activity.object.object as DislikeObject |
82 | 82 | ||
83 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) | 83 | const { video } = await getOrCreateAPVideo({ videoObject: dislike.object }) |
84 | 84 | ||
85 | return sequelizeTypescript.transaction(async t => { | 85 | return sequelizeTypescript.transaction(async t => { |
86 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | 86 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
@@ -103,10 +103,10 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU | |||
103 | async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { | 103 | async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { |
104 | const cacheFileObject = activity.object.object as CacheFileObject | 104 | const cacheFileObject = activity.object.object as CacheFileObject |
105 | 105 | ||
106 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) | 106 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) |
107 | 107 | ||
108 | return sequelizeTypescript.transaction(async t => { | 108 | return sequelizeTypescript.transaction(async t => { |
109 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | 109 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) |
110 | if (!cacheFile) { | 110 | if (!cacheFile) { |
111 | logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) | 111 | logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) |
112 | return | 112 | return |
@@ -114,7 +114,7 @@ async function processUndoCacheFile (byActor: MActorSignature, activity: Activit | |||
114 | 114 | ||
115 | if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') | 115 | if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') |
116 | 116 | ||
117 | await cacheFile.destroy() | 117 | await cacheFile.destroy({ transaction: t }) |
118 | 118 | ||
119 | if (video.isOwned()) { | 119 | if (video.isOwned()) { |
120 | // Don't resend the activity to the sender | 120 | // Don't resend the activity to the sender |
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 6df9b93b2..f40008a6b 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -1,23 +1,20 @@ | |||
1 | import { isRedundancyAccepted } from '@server/lib/redundancy' | ||
1 | import { ActivityUpdate, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' | 2 | import { ActivityUpdate, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' |
2 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' | 3 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' |
3 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' | 4 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' |
5 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' | ||
6 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' | ||
7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
4 | import { logger } from '../../../helpers/logger' | 8 | import { logger } from '../../../helpers/logger' |
5 | import { sequelizeTypescript } from '../../../initializers/database' | 9 | import { sequelizeTypescript } from '../../../initializers/database' |
6 | import { AccountModel } from '../../../models/account/account' | 10 | import { ActorModel } from '../../../models/actor/actor' |
7 | import { ActorModel } from '../../../models/activitypub/actor' | 11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
8 | import { VideoChannelModel } from '../../../models/video/video-channel' | 12 | import { MActorFull, MActorSignature } from '../../../types/models' |
9 | import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } from '../actor' | 13 | import { APActorUpdater } from '../actors/updater' |
10 | import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' | ||
11 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' | ||
12 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' | ||
13 | import { createOrUpdateCacheFile } from '../cache-file' | 14 | import { createOrUpdateCacheFile } from '../cache-file' |
15 | import { createOrUpdateVideoPlaylist } from '../playlists' | ||
14 | import { forwardVideoRelatedActivity } from '../send/utils' | 16 | import { forwardVideoRelatedActivity } from '../send/utils' |
15 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' | 17 | import { APVideoUpdater, getOrCreateAPVideo } from '../videos' |
16 | import { createOrUpdateVideoPlaylist } from '../playlist' | ||
17 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
18 | import { MActorSignature, MAccountIdActor } from '../../../types/models' | ||
19 | import { isRedundancyAccepted } from '@server/lib/redundancy' | ||
20 | import { ActorImageType } from '@shared/models' | ||
21 | 18 | ||
22 | async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { | 19 | async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { |
23 | const { activity, byActor } = options | 20 | const { activity, byActor } = options |
@@ -25,7 +22,7 @@ async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate | |||
25 | const objectType = activity.object.type | 22 | const objectType = activity.object.type |
26 | 23 | ||
27 | if (objectType === 'Video') { | 24 | if (objectType === 'Video') { |
28 | return retryTransactionWrapper(processUpdateVideo, byActor, activity) | 25 | return retryTransactionWrapper(processUpdateVideo, activity) |
29 | } | 26 | } |
30 | 27 | ||
31 | if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { | 28 | if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { |
@@ -55,7 +52,7 @@ export { | |||
55 | 52 | ||
56 | // --------------------------------------------------------------------------- | 53 | // --------------------------------------------------------------------------- |
57 | 54 | ||
58 | async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpdate) { | 55 | async function processUpdateVideo (activity: ActivityUpdate) { |
59 | const videoObject = activity.object as VideoObject | 56 | const videoObject = activity.object as VideoObject |
60 | 57 | ||
61 | if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { | 58 | if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { |
@@ -63,7 +60,7 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd | |||
63 | return undefined | 60 | return undefined |
64 | } | 61 | } |
65 | 62 | ||
66 | const { video, created } = await getOrCreateVideoAndAccountAndChannel({ | 63 | const { video, created } = await getOrCreateAPVideo({ |
67 | videoObject: videoObject.id, | 64 | videoObject: videoObject.id, |
68 | allowRefresh: false, | 65 | allowRefresh: false, |
69 | fetchType: 'all' | 66 | fetchType: 'all' |
@@ -71,20 +68,8 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd | |||
71 | // We did not have this video, it has been created so no need to update | 68 | // We did not have this video, it has been created so no need to update |
72 | if (created) return | 69 | if (created) return |
73 | 70 | ||
74 | // Load new channel | 71 | const updater = new APVideoUpdater(videoObject, video) |
75 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 72 | return updater.update(activity.to) |
76 | |||
77 | const account = actor.Account as MAccountIdActor | ||
78 | account.Actor = actor | ||
79 | |||
80 | const updateOptions = { | ||
81 | video, | ||
82 | videoObject, | ||
83 | account, | ||
84 | channel: channelActor.VideoChannel, | ||
85 | overrideTo: activity.to | ||
86 | } | ||
87 | return updateVideoFromAP(updateOptions) | ||
88 | } | 73 | } |
89 | 74 | ||
90 | async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) { | 75 | async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) { |
@@ -97,7 +82,7 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ | |||
97 | return undefined | 82 | return undefined |
98 | } | 83 | } |
99 | 84 | ||
100 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) | 85 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) |
101 | 86 | ||
102 | await sequelizeTypescript.transaction(async t => { | 87 | await sequelizeTypescript.transaction(async t => { |
103 | await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) | 88 | await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) |
@@ -111,56 +96,13 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ | |||
111 | } | 96 | } |
112 | } | 97 | } |
113 | 98 | ||
114 | async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { | 99 | async function processUpdateActor (actor: MActorFull, activity: ActivityUpdate) { |
115 | const actorAttributesToUpdate = activity.object as ActivityPubActor | 100 | const actorObject = activity.object as ActivityPubActor |
116 | 101 | ||
117 | logger.debug('Updating remote account "%s".', actorAttributesToUpdate.url) | 102 | logger.debug('Updating remote account "%s".', actorObject.url) |
118 | let accountOrChannelInstance: AccountModel | VideoChannelModel | ||
119 | let actorFieldsSave: object | ||
120 | let accountOrChannelFieldsSave: object | ||
121 | 103 | ||
122 | // Fetch icon? | 104 | const updater = new APActorUpdater(actorObject, actor) |
123 | const avatarInfo = getImageInfoIfExists(actorAttributesToUpdate, ActorImageType.AVATAR) | 105 | return updater.update() |
124 | const bannerInfo = getImageInfoIfExists(actorAttributesToUpdate, ActorImageType.BANNER) | ||
125 | |||
126 | try { | ||
127 | await sequelizeTypescript.transaction(async t => { | ||
128 | actorFieldsSave = actor.toJSON() | ||
129 | |||
130 | if (actorAttributesToUpdate.type === 'Group') accountOrChannelInstance = actor.VideoChannel | ||
131 | else accountOrChannelInstance = actor.Account | ||
132 | |||
133 | accountOrChannelFieldsSave = accountOrChannelInstance.toJSON() | ||
134 | |||
135 | await updateActorInstance(actor, actorAttributesToUpdate) | ||
136 | |||
137 | await updateActorImageInstance(actor, ActorImageType.AVATAR, avatarInfo, t) | ||
138 | await updateActorImageInstance(actor, ActorImageType.BANNER, bannerInfo, t) | ||
139 | |||
140 | await actor.save({ transaction: t }) | ||
141 | |||
142 | accountOrChannelInstance.name = actorAttributesToUpdate.name || actorAttributesToUpdate.preferredUsername | ||
143 | accountOrChannelInstance.description = actorAttributesToUpdate.summary | ||
144 | |||
145 | if (accountOrChannelInstance instanceof VideoChannelModel) accountOrChannelInstance.support = actorAttributesToUpdate.support | ||
146 | |||
147 | await accountOrChannelInstance.save({ transaction: t }) | ||
148 | }) | ||
149 | |||
150 | logger.info('Remote account %s updated', actorAttributesToUpdate.url) | ||
151 | } catch (err) { | ||
152 | if (actor !== undefined && actorFieldsSave !== undefined) { | ||
153 | resetSequelizeInstance(actor, actorFieldsSave) | ||
154 | } | ||
155 | |||
156 | if (accountOrChannelInstance !== undefined && accountOrChannelFieldsSave !== undefined) { | ||
157 | resetSequelizeInstance(accountOrChannelInstance, accountOrChannelFieldsSave) | ||
158 | } | ||
159 | |||
160 | // This is just a debug because we will retry the insert | ||
161 | logger.debug('Cannot update the remote account.', { err }) | ||
162 | throw err | ||
163 | } | ||
164 | } | 106 | } |
165 | 107 | ||
166 | async function processUpdatePlaylist (byActor: MActorSignature, activity: ActivityUpdate) { | 108 | async function processUpdatePlaylist (byActor: MActorSignature, activity: ActivityUpdate) { |
@@ -169,5 +111,5 @@ async function processUpdatePlaylist (byActor: MActorSignature, activity: Activi | |||
169 | 111 | ||
170 | if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) | 112 | if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) |
171 | 113 | ||
172 | await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) | 114 | await createOrUpdateVideoPlaylist(playlistObject, activity.to) |
173 | } | 115 | } |
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts index 84697673b..5593ee257 100644 --- a/server/lib/activitypub/process/process-view.ts +++ b/server/lib/activitypub/process/process-view.ts | |||
@@ -1,10 +1,10 @@ | |||
1 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 1 | import { getOrCreateAPVideo } from '../videos' |
2 | import { forwardVideoRelatedActivity } from '../send/utils' | 2 | import { forwardVideoRelatedActivity } from '../send/utils' |
3 | import { Redis } from '../../redis' | 3 | import { Redis } from '../../redis' |
4 | import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' | 4 | import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' |
5 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 5 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
6 | import { MActorSignature } from '../../../types/models' | 6 | import { MActorSignature } from '../../../types/models' |
7 | import { LiveManager } from '@server/lib/live-manager' | 7 | import { LiveManager } from '@server/lib/live/live-manager' |
8 | 8 | ||
9 | async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) { | 9 | async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) { |
10 | const { activity, byActor } = options | 10 | const { activity, byActor } = options |
@@ -24,12 +24,11 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct | |||
24 | ? activity.object | 24 | ? activity.object |
25 | : (activity.object as ViewObject).object | 25 | : (activity.object as ViewObject).object |
26 | 26 | ||
27 | const options = { | 27 | const { video } = await getOrCreateAPVideo({ |
28 | videoObject, | 28 | videoObject, |
29 | fetchType: 'only-video' as 'only-video', | 29 | fetchType: 'only-video', |
30 | allowRefresh: false as false | 30 | allowRefresh: false |
31 | } | 31 | }) |
32 | const { video } = await getOrCreateVideoAndAccountAndChannel(options) | ||
33 | 32 | ||
34 | if (!video.isLive) { | 33 | if (!video.isLive) { |
35 | await Redis.Instance.addVideoView(video.id) | 34 | await Redis.Instance.addVideoView(video.id) |
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index 5cef75665..02a23d098 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts | |||
@@ -1,22 +1,22 @@ | |||
1 | import { StatsManager } from '@server/lib/stat-manager' | ||
1 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' | 2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' |
2 | import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' | 3 | import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' |
3 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
5 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
6 | import { MActorDefault, MActorSignature } from '../../../types/models' | ||
7 | import { getOrCreateAPActor } from '../actors' | ||
4 | import { processAcceptActivity } from './process-accept' | 8 | import { processAcceptActivity } from './process-accept' |
5 | import { processAnnounceActivity } from './process-announce' | 9 | import { processAnnounceActivity } from './process-announce' |
6 | import { processCreateActivity } from './process-create' | 10 | import { processCreateActivity } from './process-create' |
7 | import { processDeleteActivity } from './process-delete' | 11 | import { processDeleteActivity } from './process-delete' |
12 | import { processDislikeActivity } from './process-dislike' | ||
13 | import { processFlagActivity } from './process-flag' | ||
8 | import { processFollowActivity } from './process-follow' | 14 | import { processFollowActivity } from './process-follow' |
9 | import { processLikeActivity } from './process-like' | 15 | import { processLikeActivity } from './process-like' |
10 | import { processRejectActivity } from './process-reject' | 16 | import { processRejectActivity } from './process-reject' |
11 | import { processUndoActivity } from './process-undo' | 17 | import { processUndoActivity } from './process-undo' |
12 | import { processUpdateActivity } from './process-update' | 18 | import { processUpdateActivity } from './process-update' |
13 | import { getOrCreateActorAndServerAndModel } from '../actor' | ||
14 | import { processDislikeActivity } from './process-dislike' | ||
15 | import { processFlagActivity } from './process-flag' | ||
16 | import { processViewActivity } from './process-view' | 19 | import { processViewActivity } from './process-view' |
17 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
18 | import { MActorDefault, MActorSignature } from '../../../types/models' | ||
19 | import { StatsManager } from '@server/lib/stat-manager' | ||
20 | 20 | ||
21 | const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions<Activity>) => Promise<any> } = { | 21 | const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions<Activity>) => Promise<any> } = { |
22 | Create: processCreateActivity, | 22 | Create: processCreateActivity, |
@@ -65,7 +65,7 @@ async function processActivities ( | |||
65 | continue | 65 | continue |
66 | } | 66 | } |
67 | 67 | ||
68 | const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) | 68 | const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateAPActor(actorUrl) |
69 | actorsCache[actorUrl] = byActor | 69 | actorsCache[actorUrl] = byActor |
70 | 70 | ||
71 | const activityProcessor = processActivity[activity.type] | 71 | const activityProcessor = processActivity[activity.type] |
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts index e0acced18..d31f8c10b 100644 --- a/server/lib/activitypub/send/send-delete.ts +++ b/server/lib/activitypub/send/send-delete.ts | |||
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize' | |||
2 | import { getServerActor } from '@server/models/application/application' | 2 | import { getServerActor } from '@server/models/application/application' |
3 | import { ActivityAudience, ActivityDelete } from '../../../../shared/models/activitypub' | 3 | import { ActivityAudience, ActivityDelete } from '../../../../shared/models/activitypub' |
4 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/actor/actor' |
6 | import { VideoCommentModel } from '../../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../../models/video/video-comment' |
7 | import { VideoShareModel } from '../../../models/video/video-share' | 7 | import { VideoShareModel } from '../../../models/video/video-share' |
8 | import { MActorUrl } from '../../../types/models' | 8 | import { MActorUrl } from '../../../types/models' |
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts index 9254dc7c5..153e94295 100644 --- a/server/lib/activitypub/send/send-view.ts +++ b/server/lib/activitypub/send/send-view.ts | |||
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize' | |||
2 | import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models' | 2 | import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models' |
3 | import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' | 3 | import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' |
4 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/actor/actor' |
6 | import { audiencify, getAudience } from '../audience' | 6 | import { audiencify, getAudience } from '../audience' |
7 | import { getLocalVideoViewActivityPubUrl } from '../url' | 7 | import { getLocalVideoViewActivityPubUrl } from '../url' |
8 | import { sendVideoRelatedActivity } from './utils' | 8 | import { sendVideoRelatedActivity } from './utils' |
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts index 85a9f009d..7cd8030e1 100644 --- a/server/lib/activitypub/send/utils.ts +++ b/server/lib/activitypub/send/utils.ts | |||
@@ -1,14 +1,14 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { getServerActor } from '@server/models/application/application' | ||
3 | import { ContextType } from '@shared/models/activitypub/context' | ||
2 | import { Activity, ActivityAudience } from '../../../../shared/models/activitypub' | 4 | import { Activity, ActivityAudience } from '../../../../shared/models/activitypub' |
5 | import { afterCommitIfTransaction } from '../../../helpers/database-utils' | ||
3 | import { logger } from '../../../helpers/logger' | 6 | import { logger } from '../../../helpers/logger' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 7 | import { ActorModel } from '../../../models/actor/actor' |
5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 8 | import { ActorFollowModel } from '../../../models/actor/actor-follow' |
9 | import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../types/models' | ||
6 | import { JobQueue } from '../../job-queue' | 10 | import { JobQueue } from '../../job-queue' |
7 | import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience' | 11 | import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience' |
8 | import { afterCommitIfTransaction } from '../../../helpers/database-utils' | ||
9 | import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../types/models' | ||
10 | import { getServerActor } from '@server/models/application/application' | ||
11 | import { ContextType } from '@shared/models/activitypub/context' | ||
12 | 12 | ||
13 | async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { | 13 | async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { |
14 | byActor: MActorLight | 14 | byActor: MActorLight |
@@ -22,7 +22,9 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud | |||
22 | 22 | ||
23 | // Send to origin | 23 | // Send to origin |
24 | if (video.isOwned() === false) { | 24 | if (video.isOwned() === false) { |
25 | const accountActor = (video as MVideoAccountLight).VideoChannel?.Account?.Actor || await ActorModel.loadAccountActorByVideoId(video.id) | 25 | let accountActor: MActorLight = (video as MVideoAccountLight).VideoChannel?.Account?.Actor |
26 | |||
27 | if (!accountActor) accountActor = await ActorModel.loadAccountActorByVideoId(video.id, transaction) | ||
26 | 28 | ||
27 | const audience = getRemoteVideoAudience(accountActor, actorsInvolvedInVideo) | 29 | const audience = getRemoteVideoAudience(accountActor, actorsInvolvedInVideo) |
28 | const activity = activityBuilder(audience) | 30 | const activity = activityBuilder(audience) |
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index c22fa0893..1ff01a175 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts | |||
@@ -7,7 +7,7 @@ import { doJSONRequest } from '../../helpers/requests' | |||
7 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | 7 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
8 | import { VideoShareModel } from '../../models/video/video-share' | 8 | import { VideoShareModel } from '../../models/video/video-share' |
9 | import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' | 9 | import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' |
10 | import { getOrCreateActorAndServerAndModel } from './actor' | 10 | import { getOrCreateAPActor } from './actors' |
11 | import { sendUndoAnnounce, sendVideoAnnounce } from './send' | 11 | import { sendUndoAnnounce, sendVideoAnnounce } from './send' |
12 | import { getLocalVideoAnnounceActivityPubUrl } from './url' | 12 | import { getLocalVideoAnnounceActivityPubUrl } from './url' |
13 | 13 | ||
@@ -40,23 +40,7 @@ async function changeVideoChannelShare ( | |||
40 | async function addVideoShares (shareUrls: string[], video: MVideoId) { | 40 | async function addVideoShares (shareUrls: string[], video: MVideoId) { |
41 | await Bluebird.map(shareUrls, async shareUrl => { | 41 | await Bluebird.map(shareUrls, async shareUrl => { |
42 | try { | 42 | try { |
43 | const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true }) | 43 | await addVideoShare(shareUrl, video) |
44 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | ||
45 | |||
46 | const actorUrl = getAPId(body.actor) | ||
47 | if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { | ||
48 | throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) | ||
49 | } | ||
50 | |||
51 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | ||
52 | |||
53 | const entry = { | ||
54 | actorId: actor.id, | ||
55 | videoId: video.id, | ||
56 | url: shareUrl | ||
57 | } | ||
58 | |||
59 | await VideoShareModel.upsert(entry) | ||
60 | } catch (err) { | 44 | } catch (err) { |
61 | logger.warn('Cannot add share %s.', shareUrl, { err }) | 45 | logger.warn('Cannot add share %s.', shareUrl, { err }) |
62 | } | 46 | } |
@@ -71,6 +55,26 @@ export { | |||
71 | 55 | ||
72 | // --------------------------------------------------------------------------- | 56 | // --------------------------------------------------------------------------- |
73 | 57 | ||
58 | async function addVideoShare (shareUrl: string, video: MVideoId) { | ||
59 | const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true }) | ||
60 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | ||
61 | |||
62 | const actorUrl = getAPId(body.actor) | ||
63 | if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { | ||
64 | throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) | ||
65 | } | ||
66 | |||
67 | const actor = await getOrCreateAPActor(actorUrl) | ||
68 | |||
69 | const entry = { | ||
70 | actorId: actor.id, | ||
71 | videoId: video.id, | ||
72 | url: shareUrl | ||
73 | } | ||
74 | |||
75 | await VideoShareModel.upsert(entry) | ||
76 | } | ||
77 | |||
74 | async function shareByServer (video: MVideo, t: Transaction) { | 78 | async function shareByServer (video: MVideo, t: Transaction) { |
75 | const serverActor = await getServerActor() | 79 | const serverActor = await getServerActor() |
76 | 80 | ||
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index e23e0c0e7..6b7f9504f 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -6,8 +6,8 @@ import { doJSONRequest } from '../../helpers/requests' | |||
6 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | 6 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
7 | import { VideoCommentModel } from '../../models/video/video-comment' | 7 | import { VideoCommentModel } from '../../models/video/video-comment' |
8 | import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' | 8 | import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' |
9 | import { getOrCreateActorAndServerAndModel } from './actor' | 9 | import { getOrCreateAPActor } from './actors' |
10 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | 10 | import { getOrCreateAPVideo } from './videos' |
11 | 11 | ||
12 | type ResolveThreadParams = { | 12 | type ResolveThreadParams = { |
13 | url: string | 13 | url: string |
@@ -29,10 +29,11 @@ async function addVideoComments (commentUrls: string[]) { | |||
29 | 29 | ||
30 | async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { | 30 | async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { |
31 | const { url, isVideo } = params | 31 | const { url, isVideo } = params |
32 | |||
32 | if (params.commentCreated === undefined) params.commentCreated = false | 33 | if (params.commentCreated === undefined) params.commentCreated = false |
33 | if (params.comments === undefined) params.comments = [] | 34 | if (params.comments === undefined) params.comments = [] |
34 | 35 | ||
35 | // If it is not a video, or if we don't know if it's a video | 36 | // If it is not a video, or if we don't know if it's a video, try to get the thread from DB |
36 | if (isVideo === false || isVideo === undefined) { | 37 | if (isVideo === false || isVideo === undefined) { |
37 | const result = await resolveCommentFromDB(params) | 38 | const result = await resolveCommentFromDB(params) |
38 | if (result) return result | 39 | if (result) return result |
@@ -42,7 +43,7 @@ async function resolveThread (params: ResolveThreadParams): ResolveThreadResult | |||
42 | // If it is a video, or if we don't know if it's a video | 43 | // If it is a video, or if we don't know if it's a video |
43 | if (isVideo === true || isVideo === undefined) { | 44 | if (isVideo === true || isVideo === undefined) { |
44 | // Keep await so we catch the exception | 45 | // Keep await so we catch the exception |
45 | return await tryResolveThreadFromVideo(params) | 46 | return await tryToResolveThreadFromVideo(params) |
46 | } | 47 | } |
47 | } catch (err) { | 48 | } catch (err) { |
48 | logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) | 49 | logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) |
@@ -62,34 +63,32 @@ async function resolveCommentFromDB (params: ResolveThreadParams) { | |||
62 | const { url, comments, commentCreated } = params | 63 | const { url, comments, commentCreated } = params |
63 | 64 | ||
64 | const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) | 65 | const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) |
65 | if (commentFromDatabase) { | 66 | if (!commentFromDatabase) return undefined |
66 | let parentComments = comments.concat([ commentFromDatabase ]) | ||
67 | 67 | ||
68 | // Speed up things and resolve directly the thread | 68 | let parentComments = comments.concat([ commentFromDatabase ]) |
69 | if (commentFromDatabase.InReplyToVideoComment) { | ||
70 | const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') | ||
71 | 69 | ||
72 | parentComments = parentComments.concat(data) | 70 | // Speed up things and resolve directly the thread |
73 | } | 71 | if (commentFromDatabase.InReplyToVideoComment) { |
72 | const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') | ||
74 | 73 | ||
75 | return resolveThread({ | 74 | parentComments = parentComments.concat(data) |
76 | url: commentFromDatabase.Video.url, | ||
77 | comments: parentComments, | ||
78 | isVideo: true, | ||
79 | commentCreated | ||
80 | }) | ||
81 | } | 75 | } |
82 | 76 | ||
83 | return undefined | 77 | return resolveThread({ |
78 | url: commentFromDatabase.Video.url, | ||
79 | comments: parentComments, | ||
80 | isVideo: true, | ||
81 | commentCreated | ||
82 | }) | ||
84 | } | 83 | } |
85 | 84 | ||
86 | async function tryResolveThreadFromVideo (params: ResolveThreadParams) { | 85 | async function tryToResolveThreadFromVideo (params: ResolveThreadParams) { |
87 | const { url, comments, commentCreated } = params | 86 | const { url, comments, commentCreated } = params |
88 | 87 | ||
89 | // Maybe it's a reply to a video? | 88 | // Maybe it's a reply to a video? |
90 | // If yes, it's done: we resolved all the thread | 89 | // If yes, it's done: we resolved all the thread |
91 | const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false } | 90 | const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false } |
92 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) | 91 | const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam }) |
93 | 92 | ||
94 | if (video.isOwned() && !video.hasPrivacyForFederation()) { | 93 | if (video.isOwned() && !video.hasPrivacyForFederation()) { |
95 | throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') | 94 | throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') |
@@ -148,7 +147,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) { | |||
148 | } | 147 | } |
149 | 148 | ||
150 | const actor = actorUrl | 149 | const actor = actorUrl |
151 | ? await getOrCreateActorAndServerAndModel(actorUrl, 'all') | 150 | ? await getOrCreateAPActor(actorUrl, 'all') |
152 | : null | 151 | : null |
153 | 152 | ||
154 | const comment = new VideoCommentModel({ | 153 | const comment = new VideoCommentModel({ |
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index f40c07fea..9fb97ef84 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts | |||
@@ -3,44 +3,23 @@ import { Transaction } from 'sequelize' | |||
3 | import { doJSONRequest } from '@server/helpers/requests' | 3 | import { doJSONRequest } from '@server/helpers/requests' |
4 | import { VideoRateType } from '../../../shared/models/videos' | 4 | import { VideoRateType } from '../../../shared/models/videos' |
5 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | 5 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
6 | import { logger } from '../../helpers/logger' | 6 | import { logger, loggerTagsFactory } from '../../helpers/logger' |
7 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | 7 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
8 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | 8 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' |
9 | import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' | 9 | import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' |
10 | import { getOrCreateActorAndServerAndModel } from './actor' | 10 | import { getOrCreateAPActor } from './actors' |
11 | import { sendLike, sendUndoDislike, sendUndoLike } from './send' | 11 | import { sendLike, sendUndoDislike, sendUndoLike } from './send' |
12 | import { sendDislike } from './send/send-dislike' | 12 | import { sendDislike } from './send/send-dislike' |
13 | import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' | 13 | import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' |
14 | 14 | ||
15 | const lTags = loggerTagsFactory('ap', 'video-rate', 'create') | ||
16 | |||
15 | async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { | 17 | async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { |
16 | await Bluebird.map(ratesUrl, async rateUrl => { | 18 | await Bluebird.map(ratesUrl, async rateUrl => { |
17 | try { | 19 | try { |
18 | // Fetch url | 20 | await createRate(rateUrl, video, rate) |
19 | const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true }) | ||
20 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | ||
21 | |||
22 | const actorUrl = getAPId(body.actor) | ||
23 | if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { | ||
24 | throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) | ||
25 | } | ||
26 | |||
27 | if (checkUrlsSameHost(body.id, rateUrl) !== true) { | ||
28 | throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`) | ||
29 | } | ||
30 | |||
31 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | ||
32 | |||
33 | const entry = { | ||
34 | videoId: video.id, | ||
35 | accountId: actor.Account.id, | ||
36 | type: rate, | ||
37 | url: body.id | ||
38 | } | ||
39 | |||
40 | // Video "likes"/"dislikes" will be updated by the caller | ||
41 | await AccountVideoRateModel.upsert(entry) | ||
42 | } catch (err) { | 21 | } catch (err) { |
43 | logger.warn('Cannot add rate %s.', rateUrl, { err }) | 22 | logger.info('Cannot add rate %s.', rateUrl, { err, ...lTags(rateUrl, video.uuid, video.url) }) |
44 | } | 23 | } |
45 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | 24 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) |
46 | } | 25 | } |
@@ -73,8 +52,39 @@ function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVid | |||
73 | : getVideoDislikeActivityPubUrlByLocalActor(actor, video) | 52 | : getVideoDislikeActivityPubUrlByLocalActor(actor, video) |
74 | } | 53 | } |
75 | 54 | ||
55 | // --------------------------------------------------------------------------- | ||
56 | |||
76 | export { | 57 | export { |
77 | getLocalRateUrl, | 58 | getLocalRateUrl, |
78 | createRates, | 59 | createRates, |
79 | sendVideoRateChange | 60 | sendVideoRateChange |
80 | } | 61 | } |
62 | |||
63 | // --------------------------------------------------------------------------- | ||
64 | |||
65 | async function createRate (rateUrl: string, video: MVideo, rate: VideoRateType) { | ||
66 | // Fetch url | ||
67 | const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true }) | ||
68 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | ||
69 | |||
70 | const actorUrl = getAPId(body.actor) | ||
71 | if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { | ||
72 | throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) | ||
73 | } | ||
74 | |||
75 | if (checkUrlsSameHost(body.id, rateUrl) !== true) { | ||
76 | throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`) | ||
77 | } | ||
78 | |||
79 | const actor = await getOrCreateAPActor(actorUrl) | ||
80 | |||
81 | const entry = { | ||
82 | videoId: video.id, | ||
83 | accountId: actor.Account.id, | ||
84 | type: rate, | ||
85 | url: body.id | ||
86 | } | ||
87 | |||
88 | // Video "likes"/"dislikes" will be updated by the caller | ||
89 | await AccountVideoRateModel.upsert(entry) | ||
90 | } | ||
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts deleted file mode 100644 index 127a0dd8a..000000000 --- a/server/lib/activitypub/videos.ts +++ /dev/null | |||
@@ -1,931 +0,0 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { maxBy, minBy } from 'lodash' | ||
3 | import * as magnetUtil from 'magnet-uri' | ||
4 | import { basename } from 'path' | ||
5 | import { Transaction } from 'sequelize/types' | ||
6 | import { TrackerModel } from '@server/models/server/tracker' | ||
7 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
8 | import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' | ||
9 | import { | ||
10 | ActivityHashTagObject, | ||
11 | ActivityMagnetUrlObject, | ||
12 | ActivityPlaylistSegmentHashesObject, | ||
13 | ActivityPlaylistUrlObject, | ||
14 | ActivitypubHttpFetcherPayload, | ||
15 | ActivityTagObject, | ||
16 | ActivityUrlObject, | ||
17 | ActivityVideoUrlObject | ||
18 | } from '../../../shared/index' | ||
19 | import { ActivityTrackerUrlObject, VideoObject } from '../../../shared/models/activitypub/objects' | ||
20 | import { VideoPrivacy } from '../../../shared/models/videos' | ||
21 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' | ||
22 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
23 | import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | ||
24 | import { | ||
25 | isAPVideoFileUrlMetadataObject, | ||
26 | isAPVideoTrackerUrlObject, | ||
27 | sanitizeAndCheckVideoTorrentObject | ||
28 | } from '../../helpers/custom-validators/activitypub/videos' | ||
29 | import { isArray } from '../../helpers/custom-validators/misc' | ||
30 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | ||
31 | import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' | ||
32 | import { logger } from '../../helpers/logger' | ||
33 | import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests' | ||
34 | import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video' | ||
35 | import { | ||
36 | ACTIVITY_PUB, | ||
37 | MIMETYPES, | ||
38 | P2P_MEDIA_LOADER_PEER_VERSION, | ||
39 | PREVIEWS_SIZE, | ||
40 | REMOTE_SCHEME, | ||
41 | THUMBNAILS_SIZE | ||
42 | } from '../../initializers/constants' | ||
43 | import { sequelizeTypescript } from '../../initializers/database' | ||
44 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | ||
45 | import { VideoModel } from '../../models/video/video' | ||
46 | import { VideoCaptionModel } from '../../models/video/video-caption' | ||
47 | import { VideoCommentModel } from '../../models/video/video-comment' | ||
48 | import { VideoFileModel } from '../../models/video/video-file' | ||
49 | import { VideoShareModel } from '../../models/video/video-share' | ||
50 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
51 | import { | ||
52 | MAccountIdActor, | ||
53 | MChannelAccountLight, | ||
54 | MChannelDefault, | ||
55 | MChannelId, | ||
56 | MStreamingPlaylist, | ||
57 | MStreamingPlaylistFilesVideo, | ||
58 | MStreamingPlaylistVideo, | ||
59 | MVideo, | ||
60 | MVideoAccountLight, | ||
61 | MVideoAccountLightBlacklistAllFiles, | ||
62 | MVideoAP, | ||
63 | MVideoAPWithoutCaption, | ||
64 | MVideoCaption, | ||
65 | MVideoFile, | ||
66 | MVideoFullLight, | ||
67 | MVideoId, | ||
68 | MVideoImmutable, | ||
69 | MVideoThumbnail, | ||
70 | MVideoWithHost | ||
71 | } from '../../types/models' | ||
72 | import { MThumbnail } from '../../types/models/video/thumbnail' | ||
73 | import { FilteredModelAttributes } from '../../types/sequelize' | ||
74 | import { ActorFollowScoreCache } from '../files-cache' | ||
75 | import { JobQueue } from '../job-queue' | ||
76 | import { Notifier } from '../notifier' | ||
77 | import { PeerTubeSocket } from '../peertube-socket' | ||
78 | import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail' | ||
79 | import { setVideoTags } from '../video' | ||
80 | import { autoBlacklistVideoIfNeeded } from '../video-blacklist' | ||
81 | import { generateTorrentFileName } from '../video-paths' | ||
82 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
83 | import { crawlCollectionPage } from './crawl' | ||
84 | import { sendCreateVideo, sendUpdateVideo } from './send' | ||
85 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | ||
86 | import { addVideoComments } from './video-comments' | ||
87 | import { createRates } from './video-rates' | ||
88 | |||
89 | async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) { | ||
90 | const video = videoArg as MVideoAP | ||
91 | |||
92 | if ( | ||
93 | // Check this is not a blacklisted video, or unfederated blacklisted video | ||
94 | (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) && | ||
95 | // Check the video is public/unlisted and published | ||
96 | video.hasPrivacyForFederation() && video.hasStateForFederation() | ||
97 | ) { | ||
98 | // Fetch more attributes that we will need to serialize in AP object | ||
99 | if (isArray(video.VideoCaptions) === false) { | ||
100 | video.VideoCaptions = await video.$get('VideoCaptions', { | ||
101 | attributes: [ 'filename', 'language' ], | ||
102 | transaction | ||
103 | }) | ||
104 | } | ||
105 | |||
106 | if (isNewVideo) { | ||
107 | // Now we'll add the video's meta data to our followers | ||
108 | await sendCreateVideo(video, transaction) | ||
109 | await shareVideoByServerAndChannel(video, transaction) | ||
110 | } else { | ||
111 | await sendUpdateVideo(video, transaction) | ||
112 | } | ||
113 | } | ||
114 | } | ||
115 | |||
116 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | ||
117 | logger.info('Fetching remote video %s.', videoUrl) | ||
118 | |||
119 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | ||
120 | |||
121 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | ||
122 | logger.debug('Remote video JSON is not valid.', { body }) | ||
123 | return { statusCode, videoObject: undefined } | ||
124 | } | ||
125 | |||
126 | return { statusCode, videoObject: body } | ||
127 | } | ||
128 | |||
129 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { | ||
130 | const host = video.VideoChannel.Account.Actor.Server.host | ||
131 | const path = video.getDescriptionAPIPath() | ||
132 | const url = REMOTE_SCHEME.HTTP + '://' + host + path | ||
133 | |||
134 | const { body } = await doJSONRequest<any>(url) | ||
135 | return body.description || '' | ||
136 | } | ||
137 | |||
138 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { | ||
139 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') | ||
140 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) | ||
141 | |||
142 | if (checkUrlsSameHost(channel.id, videoObject.id) !== true) { | ||
143 | throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`) | ||
144 | } | ||
145 | |||
146 | return getOrCreateActorAndServerAndModel(channel.id, 'all') | ||
147 | } | ||
148 | |||
149 | type SyncParam = { | ||
150 | likes: boolean | ||
151 | dislikes: boolean | ||
152 | shares: boolean | ||
153 | comments: boolean | ||
154 | thumbnail: boolean | ||
155 | refreshVideo?: boolean | ||
156 | } | ||
157 | async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoObject, syncParam: SyncParam) { | ||
158 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) | ||
159 | |||
160 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] | ||
161 | |||
162 | if (syncParam.likes === true) { | ||
163 | const handler = items => createRates(items, video, 'like') | ||
164 | const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate) | ||
165 | |||
166 | await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner) | ||
167 | .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes })) | ||
168 | } else { | ||
169 | jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) | ||
170 | } | ||
171 | |||
172 | if (syncParam.dislikes === true) { | ||
173 | const handler = items => createRates(items, video, 'dislike') | ||
174 | const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate) | ||
175 | |||
176 | await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner) | ||
177 | .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes })) | ||
178 | } else { | ||
179 | jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) | ||
180 | } | ||
181 | |||
182 | if (syncParam.shares === true) { | ||
183 | const handler = items => addVideoShares(items, video) | ||
184 | const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) | ||
185 | |||
186 | await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner) | ||
187 | .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares })) | ||
188 | } else { | ||
189 | jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) | ||
190 | } | ||
191 | |||
192 | if (syncParam.comments === true) { | ||
193 | const handler = items => addVideoComments(items) | ||
194 | const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) | ||
195 | |||
196 | await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner) | ||
197 | .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments })) | ||
198 | } else { | ||
199 | jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' }) | ||
200 | } | ||
201 | |||
202 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })) | ||
203 | } | ||
204 | |||
205 | type GetVideoResult <T> = Promise<{ | ||
206 | video: T | ||
207 | created: boolean | ||
208 | autoBlacklisted?: boolean | ||
209 | }> | ||
210 | |||
211 | type GetVideoParamAll = { | ||
212 | videoObject: { id: string } | string | ||
213 | syncParam?: SyncParam | ||
214 | fetchType?: 'all' | ||
215 | allowRefresh?: boolean | ||
216 | } | ||
217 | |||
218 | type GetVideoParamImmutable = { | ||
219 | videoObject: { id: string } | string | ||
220 | syncParam?: SyncParam | ||
221 | fetchType: 'only-immutable-attributes' | ||
222 | allowRefresh: false | ||
223 | } | ||
224 | |||
225 | type GetVideoParamOther = { | ||
226 | videoObject: { id: string } | string | ||
227 | syncParam?: SyncParam | ||
228 | fetchType?: 'all' | 'only-video' | ||
229 | allowRefresh?: boolean | ||
230 | } | ||
231 | |||
232 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | ||
233 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | ||
234 | function getOrCreateVideoAndAccountAndChannel ( | ||
235 | options: GetVideoParamOther | ||
236 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | ||
237 | async function getOrCreateVideoAndAccountAndChannel ( | ||
238 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | ||
239 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
240 | // Default params | ||
241 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
242 | const fetchType = options.fetchType || 'all' | ||
243 | const allowRefresh = options.allowRefresh !== false | ||
244 | |||
245 | // Get video url | ||
246 | const videoUrl = getAPId(options.videoObject) | ||
247 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | ||
248 | |||
249 | if (videoFromDatabase) { | ||
250 | // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type | ||
251 | if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { | ||
252 | const refreshOptions = { | ||
253 | video: videoFromDatabase as MVideoThumbnail, | ||
254 | fetchedType: fetchType, | ||
255 | syncParam | ||
256 | } | ||
257 | |||
258 | if (syncParam.refreshVideo === true) { | ||
259 | videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | ||
260 | } else { | ||
261 | await JobQueue.Instance.createJobWithPromise({ | ||
262 | type: 'activitypub-refresher', | ||
263 | payload: { type: 'video', url: videoFromDatabase.url } | ||
264 | }) | ||
265 | } | ||
266 | } | ||
267 | |||
268 | return { video: videoFromDatabase, created: false } | ||
269 | } | ||
270 | |||
271 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) | ||
272 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
273 | |||
274 | const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) | ||
275 | const videoChannel = actor.VideoChannel | ||
276 | |||
277 | try { | ||
278 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) | ||
279 | |||
280 | await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) | ||
281 | |||
282 | return { video: videoCreated, created: true, autoBlacklisted } | ||
283 | } catch (err) { | ||
284 | // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video | ||
285 | if (err.name === 'SequelizeUniqueConstraintError') { | ||
286 | const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) | ||
287 | if (fallbackVideo) return { video: fallbackVideo, created: false } | ||
288 | } | ||
289 | |||
290 | throw err | ||
291 | } | ||
292 | } | ||
293 | |||
294 | async function updateVideoFromAP (options: { | ||
295 | video: MVideoAccountLightBlacklistAllFiles | ||
296 | videoObject: VideoObject | ||
297 | account: MAccountIdActor | ||
298 | channel: MChannelDefault | ||
299 | overrideTo?: string[] | ||
300 | }) { | ||
301 | const { video, videoObject, account, channel, overrideTo } = options | ||
302 | |||
303 | logger.debug('Updating remote video "%s".', options.videoObject.uuid, { videoObject: options.videoObject, account, channel }) | ||
304 | |||
305 | let videoFieldsSave: any | ||
306 | const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE | ||
307 | const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED | ||
308 | |||
309 | try { | ||
310 | let thumbnailModel: MThumbnail | ||
311 | |||
312 | try { | ||
313 | thumbnailModel = await createVideoMiniatureFromUrl({ | ||
314 | downloadUrl: getThumbnailFromIcons(videoObject).url, | ||
315 | video, | ||
316 | type: ThumbnailType.MINIATURE | ||
317 | }) | ||
318 | } catch (err) { | ||
319 | logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }) | ||
320 | } | ||
321 | |||
322 | const videoUpdated = await sequelizeTypescript.transaction(async t => { | ||
323 | const sequelizeOptions = { transaction: t } | ||
324 | |||
325 | videoFieldsSave = video.toJSON() | ||
326 | |||
327 | // Check we can update the channel: we trust the remote server | ||
328 | const oldVideoChannel = video.VideoChannel | ||
329 | |||
330 | if (!oldVideoChannel.Actor.serverId || !channel.Actor.serverId) { | ||
331 | throw new Error('Cannot check old channel/new channel validity because `serverId` is null') | ||
332 | } | ||
333 | |||
334 | if (oldVideoChannel.Actor.serverId !== channel.Actor.serverId) { | ||
335 | throw new Error('New channel ' + channel.Actor.url + ' is not on the same server than new channel ' + oldVideoChannel.Actor.url) | ||
336 | } | ||
337 | |||
338 | const to = overrideTo || videoObject.to | ||
339 | const videoData = videoActivityObjectToDBAttributes(channel, videoObject, to) | ||
340 | video.name = videoData.name | ||
341 | video.uuid = videoData.uuid | ||
342 | video.url = videoData.url | ||
343 | video.category = videoData.category | ||
344 | video.licence = videoData.licence | ||
345 | video.language = videoData.language | ||
346 | video.description = videoData.description | ||
347 | video.support = videoData.support | ||
348 | video.nsfw = videoData.nsfw | ||
349 | video.commentsEnabled = videoData.commentsEnabled | ||
350 | video.downloadEnabled = videoData.downloadEnabled | ||
351 | video.waitTranscoding = videoData.waitTranscoding | ||
352 | video.state = videoData.state | ||
353 | video.duration = videoData.duration | ||
354 | video.createdAt = videoData.createdAt | ||
355 | video.publishedAt = videoData.publishedAt | ||
356 | video.originallyPublishedAt = videoData.originallyPublishedAt | ||
357 | video.privacy = videoData.privacy | ||
358 | video.channelId = videoData.channelId | ||
359 | video.views = videoData.views | ||
360 | video.isLive = videoData.isLive | ||
361 | |||
362 | // Ensures we update the updated video attribute | ||
363 | video.changed('updatedAt', true) | ||
364 | |||
365 | const videoUpdated = await video.save(sequelizeOptions) as MVideoFullLight | ||
366 | |||
367 | if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t) | ||
368 | |||
369 | const previewIcon = getPreviewFromIcons(videoObject) | ||
370 | if (videoUpdated.getPreview() && previewIcon) { | ||
371 | const previewModel = createPlaceholderThumbnail({ | ||
372 | fileUrl: previewIcon.url, | ||
373 | video, | ||
374 | type: ThumbnailType.PREVIEW, | ||
375 | size: previewIcon | ||
376 | }) | ||
377 | await videoUpdated.addAndSaveThumbnail(previewModel, t) | ||
378 | } | ||
379 | |||
380 | { | ||
381 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url) | ||
382 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) | ||
383 | |||
384 | // Remove video files that do not exist anymore | ||
385 | const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t) | ||
386 | await Promise.all(destroyTasks) | ||
387 | |||
388 | // Update or add other one | ||
389 | const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) | ||
390 | videoUpdated.VideoFiles = await Promise.all(upsertTasks) | ||
391 | } | ||
392 | |||
393 | { | ||
394 | const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles) | ||
395 | const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) | ||
396 | |||
397 | // Remove video playlists that do not exist anymore | ||
398 | const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t) | ||
399 | await Promise.all(destroyTasks) | ||
400 | |||
401 | let oldStreamingPlaylistFiles: MVideoFile[] = [] | ||
402 | for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) { | ||
403 | oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles) | ||
404 | } | ||
405 | |||
406 | videoUpdated.VideoStreamingPlaylists = [] | ||
407 | |||
408 | for (const playlistAttributes of streamingPlaylistAttributes) { | ||
409 | const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t }) | ||
410 | .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo) | ||
411 | streamingPlaylistModel.Video = videoUpdated | ||
412 | |||
413 | const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject) | ||
414 | .map(a => new VideoFileModel(a)) | ||
415 | const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t) | ||
416 | await Promise.all(destroyTasks) | ||
417 | |||
418 | // Update or add other one | ||
419 | const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) | ||
420 | streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks) | ||
421 | |||
422 | videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel) | ||
423 | } | ||
424 | } | ||
425 | |||
426 | { | ||
427 | // Update Tags | ||
428 | const tags = videoObject.tag | ||
429 | .filter(isAPHashTagObject) | ||
430 | .map(tag => tag.name) | ||
431 | await setVideoTags({ video: videoUpdated, tags, transaction: t }) | ||
432 | } | ||
433 | |||
434 | // Update trackers | ||
435 | { | ||
436 | const trackers = getTrackerUrls(videoObject, videoUpdated) | ||
437 | await setVideoTrackers({ video: videoUpdated, trackers, transaction: t }) | ||
438 | } | ||
439 | |||
440 | { | ||
441 | // Update captions | ||
442 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t) | ||
443 | |||
444 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
445 | const caption = new VideoCaptionModel({ | ||
446 | videoId: videoUpdated.id, | ||
447 | filename: VideoCaptionModel.generateCaptionName(c.identifier), | ||
448 | language: c.identifier, | ||
449 | fileUrl: c.url | ||
450 | }) as MVideoCaption | ||
451 | |||
452 | return VideoCaptionModel.insertOrReplaceLanguage(caption, t) | ||
453 | }) | ||
454 | await Promise.all(videoCaptionsPromises) | ||
455 | } | ||
456 | |||
457 | { | ||
458 | // Create or update existing live | ||
459 | if (video.isLive) { | ||
460 | const [ videoLive ] = await VideoLiveModel.upsert({ | ||
461 | saveReplay: videoObject.liveSaveReplay, | ||
462 | permanentLive: videoObject.permanentLive, | ||
463 | videoId: video.id | ||
464 | }, { transaction: t, returning: true }) | ||
465 | |||
466 | videoUpdated.VideoLive = videoLive | ||
467 | } else { // Delete existing live if it exists | ||
468 | await VideoLiveModel.destroy({ | ||
469 | where: { | ||
470 | videoId: video.id | ||
471 | }, | ||
472 | transaction: t | ||
473 | }) | ||
474 | |||
475 | videoUpdated.VideoLive = null | ||
476 | } | ||
477 | } | ||
478 | |||
479 | return videoUpdated | ||
480 | }) | ||
481 | |||
482 | await autoBlacklistVideoIfNeeded({ | ||
483 | video: videoUpdated, | ||
484 | user: undefined, | ||
485 | isRemote: true, | ||
486 | isNew: false, | ||
487 | transaction: undefined | ||
488 | }) | ||
489 | |||
490 | // Notify our users? | ||
491 | if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) | ||
492 | |||
493 | if (videoUpdated.isLive) { | ||
494 | PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated) | ||
495 | PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated) | ||
496 | } | ||
497 | |||
498 | logger.info('Remote video with uuid %s updated', videoObject.uuid) | ||
499 | |||
500 | return videoUpdated | ||
501 | } catch (err) { | ||
502 | if (video !== undefined && videoFieldsSave !== undefined) { | ||
503 | resetSequelizeInstance(video, videoFieldsSave) | ||
504 | } | ||
505 | |||
506 | // This is just a debug because we will retry the insert | ||
507 | logger.debug('Cannot update the remote video.', { err }) | ||
508 | throw err | ||
509 | } | ||
510 | } | ||
511 | |||
512 | async function refreshVideoIfNeeded (options: { | ||
513 | video: MVideoThumbnail | ||
514 | fetchedType: VideoFetchByUrlType | ||
515 | syncParam: SyncParam | ||
516 | }): Promise<MVideoThumbnail> { | ||
517 | if (!options.video.isOutdated()) return options.video | ||
518 | |||
519 | // We need more attributes if the argument video was fetched with not enough joints | ||
520 | const video = options.fetchedType === 'all' | ||
521 | ? options.video as MVideoAccountLightBlacklistAllFiles | ||
522 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
523 | |||
524 | try { | ||
525 | const { videoObject } = await fetchRemoteVideo(video.url) | ||
526 | |||
527 | if (videoObject === undefined) { | ||
528 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | ||
529 | |||
530 | await video.setAsRefreshed() | ||
531 | return video | ||
532 | } | ||
533 | |||
534 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | ||
535 | |||
536 | const updateOptions = { | ||
537 | video, | ||
538 | videoObject, | ||
539 | account: channelActor.VideoChannel.Account, | ||
540 | channel: channelActor.VideoChannel | ||
541 | } | ||
542 | await updateVideoFromAP(updateOptions) | ||
543 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
544 | |||
545 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | ||
546 | |||
547 | return video | ||
548 | } catch (err) { | ||
549 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
550 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | ||
551 | |||
552 | // Video does not exist anymore | ||
553 | await video.destroy() | ||
554 | return undefined | ||
555 | } | ||
556 | |||
557 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | ||
558 | |||
559 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | ||
560 | |||
561 | // Don't refresh in loop | ||
562 | await video.setAsRefreshed() | ||
563 | return video | ||
564 | } | ||
565 | } | ||
566 | |||
567 | export { | ||
568 | updateVideoFromAP, | ||
569 | refreshVideoIfNeeded, | ||
570 | federateVideoIfNeeded, | ||
571 | fetchRemoteVideo, | ||
572 | getOrCreateVideoAndAccountAndChannel, | ||
573 | fetchRemoteVideoDescription, | ||
574 | getOrCreateVideoChannelFromVideoObject | ||
575 | } | ||
576 | |||
577 | // --------------------------------------------------------------------------- | ||
578 | |||
579 | function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { | ||
580 | const urlMediaType = url.mediaType | ||
581 | |||
582 | return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/') | ||
583 | } | ||
584 | |||
585 | function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject { | ||
586 | return url && url.mediaType === 'application/x-mpegURL' | ||
587 | } | ||
588 | |||
589 | function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { | ||
590 | return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json' | ||
591 | } | ||
592 | |||
593 | function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject { | ||
594 | return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' | ||
595 | } | ||
596 | |||
597 | function isAPHashTagObject (url: any): url is ActivityHashTagObject { | ||
598 | return url && url.type === 'Hashtag' | ||
599 | } | ||
600 | |||
601 | async function createVideo (videoObject: VideoObject, channel: MChannelAccountLight, waitThumbnail = false) { | ||
602 | logger.debug('Adding remote video %s.', videoObject.id) | ||
603 | |||
604 | const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to) | ||
605 | const video = VideoModel.build(videoData) as MVideoThumbnail | ||
606 | |||
607 | const promiseThumbnail = createVideoMiniatureFromUrl({ | ||
608 | downloadUrl: getThumbnailFromIcons(videoObject).url, | ||
609 | video, | ||
610 | type: ThumbnailType.MINIATURE | ||
611 | }).catch(err => { | ||
612 | logger.error('Cannot create miniature from url.', { err }) | ||
613 | return undefined | ||
614 | }) | ||
615 | |||
616 | let thumbnailModel: MThumbnail | ||
617 | if (waitThumbnail === true) { | ||
618 | thumbnailModel = await promiseThumbnail | ||
619 | } | ||
620 | |||
621 | const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { | ||
622 | try { | ||
623 | const sequelizeOptions = { transaction: t } | ||
624 | |||
625 | const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight | ||
626 | videoCreated.VideoChannel = channel | ||
627 | |||
628 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | ||
629 | |||
630 | const previewIcon = getPreviewFromIcons(videoObject) | ||
631 | if (previewIcon) { | ||
632 | const previewModel = createPlaceholderThumbnail({ | ||
633 | fileUrl: previewIcon.url, | ||
634 | video: videoCreated, | ||
635 | type: ThumbnailType.PREVIEW, | ||
636 | size: previewIcon | ||
637 | }) | ||
638 | |||
639 | await videoCreated.addAndSaveThumbnail(previewModel, t) | ||
640 | } | ||
641 | |||
642 | // Process files | ||
643 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url) | ||
644 | |||
645 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | ||
646 | const videoFiles = await Promise.all(videoFilePromises) | ||
647 | |||
648 | const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles) | ||
649 | videoCreated.VideoStreamingPlaylists = [] | ||
650 | |||
651 | for (const playlistAttributes of streamingPlaylistsAttributes) { | ||
652 | const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo | ||
653 | playlist.Video = videoCreated | ||
654 | |||
655 | const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject) | ||
656 | const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t })) | ||
657 | playlist.VideoFiles = await Promise.all(videoFilePromises) | ||
658 | |||
659 | videoCreated.VideoStreamingPlaylists.push(playlist) | ||
660 | } | ||
661 | |||
662 | // Process tags | ||
663 | const tags = videoObject.tag | ||
664 | .filter(isAPHashTagObject) | ||
665 | .map(t => t.name) | ||
666 | await setVideoTags({ video: videoCreated, tags, transaction: t }) | ||
667 | |||
668 | // Process captions | ||
669 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
670 | const caption = new VideoCaptionModel({ | ||
671 | videoId: videoCreated.id, | ||
672 | filename: VideoCaptionModel.generateCaptionName(c.identifier), | ||
673 | language: c.identifier, | ||
674 | fileUrl: c.url | ||
675 | }) as MVideoCaption | ||
676 | |||
677 | return VideoCaptionModel.insertOrReplaceLanguage(caption, t) | ||
678 | }) | ||
679 | await Promise.all(videoCaptionsPromises) | ||
680 | |||
681 | // Process trackers | ||
682 | { | ||
683 | const trackers = getTrackerUrls(videoObject, videoCreated) | ||
684 | await setVideoTrackers({ video: videoCreated, trackers, transaction: t }) | ||
685 | } | ||
686 | |||
687 | videoCreated.VideoFiles = videoFiles | ||
688 | |||
689 | if (videoCreated.isLive) { | ||
690 | const videoLive = new VideoLiveModel({ | ||
691 | streamKey: null, | ||
692 | saveReplay: videoObject.liveSaveReplay, | ||
693 | permanentLive: videoObject.permanentLive, | ||
694 | videoId: videoCreated.id | ||
695 | }) | ||
696 | |||
697 | videoCreated.VideoLive = await videoLive.save({ transaction: t }) | ||
698 | } | ||
699 | |||
700 | // We added a video in this channel, set it as updated | ||
701 | await channel.setAsUpdated(t) | ||
702 | |||
703 | const autoBlacklisted = await autoBlacklistVideoIfNeeded({ | ||
704 | video: videoCreated, | ||
705 | user: undefined, | ||
706 | isRemote: true, | ||
707 | isNew: true, | ||
708 | transaction: t | ||
709 | }) | ||
710 | |||
711 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) | ||
712 | |||
713 | return { autoBlacklisted, videoCreated } | ||
714 | } catch (err) { | ||
715 | // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released | ||
716 | // Remove thumbnail | ||
717 | if (thumbnailModel) await thumbnailModel.removeThumbnail() | ||
718 | |||
719 | throw err | ||
720 | } | ||
721 | }) | ||
722 | |||
723 | if (waitThumbnail === false) { | ||
724 | // Error is already caught above | ||
725 | // eslint-disable-next-line @typescript-eslint/no-floating-promises | ||
726 | promiseThumbnail.then(thumbnailModel => { | ||
727 | if (!thumbnailModel) return | ||
728 | |||
729 | thumbnailModel = videoCreated.id | ||
730 | |||
731 | return thumbnailModel.save() | ||
732 | }) | ||
733 | } | ||
734 | |||
735 | return { autoBlacklisted, videoCreated } | ||
736 | } | ||
737 | |||
738 | function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { | ||
739 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | ||
740 | ? VideoPrivacy.PUBLIC | ||
741 | : VideoPrivacy.UNLISTED | ||
742 | |||
743 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
744 | const language = videoObject.language?.identifier | ||
745 | |||
746 | const category = videoObject.category | ||
747 | ? parseInt(videoObject.category.identifier, 10) | ||
748 | : undefined | ||
749 | |||
750 | const licence = videoObject.licence | ||
751 | ? parseInt(videoObject.licence.identifier, 10) | ||
752 | : undefined | ||
753 | |||
754 | const description = videoObject.content || null | ||
755 | const support = videoObject.support || null | ||
756 | |||
757 | return { | ||
758 | name: videoObject.name, | ||
759 | uuid: videoObject.uuid, | ||
760 | url: videoObject.id, | ||
761 | category, | ||
762 | licence, | ||
763 | language, | ||
764 | description, | ||
765 | support, | ||
766 | nsfw: videoObject.sensitive, | ||
767 | commentsEnabled: videoObject.commentsEnabled, | ||
768 | downloadEnabled: videoObject.downloadEnabled, | ||
769 | waitTranscoding: videoObject.waitTranscoding, | ||
770 | isLive: videoObject.isLiveBroadcast, | ||
771 | state: videoObject.state, | ||
772 | channelId: videoChannel.id, | ||
773 | duration: parseInt(duration, 10), | ||
774 | createdAt: new Date(videoObject.published), | ||
775 | publishedAt: new Date(videoObject.published), | ||
776 | |||
777 | originallyPublishedAt: videoObject.originallyPublishedAt | ||
778 | ? new Date(videoObject.originallyPublishedAt) | ||
779 | : null, | ||
780 | |||
781 | updatedAt: new Date(videoObject.updated), | ||
782 | views: videoObject.views, | ||
783 | likes: 0, | ||
784 | dislikes: 0, | ||
785 | remote: true, | ||
786 | privacy | ||
787 | } | ||
788 | } | ||
789 | |||
790 | function videoFileActivityUrlToDBAttributes ( | ||
791 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, | ||
792 | urls: (ActivityTagObject | ActivityUrlObject)[] | ||
793 | ) { | ||
794 | const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
795 | |||
796 | if (fileUrls.length === 0) return [] | ||
797 | |||
798 | const attributes: FilteredModelAttributes<VideoFileModel>[] = [] | ||
799 | for (const fileUrl of fileUrls) { | ||
800 | // Fetch associated magnet uri | ||
801 | const magnet = urls.filter(isAPMagnetUrlObject) | ||
802 | .find(u => u.height === fileUrl.height) | ||
803 | |||
804 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
805 | |||
806 | const parsed = magnetUtil.decode(magnet.href) | ||
807 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
808 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
809 | } | ||
810 | |||
811 | const torrentUrl = Array.isArray(parsed.xs) | ||
812 | ? parsed.xs[0] | ||
813 | : parsed.xs | ||
814 | |||
815 | // Fetch associated metadata url, if any | ||
816 | const metadata = urls.filter(isAPVideoFileUrlMetadataObject) | ||
817 | .find(u => { | ||
818 | return u.height === fileUrl.height && | ||
819 | u.fps === fileUrl.fps && | ||
820 | u.rel.includes(fileUrl.mediaType) | ||
821 | }) | ||
822 | |||
823 | const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) | ||
824 | const resolution = fileUrl.height | ||
825 | const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id | ||
826 | const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null | ||
827 | |||
828 | const attribute = { | ||
829 | extname, | ||
830 | infoHash: parsed.infoHash, | ||
831 | resolution, | ||
832 | size: fileUrl.size, | ||
833 | fps: fileUrl.fps || -1, | ||
834 | metadataUrl: metadata?.href, | ||
835 | |||
836 | // Use the name of the remote file because we don't proxify video file requests | ||
837 | filename: basename(fileUrl.href), | ||
838 | fileUrl: fileUrl.href, | ||
839 | |||
840 | torrentUrl, | ||
841 | // Use our own torrent name since we proxify torrent requests | ||
842 | torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), | ||
843 | |||
844 | // This is a video file owned by a video or by a streaming playlist | ||
845 | videoId, | ||
846 | videoStreamingPlaylistId | ||
847 | } | ||
848 | |||
849 | attributes.push(attribute) | ||
850 | } | ||
851 | |||
852 | return attributes | ||
853 | } | ||
854 | |||
855 | function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) { | ||
856 | const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] | ||
857 | if (playlistUrls.length === 0) return [] | ||
858 | |||
859 | const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = [] | ||
860 | for (const playlistUrlObject of playlistUrls) { | ||
861 | const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject) | ||
862 | |||
863 | let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
864 | |||
865 | // FIXME: backward compatibility introduced in v2.1.0 | ||
866 | if (files.length === 0) files = videoFiles | ||
867 | |||
868 | if (!segmentsSha256UrlObject) { | ||
869 | logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
870 | continue | ||
871 | } | ||
872 | |||
873 | const attribute = { | ||
874 | type: VideoStreamingPlaylistType.HLS, | ||
875 | playlistUrl: playlistUrlObject.href, | ||
876 | segmentsSha256Url: segmentsSha256UrlObject.href, | ||
877 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), | ||
878 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
879 | videoId: video.id, | ||
880 | tagAPObject: playlistUrlObject.tag | ||
881 | } | ||
882 | |||
883 | attributes.push(attribute) | ||
884 | } | ||
885 | |||
886 | return attributes | ||
887 | } | ||
888 | |||
889 | function getThumbnailFromIcons (videoObject: VideoObject) { | ||
890 | let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) | ||
891 | // Fallback if there are not valid icons | ||
892 | if (validIcons.length === 0) validIcons = videoObject.icon | ||
893 | |||
894 | return minBy(validIcons, 'width') | ||
895 | } | ||
896 | |||
897 | function getPreviewFromIcons (videoObject: VideoObject) { | ||
898 | const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth) | ||
899 | |||
900 | return maxBy(validIcons, 'width') | ||
901 | } | ||
902 | |||
903 | function getTrackerUrls (object: VideoObject, video: MVideoWithHost) { | ||
904 | let wsFound = false | ||
905 | |||
906 | const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u)) | ||
907 | .map((u: ActivityTrackerUrlObject) => { | ||
908 | if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true | ||
909 | |||
910 | return u.href | ||
911 | }) | ||
912 | |||
913 | if (wsFound) return trackers | ||
914 | |||
915 | return [ | ||
916 | buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS), | ||
917 | buildRemoteVideoBaseUrl(video, '/tracker/announce') | ||
918 | ] | ||
919 | } | ||
920 | |||
921 | async function setVideoTrackers (options: { | ||
922 | video: MVideo | ||
923 | trackers: string[] | ||
924 | transaction?: Transaction | ||
925 | }) { | ||
926 | const { video, trackers, transaction } = options | ||
927 | |||
928 | const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction) | ||
929 | |||
930 | await video.$set('Trackers', trackerInstances, { transaction }) | ||
931 | } | ||
diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts new file mode 100644 index 000000000..bd0c54b0c --- /dev/null +++ b/server/lib/activitypub/videos/federate.ts | |||
@@ -0,0 +1,36 @@ | |||
1 | import { Transaction } from 'sequelize/types' | ||
2 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
3 | import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models' | ||
4 | import { sendCreateVideo, sendUpdateVideo } from '../send' | ||
5 | import { shareVideoByServerAndChannel } from '../share' | ||
6 | |||
7 | async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) { | ||
8 | const video = videoArg as MVideoAP | ||
9 | |||
10 | if ( | ||
11 | // Check this is not a blacklisted video, or unfederated blacklisted video | ||
12 | (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) && | ||
13 | // Check the video is public/unlisted and published | ||
14 | video.hasPrivacyForFederation() && video.hasStateForFederation() | ||
15 | ) { | ||
16 | // Fetch more attributes that we will need to serialize in AP object | ||
17 | if (isArray(video.VideoCaptions) === false) { | ||
18 | video.VideoCaptions = await video.$get('VideoCaptions', { | ||
19 | attributes: [ 'filename', 'language' ], | ||
20 | transaction | ||
21 | }) | ||
22 | } | ||
23 | |||
24 | if (isNewVideo) { | ||
25 | // Now we'll add the video's meta data to our followers | ||
26 | await sendCreateVideo(video, transaction) | ||
27 | await shareVideoByServerAndChannel(video, transaction) | ||
28 | } else { | ||
29 | await sendUpdateVideo(video, transaction) | ||
30 | } | ||
31 | } | ||
32 | } | ||
33 | |||
34 | export { | ||
35 | federateVideoIfNeeded | ||
36 | } | ||
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts new file mode 100644 index 000000000..f3e2f0625 --- /dev/null +++ b/server/lib/activitypub/videos/get.ts | |||
@@ -0,0 +1,113 @@ | |||
1 | import { getAPId } from '@server/helpers/activitypub' | ||
2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
3 | import { JobQueue } from '@server/lib/job-queue' | ||
4 | import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders' | ||
5 | import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' | ||
6 | import { APObject } from '@shared/models' | ||
7 | import { refreshVideoIfNeeded } from './refresh' | ||
8 | import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' | ||
9 | |||
10 | type GetVideoResult <T> = Promise<{ | ||
11 | video: T | ||
12 | created: boolean | ||
13 | autoBlacklisted?: boolean | ||
14 | }> | ||
15 | |||
16 | type GetVideoParamAll = { | ||
17 | videoObject: APObject | ||
18 | syncParam?: SyncParam | ||
19 | fetchType?: 'all' | ||
20 | allowRefresh?: boolean | ||
21 | } | ||
22 | |||
23 | type GetVideoParamImmutable = { | ||
24 | videoObject: APObject | ||
25 | syncParam?: SyncParam | ||
26 | fetchType: 'only-immutable-attributes' | ||
27 | allowRefresh: false | ||
28 | } | ||
29 | |||
30 | type GetVideoParamOther = { | ||
31 | videoObject: APObject | ||
32 | syncParam?: SyncParam | ||
33 | fetchType?: 'all' | 'only-video' | ||
34 | allowRefresh?: boolean | ||
35 | } | ||
36 | |||
37 | function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | ||
38 | function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | ||
39 | function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | ||
40 | |||
41 | async function getOrCreateAPVideo ( | ||
42 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | ||
43 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
44 | // Default params | ||
45 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
46 | const fetchType = options.fetchType || 'all' | ||
47 | const allowRefresh = options.allowRefresh !== false | ||
48 | |||
49 | // Get video url | ||
50 | const videoUrl = getAPId(options.videoObject) | ||
51 | let videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) | ||
52 | |||
53 | if (videoFromDatabase) { | ||
54 | if (allowRefresh === true) { | ||
55 | // Typings ensure allowRefresh === false in only-immutable-attributes fetch type | ||
56 | videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam) | ||
57 | } | ||
58 | |||
59 | return { video: videoFromDatabase, created: false } | ||
60 | } | ||
61 | |||
62 | const { videoObject } = await fetchRemoteVideo(videoUrl) | ||
63 | if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
64 | |||
65 | // videoUrl is just an alias/rediraction, so process object id instead | ||
66 | if (videoObject.id !== videoUrl) return getOrCreateAPVideo({ ...options, fetchType: 'all', videoObject }) | ||
67 | |||
68 | try { | ||
69 | const creator = new APVideoCreator(videoObject) | ||
70 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) | ||
71 | |||
72 | await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) | ||
73 | |||
74 | return { video: videoCreated, created: true, autoBlacklisted } | ||
75 | } catch (err) { | ||
76 | // Maybe a concurrent getOrCreateAPVideo call created this video | ||
77 | if (err.name === 'SequelizeUniqueConstraintError') { | ||
78 | const alreadyCreatedVideo = await loadVideoByUrl(videoUrl, fetchType) | ||
79 | if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false } | ||
80 | } | ||
81 | |||
82 | throw err | ||
83 | } | ||
84 | } | ||
85 | |||
86 | // --------------------------------------------------------------------------- | ||
87 | |||
88 | export { | ||
89 | getOrCreateAPVideo | ||
90 | } | ||
91 | |||
92 | // --------------------------------------------------------------------------- | ||
93 | |||
94 | async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoLoadByUrlType, syncParam: SyncParam) { | ||
95 | if (!video.isOutdated()) return video | ||
96 | |||
97 | const refreshOptions = { | ||
98 | video, | ||
99 | fetchedType: fetchType, | ||
100 | syncParam | ||
101 | } | ||
102 | |||
103 | if (syncParam.refreshVideo === true) { | ||
104 | return refreshVideoIfNeeded(refreshOptions) | ||
105 | } | ||
106 | |||
107 | await JobQueue.Instance.createJobWithPromise({ | ||
108 | type: 'activitypub-refresher', | ||
109 | payload: { type: 'video', url: video.url } | ||
110 | }) | ||
111 | |||
112 | return video | ||
113 | } | ||
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts new file mode 100644 index 000000000..b22062598 --- /dev/null +++ b/server/lib/activitypub/videos/index.ts | |||
@@ -0,0 +1,4 @@ | |||
1 | export * from './federate' | ||
2 | export * from './get' | ||
3 | export * from './refresh' | ||
4 | export * from './updater' | ||
diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts new file mode 100644 index 000000000..a7b82f286 --- /dev/null +++ b/server/lib/activitypub/videos/refresh.ts | |||
@@ -0,0 +1,68 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
2 | import { PeerTubeRequestError } from '@server/helpers/requests' | ||
3 | import { ActorFollowScoreCache } from '@server/lib/files-cache' | ||
4 | import { VideoLoadByUrlType } from '@server/lib/model-loaders' | ||
5 | import { VideoModel } from '@server/models/video/video' | ||
6 | import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models' | ||
7 | import { HttpStatusCode } from '@shared/core-utils' | ||
8 | import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' | ||
9 | import { APVideoUpdater } from './updater' | ||
10 | |||
11 | async function refreshVideoIfNeeded (options: { | ||
12 | video: MVideoThumbnail | ||
13 | fetchedType: VideoLoadByUrlType | ||
14 | syncParam: SyncParam | ||
15 | }): Promise<MVideoThumbnail> { | ||
16 | if (!options.video.isOutdated()) return options.video | ||
17 | |||
18 | // We need more attributes if the argument video was fetched with not enough joints | ||
19 | const video = options.fetchedType === 'all' | ||
20 | ? options.video as MVideoAccountLightBlacklistAllFiles | ||
21 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
22 | |||
23 | const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url) | ||
24 | |||
25 | logger.info('Refreshing video %s.', video.url, lTags()) | ||
26 | |||
27 | try { | ||
28 | const { videoObject } = await fetchRemoteVideo(video.url) | ||
29 | |||
30 | if (videoObject === undefined) { | ||
31 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags()) | ||
32 | |||
33 | await video.setAsRefreshed() | ||
34 | return video | ||
35 | } | ||
36 | |||
37 | const videoUpdater = new APVideoUpdater(videoObject, video) | ||
38 | await videoUpdater.update() | ||
39 | |||
40 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
41 | |||
42 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | ||
43 | |||
44 | return video | ||
45 | } catch (err) { | ||
46 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
47 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags()) | ||
48 | |||
49 | // Video does not exist anymore | ||
50 | await video.destroy() | ||
51 | return undefined | ||
52 | } | ||
53 | |||
54 | logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags() }) | ||
55 | |||
56 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | ||
57 | |||
58 | // Don't refresh in loop | ||
59 | await video.setAsRefreshed() | ||
60 | return video | ||
61 | } | ||
62 | } | ||
63 | |||
64 | // --------------------------------------------------------------------------- | ||
65 | |||
66 | export { | ||
67 | refreshVideoIfNeeded | ||
68 | } | ||
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts new file mode 100644 index 000000000..e89c94bcd --- /dev/null +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts | |||
@@ -0,0 +1,173 @@ | |||
1 | import { Transaction } from 'sequelize/types' | ||
2 | import { checkUrlsSameHost } from '@server/helpers/activitypub' | ||
3 | import { deleteNonExistingModels } from '@server/helpers/database-utils' | ||
4 | import { logger, LoggerTagsFn } from '@server/helpers/logger' | ||
5 | import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' | ||
6 | import { setVideoTags } from '@server/lib/video' | ||
7 | import { VideoCaptionModel } from '@server/models/video/video-caption' | ||
8 | import { VideoFileModel } from '@server/models/video/video-file' | ||
9 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
10 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
11 | import { MStreamingPlaylistFilesVideo, MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models' | ||
12 | import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' | ||
13 | import { getOrCreateAPActor } from '../../actors' | ||
14 | import { | ||
15 | getCaptionAttributesFromObject, | ||
16 | getFileAttributesFromUrl, | ||
17 | getLiveAttributesFromObject, | ||
18 | getPreviewFromIcons, | ||
19 | getStreamingPlaylistAttributesFromObject, | ||
20 | getTagsFromObject, | ||
21 | getThumbnailFromIcons | ||
22 | } from './object-to-model-attributes' | ||
23 | import { getTrackerUrls, setVideoTrackers } from './trackers' | ||
24 | |||
25 | export abstract class APVideoAbstractBuilder { | ||
26 | protected abstract videoObject: VideoObject | ||
27 | protected abstract lTags: LoggerTagsFn | ||
28 | |||
29 | protected async getOrCreateVideoChannelFromVideoObject () { | ||
30 | const channel = this.videoObject.attributedTo.find(a => a.type === 'Group') | ||
31 | if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url) | ||
32 | |||
33 | if (checkUrlsSameHost(channel.id, this.videoObject.id) !== true) { | ||
34 | throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${this.videoObject.id}`) | ||
35 | } | ||
36 | |||
37 | return getOrCreateAPActor(channel.id, 'all') | ||
38 | } | ||
39 | |||
40 | protected tryToGenerateThumbnail (video: MVideoThumbnail): Promise<MThumbnail> { | ||
41 | return updateVideoMiniatureFromUrl({ | ||
42 | downloadUrl: getThumbnailFromIcons(this.videoObject).url, | ||
43 | video, | ||
44 | type: ThumbnailType.MINIATURE | ||
45 | }).catch(err => { | ||
46 | logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err, ...this.lTags() }) | ||
47 | |||
48 | return undefined | ||
49 | }) | ||
50 | } | ||
51 | |||
52 | protected async setPreview (video: MVideoFullLight, t?: Transaction) { | ||
53 | // Don't fetch the preview that could be big, create a placeholder instead | ||
54 | const previewIcon = getPreviewFromIcons(this.videoObject) | ||
55 | if (!previewIcon) return | ||
56 | |||
57 | const previewModel = updatePlaceholderThumbnail({ | ||
58 | fileUrl: previewIcon.url, | ||
59 | video, | ||
60 | type: ThumbnailType.PREVIEW, | ||
61 | size: previewIcon | ||
62 | }) | ||
63 | |||
64 | await video.addAndSaveThumbnail(previewModel, t) | ||
65 | } | ||
66 | |||
67 | protected async setTags (video: MVideoFullLight, t: Transaction) { | ||
68 | const tags = getTagsFromObject(this.videoObject) | ||
69 | await setVideoTags({ video, tags, transaction: t }) | ||
70 | } | ||
71 | |||
72 | protected async setTrackers (video: MVideoFullLight, t: Transaction) { | ||
73 | const trackers = getTrackerUrls(this.videoObject, video) | ||
74 | await setVideoTrackers({ video, trackers, transaction: t }) | ||
75 | } | ||
76 | |||
77 | protected async insertOrReplaceCaptions (video: MVideoFullLight, t: Transaction) { | ||
78 | const existingCaptions = await VideoCaptionModel.listVideoCaptions(video.id, t) | ||
79 | |||
80 | let captionsToCreate = getCaptionAttributesFromObject(video, this.videoObject) | ||
81 | .map(a => new VideoCaptionModel(a) as MVideoCaption) | ||
82 | |||
83 | for (const existingCaption of existingCaptions) { | ||
84 | // Only keep captions that do not already exist | ||
85 | const filtered = captionsToCreate.filter(c => !c.isEqual(existingCaption)) | ||
86 | |||
87 | // This caption already exists, we don't need to destroy and create it | ||
88 | if (filtered.length !== captionsToCreate.length) { | ||
89 | captionsToCreate = filtered | ||
90 | continue | ||
91 | } | ||
92 | |||
93 | // Destroy this caption that does not exist anymore | ||
94 | await existingCaption.destroy({ transaction: t }) | ||
95 | } | ||
96 | |||
97 | for (const captionToCreate of captionsToCreate) { | ||
98 | await captionToCreate.save({ transaction: t }) | ||
99 | } | ||
100 | } | ||
101 | |||
102 | protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { | ||
103 | const attributes = getLiveAttributesFromObject(video, this.videoObject) | ||
104 | const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) | ||
105 | |||
106 | video.VideoLive = videoLive | ||
107 | } | ||
108 | |||
109 | protected async setWebTorrentFiles (video: MVideoFullLight, t: Transaction) { | ||
110 | const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url) | ||
111 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) | ||
112 | |||
113 | // Remove video files that do not exist anymore | ||
114 | const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t) | ||
115 | await Promise.all(destroyTasks) | ||
116 | |||
117 | // Update or add other one | ||
118 | const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) | ||
119 | video.VideoFiles = await Promise.all(upsertTasks) | ||
120 | } | ||
121 | |||
122 | protected async setStreamingPlaylists (video: MVideoFullLight, t: Transaction) { | ||
123 | const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject, video.VideoFiles || []) | ||
124 | const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) | ||
125 | |||
126 | // Remove video playlists that do not exist anymore | ||
127 | const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t) | ||
128 | await Promise.all(destroyTasks) | ||
129 | |||
130 | video.VideoStreamingPlaylists = [] | ||
131 | |||
132 | for (const playlistAttributes of streamingPlaylistAttributes) { | ||
133 | |||
134 | const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) | ||
135 | streamingPlaylistModel.Video = video | ||
136 | |||
137 | await this.setStreamingPlaylistFiles(video, streamingPlaylistModel, playlistAttributes.tagAPObject, t) | ||
138 | |||
139 | video.VideoStreamingPlaylists.push(streamingPlaylistModel) | ||
140 | } | ||
141 | } | ||
142 | |||
143 | private async insertOrReplaceStreamingPlaylist (attributes: VideoStreamingPlaylistModel['_creationAttributes'], t: Transaction) { | ||
144 | const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) | ||
145 | |||
146 | return streamingPlaylist as MStreamingPlaylistFilesVideo | ||
147 | } | ||
148 | |||
149 | private getStreamingPlaylistFiles (video: MVideoFullLight, type: VideoStreamingPlaylistType) { | ||
150 | const playlist = video.VideoStreamingPlaylists.find(s => s.type === type) | ||
151 | if (!playlist) return [] | ||
152 | |||
153 | return playlist.VideoFiles | ||
154 | } | ||
155 | |||
156 | private async setStreamingPlaylistFiles ( | ||
157 | video: MVideoFullLight, | ||
158 | playlistModel: MStreamingPlaylistFilesVideo, | ||
159 | tagObjects: ActivityTagObject[], | ||
160 | t: Transaction | ||
161 | ) { | ||
162 | const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(video, playlistModel.type) | ||
163 | |||
164 | const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) | ||
165 | |||
166 | const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t) | ||
167 | await Promise.all(destroyTasks) | ||
168 | |||
169 | // Update or add other one | ||
170 | const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) | ||
171 | playlistModel.VideoFiles = await Promise.all(upsertTasks) | ||
172 | } | ||
173 | } | ||
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts new file mode 100644 index 000000000..ad3b88936 --- /dev/null +++ b/server/lib/activitypub/videos/shared/creator.ts | |||
@@ -0,0 +1,88 @@ | |||
1 | |||
2 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | ||
3 | import { sequelizeTypescript } from '@server/initializers/database' | ||
4 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | ||
5 | import { VideoModel } from '@server/models/video/video' | ||
6 | import { MThumbnail, MVideoFullLight, MVideoThumbnail } from '@server/types/models' | ||
7 | import { VideoObject } from '@shared/models' | ||
8 | import { APVideoAbstractBuilder } from './abstract-builder' | ||
9 | import { getVideoAttributesFromObject } from './object-to-model-attributes' | ||
10 | |||
11 | export class APVideoCreator extends APVideoAbstractBuilder { | ||
12 | protected lTags: LoggerTagsFn | ||
13 | |||
14 | constructor (protected readonly videoObject: VideoObject) { | ||
15 | super() | ||
16 | |||
17 | this.lTags = loggerTagsFactory('ap', 'video', 'create', this.videoObject.uuid, this.videoObject.id) | ||
18 | } | ||
19 | |||
20 | async create (waitThumbnail = false) { | ||
21 | logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags()) | ||
22 | |||
23 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() | ||
24 | const channel = channelActor.VideoChannel | ||
25 | |||
26 | const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) | ||
27 | const video = VideoModel.build(videoData) as MVideoThumbnail | ||
28 | |||
29 | const promiseThumbnail = this.tryToGenerateThumbnail(video) | ||
30 | |||
31 | let thumbnailModel: MThumbnail | ||
32 | if (waitThumbnail === true) { | ||
33 | thumbnailModel = await promiseThumbnail | ||
34 | } | ||
35 | |||
36 | const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { | ||
37 | try { | ||
38 | const videoCreated = await video.save({ transaction: t }) as MVideoFullLight | ||
39 | videoCreated.VideoChannel = channel | ||
40 | |||
41 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | ||
42 | |||
43 | await this.setPreview(videoCreated, t) | ||
44 | await this.setWebTorrentFiles(videoCreated, t) | ||
45 | await this.setStreamingPlaylists(videoCreated, t) | ||
46 | await this.setTags(videoCreated, t) | ||
47 | await this.setTrackers(videoCreated, t) | ||
48 | await this.insertOrReplaceCaptions(videoCreated, t) | ||
49 | await this.insertOrReplaceLive(videoCreated, t) | ||
50 | |||
51 | // We added a video in this channel, set it as updated | ||
52 | await channel.setAsUpdated(t) | ||
53 | |||
54 | const autoBlacklisted = await autoBlacklistVideoIfNeeded({ | ||
55 | video: videoCreated, | ||
56 | user: undefined, | ||
57 | isRemote: true, | ||
58 | isNew: true, | ||
59 | transaction: t | ||
60 | }) | ||
61 | |||
62 | logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) | ||
63 | |||
64 | return { autoBlacklisted, videoCreated } | ||
65 | } catch (err) { | ||
66 | // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released | ||
67 | // Remove thumbnail | ||
68 | if (thumbnailModel) await thumbnailModel.removeThumbnail() | ||
69 | |||
70 | throw err | ||
71 | } | ||
72 | }) | ||
73 | |||
74 | if (waitThumbnail === false) { | ||
75 | // Error is already caught above | ||
76 | // eslint-disable-next-line @typescript-eslint/no-floating-promises | ||
77 | promiseThumbnail.then(thumbnailModel => { | ||
78 | if (!thumbnailModel) return | ||
79 | |||
80 | thumbnailModel = videoCreated.id | ||
81 | |||
82 | return thumbnailModel.save() | ||
83 | }) | ||
84 | } | ||
85 | |||
86 | return { autoBlacklisted, videoCreated } | ||
87 | } | ||
88 | } | ||
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts new file mode 100644 index 000000000..951403493 --- /dev/null +++ b/server/lib/activitypub/videos/shared/index.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | export * from './abstract-builder' | ||
2 | export * from './creator' | ||
3 | export * from './object-to-model-attributes' | ||
4 | export * from './trackers' | ||
5 | export * from './url-to-object' | ||
6 | export * from './video-sync-attributes' | ||
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts new file mode 100644 index 000000000..85548428c --- /dev/null +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts | |||
@@ -0,0 +1,256 @@ | |||
1 | import { maxBy, minBy } from 'lodash' | ||
2 | import * as magnetUtil from 'magnet-uri' | ||
3 | import { basename } from 'path' | ||
4 | import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
5 | import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos' | ||
6 | import { logger } from '@server/helpers/logger' | ||
7 | import { getExtFromMimetype } from '@server/helpers/video' | ||
8 | import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' | ||
9 | import { generateTorrentFileName } from '@server/lib/video-paths' | ||
10 | import { VideoFileModel } from '@server/models/video/video-file' | ||
11 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
12 | import { FilteredModelAttributes } from '@server/types' | ||
13 | import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models' | ||
14 | import { | ||
15 | ActivityHashTagObject, | ||
16 | ActivityMagnetUrlObject, | ||
17 | ActivityPlaylistSegmentHashesObject, | ||
18 | ActivityPlaylistUrlObject, | ||
19 | ActivityTagObject, | ||
20 | ActivityUrlObject, | ||
21 | ActivityVideoUrlObject, | ||
22 | VideoObject, | ||
23 | VideoPrivacy, | ||
24 | VideoStreamingPlaylistType | ||
25 | } from '@shared/models' | ||
26 | import { VideoCaptionModel } from '@server/models/video/video-caption' | ||
27 | |||
28 | function getThumbnailFromIcons (videoObject: VideoObject) { | ||
29 | let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) | ||
30 | // Fallback if there are not valid icons | ||
31 | if (validIcons.length === 0) validIcons = videoObject.icon | ||
32 | |||
33 | return minBy(validIcons, 'width') | ||
34 | } | ||
35 | |||
36 | function getPreviewFromIcons (videoObject: VideoObject) { | ||
37 | const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth) | ||
38 | |||
39 | return maxBy(validIcons, 'width') | ||
40 | } | ||
41 | |||
42 | function getTagsFromObject (videoObject: VideoObject) { | ||
43 | return videoObject.tag | ||
44 | .filter(isAPHashTagObject) | ||
45 | .map(t => t.name) | ||
46 | } | ||
47 | |||
48 | function getFileAttributesFromUrl ( | ||
49 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, | ||
50 | urls: (ActivityTagObject | ActivityUrlObject)[] | ||
51 | ) { | ||
52 | const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
53 | |||
54 | if (fileUrls.length === 0) return [] | ||
55 | |||
56 | const attributes: FilteredModelAttributes<VideoFileModel>[] = [] | ||
57 | for (const fileUrl of fileUrls) { | ||
58 | // Fetch associated magnet uri | ||
59 | const magnet = urls.filter(isAPMagnetUrlObject) | ||
60 | .find(u => u.height === fileUrl.height) | ||
61 | |||
62 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
63 | |||
64 | const parsed = magnetUtil.decode(magnet.href) | ||
65 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
66 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
67 | } | ||
68 | |||
69 | const torrentUrl = Array.isArray(parsed.xs) | ||
70 | ? parsed.xs[0] | ||
71 | : parsed.xs | ||
72 | |||
73 | // Fetch associated metadata url, if any | ||
74 | const metadata = urls.filter(isAPVideoFileUrlMetadataObject) | ||
75 | .find(u => { | ||
76 | return u.height === fileUrl.height && | ||
77 | u.fps === fileUrl.fps && | ||
78 | u.rel.includes(fileUrl.mediaType) | ||
79 | }) | ||
80 | |||
81 | const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) | ||
82 | const resolution = fileUrl.height | ||
83 | const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id | ||
84 | const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null | ||
85 | |||
86 | const attribute = { | ||
87 | extname, | ||
88 | infoHash: parsed.infoHash, | ||
89 | resolution, | ||
90 | size: fileUrl.size, | ||
91 | fps: fileUrl.fps || -1, | ||
92 | metadataUrl: metadata?.href, | ||
93 | |||
94 | // Use the name of the remote file because we don't proxify video file requests | ||
95 | filename: basename(fileUrl.href), | ||
96 | fileUrl: fileUrl.href, | ||
97 | |||
98 | torrentUrl, | ||
99 | // Use our own torrent name since we proxify torrent requests | ||
100 | torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), | ||
101 | |||
102 | // This is a video file owned by a video or by a streaming playlist | ||
103 | videoId, | ||
104 | videoStreamingPlaylistId | ||
105 | } | ||
106 | |||
107 | attributes.push(attribute) | ||
108 | } | ||
109 | |||
110 | return attributes | ||
111 | } | ||
112 | |||
113 | function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) { | ||
114 | const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] | ||
115 | if (playlistUrls.length === 0) return [] | ||
116 | |||
117 | const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = [] | ||
118 | for (const playlistUrlObject of playlistUrls) { | ||
119 | const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject) | ||
120 | |||
121 | let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
122 | |||
123 | // FIXME: backward compatibility introduced in v2.1.0 | ||
124 | if (files.length === 0) files = videoFiles | ||
125 | |||
126 | if (!segmentsSha256UrlObject) { | ||
127 | logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
128 | continue | ||
129 | } | ||
130 | |||
131 | const attribute = { | ||
132 | type: VideoStreamingPlaylistType.HLS, | ||
133 | playlistUrl: playlistUrlObject.href, | ||
134 | segmentsSha256Url: segmentsSha256UrlObject.href, | ||
135 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), | ||
136 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
137 | videoId: video.id, | ||
138 | |||
139 | tagAPObject: playlistUrlObject.tag | ||
140 | } | ||
141 | |||
142 | attributes.push(attribute) | ||
143 | } | ||
144 | |||
145 | return attributes | ||
146 | } | ||
147 | |||
148 | function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) { | ||
149 | return { | ||
150 | saveReplay: videoObject.liveSaveReplay, | ||
151 | permanentLive: videoObject.permanentLive, | ||
152 | videoId: video.id | ||
153 | } | ||
154 | } | ||
155 | |||
156 | function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObject) { | ||
157 | return videoObject.subtitleLanguage.map(c => ({ | ||
158 | videoId: video.id, | ||
159 | filename: VideoCaptionModel.generateCaptionName(c.identifier), | ||
160 | language: c.identifier, | ||
161 | fileUrl: c.url | ||
162 | })) | ||
163 | } | ||
164 | |||
165 | function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { | ||
166 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | ||
167 | ? VideoPrivacy.PUBLIC | ||
168 | : VideoPrivacy.UNLISTED | ||
169 | |||
170 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
171 | const language = videoObject.language?.identifier | ||
172 | |||
173 | const category = videoObject.category | ||
174 | ? parseInt(videoObject.category.identifier, 10) | ||
175 | : undefined | ||
176 | |||
177 | const licence = videoObject.licence | ||
178 | ? parseInt(videoObject.licence.identifier, 10) | ||
179 | : undefined | ||
180 | |||
181 | const description = videoObject.content || null | ||
182 | const support = videoObject.support || null | ||
183 | |||
184 | return { | ||
185 | name: videoObject.name, | ||
186 | uuid: videoObject.uuid, | ||
187 | url: videoObject.id, | ||
188 | category, | ||
189 | licence, | ||
190 | language, | ||
191 | description, | ||
192 | support, | ||
193 | nsfw: videoObject.sensitive, | ||
194 | commentsEnabled: videoObject.commentsEnabled, | ||
195 | downloadEnabled: videoObject.downloadEnabled, | ||
196 | waitTranscoding: videoObject.waitTranscoding, | ||
197 | isLive: videoObject.isLiveBroadcast, | ||
198 | state: videoObject.state, | ||
199 | channelId: videoChannel.id, | ||
200 | duration: parseInt(duration, 10), | ||
201 | createdAt: new Date(videoObject.published), | ||
202 | publishedAt: new Date(videoObject.published), | ||
203 | |||
204 | originallyPublishedAt: videoObject.originallyPublishedAt | ||
205 | ? new Date(videoObject.originallyPublishedAt) | ||
206 | : null, | ||
207 | |||
208 | updatedAt: new Date(videoObject.updated), | ||
209 | views: videoObject.views, | ||
210 | likes: 0, | ||
211 | dislikes: 0, | ||
212 | remote: true, | ||
213 | privacy | ||
214 | } | ||
215 | } | ||
216 | |||
217 | // --------------------------------------------------------------------------- | ||
218 | |||
219 | export { | ||
220 | getThumbnailFromIcons, | ||
221 | getPreviewFromIcons, | ||
222 | |||
223 | getTagsFromObject, | ||
224 | |||
225 | getFileAttributesFromUrl, | ||
226 | getStreamingPlaylistAttributesFromObject, | ||
227 | |||
228 | getLiveAttributesFromObject, | ||
229 | getCaptionAttributesFromObject, | ||
230 | |||
231 | getVideoAttributesFromObject | ||
232 | } | ||
233 | |||
234 | // --------------------------------------------------------------------------- | ||
235 | |||
236 | function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { | ||
237 | const urlMediaType = url.mediaType | ||
238 | |||
239 | return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/') | ||
240 | } | ||
241 | |||
242 | function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject { | ||
243 | return url && url.mediaType === 'application/x-mpegURL' | ||
244 | } | ||
245 | |||
246 | function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { | ||
247 | return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json' | ||
248 | } | ||
249 | |||
250 | function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject { | ||
251 | return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' | ||
252 | } | ||
253 | |||
254 | function isAPHashTagObject (url: any): url is ActivityHashTagObject { | ||
255 | return url && url.type === 'Hashtag' | ||
256 | } | ||
diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts new file mode 100644 index 000000000..1c5fc4f84 --- /dev/null +++ b/server/lib/activitypub/videos/shared/trackers.ts | |||
@@ -0,0 +1,43 @@ | |||
1 | import { Transaction } from 'sequelize/types' | ||
2 | import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' | ||
3 | import { isAPVideoTrackerUrlObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
4 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
5 | import { REMOTE_SCHEME } from '@server/initializers/constants' | ||
6 | import { TrackerModel } from '@server/models/server/tracker' | ||
7 | import { MVideo, MVideoWithHost } from '@server/types/models' | ||
8 | import { ActivityTrackerUrlObject, VideoObject } from '@shared/models' | ||
9 | |||
10 | function getTrackerUrls (object: VideoObject, video: MVideoWithHost) { | ||
11 | let wsFound = false | ||
12 | |||
13 | const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u)) | ||
14 | .map((u: ActivityTrackerUrlObject) => { | ||
15 | if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true | ||
16 | |||
17 | return u.href | ||
18 | }) | ||
19 | |||
20 | if (wsFound) return trackers | ||
21 | |||
22 | return [ | ||
23 | buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS), | ||
24 | buildRemoteVideoBaseUrl(video, '/tracker/announce') | ||
25 | ] | ||
26 | } | ||
27 | |||
28 | async function setVideoTrackers (options: { | ||
29 | video: MVideo | ||
30 | trackers: string[] | ||
31 | transaction: Transaction | ||
32 | }) { | ||
33 | const { video, trackers, transaction } = options | ||
34 | |||
35 | const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction) | ||
36 | |||
37 | await video.$set('Trackers', trackerInstances, { transaction }) | ||
38 | } | ||
39 | |||
40 | export { | ||
41 | getTrackerUrls, | ||
42 | setVideoTrackers | ||
43 | } | ||
diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts new file mode 100644 index 000000000..dba3e9480 --- /dev/null +++ b/server/lib/activitypub/videos/shared/url-to-object.ts | |||
@@ -0,0 +1,25 @@ | |||
1 | import { checkUrlsSameHost } from '@server/helpers/activitypub' | ||
2 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
4 | import { doJSONRequest } from '@server/helpers/requests' | ||
5 | import { VideoObject } from '@shared/models' | ||
6 | |||
7 | const lTags = loggerTagsFactory('ap', 'video') | ||
8 | |||
9 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | ||
10 | logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl)) | ||
11 | |||
12 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | ||
13 | |||
14 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | ||
15 | logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) }) | ||
16 | |||
17 | return { statusCode, videoObject: undefined } | ||
18 | } | ||
19 | |||
20 | return { statusCode, videoObject: body } | ||
21 | } | ||
22 | |||
23 | export { | ||
24 | fetchRemoteVideo | ||
25 | } | ||
diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts new file mode 100644 index 000000000..c4e101005 --- /dev/null +++ b/server/lib/activitypub/videos/shared/video-sync-attributes.ts | |||
@@ -0,0 +1,94 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
2 | import { JobQueue } from '@server/lib/job-queue' | ||
3 | import { AccountVideoRateModel } from '@server/models/account/account-video-rate' | ||
4 | import { VideoCommentModel } from '@server/models/video/video-comment' | ||
5 | import { VideoShareModel } from '@server/models/video/video-share' | ||
6 | import { MVideo } from '@server/types/models' | ||
7 | import { ActivitypubHttpFetcherPayload, VideoObject } from '@shared/models' | ||
8 | import { crawlCollectionPage } from '../../crawl' | ||
9 | import { addVideoShares } from '../../share' | ||
10 | import { addVideoComments } from '../../video-comments' | ||
11 | import { createRates } from '../../video-rates' | ||
12 | |||
13 | const lTags = loggerTagsFactory('ap', 'video') | ||
14 | |||
15 | type SyncParam = { | ||
16 | likes: boolean | ||
17 | dislikes: boolean | ||
18 | shares: boolean | ||
19 | comments: boolean | ||
20 | thumbnail: boolean | ||
21 | refreshVideo?: boolean | ||
22 | } | ||
23 | |||
24 | async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoObject, syncParam: SyncParam) { | ||
25 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) | ||
26 | |||
27 | await syncRates('like', video, fetchedVideo, syncParam.likes) | ||
28 | await syncRates('dislike', video, fetchedVideo, syncParam.dislikes) | ||
29 | |||
30 | await syncShares(video, fetchedVideo, syncParam.shares) | ||
31 | |||
32 | await syncComments(video, fetchedVideo, syncParam.comments) | ||
33 | } | ||
34 | |||
35 | // --------------------------------------------------------------------------- | ||
36 | |||
37 | export { | ||
38 | SyncParam, | ||
39 | syncVideoExternalAttributes | ||
40 | } | ||
41 | |||
42 | // --------------------------------------------------------------------------- | ||
43 | |||
44 | function createJob (payload: ActivitypubHttpFetcherPayload) { | ||
45 | return JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) | ||
46 | } | ||
47 | |||
48 | function syncRates (type: 'like' | 'dislike', video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { | ||
49 | const uri = type === 'like' | ||
50 | ? fetchedVideo.likes | ||
51 | : fetchedVideo.dislikes | ||
52 | |||
53 | if (!isSync) { | ||
54 | const jobType = type === 'like' | ||
55 | ? 'video-likes' | ||
56 | : 'video-dislikes' | ||
57 | |||
58 | return createJob({ uri, videoId: video.id, type: jobType }) | ||
59 | } | ||
60 | |||
61 | const handler = items => createRates(items, video, type) | ||
62 | const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, type, crawlStartDate) | ||
63 | |||
64 | return crawlCollectionPage<string>(uri, handler, cleaner) | ||
65 | .catch(err => logger.error('Cannot add rate of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) | ||
66 | } | ||
67 | |||
68 | function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { | ||
69 | const uri = fetchedVideo.shares | ||
70 | |||
71 | if (!isSync) { | ||
72 | return createJob({ uri, videoId: video.id, type: 'video-shares' }) | ||
73 | } | ||
74 | |||
75 | const handler = items => addVideoShares(items, video) | ||
76 | const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) | ||
77 | |||
78 | return crawlCollectionPage<string>(uri, handler, cleaner) | ||
79 | .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) | ||
80 | } | ||
81 | |||
82 | function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { | ||
83 | const uri = fetchedVideo.comments | ||
84 | |||
85 | if (!isSync) { | ||
86 | return createJob({ uri, videoId: video.id, type: 'video-comments' }) | ||
87 | } | ||
88 | |||
89 | const handler = items => addVideoComments(items) | ||
90 | const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) | ||
91 | |||
92 | return crawlCollectionPage<string>(uri, handler, cleaner) | ||
93 | .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) | ||
94 | } | ||
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts new file mode 100644 index 000000000..157569414 --- /dev/null +++ b/server/lib/activitypub/videos/updater.ts | |||
@@ -0,0 +1,166 @@ | |||
1 | import { Transaction } from 'sequelize/types' | ||
2 | import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' | ||
3 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | ||
4 | import { Notifier } from '@server/lib/notifier' | ||
5 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | ||
6 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | ||
7 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
8 | import { MActor, MChannelAccountLight, MChannelId, MVideoAccountLightBlacklistAllFiles, MVideoFullLight } from '@server/types/models' | ||
9 | import { VideoObject, VideoPrivacy } from '@shared/models' | ||
10 | import { APVideoAbstractBuilder, getVideoAttributesFromObject } from './shared' | ||
11 | |||
12 | export class APVideoUpdater extends APVideoAbstractBuilder { | ||
13 | private readonly wasPrivateVideo: boolean | ||
14 | private readonly wasUnlistedVideo: boolean | ||
15 | |||
16 | private readonly videoFieldsSave: any | ||
17 | |||
18 | private readonly oldVideoChannel: MChannelAccountLight | ||
19 | |||
20 | protected lTags: LoggerTagsFn | ||
21 | |||
22 | constructor ( | ||
23 | protected readonly videoObject: VideoObject, | ||
24 | private readonly video: MVideoAccountLightBlacklistAllFiles | ||
25 | ) { | ||
26 | super() | ||
27 | |||
28 | this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE | ||
29 | this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED | ||
30 | |||
31 | this.oldVideoChannel = this.video.VideoChannel | ||
32 | |||
33 | this.videoFieldsSave = this.video.toJSON() | ||
34 | |||
35 | this.lTags = loggerTagsFactory('ap', 'video', 'update', video.uuid, video.url) | ||
36 | } | ||
37 | |||
38 | async update (overrideTo?: string[]) { | ||
39 | logger.debug( | ||
40 | 'Updating remote video "%s".', this.videoObject.uuid, | ||
41 | { videoObject: this.videoObject, ...this.lTags() } | ||
42 | ) | ||
43 | |||
44 | try { | ||
45 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() | ||
46 | |||
47 | const thumbnailModel = await this.tryToGenerateThumbnail(this.video) | ||
48 | |||
49 | this.checkChannelUpdateOrThrow(channelActor) | ||
50 | |||
51 | const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo) | ||
52 | |||
53 | if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel) | ||
54 | |||
55 | await runInReadCommittedTransaction(async t => { | ||
56 | await this.setWebTorrentFiles(videoUpdated, t) | ||
57 | await this.setStreamingPlaylists(videoUpdated, t) | ||
58 | }) | ||
59 | |||
60 | await Promise.all([ | ||
61 | runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), | ||
62 | runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), | ||
63 | this.setOrDeleteLive(videoUpdated), | ||
64 | this.setPreview(videoUpdated) | ||
65 | ]) | ||
66 | |||
67 | await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) | ||
68 | |||
69 | await autoBlacklistVideoIfNeeded({ | ||
70 | video: videoUpdated, | ||
71 | user: undefined, | ||
72 | isRemote: true, | ||
73 | isNew: false, | ||
74 | transaction: undefined | ||
75 | }) | ||
76 | |||
77 | // Notify our users? | ||
78 | if (this.wasPrivateVideo || this.wasUnlistedVideo) { | ||
79 | Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) | ||
80 | } | ||
81 | |||
82 | if (videoUpdated.isLive) { | ||
83 | PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated) | ||
84 | PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated) | ||
85 | } | ||
86 | |||
87 | logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags()) | ||
88 | |||
89 | return videoUpdated | ||
90 | } catch (err) { | ||
91 | this.catchUpdateError(err) | ||
92 | } | ||
93 | } | ||
94 | |||
95 | // Check we can update the channel: we trust the remote server | ||
96 | private checkChannelUpdateOrThrow (newChannelActor: MActor) { | ||
97 | if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) { | ||
98 | throw new Error('Cannot check old channel/new channel validity because `serverId` is null') | ||
99 | } | ||
100 | |||
101 | if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) { | ||
102 | throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`) | ||
103 | } | ||
104 | } | ||
105 | |||
106 | private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) { | ||
107 | const to = overrideTo || this.videoObject.to | ||
108 | const videoData = getVideoAttributesFromObject(channel, this.videoObject, to) | ||
109 | this.video.name = videoData.name | ||
110 | this.video.uuid = videoData.uuid | ||
111 | this.video.url = videoData.url | ||
112 | this.video.category = videoData.category | ||
113 | this.video.licence = videoData.licence | ||
114 | this.video.language = videoData.language | ||
115 | this.video.description = videoData.description | ||
116 | this.video.support = videoData.support | ||
117 | this.video.nsfw = videoData.nsfw | ||
118 | this.video.commentsEnabled = videoData.commentsEnabled | ||
119 | this.video.downloadEnabled = videoData.downloadEnabled | ||
120 | this.video.waitTranscoding = videoData.waitTranscoding | ||
121 | this.video.state = videoData.state | ||
122 | this.video.duration = videoData.duration | ||
123 | this.video.createdAt = videoData.createdAt | ||
124 | this.video.publishedAt = videoData.publishedAt | ||
125 | this.video.originallyPublishedAt = videoData.originallyPublishedAt | ||
126 | this.video.privacy = videoData.privacy | ||
127 | this.video.channelId = videoData.channelId | ||
128 | this.video.views = videoData.views | ||
129 | this.video.isLive = videoData.isLive | ||
130 | |||
131 | // Ensures we update the updatedAt attribute, even if main attributes did not change | ||
132 | this.video.changed('updatedAt', true) | ||
133 | |||
134 | return this.video.save({ transaction }) as Promise<MVideoFullLight> | ||
135 | } | ||
136 | |||
137 | private async setCaptions (videoUpdated: MVideoFullLight, t: Transaction) { | ||
138 | await this.insertOrReplaceCaptions(videoUpdated, t) | ||
139 | } | ||
140 | |||
141 | private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { | ||
142 | if (!this.video.isLive) return | ||
143 | |||
144 | if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction) | ||
145 | |||
146 | // Delete existing live if it exists | ||
147 | await VideoLiveModel.destroy({ | ||
148 | where: { | ||
149 | videoId: this.video.id | ||
150 | }, | ||
151 | transaction | ||
152 | }) | ||
153 | |||
154 | videoUpdated.VideoLive = null | ||
155 | } | ||
156 | |||
157 | private catchUpdateError (err: Error) { | ||
158 | if (this.video !== undefined && this.videoFieldsSave !== undefined) { | ||
159 | resetSequelizeInstance(this.video, this.videoFieldsSave) | ||
160 | } | ||
161 | |||
162 | // This is just a debug because we will retry the insert | ||
163 | logger.debug('Cannot update the remote video.', { err, ...this.lTags() }) | ||
164 | throw err | ||
165 | } | ||
166 | } | ||
diff --git a/server/lib/auth/oauth-model.ts b/server/lib/auth/oauth-model.ts index b9c69eb2d..ae728d080 100644 --- a/server/lib/auth/oauth-model.ts +++ b/server/lib/auth/oauth-model.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { AccessDeniedError } from 'oauth2-server' | 2 | import { AccessDeniedError } from 'oauth2-server' |
3 | import { PluginManager } from '@server/lib/plugins/plugin-manager' | 3 | import { PluginManager } from '@server/lib/plugins/plugin-manager' |
4 | import { ActorModel } from '@server/models/activitypub/actor' | 4 | import { ActorModel } from '@server/models/actor/actor' |
5 | import { MOAuthClient } from '@server/types/models' | 5 | import { MOAuthClient } from '@server/types/models' |
6 | import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token' | 6 | import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token' |
7 | import { MUser } from '@server/types/models/user/user' | 7 | import { MUser } from '@server/types/models/user/user' |
@@ -9,7 +9,7 @@ import { UserAdminFlag } from '@shared/models/users/user-flag.model' | |||
9 | import { UserRole } from '@shared/models/users/user-role' | 9 | import { UserRole } from '@shared/models/users/user-role' |
10 | import { logger } from '../../helpers/logger' | 10 | import { logger } from '../../helpers/logger' |
11 | import { CONFIG } from '../../initializers/config' | 11 | import { CONFIG } from '../../initializers/config' |
12 | import { UserModel } from '../../models/account/user' | 12 | import { UserModel } from '../../models/user/user' |
13 | import { OAuthClientModel } from '../../models/oauth/oauth-client' | 13 | import { OAuthClientModel } from '../../models/oauth/oauth-client' |
14 | import { OAuthTokenModel } from '../../models/oauth/oauth-token' | 14 | import { OAuthTokenModel } from '../../models/oauth/oauth-token' |
15 | import { createUserAccountAndChannelAndPlaylist } from '../user' | 15 | import { createUserAccountAndChannelAndPlaylist } from '../user' |
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 203bd3893..72194416d 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -2,12 +2,14 @@ import * as express from 'express' | |||
2 | import { readFile } from 'fs-extra' | 2 | import { readFile } from 'fs-extra' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import validator from 'validator' | 4 | import validator from 'validator' |
5 | import { escapeHTML } from '@shared/core-utils/renderer' | ||
6 | import { HTMLServerConfig } from '@shared/models' | ||
5 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n' | 7 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n' |
6 | import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' | 8 | import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' |
7 | import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos' | 9 | import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos' |
8 | import { isTestInstance, sha256 } from '../helpers/core-utils' | 10 | import { isTestInstance, sha256 } from '../helpers/core-utils' |
9 | import { escapeHTML } from '@shared/core-utils/renderer' | ||
10 | import { logger } from '../helpers/logger' | 11 | import { logger } from '../helpers/logger' |
12 | import { mdToPlainText } from '../helpers/markdown' | ||
11 | import { CONFIG } from '../initializers/config' | 13 | import { CONFIG } from '../initializers/config' |
12 | import { | 14 | import { |
13 | ACCEPT_HEADERS, | 15 | ACCEPT_HEADERS, |
@@ -19,12 +21,13 @@ import { | |||
19 | WEBSERVER | 21 | WEBSERVER |
20 | } from '../initializers/constants' | 22 | } from '../initializers/constants' |
21 | import { AccountModel } from '../models/account/account' | 23 | import { AccountModel } from '../models/account/account' |
24 | import { getActivityStreamDuration } from '../models/video/formatter/video-format-utils' | ||
22 | import { VideoModel } from '../models/video/video' | 25 | import { VideoModel } from '../models/video/video' |
23 | import { VideoChannelModel } from '../models/video/video-channel' | 26 | import { VideoChannelModel } from '../models/video/video-channel' |
24 | import { getActivityStreamDuration } from '../models/video/video-format-utils' | ||
25 | import { VideoPlaylistModel } from '../models/video/video-playlist' | 27 | import { VideoPlaylistModel } from '../models/video/video-playlist' |
26 | import { MAccountActor, MChannelActor } from '../types/models' | 28 | import { MAccountActor, MChannelActor } from '../types/models' |
27 | import { mdToPlainText } from '../helpers/markdown' | 29 | import { ServerConfigManager } from './server-config-manager' |
30 | import { toCompleteUUID } from '@server/helpers/custom-validators/misc' | ||
28 | 31 | ||
29 | type Tags = { | 32 | type Tags = { |
30 | ogType: string | 33 | ogType: string |
@@ -76,7 +79,9 @@ class ClientHtml { | |||
76 | return customHtml | 79 | return customHtml |
77 | } | 80 | } |
78 | 81 | ||
79 | static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { | 82 | static async getWatchHTMLPage (videoIdArg: string, req: express.Request, res: express.Response) { |
83 | const videoId = toCompleteUUID(videoIdArg) | ||
84 | |||
80 | // Let Angular application handle errors | 85 | // Let Angular application handle errors |
81 | if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) { | 86 | if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) { |
82 | res.status(HttpStatusCode.NOT_FOUND_404) | 87 | res.status(HttpStatusCode.NOT_FOUND_404) |
@@ -134,7 +139,9 @@ class ClientHtml { | |||
134 | return customHtml | 139 | return customHtml |
135 | } | 140 | } |
136 | 141 | ||
137 | static async getWatchPlaylistHTMLPage (videoPlaylistId: string, req: express.Request, res: express.Response) { | 142 | static async getWatchPlaylistHTMLPage (videoPlaylistIdArg: string, req: express.Request, res: express.Response) { |
143 | const videoPlaylistId = toCompleteUUID(videoPlaylistIdArg) | ||
144 | |||
138 | // Let Angular application handle errors | 145 | // Let Angular application handle errors |
139 | if (!validator.isInt(videoPlaylistId) && !validator.isUUID(videoPlaylistId, 4)) { | 146 | if (!validator.isInt(videoPlaylistId) && !validator.isUUID(videoPlaylistId, 4)) { |
140 | res.status(HttpStatusCode.NOT_FOUND_404) | 147 | res.status(HttpStatusCode.NOT_FOUND_404) |
@@ -196,11 +203,22 @@ class ClientHtml { | |||
196 | } | 203 | } |
197 | 204 | ||
198 | static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { | 205 | static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { |
199 | return this.getAccountOrChannelHTMLPage(() => AccountModel.loadByNameWithHost(nameWithHost), req, res) | 206 | const accountModelPromise = AccountModel.loadByNameWithHost(nameWithHost) |
207 | return this.getAccountOrChannelHTMLPage(() => accountModelPromise, req, res) | ||
200 | } | 208 | } |
201 | 209 | ||
202 | static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { | 210 | static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { |
203 | return this.getAccountOrChannelHTMLPage(() => VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost), req, res) | 211 | const videoChannelModelPromise = VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) |
212 | return this.getAccountOrChannelHTMLPage(() => videoChannelModelPromise, req, res) | ||
213 | } | ||
214 | |||
215 | static async getActorHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { | ||
216 | const [ account, channel ] = await Promise.all([ | ||
217 | AccountModel.loadByNameWithHost(nameWithHost), | ||
218 | VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) | ||
219 | ]) | ||
220 | |||
221 | return this.getAccountOrChannelHTMLPage(() => Promise.resolve(account || channel), req, res) | ||
204 | } | 222 | } |
205 | 223 | ||
206 | static async getEmbedHTML () { | 224 | static async getEmbedHTML () { |
@@ -209,11 +227,14 @@ class ClientHtml { | |||
209 | if (!isTestInstance() && ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] | 227 | if (!isTestInstance() && ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] |
210 | 228 | ||
211 | const buffer = await readFile(path) | 229 | const buffer = await readFile(path) |
230 | const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() | ||
212 | 231 | ||
213 | let html = buffer.toString() | 232 | let html = buffer.toString() |
214 | html = await ClientHtml.addAsyncPluginCSS(html) | 233 | html = await ClientHtml.addAsyncPluginCSS(html) |
215 | html = ClientHtml.addCustomCSS(html) | 234 | html = ClientHtml.addCustomCSS(html) |
216 | html = ClientHtml.addTitleTag(html) | 235 | html = ClientHtml.addTitleTag(html) |
236 | html = ClientHtml.addDescriptionTag(html) | ||
237 | html = ClientHtml.addServerConfig(html, serverConfig) | ||
217 | 238 | ||
218 | ClientHtml.htmlCache[path] = html | 239 | ClientHtml.htmlCache[path] = html |
219 | 240 | ||
@@ -275,6 +296,7 @@ class ClientHtml { | |||
275 | if (!isTestInstance() && ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] | 296 | if (!isTestInstance() && ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] |
276 | 297 | ||
277 | const buffer = await readFile(path) | 298 | const buffer = await readFile(path) |
299 | const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() | ||
278 | 300 | ||
279 | let html = buffer.toString() | 301 | let html = buffer.toString() |
280 | 302 | ||
@@ -283,6 +305,7 @@ class ClientHtml { | |||
283 | html = ClientHtml.addFaviconContentHash(html) | 305 | html = ClientHtml.addFaviconContentHash(html) |
284 | html = ClientHtml.addLogoContentHash(html) | 306 | html = ClientHtml.addLogoContentHash(html) |
285 | html = ClientHtml.addCustomCSS(html) | 307 | html = ClientHtml.addCustomCSS(html) |
308 | html = ClientHtml.addServerConfig(html, serverConfig) | ||
286 | html = await ClientHtml.addAsyncPluginCSS(html) | 309 | html = await ClientHtml.addAsyncPluginCSS(html) |
287 | 310 | ||
288 | ClientHtml.htmlCache[path] = html | 311 | ClientHtml.htmlCache[path] = html |
@@ -355,6 +378,13 @@ class ClientHtml { | |||
355 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) | 378 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) |
356 | } | 379 | } |
357 | 380 | ||
381 | private static addServerConfig (htmlStringPage: string, serverConfig: HTMLServerConfig) { | ||
382 | const serverConfigString = JSON.stringify(serverConfig) | ||
383 | const configScriptTag = `<script type="application/javascript">window.PeerTubeServerConfig = '${serverConfigString}'</script>` | ||
384 | |||
385 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.SERVER_CONFIG, configScriptTag) | ||
386 | } | ||
387 | |||
358 | private static async addAsyncPluginCSS (htmlStringPage: string) { | 388 | private static async addAsyncPluginCSS (htmlStringPage: string) { |
359 | const globalCSSContent = await readFile(PLUGIN_GLOBAL_CSS_PATH) | 389 | const globalCSSContent = await readFile(PLUGIN_GLOBAL_CSS_PATH) |
360 | if (globalCSSContent.byteLength === 0) return htmlStringPage | 390 | if (globalCSSContent.byteLength === 0) return htmlStringPage |
@@ -524,11 +554,11 @@ async function serveIndexHTML (req: express.Request, res: express.Response) { | |||
524 | return | 554 | return |
525 | } catch (err) { | 555 | } catch (err) { |
526 | logger.error('Cannot generate HTML page.', err) | 556 | logger.error('Cannot generate HTML page.', err) |
527 | return res.sendStatus(HttpStatusCode.INTERNAL_SERVER_ERROR_500) | 557 | return res.status(HttpStatusCode.INTERNAL_SERVER_ERROR_500).end() |
528 | } | 558 | } |
529 | } | 559 | } |
530 | 560 | ||
531 | return res.sendStatus(HttpStatusCode.NOT_ACCEPTABLE_406) | 561 | return res.status(HttpStatusCode.NOT_ACCEPTABLE_406).end() |
532 | } | 562 | } |
533 | 563 | ||
534 | // --------------------------------------------------------------------------- | 564 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/config.ts b/server/lib/config.ts deleted file mode 100644 index b4c4c9299..000000000 --- a/server/lib/config.ts +++ /dev/null | |||
@@ -1,255 +0,0 @@ | |||
1 | import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/helpers/signup' | ||
2 | import { getServerCommit } from '@server/helpers/utils' | ||
3 | import { CONFIG, isEmailEnabled } from '@server/initializers/config' | ||
4 | import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants' | ||
5 | import { RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models' | ||
6 | import { Hooks } from './plugins/hooks' | ||
7 | import { PluginManager } from './plugins/plugin-manager' | ||
8 | import { getThemeOrDefault } from './plugins/theme-utils' | ||
9 | import { getEnabledResolutions } from './video-transcoding' | ||
10 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | ||
11 | |||
12 | let serverCommit: string | ||
13 | |||
14 | async function getServerConfig (ip?: string): Promise<ServerConfig> { | ||
15 | if (serverCommit === undefined) serverCommit = await getServerCommit() | ||
16 | |||
17 | const { allowed } = await Hooks.wrapPromiseFun( | ||
18 | isSignupAllowed, | ||
19 | { | ||
20 | ip | ||
21 | }, | ||
22 | 'filter:api.user.signup.allowed.result' | ||
23 | ) | ||
24 | |||
25 | const allowedForCurrentIP = isSignupAllowedForCurrentIP(ip) | ||
26 | const defaultTheme = getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) | ||
27 | |||
28 | return { | ||
29 | instance: { | ||
30 | name: CONFIG.INSTANCE.NAME, | ||
31 | shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, | ||
32 | isNSFW: CONFIG.INSTANCE.IS_NSFW, | ||
33 | defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, | ||
34 | defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE, | ||
35 | customizations: { | ||
36 | javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT, | ||
37 | css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS | ||
38 | } | ||
39 | }, | ||
40 | search: { | ||
41 | remoteUri: { | ||
42 | users: CONFIG.SEARCH.REMOTE_URI.USERS, | ||
43 | anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS | ||
44 | }, | ||
45 | searchIndex: { | ||
46 | enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, | ||
47 | url: CONFIG.SEARCH.SEARCH_INDEX.URL, | ||
48 | disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, | ||
49 | isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH | ||
50 | } | ||
51 | }, | ||
52 | plugin: { | ||
53 | registered: getRegisteredPlugins(), | ||
54 | registeredExternalAuths: getExternalAuthsPlugins(), | ||
55 | registeredIdAndPassAuths: getIdAndPassAuthPlugins() | ||
56 | }, | ||
57 | theme: { | ||
58 | registered: getRegisteredThemes(), | ||
59 | default: defaultTheme | ||
60 | }, | ||
61 | email: { | ||
62 | enabled: isEmailEnabled() | ||
63 | }, | ||
64 | contactForm: { | ||
65 | enabled: CONFIG.CONTACT_FORM.ENABLED | ||
66 | }, | ||
67 | serverVersion: PEERTUBE_VERSION, | ||
68 | serverCommit, | ||
69 | signup: { | ||
70 | allowed, | ||
71 | allowedForCurrentIP, | ||
72 | requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION | ||
73 | }, | ||
74 | transcoding: { | ||
75 | hls: { | ||
76 | enabled: CONFIG.TRANSCODING.HLS.ENABLED | ||
77 | }, | ||
78 | webtorrent: { | ||
79 | enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED | ||
80 | }, | ||
81 | enabledResolutions: getEnabledResolutions('vod'), | ||
82 | profile: CONFIG.TRANSCODING.PROFILE, | ||
83 | availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod') | ||
84 | }, | ||
85 | live: { | ||
86 | enabled: CONFIG.LIVE.ENABLED, | ||
87 | |||
88 | allowReplay: CONFIG.LIVE.ALLOW_REPLAY, | ||
89 | maxDuration: CONFIG.LIVE.MAX_DURATION, | ||
90 | maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES, | ||
91 | maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, | ||
92 | |||
93 | transcoding: { | ||
94 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, | ||
95 | enabledResolutions: getEnabledResolutions('live'), | ||
96 | profile: CONFIG.LIVE.TRANSCODING.PROFILE, | ||
97 | availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') | ||
98 | }, | ||
99 | |||
100 | rtmp: { | ||
101 | port: CONFIG.LIVE.RTMP.PORT | ||
102 | } | ||
103 | }, | ||
104 | import: { | ||
105 | videos: { | ||
106 | http: { | ||
107 | enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED | ||
108 | }, | ||
109 | torrent: { | ||
110 | enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED | ||
111 | } | ||
112 | } | ||
113 | }, | ||
114 | autoBlacklist: { | ||
115 | videos: { | ||
116 | ofUsers: { | ||
117 | enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED | ||
118 | } | ||
119 | } | ||
120 | }, | ||
121 | avatar: { | ||
122 | file: { | ||
123 | size: { | ||
124 | max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max | ||
125 | }, | ||
126 | extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME | ||
127 | } | ||
128 | }, | ||
129 | banner: { | ||
130 | file: { | ||
131 | size: { | ||
132 | max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max | ||
133 | }, | ||
134 | extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME | ||
135 | } | ||
136 | }, | ||
137 | video: { | ||
138 | image: { | ||
139 | extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME, | ||
140 | size: { | ||
141 | max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max | ||
142 | } | ||
143 | }, | ||
144 | file: { | ||
145 | extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME | ||
146 | } | ||
147 | }, | ||
148 | videoCaption: { | ||
149 | file: { | ||
150 | size: { | ||
151 | max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max | ||
152 | }, | ||
153 | extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME | ||
154 | } | ||
155 | }, | ||
156 | user: { | ||
157 | videoQuota: CONFIG.USER.VIDEO_QUOTA, | ||
158 | videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY | ||
159 | }, | ||
160 | trending: { | ||
161 | videos: { | ||
162 | intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS, | ||
163 | algorithms: { | ||
164 | enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED, | ||
165 | default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT | ||
166 | } | ||
167 | } | ||
168 | }, | ||
169 | tracker: { | ||
170 | enabled: CONFIG.TRACKER.ENABLED | ||
171 | }, | ||
172 | |||
173 | followings: { | ||
174 | instance: { | ||
175 | autoFollowIndex: { | ||
176 | indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL | ||
177 | } | ||
178 | } | ||
179 | }, | ||
180 | |||
181 | broadcastMessage: { | ||
182 | enabled: CONFIG.BROADCAST_MESSAGE.ENABLED, | ||
183 | message: CONFIG.BROADCAST_MESSAGE.MESSAGE, | ||
184 | level: CONFIG.BROADCAST_MESSAGE.LEVEL, | ||
185 | dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE | ||
186 | } | ||
187 | } | ||
188 | } | ||
189 | |||
190 | function getRegisteredThemes () { | ||
191 | return PluginManager.Instance.getRegisteredThemes() | ||
192 | .map(t => ({ | ||
193 | name: t.name, | ||
194 | version: t.version, | ||
195 | description: t.description, | ||
196 | css: t.css, | ||
197 | clientScripts: t.clientScripts | ||
198 | })) | ||
199 | } | ||
200 | |||
201 | function getRegisteredPlugins () { | ||
202 | return PluginManager.Instance.getRegisteredPlugins() | ||
203 | .map(p => ({ | ||
204 | name: p.name, | ||
205 | version: p.version, | ||
206 | description: p.description, | ||
207 | clientScripts: p.clientScripts | ||
208 | })) | ||
209 | } | ||
210 | |||
211 | // --------------------------------------------------------------------------- | ||
212 | |||
213 | export { | ||
214 | getServerConfig, | ||
215 | getRegisteredThemes, | ||
216 | getRegisteredPlugins | ||
217 | } | ||
218 | |||
219 | // --------------------------------------------------------------------------- | ||
220 | |||
221 | function getIdAndPassAuthPlugins () { | ||
222 | const result: RegisteredIdAndPassAuthConfig[] = [] | ||
223 | |||
224 | for (const p of PluginManager.Instance.getIdAndPassAuths()) { | ||
225 | for (const auth of p.idAndPassAuths) { | ||
226 | result.push({ | ||
227 | npmName: p.npmName, | ||
228 | name: p.name, | ||
229 | version: p.version, | ||
230 | authName: auth.authName, | ||
231 | weight: auth.getWeight() | ||
232 | }) | ||
233 | } | ||
234 | } | ||
235 | |||
236 | return result | ||
237 | } | ||
238 | |||
239 | function getExternalAuthsPlugins () { | ||
240 | const result: RegisteredExternalAuthConfig[] = [] | ||
241 | |||
242 | for (const p of PluginManager.Instance.getExternalAuths()) { | ||
243 | for (const auth of p.externalAuths) { | ||
244 | result.push({ | ||
245 | npmName: p.npmName, | ||
246 | name: p.name, | ||
247 | version: p.version, | ||
248 | authName: auth.authName, | ||
249 | authDisplayName: auth.authDisplayName() | ||
250 | }) | ||
251 | } | ||
252 | } | ||
253 | |||
254 | return result | ||
255 | } | ||
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts index 82c95be80..f896d7af4 100644 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ b/server/lib/job-queue/handlers/activitypub-follow.ts | |||
@@ -1,18 +1,17 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url' |
3 | import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants' | 3 | import { ActivitypubFollowPayload } from '@shared/models' |
4 | import { sendFollow } from '../../activitypub/send' | ||
5 | import { sanitizeHost } from '../../../helpers/core-utils' | 4 | import { sanitizeHost } from '../../../helpers/core-utils' |
6 | import { loadActorUrlOrGetFromWebfinger } from '../../../helpers/webfinger' | ||
7 | import { getOrCreateActorAndServerAndModel } from '../../activitypub/actor' | ||
8 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 5 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
9 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 6 | import { logger } from '../../../helpers/logger' |
10 | import { ActorModel } from '../../../models/activitypub/actor' | 7 | import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants' |
11 | import { Notifier } from '../../notifier' | ||
12 | import { sequelizeTypescript } from '../../../initializers/database' | 8 | import { sequelizeTypescript } from '../../../initializers/database' |
9 | import { ActorModel } from '../../../models/actor/actor' | ||
10 | import { ActorFollowModel } from '../../../models/actor/actor-follow' | ||
13 | import { MActor, MActorFollowActors, MActorFull } from '../../../types/models' | 11 | import { MActor, MActorFollowActors, MActorFull } from '../../../types/models' |
14 | import { ActivitypubFollowPayload } from '@shared/models' | 12 | import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors' |
15 | import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url' | 13 | import { sendFollow } from '../../activitypub/send' |
14 | import { Notifier } from '../../notifier' | ||
16 | 15 | ||
17 | async function processActivityPubFollow (job: Bull.Job) { | 16 | async function processActivityPubFollow (job: Bull.Job) { |
18 | const payload = job.data as ActivitypubFollowPayload | 17 | const payload = job.data as ActivitypubFollowPayload |
@@ -26,7 +25,7 @@ async function processActivityPubFollow (job: Bull.Job) { | |||
26 | } else { | 25 | } else { |
27 | const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) | 26 | const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) |
28 | const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost) | 27 | const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost) |
29 | targetActor = await getOrCreateActorAndServerAndModel(actorUrl, 'all') | 28 | targetActor = await getOrCreateAPActor(actorUrl, 'all') |
30 | } | 29 | } |
31 | 30 | ||
32 | if (payload.assertIsChannel && !targetActor.VideoChannel) { | 31 | if (payload.assertIsChannel && !targetActor.VideoChannel) { |
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index c69ff9e83..d4b328635 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts | |||
@@ -3,7 +3,7 @@ import * as Bull from 'bull' | |||
3 | import { ActivitypubHttpBroadcastPayload } from '@shared/models' | 3 | import { ActivitypubHttpBroadcastPayload } from '@shared/models' |
4 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
5 | import { doRequest } from '../../../helpers/requests' | 5 | import { doRequest } from '../../../helpers/requests' |
6 | import { BROADCAST_CONCURRENCY, REQUEST_TIMEOUT } from '../../../initializers/constants' | 6 | import { BROADCAST_CONCURRENCY } from '../../../initializers/constants' |
7 | import { ActorFollowScoreCache } from '../../files-cache' | 7 | import { ActorFollowScoreCache } from '../../files-cache' |
8 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' | 8 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' |
9 | 9 | ||
@@ -19,7 +19,6 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { | |||
19 | method: 'POST' as 'POST', | 19 | method: 'POST' as 'POST', |
20 | json: body, | 20 | json: body, |
21 | httpSignature: httpSignatureOptions, | 21 | httpSignature: httpSignatureOptions, |
22 | timeout: REQUEST_TIMEOUT, | ||
23 | headers: buildGlobalHeaders(body) | 22 | headers: buildGlobalHeaders(body) |
24 | } | 23 | } |
25 | 24 | ||
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index e210ac3ef..ab9675cae 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts | |||
@@ -1,14 +1,13 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models' | 2 | import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models' |
3 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
4 | import { AccountModel } from '../../../models/account/account' | ||
5 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 4 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
6 | import { VideoModel } from '../../../models/video/video' | 5 | import { VideoModel } from '../../../models/video/video' |
7 | import { VideoCommentModel } from '../../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../../models/video/video-comment' |
8 | import { VideoShareModel } from '../../../models/video/video-share' | 7 | import { VideoShareModel } from '../../../models/video/video-share' |
9 | import { MAccountDefault, MVideoFullLight } from '../../../types/models' | 8 | import { MVideoFullLight } from '../../../types/models' |
10 | import { crawlCollectionPage } from '../../activitypub/crawl' | 9 | import { crawlCollectionPage } from '../../activitypub/crawl' |
11 | import { createAccountPlaylists } from '../../activitypub/playlist' | 10 | import { createAccountPlaylists } from '../../activitypub/playlists' |
12 | import { processActivities } from '../../activitypub/process' | 11 | import { processActivities } from '../../activitypub/process' |
13 | import { addVideoShares } from '../../activitypub/share' | 12 | import { addVideoShares } from '../../activitypub/share' |
14 | import { addVideoComments } from '../../activitypub/video-comments' | 13 | import { addVideoComments } from '../../activitypub/video-comments' |
@@ -22,16 +21,13 @@ async function processActivityPubHttpFetcher (job: Bull.Job) { | |||
22 | let video: MVideoFullLight | 21 | let video: MVideoFullLight |
23 | if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) | 22 | if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) |
24 | 23 | ||
25 | let account: MAccountDefault | ||
26 | if (payload.accountId) account = await AccountModel.load(payload.accountId) | ||
27 | |||
28 | const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { | 24 | const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { |
29 | 'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }), | 25 | 'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }), |
30 | 'video-likes': items => createRates(items, video, 'like'), | 26 | 'video-likes': items => createRates(items, video, 'like'), |
31 | 'video-dislikes': items => createRates(items, video, 'dislike'), | 27 | 'video-dislikes': items => createRates(items, video, 'dislike'), |
32 | 'video-shares': items => addVideoShares(items, video), | 28 | 'video-shares': items => addVideoShares(items, video), |
33 | 'video-comments': items => addVideoComments(items), | 29 | 'video-comments': items => addVideoComments(items), |
34 | 'account-playlists': items => createAccountPlaylists(items, account) | 30 | 'account-playlists': items => createAccountPlaylists(items) |
35 | } | 31 | } |
36 | 32 | ||
37 | const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = { | 33 | const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = { |
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index 585dad671..9e561c6b7 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts | |||
@@ -2,7 +2,6 @@ import * as Bull from 'bull' | |||
2 | import { ActivitypubHttpUnicastPayload } from '@shared/models' | 2 | import { ActivitypubHttpUnicastPayload } from '@shared/models' |
3 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
4 | import { doRequest } from '../../../helpers/requests' | 4 | import { doRequest } from '../../../helpers/requests' |
5 | import { REQUEST_TIMEOUT } from '../../../initializers/constants' | ||
6 | import { ActorFollowScoreCache } from '../../files-cache' | 5 | import { ActorFollowScoreCache } from '../../files-cache' |
7 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' | 6 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' |
8 | 7 | ||
@@ -19,7 +18,6 @@ async function processActivityPubHttpUnicast (job: Bull.Job) { | |||
19 | method: 'POST' as 'POST', | 18 | method: 'POST' as 'POST', |
20 | json: body, | 19 | json: body, |
21 | httpSignature: httpSignatureOptions, | 20 | httpSignature: httpSignatureOptions, |
22 | timeout: REQUEST_TIMEOUT, | ||
23 | headers: buildGlobalHeaders(body) | 21 | headers: buildGlobalHeaders(body) |
24 | } | 22 | } |
25 | 23 | ||
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 666e56868..d97e50ebc 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts | |||
@@ -1,12 +1,12 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists' | ||
3 | import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
4 | import { loadVideoByUrl } from '@server/lib/model-loaders' | ||
5 | import { RefreshPayload } from '@shared/models' | ||
2 | import { logger } from '../../../helpers/logger' | 6 | import { logger } from '../../../helpers/logger' |
3 | import { fetchVideoByUrl } from '../../../helpers/video' | 7 | import { ActorModel } from '../../../models/actor/actor' |
4 | import { refreshActorIfNeeded } from '../../activitypub/actor' | ||
5 | import { refreshVideoIfNeeded } from '../../activitypub/videos' | ||
6 | import { ActorModel } from '../../../models/activitypub/actor' | ||
7 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | 8 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' |
8 | import { RefreshPayload } from '@shared/models' | 9 | import { refreshActorIfNeeded } from '../../activitypub/actors' |
9 | import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist' | ||
10 | 10 | ||
11 | async function refreshAPObject (job: Bull.Job) { | 11 | async function refreshAPObject (job: Bull.Job) { |
12 | const payload = job.data as RefreshPayload | 12 | const payload = job.data as RefreshPayload |
@@ -30,7 +30,7 @@ async function refreshVideo (videoUrl: string) { | |||
30 | const fetchType = 'all' as 'all' | 30 | const fetchType = 'all' as 'all' |
31 | const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } | 31 | const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } |
32 | 32 | ||
33 | const videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | 33 | const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) |
34 | if (videoFromDatabase) { | 34 | if (videoFromDatabase) { |
35 | const refreshOptions = { | 35 | const refreshOptions = { |
36 | video: videoFromDatabase, | 36 | video: videoFromDatabase, |
@@ -47,7 +47,7 @@ async function refreshActor (actorUrl: string) { | |||
47 | const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) | 47 | const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) |
48 | 48 | ||
49 | if (actor) { | 49 | if (actor) { |
50 | await refreshActorIfNeeded(actor, fetchType) | 50 | await refreshActorIfNeeded({ actor, fetchedType: fetchType }) |
51 | } | 51 | } |
52 | } | 52 | } |
53 | 53 | ||
diff --git a/server/lib/job-queue/handlers/actor-keys.ts b/server/lib/job-queue/handlers/actor-keys.ts index 125307843..60ac61afd 100644 --- a/server/lib/job-queue/handlers/actor-keys.ts +++ b/server/lib/job-queue/handlers/actor-keys.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { generateAndSaveActorKeys } from '@server/lib/activitypub/actor' | 2 | import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors' |
3 | import { ActorModel } from '@server/models/activitypub/actor' | 3 | import { ActorModel } from '@server/models/actor/actor' |
4 | import { ActorKeysPayload } from '@shared/models' | 4 | import { ActorKeysPayload } from '@shared/models' |
5 | import { logger } from '../../../helpers/logger' | 5 | import { logger } from '../../../helpers/logger' |
6 | 6 | ||
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index e8a91450d..37e7c1fad 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts | |||
@@ -1,10 +1,10 @@ | |||
1 | import { buildDigest } from '@server/helpers/peertube-crypto' | ||
2 | import { getServerActor } from '@server/models/application/application' | ||
3 | import { ContextType } from '@shared/models/activitypub/context' | ||
1 | import { buildSignedActivity } from '../../../../helpers/activitypub' | 4 | import { buildSignedActivity } from '../../../../helpers/activitypub' |
2 | import { ActorModel } from '../../../../models/activitypub/actor' | ||
3 | import { ACTIVITY_PUB, HTTP_SIGNATURE } from '../../../../initializers/constants' | 5 | import { ACTIVITY_PUB, HTTP_SIGNATURE } from '../../../../initializers/constants' |
6 | import { ActorModel } from '../../../../models/actor/actor' | ||
4 | import { MActor } from '../../../../types/models' | 7 | import { MActor } from '../../../../types/models' |
5 | import { getServerActor } from '@server/models/application/application' | ||
6 | import { buildDigest } from '@server/helpers/peertube-crypto' | ||
7 | import { ContextType } from '@shared/models/activitypub/context' | ||
8 | 8 | ||
9 | type Payload <T> = { body: T, contextType?: ContextType, signatureActorId?: number } | 9 | type Payload <T> = { body: T, contextType?: ContextType, signatureActorId?: number } |
10 | 10 | ||
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 71f2cafcd..187cb652e 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts | |||
@@ -1,9 +1,9 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { copy, stat } from 'fs-extra' | 2 | import { copy, stat } from 'fs-extra' |
3 | import { extname } from 'path' | 3 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
5 | import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 5 | import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' |
6 | import { UserModel } from '@server/models/account/user' | 6 | import { UserModel } from '@server/models/user/user' |
7 | import { MVideoFullLight } from '@server/types/models' | 7 | import { MVideoFullLight } from '@server/types/models' |
8 | import { VideoFileImportPayload } from '@shared/models' | 8 | import { VideoFileImportPayload } from '@shared/models' |
9 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' | 9 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' |
@@ -55,7 +55,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { | |||
55 | const { size } = await stat(inputFilePath) | 55 | const { size } = await stat(inputFilePath) |
56 | const fps = await getVideoFileFPS(inputFilePath) | 56 | const fps = await getVideoFileFPS(inputFilePath) |
57 | 57 | ||
58 | const fileExt = extname(inputFilePath) | 58 | const fileExt = getLowercaseExtension(inputFilePath) |
59 | 59 | ||
60 | const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === videoFileResolution) | 60 | const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === videoFileResolution) |
61 | 61 | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index ed2c5eac0..55498003d 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -1,9 +1,11 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { move, remove, stat } from 'fs-extra' | 2 | import { move, remove, stat } from 'fs-extra' |
3 | import { extname } from 'path' | 3 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
4 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | 4 | import { retryTransactionWrapper } from '@server/helpers/database-utils' |
5 | import { YoutubeDL } from '@server/helpers/youtube-dl' | ||
5 | import { isPostImportVideoAccepted } from '@server/lib/moderation' | 6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' |
6 | import { Hooks } from '@server/lib/plugins/hooks' | 7 | import { Hooks } from '@server/lib/plugins/hooks' |
8 | import { ServerConfigManager } from '@server/lib/server-config-manager' | ||
7 | import { isAbleToUploadVideo } from '@server/lib/user' | 9 | import { isAbleToUploadVideo } from '@server/lib/user' |
8 | import { addOptimizeOrMergeAudioJob } from '@server/lib/video' | 10 | import { addOptimizeOrMergeAudioJob } from '@server/lib/video' |
9 | import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 11 | import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' |
@@ -23,7 +25,6 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro | |||
23 | import { logger } from '../../../helpers/logger' | 25 | import { logger } from '../../../helpers/logger' |
24 | import { getSecureTorrentName } from '../../../helpers/utils' | 26 | import { getSecureTorrentName } from '../../../helpers/utils' |
25 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 27 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
26 | import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' | ||
27 | import { CONFIG } from '../../../initializers/config' | 28 | import { CONFIG } from '../../../initializers/config' |
28 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' | 29 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' |
29 | import { sequelizeTypescript } from '../../../initializers/database' | 30 | import { sequelizeTypescript } from '../../../initializers/database' |
@@ -75,8 +76,10 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub | |||
75 | videoImportId: videoImport.id | 76 | videoImportId: videoImport.id |
76 | } | 77 | } |
77 | 78 | ||
79 | const youtubeDL = new YoutubeDL(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod')) | ||
80 | |||
78 | return processFile( | 81 | return processFile( |
79 | () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), | 82 | () => youtubeDL.downloadYoutubeDLVideo(payload.fileExt, VIDEO_IMPORT_TIMEOUT), |
80 | videoImport, | 83 | videoImport, |
81 | options | 84 | options |
82 | ) | 85 | ) |
@@ -116,7 +119,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
116 | const duration = await getDurationFromVideoFile(tempVideoPath) | 119 | const duration = await getDurationFromVideoFile(tempVideoPath) |
117 | 120 | ||
118 | // Prepare video file object for creation in database | 121 | // Prepare video file object for creation in database |
119 | const fileExt = extname(tempVideoPath) | 122 | const fileExt = getLowercaseExtension(tempVideoPath) |
120 | const videoFileData = { | 123 | const videoFileData = { |
121 | extname: fileExt, | 124 | extname: fileExt, |
122 | resolution: videoFileResolution, | 125 | resolution: videoFileResolution, |
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index d57202ca5..9eba41bf8 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts | |||
@@ -3,16 +3,16 @@ import { pathExists, readdir, remove } from 'fs-extra' | |||
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' | 4 | import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' |
5 | import { VIDEO_LIVE } from '@server/initializers/constants' | 5 | import { VIDEO_LIVE } from '@server/initializers/constants' |
6 | import { LiveManager } from '@server/lib/live-manager' | 6 | import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' |
7 | import { generateVideoMiniature } from '@server/lib/thumbnail' | 7 | import { generateVideoMiniature } from '@server/lib/thumbnail' |
8 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' | ||
8 | import { publishAndFederateIfNeeded } from '@server/lib/video' | 9 | import { publishAndFederateIfNeeded } from '@server/lib/video' |
9 | import { getHLSDirectory } from '@server/lib/video-paths' | 10 | import { getHLSDirectory } from '@server/lib/video-paths' |
10 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/video-transcoding' | ||
11 | import { VideoModel } from '@server/models/video/video' | 11 | import { VideoModel } from '@server/models/video/video' |
12 | import { VideoFileModel } from '@server/models/video/video-file' | 12 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { VideoLiveModel } from '@server/models/video/video-live' | 13 | import { VideoLiveModel } from '@server/models/video/video-live' |
14 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 14 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
15 | import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models' | 15 | import { MVideo, MVideoLive } from '@server/types/models' |
16 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' | 16 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' |
17 | import { logger } from '../../../helpers/logger' | 17 | import { logger } from '../../../helpers/logger' |
18 | 18 | ||
@@ -37,7 +37,7 @@ async function processVideoLiveEnding (job: Bull.Job) { | |||
37 | return | 37 | return |
38 | } | 38 | } |
39 | 39 | ||
40 | LiveManager.Instance.cleanupShaSegments(video.uuid) | 40 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) |
41 | 41 | ||
42 | if (live.saveReplay !== true) { | 42 | if (live.saveReplay !== true) { |
43 | return cleanupLive(video, streamingPlaylist) | 43 | return cleanupLive(video, streamingPlaylist) |
@@ -46,19 +46,10 @@ async function processVideoLiveEnding (job: Bull.Job) { | |||
46 | return saveLive(video, live) | 46 | return saveLive(video, live) |
47 | } | 47 | } |
48 | 48 | ||
49 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | ||
50 | const hlsDirectory = getHLSDirectory(video) | ||
51 | |||
52 | await remove(hlsDirectory) | ||
53 | |||
54 | await streamingPlaylist.destroy() | ||
55 | } | ||
56 | |||
57 | // --------------------------------------------------------------------------- | 49 | // --------------------------------------------------------------------------- |
58 | 50 | ||
59 | export { | 51 | export { |
60 | processVideoLiveEnding, | 52 | processVideoLiveEnding |
61 | cleanupLive | ||
62 | } | 53 | } |
63 | 54 | ||
64 | // --------------------------------------------------------------------------- | 55 | // --------------------------------------------------------------------------- |
@@ -94,7 +85,7 @@ async function saveLive (video: MVideo, live: MVideoLive) { | |||
94 | let durationDone = false | 85 | let durationDone = false |
95 | 86 | ||
96 | for (const playlistFile of playlistFiles) { | 87 | for (const playlistFile of playlistFiles) { |
97 | const concatenatedTsFile = LiveManager.Instance.buildConcatenatedName(playlistFile) | 88 | const concatenatedTsFile = buildConcatenatedName(playlistFile) |
98 | const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) | 89 | const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) |
99 | 90 | ||
100 | const probe = await ffprobePromise(concatenatedTsFilePath) | 91 | const probe = await ffprobePromise(concatenatedTsFilePath) |
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 010b95b05..f5ba6f435 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -2,7 +2,7 @@ import * as Bull from 'bull' | |||
2 | import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' | 2 | import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' |
3 | import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' | 3 | import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' |
4 | import { getVideoFilePath } from '@server/lib/video-paths' | 4 | import { getVideoFilePath } from '@server/lib/video-paths' |
5 | import { UserModel } from '@server/models/account/user' | 5 | import { UserModel } from '@server/models/user/user' |
6 | import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' | 6 | import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' |
7 | import { | 7 | import { |
8 | HLSTranscodingPayload, | 8 | HLSTranscodingPayload, |
@@ -15,7 +15,6 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' | |||
15 | import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils' | 15 | import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils' |
16 | import { logger } from '../../../helpers/logger' | 16 | import { logger } from '../../../helpers/logger' |
17 | import { CONFIG } from '../../../initializers/config' | 17 | import { CONFIG } from '../../../initializers/config' |
18 | import { sequelizeTypescript } from '../../../initializers/database' | ||
19 | import { VideoModel } from '../../../models/video/video' | 18 | import { VideoModel } from '../../../models/video/video' |
20 | import { federateVideoIfNeeded } from '../../activitypub/videos' | 19 | import { federateVideoIfNeeded } from '../../activitypub/videos' |
21 | import { Notifier } from '../../notifier' | 20 | import { Notifier } from '../../notifier' |
@@ -24,7 +23,7 @@ import { | |||
24 | mergeAudioVideofile, | 23 | mergeAudioVideofile, |
25 | optimizeOriginalVideofile, | 24 | optimizeOriginalVideofile, |
26 | transcodeNewWebTorrentResolution | 25 | transcodeNewWebTorrentResolution |
27 | } from '../../video-transcoding' | 26 | } from '../../transcoding/video-transcoding' |
28 | import { JobQueue } from '../job-queue' | 27 | import { JobQueue } from '../job-queue' |
29 | 28 | ||
30 | type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> | 29 | type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> |
@@ -151,35 +150,31 @@ async function onVideoFileOptimizer ( | |||
151 | // Outside the transaction (IO on disk) | 150 | // Outside the transaction (IO on disk) |
152 | const { videoFileResolution, isPortraitMode } = await videoArg.getMaxQualityResolution() | 151 | const { videoFileResolution, isPortraitMode } = await videoArg.getMaxQualityResolution() |
153 | 152 | ||
154 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { | 153 | // Maybe the video changed in database, refresh it |
155 | // Maybe the video changed in database, refresh it | 154 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid) |
156 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t) | 155 | // Video does not exist anymore |
157 | // Video does not exist anymore | 156 | if (!videoDatabase) return undefined |
158 | if (!videoDatabase) return undefined | ||
159 | |||
160 | let videoPublished = false | ||
161 | |||
162 | // Generate HLS version of the original file | ||
163 | const originalFileHLSPayload = Object.assign({}, payload, { | ||
164 | isPortraitMode, | ||
165 | resolution: videoDatabase.getMaxQualityFile().resolution, | ||
166 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues | ||
167 | copyCodecs: transcodeType !== 'quick-transcode', | ||
168 | isMaxQuality: true | ||
169 | }) | ||
170 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) | ||
171 | |||
172 | const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, videoFileResolution, isPortraitMode, 'webtorrent') | ||
173 | |||
174 | if (!hasHls && !hasNewResolutions) { | ||
175 | // No transcoding to do, it's now published | ||
176 | videoPublished = await videoDatabase.publishIfNeededAndSave(t) | ||
177 | } | ||
178 | 157 | ||
179 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) | 158 | let videoPublished = false |
180 | 159 | ||
181 | return { videoDatabase, videoPublished } | 160 | // Generate HLS version of the original file |
161 | const originalFileHLSPayload = Object.assign({}, payload, { | ||
162 | isPortraitMode, | ||
163 | resolution: videoDatabase.getMaxQualityFile().resolution, | ||
164 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues | ||
165 | copyCodecs: transcodeType !== 'quick-transcode', | ||
166 | isMaxQuality: true | ||
182 | }) | 167 | }) |
168 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) | ||
169 | |||
170 | const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, videoFileResolution, isPortraitMode, 'webtorrent') | ||
171 | |||
172 | if (!hasHls && !hasNewResolutions) { | ||
173 | // No transcoding to do, it's now published | ||
174 | videoPublished = await videoDatabase.publishIfNeededAndSave(undefined) | ||
175 | } | ||
176 | |||
177 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo) | ||
183 | 178 | ||
184 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) | 179 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) |
185 | if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) | 180 | if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) |
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts index 897235ec0..86d0a271f 100644 --- a/server/lib/job-queue/handlers/video-views.ts +++ b/server/lib/job-queue/handlers/video-views.ts | |||
@@ -36,8 +36,8 @@ async function processVideosViews () { | |||
36 | } | 36 | } |
37 | 37 | ||
38 | await VideoViewModel.create({ | 38 | await VideoViewModel.create({ |
39 | startDate, | 39 | startDate: new Date(startDate), |
40 | endDate, | 40 | endDate: new Date(endDate), |
41 | views, | 41 | views, |
42 | videoId | 42 | videoId |
43 | }) | 43 | }) |
diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts deleted file mode 100644 index 66b5d119b..000000000 --- a/server/lib/live-manager.ts +++ /dev/null | |||
@@ -1,621 +0,0 @@ | |||
1 | |||
2 | import * as Bluebird from 'bluebird' | ||
3 | import * as chokidar from 'chokidar' | ||
4 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
5 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' | ||
6 | import { createServer, Server } from 'net' | ||
7 | import { basename, join } from 'path' | ||
8 | import { isTestInstance } from '@server/helpers/core-utils' | ||
9 | import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils' | ||
10 | import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' | ||
11 | import { logger } from '@server/helpers/logger' | ||
12 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' | ||
13 | import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants' | ||
14 | import { UserModel } from '@server/models/account/user' | ||
15 | import { VideoModel } from '@server/models/video/video' | ||
16 | import { VideoFileModel } from '@server/models/video/video-file' | ||
17 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
18 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
19 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models' | ||
20 | import { VideoState, VideoStreamingPlaylistType } from '@shared/models' | ||
21 | import { federateVideoIfNeeded } from './activitypub/videos' | ||
22 | import { buildSha256Segment } from './hls' | ||
23 | import { JobQueue } from './job-queue' | ||
24 | import { cleanupLive } from './job-queue/handlers/video-live-ending' | ||
25 | import { PeerTubeSocket } from './peertube-socket' | ||
26 | import { isAbleToUploadVideo } from './user' | ||
27 | import { getHLSDirectory } from './video-paths' | ||
28 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | ||
29 | |||
30 | import memoizee = require('memoizee') | ||
31 | const NodeRtmpSession = require('node-media-server/node_rtmp_session') | ||
32 | const context = require('node-media-server/node_core_ctx') | ||
33 | const nodeMediaServerLogger = require('node-media-server/node_core_logger') | ||
34 | |||
35 | // Disable node media server logs | ||
36 | nodeMediaServerLogger.setLogType(0) | ||
37 | |||
38 | const config = { | ||
39 | rtmp: { | ||
40 | port: CONFIG.LIVE.RTMP.PORT, | ||
41 | chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, | ||
42 | gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, | ||
43 | ping: VIDEO_LIVE.RTMP.PING, | ||
44 | ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT | ||
45 | }, | ||
46 | transcoding: { | ||
47 | ffmpeg: 'ffmpeg' | ||
48 | } | ||
49 | } | ||
50 | |||
51 | class LiveManager { | ||
52 | |||
53 | private static instance: LiveManager | ||
54 | |||
55 | private readonly transSessions = new Map<string, FfmpegCommand>() | ||
56 | private readonly videoSessions = new Map<number, string>() | ||
57 | // Values are Date().getTime() | ||
58 | private readonly watchersPerVideo = new Map<number, number[]>() | ||
59 | private readonly segmentsSha256 = new Map<string, Map<string, string>>() | ||
60 | private readonly livesPerUser = new Map<number, { liveId: number, videoId: number, size: number }[]>() | ||
61 | |||
62 | private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { | ||
63 | return isAbleToUploadVideo(userId, 1000) | ||
64 | }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) | ||
65 | |||
66 | private readonly hasClientSocketsInBadHealthWithCache = memoizee((sessionId: string) => { | ||
67 | return this.hasClientSocketsInBadHealth(sessionId) | ||
68 | }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) | ||
69 | |||
70 | private rtmpServer: Server | ||
71 | |||
72 | private constructor () { | ||
73 | } | ||
74 | |||
75 | init () { | ||
76 | const events = this.getContext().nodeEvent | ||
77 | events.on('postPublish', (sessionId: string, streamPath: string) => { | ||
78 | logger.debug('RTMP received stream', { id: sessionId, streamPath }) | ||
79 | |||
80 | const splittedPath = streamPath.split('/') | ||
81 | if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { | ||
82 | logger.warn('Live path is incorrect.', { streamPath }) | ||
83 | return this.abortSession(sessionId) | ||
84 | } | ||
85 | |||
86 | this.handleSession(sessionId, streamPath, splittedPath[2]) | ||
87 | .catch(err => logger.error('Cannot handle sessions.', { err })) | ||
88 | }) | ||
89 | |||
90 | events.on('donePublish', sessionId => { | ||
91 | logger.info('Live session ended.', { sessionId }) | ||
92 | }) | ||
93 | |||
94 | registerConfigChangedHandler(() => { | ||
95 | if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) { | ||
96 | this.run() | ||
97 | return | ||
98 | } | ||
99 | |||
100 | if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) { | ||
101 | this.stop() | ||
102 | } | ||
103 | }) | ||
104 | |||
105 | // Cleanup broken lives, that were terminated by a server restart for example | ||
106 | this.handleBrokenLives() | ||
107 | .catch(err => logger.error('Cannot handle broken lives.', { err })) | ||
108 | |||
109 | setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE) | ||
110 | } | ||
111 | |||
112 | run () { | ||
113 | logger.info('Running RTMP server on port %d', config.rtmp.port) | ||
114 | |||
115 | this.rtmpServer = createServer(socket => { | ||
116 | const session = new NodeRtmpSession(config, socket) | ||
117 | |||
118 | session.run() | ||
119 | }) | ||
120 | |||
121 | this.rtmpServer.on('error', err => { | ||
122 | logger.error('Cannot run RTMP server.', { err }) | ||
123 | }) | ||
124 | |||
125 | this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT) | ||
126 | } | ||
127 | |||
128 | stop () { | ||
129 | logger.info('Stopping RTMP server.') | ||
130 | |||
131 | this.rtmpServer.close() | ||
132 | this.rtmpServer = undefined | ||
133 | |||
134 | // Sessions is an object | ||
135 | this.getContext().sessions.forEach((session: any) => { | ||
136 | if (session instanceof NodeRtmpSession) { | ||
137 | session.stop() | ||
138 | } | ||
139 | }) | ||
140 | } | ||
141 | |||
142 | isRunning () { | ||
143 | return !!this.rtmpServer | ||
144 | } | ||
145 | |||
146 | getSegmentsSha256 (videoUUID: string) { | ||
147 | return this.segmentsSha256.get(videoUUID) | ||
148 | } | ||
149 | |||
150 | stopSessionOf (videoId: number) { | ||
151 | const sessionId = this.videoSessions.get(videoId) | ||
152 | if (!sessionId) return | ||
153 | |||
154 | this.videoSessions.delete(videoId) | ||
155 | this.abortSession(sessionId) | ||
156 | } | ||
157 | |||
158 | getLiveQuotaUsedByUser (userId: number) { | ||
159 | const currentLives = this.livesPerUser.get(userId) | ||
160 | if (!currentLives) return 0 | ||
161 | |||
162 | return currentLives.reduce((sum, obj) => sum + obj.size, 0) | ||
163 | } | ||
164 | |||
165 | addViewTo (videoId: number) { | ||
166 | if (this.videoSessions.has(videoId) === false) return | ||
167 | |||
168 | let watchers = this.watchersPerVideo.get(videoId) | ||
169 | |||
170 | if (!watchers) { | ||
171 | watchers = [] | ||
172 | this.watchersPerVideo.set(videoId, watchers) | ||
173 | } | ||
174 | |||
175 | watchers.push(new Date().getTime()) | ||
176 | } | ||
177 | |||
178 | cleanupShaSegments (videoUUID: string) { | ||
179 | this.segmentsSha256.delete(videoUUID) | ||
180 | } | ||
181 | |||
182 | addSegmentToReplay (hlsVideoPath: string, segmentPath: string) { | ||
183 | const segmentName = basename(segmentPath) | ||
184 | const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, this.buildConcatenatedName(segmentName)) | ||
185 | |||
186 | return readFile(segmentPath) | ||
187 | .then(data => appendFile(dest, data)) | ||
188 | .catch(err => logger.error('Cannot copy segment %s to repay directory.', segmentPath, { err })) | ||
189 | } | ||
190 | |||
191 | buildConcatenatedName (segmentOrPlaylistPath: string) { | ||
192 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | ||
193 | |||
194 | return 'concat-' + num[1] + '.ts' | ||
195 | } | ||
196 | |||
197 | private processSegments (hlsVideoPath: string, videoUUID: string, videoLive: MVideoLive, segmentPaths: string[]) { | ||
198 | Bluebird.mapSeries(segmentPaths, async previousSegment => { | ||
199 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | ||
200 | await this.addSegmentSha(videoUUID, previousSegment) | ||
201 | |||
202 | if (videoLive.saveReplay) { | ||
203 | await this.addSegmentToReplay(hlsVideoPath, previousSegment) | ||
204 | } | ||
205 | }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err })) | ||
206 | } | ||
207 | |||
208 | private getContext () { | ||
209 | return context | ||
210 | } | ||
211 | |||
212 | private abortSession (id: string) { | ||
213 | const session = this.getContext().sessions.get(id) | ||
214 | if (session) { | ||
215 | session.stop() | ||
216 | this.getContext().sessions.delete(id) | ||
217 | } | ||
218 | |||
219 | const transSession = this.transSessions.get(id) | ||
220 | if (transSession) { | ||
221 | transSession.kill('SIGINT') | ||
222 | this.transSessions.delete(id) | ||
223 | } | ||
224 | } | ||
225 | |||
226 | private async handleSession (sessionId: string, streamPath: string, streamKey: string) { | ||
227 | const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) | ||
228 | if (!videoLive) { | ||
229 | logger.warn('Unknown live video with stream key %s.', streamKey) | ||
230 | return this.abortSession(sessionId) | ||
231 | } | ||
232 | |||
233 | const video = videoLive.Video | ||
234 | if (video.isBlacklisted()) { | ||
235 | logger.warn('Video is blacklisted. Refusing stream %s.', streamKey) | ||
236 | return this.abortSession(sessionId) | ||
237 | } | ||
238 | |||
239 | // Cleanup old potential live files (could happen with a permanent live) | ||
240 | this.cleanupShaSegments(video.uuid) | ||
241 | |||
242 | const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) | ||
243 | if (oldStreamingPlaylist) { | ||
244 | await cleanupLive(video, oldStreamingPlaylist) | ||
245 | } | ||
246 | |||
247 | this.videoSessions.set(video.id, sessionId) | ||
248 | |||
249 | const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) | ||
250 | |||
251 | const session = this.getContext().sessions.get(sessionId) | ||
252 | const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath | ||
253 | |||
254 | const [ resolutionResult, fps ] = await Promise.all([ | ||
255 | getVideoFileResolution(rtmpUrl), | ||
256 | getVideoFileFPS(rtmpUrl) | ||
257 | ]) | ||
258 | |||
259 | const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED | ||
260 | ? computeResolutionsToTranscode(resolutionResult.videoFileResolution, 'live') | ||
261 | : [] | ||
262 | |||
263 | const allResolutions = resolutionsEnabled.concat([ session.videoHeight ]) | ||
264 | |||
265 | logger.info('Will mux/transcode live video of original resolution %d.', session.videoHeight, { allResolutions }) | ||
266 | |||
267 | const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ | ||
268 | videoId: video.id, | ||
269 | playlistUrl, | ||
270 | segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), | ||
271 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions), | ||
272 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
273 | |||
274 | type: VideoStreamingPlaylistType.HLS | ||
275 | }, { returning: true }) as [ MStreamingPlaylist, boolean ] | ||
276 | |||
277 | return this.runMuxing({ | ||
278 | sessionId, | ||
279 | videoLive, | ||
280 | playlist: Object.assign(videoStreamingPlaylist, { Video: video }), | ||
281 | rtmpUrl, | ||
282 | fps, | ||
283 | allResolutions | ||
284 | }) | ||
285 | } | ||
286 | |||
287 | private async runMuxing (options: { | ||
288 | sessionId: string | ||
289 | videoLive: MVideoLiveVideo | ||
290 | playlist: MStreamingPlaylistVideo | ||
291 | rtmpUrl: string | ||
292 | fps: number | ||
293 | allResolutions: number[] | ||
294 | }) { | ||
295 | const { sessionId, videoLive, playlist, allResolutions, fps, rtmpUrl } = options | ||
296 | const startStreamDateTime = new Date().getTime() | ||
297 | |||
298 | const user = await UserModel.loadByLiveId(videoLive.id) | ||
299 | if (!this.livesPerUser.has(user.id)) { | ||
300 | this.livesPerUser.set(user.id, []) | ||
301 | } | ||
302 | |||
303 | const currentUserLive = { liveId: videoLive.id, videoId: videoLive.videoId, size: 0 } | ||
304 | const livesOfUser = this.livesPerUser.get(user.id) | ||
305 | livesOfUser.push(currentUserLive) | ||
306 | |||
307 | for (let i = 0; i < allResolutions.length; i++) { | ||
308 | const resolution = allResolutions[i] | ||
309 | |||
310 | const file = new VideoFileModel({ | ||
311 | resolution, | ||
312 | size: -1, | ||
313 | extname: '.ts', | ||
314 | infoHash: null, | ||
315 | fps, | ||
316 | videoStreamingPlaylistId: playlist.id | ||
317 | }) | ||
318 | |||
319 | VideoFileModel.customUpsert(file, 'streaming-playlist', null) | ||
320 | .catch(err => logger.error('Cannot create file for live streaming.', { err })) | ||
321 | } | ||
322 | |||
323 | const outPath = getHLSDirectory(videoLive.Video) | ||
324 | await ensureDir(outPath) | ||
325 | |||
326 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) | ||
327 | |||
328 | if (videoLive.saveReplay === true) { | ||
329 | await ensureDir(replayDirectory) | ||
330 | } | ||
331 | |||
332 | const videoUUID = videoLive.Video.uuid | ||
333 | |||
334 | const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED | ||
335 | ? await getLiveTranscodingCommand({ | ||
336 | rtmpUrl, | ||
337 | outPath, | ||
338 | resolutions: allResolutions, | ||
339 | fps, | ||
340 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
341 | profile: CONFIG.LIVE.TRANSCODING.PROFILE | ||
342 | }) | ||
343 | : getLiveMuxingCommand(rtmpUrl, outPath) | ||
344 | |||
345 | logger.info('Running live muxing/transcoding for %s.', videoUUID) | ||
346 | this.transSessions.set(sessionId, ffmpegExec) | ||
347 | |||
348 | const tsWatcher = chokidar.watch(outPath + '/*.ts') | ||
349 | |||
350 | const segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} | ||
351 | const playlistIdMatcher = /^([\d+])-/ | ||
352 | |||
353 | const addHandler = segmentPath => { | ||
354 | logger.debug('Live add handler of %s.', segmentPath) | ||
355 | |||
356 | const playlistId = basename(segmentPath).match(playlistIdMatcher)[0] | ||
357 | |||
358 | const segmentsToProcess = segmentsToProcessPerPlaylist[playlistId] || [] | ||
359 | this.processSegments(outPath, videoUUID, videoLive, segmentsToProcess) | ||
360 | |||
361 | segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] | ||
362 | |||
363 | if (this.hasClientSocketsInBadHealthWithCache(sessionId)) { | ||
364 | logger.error( | ||
365 | 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + | ||
366 | ' Stopping session of video %s.', videoUUID) | ||
367 | |||
368 | this.stopSessionOf(videoLive.videoId) | ||
369 | return | ||
370 | } | ||
371 | |||
372 | // Duration constraint check | ||
373 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { | ||
374 | logger.info('Stopping session of %s: max duration exceeded.', videoUUID) | ||
375 | |||
376 | this.stopSessionOf(videoLive.videoId) | ||
377 | return | ||
378 | } | ||
379 | |||
380 | // Check user quota if the user enabled replay saving | ||
381 | if (videoLive.saveReplay === true) { | ||
382 | stat(segmentPath) | ||
383 | .then(segmentStat => { | ||
384 | currentUserLive.size += segmentStat.size | ||
385 | }) | ||
386 | .then(() => this.isQuotaConstraintValid(user, videoLive)) | ||
387 | .then(quotaValid => { | ||
388 | if (quotaValid !== true) { | ||
389 | logger.info('Stopping session of %s: user quota exceeded.', videoUUID) | ||
390 | |||
391 | this.stopSessionOf(videoLive.videoId) | ||
392 | } | ||
393 | }) | ||
394 | .catch(err => logger.error('Cannot stat %s or check quota of %d.', segmentPath, user.id, { err })) | ||
395 | } | ||
396 | } | ||
397 | |||
398 | const deleteHandler = segmentPath => this.removeSegmentSha(videoUUID, segmentPath) | ||
399 | |||
400 | tsWatcher.on('add', p => addHandler(p)) | ||
401 | tsWatcher.on('unlink', p => deleteHandler(p)) | ||
402 | |||
403 | const masterWatcher = chokidar.watch(outPath + '/master.m3u8') | ||
404 | masterWatcher.on('add', async () => { | ||
405 | try { | ||
406 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoLive.videoId) | ||
407 | |||
408 | video.state = VideoState.PUBLISHED | ||
409 | await video.save() | ||
410 | videoLive.Video = video | ||
411 | |||
412 | setTimeout(() => { | ||
413 | federateVideoIfNeeded(video, false) | ||
414 | .catch(err => logger.error('Cannot federate live video %s.', video.url, { err })) | ||
415 | |||
416 | PeerTubeSocket.Instance.sendVideoLiveNewState(video) | ||
417 | }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) | ||
418 | |||
419 | } catch (err) { | ||
420 | logger.error('Cannot save/federate live video %d.', videoLive.videoId, { err }) | ||
421 | } finally { | ||
422 | masterWatcher.close() | ||
423 | .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err })) | ||
424 | } | ||
425 | }) | ||
426 | |||
427 | const onFFmpegEnded = () => { | ||
428 | logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', rtmpUrl) | ||
429 | |||
430 | this.transSessions.delete(sessionId) | ||
431 | |||
432 | this.watchersPerVideo.delete(videoLive.videoId) | ||
433 | this.videoSessions.delete(videoLive.videoId) | ||
434 | |||
435 | const newLivesPerUser = this.livesPerUser.get(user.id) | ||
436 | .filter(o => o.liveId !== videoLive.id) | ||
437 | this.livesPerUser.set(user.id, newLivesPerUser) | ||
438 | |||
439 | setTimeout(() => { | ||
440 | // Wait latest segments generation, and close watchers | ||
441 | |||
442 | Promise.all([ tsWatcher.close(), masterWatcher.close() ]) | ||
443 | .then(() => { | ||
444 | // Process remaining segments hash | ||
445 | for (const key of Object.keys(segmentsToProcessPerPlaylist)) { | ||
446 | this.processSegments(outPath, videoUUID, videoLive, segmentsToProcessPerPlaylist[key]) | ||
447 | } | ||
448 | }) | ||
449 | .catch(err => logger.error('Cannot close watchers of %s or process remaining hash segments.', outPath, { err })) | ||
450 | |||
451 | this.onEndTransmuxing(videoLive.Video.id) | ||
452 | .catch(err => logger.error('Error in closed transmuxing.', { err })) | ||
453 | }, 1000) | ||
454 | } | ||
455 | |||
456 | ffmpegExec.on('error', (err, stdout, stderr) => { | ||
457 | onFFmpegEnded() | ||
458 | |||
459 | // Don't care that we killed the ffmpeg process | ||
460 | if (err?.message?.includes('Exiting normally')) return | ||
461 | |||
462 | logger.error('Live transcoding error.', { err, stdout, stderr }) | ||
463 | |||
464 | this.abortSession(sessionId) | ||
465 | }) | ||
466 | |||
467 | ffmpegExec.on('end', () => onFFmpegEnded()) | ||
468 | |||
469 | ffmpegExec.run() | ||
470 | } | ||
471 | |||
472 | private async onEndTransmuxing (videoId: number, cleanupNow = false) { | ||
473 | try { | ||
474 | const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
475 | if (!fullVideo) return | ||
476 | |||
477 | const live = await VideoLiveModel.loadByVideoId(videoId) | ||
478 | |||
479 | if (!live.permanentLive) { | ||
480 | JobQueue.Instance.createJob({ | ||
481 | type: 'video-live-ending', | ||
482 | payload: { | ||
483 | videoId: fullVideo.id | ||
484 | } | ||
485 | }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY }) | ||
486 | |||
487 | fullVideo.state = VideoState.LIVE_ENDED | ||
488 | } else { | ||
489 | fullVideo.state = VideoState.WAITING_FOR_LIVE | ||
490 | } | ||
491 | |||
492 | await fullVideo.save() | ||
493 | |||
494 | PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) | ||
495 | |||
496 | await federateVideoIfNeeded(fullVideo, false) | ||
497 | } catch (err) { | ||
498 | logger.error('Cannot save/federate new video state of live streaming of video id %d.', videoId, { err }) | ||
499 | } | ||
500 | } | ||
501 | |||
502 | private async addSegmentSha (videoUUID: string, segmentPath: string) { | ||
503 | const segmentName = basename(segmentPath) | ||
504 | logger.debug('Adding live sha segment %s.', segmentPath) | ||
505 | |||
506 | const shaResult = await buildSha256Segment(segmentPath) | ||
507 | |||
508 | if (!this.segmentsSha256.has(videoUUID)) { | ||
509 | this.segmentsSha256.set(videoUUID, new Map()) | ||
510 | } | ||
511 | |||
512 | const filesMap = this.segmentsSha256.get(videoUUID) | ||
513 | filesMap.set(segmentName, shaResult) | ||
514 | } | ||
515 | |||
516 | private removeSegmentSha (videoUUID: string, segmentPath: string) { | ||
517 | const segmentName = basename(segmentPath) | ||
518 | |||
519 | logger.debug('Removing live sha segment %s.', segmentPath) | ||
520 | |||
521 | const filesMap = this.segmentsSha256.get(videoUUID) | ||
522 | if (!filesMap) { | ||
523 | logger.warn('Unknown files map to remove sha for %s.', videoUUID) | ||
524 | return | ||
525 | } | ||
526 | |||
527 | if (!filesMap.has(segmentName)) { | ||
528 | logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath) | ||
529 | return | ||
530 | } | ||
531 | |||
532 | filesMap.delete(segmentName) | ||
533 | } | ||
534 | |||
535 | private isDurationConstraintValid (streamingStartTime: number) { | ||
536 | const maxDuration = CONFIG.LIVE.MAX_DURATION | ||
537 | // No limit | ||
538 | if (maxDuration < 0) return true | ||
539 | |||
540 | const now = new Date().getTime() | ||
541 | const max = streamingStartTime + maxDuration | ||
542 | |||
543 | return now <= max | ||
544 | } | ||
545 | |||
546 | private hasClientSocketsInBadHealth (sessionId: string) { | ||
547 | const rtmpSession = this.getContext().sessions.get(sessionId) | ||
548 | |||
549 | if (!rtmpSession) { | ||
550 | logger.warn('Cannot get session %s to check players socket health.', sessionId) | ||
551 | return | ||
552 | } | ||
553 | |||
554 | for (const playerSessionId of rtmpSession.players) { | ||
555 | const playerSession = this.getContext().sessions.get(playerSessionId) | ||
556 | |||
557 | if (!playerSession) { | ||
558 | logger.error('Cannot get player session %s to check socket health.', playerSession) | ||
559 | continue | ||
560 | } | ||
561 | |||
562 | if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { | ||
563 | return true | ||
564 | } | ||
565 | } | ||
566 | |||
567 | return false | ||
568 | } | ||
569 | |||
570 | private async isQuotaConstraintValid (user: MUserId, live: MVideoLive) { | ||
571 | if (live.saveReplay !== true) return true | ||
572 | |||
573 | return this.isAbleToUploadVideoWithCache(user.id) | ||
574 | } | ||
575 | |||
576 | private async updateLiveViews () { | ||
577 | if (!this.isRunning()) return | ||
578 | |||
579 | if (!isTestInstance()) logger.info('Updating live video views.') | ||
580 | |||
581 | for (const videoId of this.watchersPerVideo.keys()) { | ||
582 | const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE | ||
583 | |||
584 | const watchers = this.watchersPerVideo.get(videoId) | ||
585 | |||
586 | const numWatchers = watchers.length | ||
587 | |||
588 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
589 | video.views = numWatchers | ||
590 | await video.save() | ||
591 | |||
592 | await federateVideoIfNeeded(video, false) | ||
593 | |||
594 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video) | ||
595 | |||
596 | // Only keep not expired watchers | ||
597 | const newWatchers = watchers.filter(w => w > notBefore) | ||
598 | this.watchersPerVideo.set(videoId, newWatchers) | ||
599 | |||
600 | logger.debug('New live video views for %s is %d.', video.url, numWatchers) | ||
601 | } | ||
602 | } | ||
603 | |||
604 | private async handleBrokenLives () { | ||
605 | const videoIds = await VideoModel.listPublishedLiveIds() | ||
606 | |||
607 | for (const id of videoIds) { | ||
608 | await this.onEndTransmuxing(id, true) | ||
609 | } | ||
610 | } | ||
611 | |||
612 | static get Instance () { | ||
613 | return this.instance || (this.instance = new this()) | ||
614 | } | ||
615 | } | ||
616 | |||
617 | // --------------------------------------------------------------------------- | ||
618 | |||
619 | export { | ||
620 | LiveManager | ||
621 | } | ||
diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts new file mode 100644 index 000000000..8b46800da --- /dev/null +++ b/server/lib/live/index.ts | |||
@@ -0,0 +1,4 @@ | |||
1 | export * from './live-manager' | ||
2 | export * from './live-quota-store' | ||
3 | export * from './live-segment-sha-store' | ||
4 | export * from './live-utils' | ||
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts new file mode 100644 index 000000000..014cd3fcf --- /dev/null +++ b/server/lib/live/live-manager.ts | |||
@@ -0,0 +1,419 @@ | |||
1 | |||
2 | import { createServer, Server } from 'net' | ||
3 | import { isTestInstance } from '@server/helpers/core-utils' | ||
4 | import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' | ||
5 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
6 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' | ||
7 | import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants' | ||
8 | import { UserModel } from '@server/models/user/user' | ||
9 | import { VideoModel } from '@server/models/video/video' | ||
10 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
11 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
12 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' | ||
13 | import { VideoState, VideoStreamingPlaylistType } from '@shared/models' | ||
14 | import { federateVideoIfNeeded } from '../activitypub/videos' | ||
15 | import { JobQueue } from '../job-queue' | ||
16 | import { PeerTubeSocket } from '../peertube-socket' | ||
17 | import { LiveQuotaStore } from './live-quota-store' | ||
18 | import { LiveSegmentShaStore } from './live-segment-sha-store' | ||
19 | import { cleanupLive } from './live-utils' | ||
20 | import { MuxingSession } from './shared' | ||
21 | |||
22 | const NodeRtmpSession = require('node-media-server/node_rtmp_session') | ||
23 | const context = require('node-media-server/node_core_ctx') | ||
24 | const nodeMediaServerLogger = require('node-media-server/node_core_logger') | ||
25 | |||
26 | // Disable node media server logs | ||
27 | nodeMediaServerLogger.setLogType(0) | ||
28 | |||
29 | const config = { | ||
30 | rtmp: { | ||
31 | port: CONFIG.LIVE.RTMP.PORT, | ||
32 | chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, | ||
33 | gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, | ||
34 | ping: VIDEO_LIVE.RTMP.PING, | ||
35 | ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT | ||
36 | }, | ||
37 | transcoding: { | ||
38 | ffmpeg: 'ffmpeg' | ||
39 | } | ||
40 | } | ||
41 | |||
42 | const lTags = loggerTagsFactory('live') | ||
43 | |||
44 | class LiveManager { | ||
45 | |||
46 | private static instance: LiveManager | ||
47 | |||
48 | private readonly muxingSessions = new Map<string, MuxingSession>() | ||
49 | private readonly videoSessions = new Map<number, string>() | ||
50 | // Values are Date().getTime() | ||
51 | private readonly watchersPerVideo = new Map<number, number[]>() | ||
52 | |||
53 | private rtmpServer: Server | ||
54 | |||
55 | private constructor () { | ||
56 | } | ||
57 | |||
58 | init () { | ||
59 | const events = this.getContext().nodeEvent | ||
60 | events.on('postPublish', (sessionId: string, streamPath: string) => { | ||
61 | logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) }) | ||
62 | |||
63 | const splittedPath = streamPath.split('/') | ||
64 | if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { | ||
65 | logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) }) | ||
66 | return this.abortSession(sessionId) | ||
67 | } | ||
68 | |||
69 | this.handleSession(sessionId, streamPath, splittedPath[2]) | ||
70 | .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) })) | ||
71 | }) | ||
72 | |||
73 | events.on('donePublish', sessionId => { | ||
74 | logger.info('Live session ended.', { sessionId, ...lTags(sessionId) }) | ||
75 | }) | ||
76 | |||
77 | registerConfigChangedHandler(() => { | ||
78 | if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) { | ||
79 | this.run() | ||
80 | return | ||
81 | } | ||
82 | |||
83 | if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) { | ||
84 | this.stop() | ||
85 | } | ||
86 | }) | ||
87 | |||
88 | // Cleanup broken lives, that were terminated by a server restart for example | ||
89 | this.handleBrokenLives() | ||
90 | .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() })) | ||
91 | |||
92 | setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE) | ||
93 | } | ||
94 | |||
95 | run () { | ||
96 | logger.info('Running RTMP server on port %d', config.rtmp.port, lTags()) | ||
97 | |||
98 | this.rtmpServer = createServer(socket => { | ||
99 | const session = new NodeRtmpSession(config, socket) | ||
100 | |||
101 | session.run() | ||
102 | }) | ||
103 | |||
104 | this.rtmpServer.on('error', err => { | ||
105 | logger.error('Cannot run RTMP server.', { err, ...lTags() }) | ||
106 | }) | ||
107 | |||
108 | this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT) | ||
109 | } | ||
110 | |||
111 | stop () { | ||
112 | logger.info('Stopping RTMP server.', lTags()) | ||
113 | |||
114 | this.rtmpServer.close() | ||
115 | this.rtmpServer = undefined | ||
116 | |||
117 | // Sessions is an object | ||
118 | this.getContext().sessions.forEach((session: any) => { | ||
119 | if (session instanceof NodeRtmpSession) { | ||
120 | session.stop() | ||
121 | } | ||
122 | }) | ||
123 | } | ||
124 | |||
125 | isRunning () { | ||
126 | return !!this.rtmpServer | ||
127 | } | ||
128 | |||
129 | stopSessionOf (videoId: number) { | ||
130 | const sessionId = this.videoSessions.get(videoId) | ||
131 | if (!sessionId) return | ||
132 | |||
133 | this.videoSessions.delete(videoId) | ||
134 | this.abortSession(sessionId) | ||
135 | } | ||
136 | |||
137 | addViewTo (videoId: number) { | ||
138 | if (this.videoSessions.has(videoId) === false) return | ||
139 | |||
140 | let watchers = this.watchersPerVideo.get(videoId) | ||
141 | |||
142 | if (!watchers) { | ||
143 | watchers = [] | ||
144 | this.watchersPerVideo.set(videoId, watchers) | ||
145 | } | ||
146 | |||
147 | watchers.push(new Date().getTime()) | ||
148 | } | ||
149 | |||
150 | private getContext () { | ||
151 | return context | ||
152 | } | ||
153 | |||
154 | private abortSession (sessionId: string) { | ||
155 | const session = this.getContext().sessions.get(sessionId) | ||
156 | if (session) { | ||
157 | session.stop() | ||
158 | this.getContext().sessions.delete(sessionId) | ||
159 | } | ||
160 | |||
161 | const muxingSession = this.muxingSessions.get(sessionId) | ||
162 | if (muxingSession) { | ||
163 | // Muxing session will fire and event so we correctly cleanup the session | ||
164 | muxingSession.abort() | ||
165 | |||
166 | this.muxingSessions.delete(sessionId) | ||
167 | } | ||
168 | } | ||
169 | |||
170 | private async handleSession (sessionId: string, streamPath: string, streamKey: string) { | ||
171 | const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) | ||
172 | if (!videoLive) { | ||
173 | logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId)) | ||
174 | return this.abortSession(sessionId) | ||
175 | } | ||
176 | |||
177 | const video = videoLive.Video | ||
178 | if (video.isBlacklisted()) { | ||
179 | logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid)) | ||
180 | return this.abortSession(sessionId) | ||
181 | } | ||
182 | |||
183 | // Cleanup old potential live files (could happen with a permanent live) | ||
184 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) | ||
185 | |||
186 | const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) | ||
187 | if (oldStreamingPlaylist) { | ||
188 | await cleanupLive(video, oldStreamingPlaylist) | ||
189 | } | ||
190 | |||
191 | this.videoSessions.set(video.id, sessionId) | ||
192 | |||
193 | const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath | ||
194 | |||
195 | const [ { videoFileResolution }, fps ] = await Promise.all([ | ||
196 | getVideoFileResolution(rtmpUrl), | ||
197 | getVideoFileFPS(rtmpUrl) | ||
198 | ]) | ||
199 | |||
200 | const allResolutions = this.buildAllResolutionsToTranscode(videoFileResolution) | ||
201 | |||
202 | logger.info( | ||
203 | 'Will mux/transcode live video of original resolution %d.', videoFileResolution, | ||
204 | { allResolutions, ...lTags(sessionId, video.uuid) } | ||
205 | ) | ||
206 | |||
207 | const streamingPlaylist = await this.createLivePlaylist(video, allResolutions) | ||
208 | |||
209 | return this.runMuxingSession({ | ||
210 | sessionId, | ||
211 | videoLive, | ||
212 | streamingPlaylist, | ||
213 | rtmpUrl, | ||
214 | fps, | ||
215 | allResolutions | ||
216 | }) | ||
217 | } | ||
218 | |||
219 | private async runMuxingSession (options: { | ||
220 | sessionId: string | ||
221 | videoLive: MVideoLiveVideo | ||
222 | streamingPlaylist: MStreamingPlaylistVideo | ||
223 | rtmpUrl: string | ||
224 | fps: number | ||
225 | allResolutions: number[] | ||
226 | }) { | ||
227 | const { sessionId, videoLive, streamingPlaylist, allResolutions, fps, rtmpUrl } = options | ||
228 | const videoUUID = videoLive.Video.uuid | ||
229 | const localLTags = lTags(sessionId, videoUUID) | ||
230 | |||
231 | const user = await UserModel.loadByLiveId(videoLive.id) | ||
232 | LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id) | ||
233 | |||
234 | const muxingSession = new MuxingSession({ | ||
235 | context: this.getContext(), | ||
236 | user, | ||
237 | sessionId, | ||
238 | videoLive, | ||
239 | streamingPlaylist, | ||
240 | rtmpUrl, | ||
241 | fps, | ||
242 | allResolutions | ||
243 | }) | ||
244 | |||
245 | muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) | ||
246 | |||
247 | muxingSession.on('bad-socket-health', ({ videoId }) => { | ||
248 | logger.error( | ||
249 | 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + | ||
250 | ' Stopping session of video %s.', videoUUID, | ||
251 | localLTags | ||
252 | ) | ||
253 | |||
254 | this.stopSessionOf(videoId) | ||
255 | }) | ||
256 | |||
257 | muxingSession.on('duration-exceeded', ({ videoId }) => { | ||
258 | logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) | ||
259 | |||
260 | this.stopSessionOf(videoId) | ||
261 | }) | ||
262 | |||
263 | muxingSession.on('quota-exceeded', ({ videoId }) => { | ||
264 | logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) | ||
265 | |||
266 | this.stopSessionOf(videoId) | ||
267 | }) | ||
268 | |||
269 | muxingSession.on('ffmpeg-error', ({ sessionId }) => this.abortSession(sessionId)) | ||
270 | muxingSession.on('ffmpeg-end', ({ videoId }) => { | ||
271 | this.onMuxingFFmpegEnd(videoId) | ||
272 | }) | ||
273 | |||
274 | muxingSession.on('after-cleanup', ({ videoId }) => { | ||
275 | this.muxingSessions.delete(sessionId) | ||
276 | |||
277 | muxingSession.destroy() | ||
278 | |||
279 | return this.onAfterMuxingCleanup(videoId) | ||
280 | .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) | ||
281 | }) | ||
282 | |||
283 | this.muxingSessions.set(sessionId, muxingSession) | ||
284 | |||
285 | muxingSession.runMuxing() | ||
286 | .catch(err => { | ||
287 | logger.error('Cannot run muxing.', { err, ...localLTags }) | ||
288 | this.abortSession(sessionId) | ||
289 | }) | ||
290 | } | ||
291 | |||
292 | private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) { | ||
293 | const videoId = live.videoId | ||
294 | |||
295 | try { | ||
296 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
297 | |||
298 | logger.info('Will publish and federate live %s.', video.url, localLTags) | ||
299 | |||
300 | video.state = VideoState.PUBLISHED | ||
301 | await video.save() | ||
302 | |||
303 | live.Video = video | ||
304 | |||
305 | setTimeout(() => { | ||
306 | federateVideoIfNeeded(video, false) | ||
307 | .catch(err => logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags })) | ||
308 | |||
309 | PeerTubeSocket.Instance.sendVideoLiveNewState(video) | ||
310 | }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) | ||
311 | } catch (err) { | ||
312 | logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags }) | ||
313 | } | ||
314 | } | ||
315 | |||
316 | private onMuxingFFmpegEnd (videoId: number) { | ||
317 | this.watchersPerVideo.delete(videoId) | ||
318 | this.videoSessions.delete(videoId) | ||
319 | } | ||
320 | |||
321 | private async onAfterMuxingCleanup (videoUUID: string, cleanupNow = false) { | ||
322 | try { | ||
323 | const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoUUID) | ||
324 | if (!fullVideo) return | ||
325 | |||
326 | const live = await VideoLiveModel.loadByVideoId(fullVideo.id) | ||
327 | |||
328 | if (!live.permanentLive) { | ||
329 | JobQueue.Instance.createJob({ | ||
330 | type: 'video-live-ending', | ||
331 | payload: { | ||
332 | videoId: fullVideo.id | ||
333 | } | ||
334 | }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY }) | ||
335 | |||
336 | fullVideo.state = VideoState.LIVE_ENDED | ||
337 | } else { | ||
338 | fullVideo.state = VideoState.WAITING_FOR_LIVE | ||
339 | } | ||
340 | |||
341 | await fullVideo.save() | ||
342 | |||
343 | PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) | ||
344 | |||
345 | await federateVideoIfNeeded(fullVideo, false) | ||
346 | } catch (err) { | ||
347 | logger.error('Cannot save/federate new video state of live streaming of video %d.', videoUUID, { err, ...lTags(videoUUID) }) | ||
348 | } | ||
349 | } | ||
350 | |||
351 | private async updateLiveViews () { | ||
352 | if (!this.isRunning()) return | ||
353 | |||
354 | if (!isTestInstance()) logger.info('Updating live video views.', lTags()) | ||
355 | |||
356 | for (const videoId of this.watchersPerVideo.keys()) { | ||
357 | const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE | ||
358 | |||
359 | const watchers = this.watchersPerVideo.get(videoId) | ||
360 | |||
361 | const numWatchers = watchers.length | ||
362 | |||
363 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
364 | video.views = numWatchers | ||
365 | await video.save() | ||
366 | |||
367 | await federateVideoIfNeeded(video, false) | ||
368 | |||
369 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video) | ||
370 | |||
371 | // Only keep not expired watchers | ||
372 | const newWatchers = watchers.filter(w => w > notBefore) | ||
373 | this.watchersPerVideo.set(videoId, newWatchers) | ||
374 | |||
375 | logger.debug('New live video views for %s is %d.', video.url, numWatchers, lTags()) | ||
376 | } | ||
377 | } | ||
378 | |||
379 | private async handleBrokenLives () { | ||
380 | const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() | ||
381 | |||
382 | for (const uuid of videoUUIDs) { | ||
383 | await this.onAfterMuxingCleanup(uuid, true) | ||
384 | } | ||
385 | } | ||
386 | |||
387 | private buildAllResolutionsToTranscode (originResolution: number) { | ||
388 | const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED | ||
389 | ? computeResolutionsToTranscode(originResolution, 'live') | ||
390 | : [] | ||
391 | |||
392 | return resolutionsEnabled.concat([ originResolution ]) | ||
393 | } | ||
394 | |||
395 | private async createLivePlaylist (video: MVideo, allResolutions: number[]) { | ||
396 | const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) | ||
397 | const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ | ||
398 | videoId: video.id, | ||
399 | playlistUrl, | ||
400 | segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), | ||
401 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions), | ||
402 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
403 | |||
404 | type: VideoStreamingPlaylistType.HLS | ||
405 | }, { returning: true }) as [ MStreamingPlaylist, boolean ] | ||
406 | |||
407 | return Object.assign(videoStreamingPlaylist, { Video: video }) | ||
408 | } | ||
409 | |||
410 | static get Instance () { | ||
411 | return this.instance || (this.instance = new this()) | ||
412 | } | ||
413 | } | ||
414 | |||
415 | // --------------------------------------------------------------------------- | ||
416 | |||
417 | export { | ||
418 | LiveManager | ||
419 | } | ||
diff --git a/server/lib/live/live-quota-store.ts b/server/lib/live/live-quota-store.ts new file mode 100644 index 000000000..8ceccde98 --- /dev/null +++ b/server/lib/live/live-quota-store.ts | |||
@@ -0,0 +1,48 @@ | |||
1 | class LiveQuotaStore { | ||
2 | |||
3 | private static instance: LiveQuotaStore | ||
4 | |||
5 | private readonly livesPerUser = new Map<number, { liveId: number, size: number }[]>() | ||
6 | |||
7 | private constructor () { | ||
8 | } | ||
9 | |||
10 | addNewLive (userId: number, liveId: number) { | ||
11 | if (!this.livesPerUser.has(userId)) { | ||
12 | this.livesPerUser.set(userId, []) | ||
13 | } | ||
14 | |||
15 | const currentUserLive = { liveId, size: 0 } | ||
16 | const livesOfUser = this.livesPerUser.get(userId) | ||
17 | livesOfUser.push(currentUserLive) | ||
18 | } | ||
19 | |||
20 | removeLive (userId: number, liveId: number) { | ||
21 | const newLivesPerUser = this.livesPerUser.get(userId) | ||
22 | .filter(o => o.liveId !== liveId) | ||
23 | |||
24 | this.livesPerUser.set(userId, newLivesPerUser) | ||
25 | } | ||
26 | |||
27 | addQuotaTo (userId: number, liveId: number, size: number) { | ||
28 | const lives = this.livesPerUser.get(userId) | ||
29 | const live = lives.find(l => l.liveId === liveId) | ||
30 | |||
31 | live.size += size | ||
32 | } | ||
33 | |||
34 | getLiveQuotaOf (userId: number) { | ||
35 | const currentLives = this.livesPerUser.get(userId) | ||
36 | if (!currentLives) return 0 | ||
37 | |||
38 | return currentLives.reduce((sum, obj) => sum + obj.size, 0) | ||
39 | } | ||
40 | |||
41 | static get Instance () { | ||
42 | return this.instance || (this.instance = new this()) | ||
43 | } | ||
44 | } | ||
45 | |||
46 | export { | ||
47 | LiveQuotaStore | ||
48 | } | ||
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts new file mode 100644 index 000000000..4af6f3ebf --- /dev/null +++ b/server/lib/live/live-segment-sha-store.ts | |||
@@ -0,0 +1,64 @@ | |||
1 | import { basename } from 'path' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { buildSha256Segment } from '../hls' | ||
4 | |||
5 | const lTags = loggerTagsFactory('live') | ||
6 | |||
7 | class LiveSegmentShaStore { | ||
8 | |||
9 | private static instance: LiveSegmentShaStore | ||
10 | |||
11 | private readonly segmentsSha256 = new Map<string, Map<string, string>>() | ||
12 | |||
13 | private constructor () { | ||
14 | } | ||
15 | |||
16 | getSegmentsSha256 (videoUUID: string) { | ||
17 | return this.segmentsSha256.get(videoUUID) | ||
18 | } | ||
19 | |||
20 | async addSegmentSha (videoUUID: string, segmentPath: string) { | ||
21 | const segmentName = basename(segmentPath) | ||
22 | logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID)) | ||
23 | |||
24 | const shaResult = await buildSha256Segment(segmentPath) | ||
25 | |||
26 | if (!this.segmentsSha256.has(videoUUID)) { | ||
27 | this.segmentsSha256.set(videoUUID, new Map()) | ||
28 | } | ||
29 | |||
30 | const filesMap = this.segmentsSha256.get(videoUUID) | ||
31 | filesMap.set(segmentName, shaResult) | ||
32 | } | ||
33 | |||
34 | removeSegmentSha (videoUUID: string, segmentPath: string) { | ||
35 | const segmentName = basename(segmentPath) | ||
36 | |||
37 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID)) | ||
38 | |||
39 | const filesMap = this.segmentsSha256.get(videoUUID) | ||
40 | if (!filesMap) { | ||
41 | logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID)) | ||
42 | return | ||
43 | } | ||
44 | |||
45 | if (!filesMap.has(segmentName)) { | ||
46 | logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID)) | ||
47 | return | ||
48 | } | ||
49 | |||
50 | filesMap.delete(segmentName) | ||
51 | } | ||
52 | |||
53 | cleanupShaSegments (videoUUID: string) { | ||
54 | this.segmentsSha256.delete(videoUUID) | ||
55 | } | ||
56 | |||
57 | static get Instance () { | ||
58 | return this.instance || (this.instance = new this()) | ||
59 | } | ||
60 | } | ||
61 | |||
62 | export { | ||
63 | LiveSegmentShaStore | ||
64 | } | ||
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts new file mode 100644 index 000000000..e4526c7a5 --- /dev/null +++ b/server/lib/live/live-utils.ts | |||
@@ -0,0 +1,23 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { basename } from 'path' | ||
3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | ||
4 | import { getHLSDirectory } from '../video-paths' | ||
5 | |||
6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | ||
7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | ||
8 | |||
9 | return 'concat-' + num[1] + '.ts' | ||
10 | } | ||
11 | |||
12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | ||
13 | const hlsDirectory = getHLSDirectory(video) | ||
14 | |||
15 | await remove(hlsDirectory) | ||
16 | |||
17 | await streamingPlaylist.destroy() | ||
18 | } | ||
19 | |||
20 | export { | ||
21 | cleanupLive, | ||
22 | buildConcatenatedName | ||
23 | } | ||
diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts new file mode 100644 index 000000000..c4d1b59ec --- /dev/null +++ b/server/lib/live/shared/index.ts | |||
@@ -0,0 +1 @@ | |||
export * from './muxing-session' | |||
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts new file mode 100644 index 000000000..26467f060 --- /dev/null +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -0,0 +1,346 @@ | |||
1 | |||
2 | import * as Bluebird from 'bluebird' | ||
3 | import * as chokidar from 'chokidar' | ||
4 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
5 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' | ||
6 | import { basename, join } from 'path' | ||
7 | import { EventEmitter } from 'stream' | ||
8 | import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils' | ||
9 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | ||
10 | import { CONFIG } from '@server/initializers/config' | ||
11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | ||
12 | import { VideoFileModel } from '@server/models/video/video-file' | ||
13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | ||
14 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' | ||
15 | import { isAbleToUploadVideo } from '../../user' | ||
16 | import { getHLSDirectory } from '../../video-paths' | ||
17 | import { LiveQuotaStore } from '../live-quota-store' | ||
18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | ||
19 | import { buildConcatenatedName } from '../live-utils' | ||
20 | |||
21 | import memoizee = require('memoizee') | ||
22 | |||
23 | interface MuxingSessionEvents { | ||
24 | 'master-playlist-created': ({ videoId: number }) => void | ||
25 | |||
26 | 'bad-socket-health': ({ videoId: number }) => void | ||
27 | 'duration-exceeded': ({ videoId: number }) => void | ||
28 | 'quota-exceeded': ({ videoId: number }) => void | ||
29 | |||
30 | 'ffmpeg-end': ({ videoId: number }) => void | ||
31 | 'ffmpeg-error': ({ sessionId: string }) => void | ||
32 | |||
33 | 'after-cleanup': ({ videoId: number }) => void | ||
34 | } | ||
35 | |||
36 | declare interface MuxingSession { | ||
37 | on<U extends keyof MuxingSessionEvents>( | ||
38 | event: U, listener: MuxingSessionEvents[U] | ||
39 | ): this | ||
40 | |||
41 | emit<U extends keyof MuxingSessionEvents>( | ||
42 | event: U, ...args: Parameters<MuxingSessionEvents[U]> | ||
43 | ): boolean | ||
44 | } | ||
45 | |||
46 | class MuxingSession extends EventEmitter { | ||
47 | |||
48 | private ffmpegCommand: FfmpegCommand | ||
49 | |||
50 | private readonly context: any | ||
51 | private readonly user: MUserId | ||
52 | private readonly sessionId: string | ||
53 | private readonly videoLive: MVideoLiveVideo | ||
54 | private readonly streamingPlaylist: MStreamingPlaylistVideo | ||
55 | private readonly rtmpUrl: string | ||
56 | private readonly fps: number | ||
57 | private readonly allResolutions: number[] | ||
58 | |||
59 | private readonly videoId: number | ||
60 | private readonly videoUUID: string | ||
61 | private readonly saveReplay: boolean | ||
62 | |||
63 | private readonly lTags: LoggerTagsFn | ||
64 | |||
65 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} | ||
66 | |||
67 | private tsWatcher: chokidar.FSWatcher | ||
68 | private masterWatcher: chokidar.FSWatcher | ||
69 | |||
70 | private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { | ||
71 | return isAbleToUploadVideo(userId, 1000) | ||
72 | }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) | ||
73 | |||
74 | private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { | ||
75 | return this.hasClientSocketInBadHealth(sessionId) | ||
76 | }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) | ||
77 | |||
78 | constructor (options: { | ||
79 | context: any | ||
80 | user: MUserId | ||
81 | sessionId: string | ||
82 | videoLive: MVideoLiveVideo | ||
83 | streamingPlaylist: MStreamingPlaylistVideo | ||
84 | rtmpUrl: string | ||
85 | fps: number | ||
86 | allResolutions: number[] | ||
87 | }) { | ||
88 | super() | ||
89 | |||
90 | this.context = options.context | ||
91 | this.user = options.user | ||
92 | this.sessionId = options.sessionId | ||
93 | this.videoLive = options.videoLive | ||
94 | this.streamingPlaylist = options.streamingPlaylist | ||
95 | this.rtmpUrl = options.rtmpUrl | ||
96 | this.fps = options.fps | ||
97 | this.allResolutions = options.allResolutions | ||
98 | |||
99 | this.videoId = this.videoLive.Video.id | ||
100 | this.videoUUID = this.videoLive.Video.uuid | ||
101 | |||
102 | this.saveReplay = this.videoLive.saveReplay | ||
103 | |||
104 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) | ||
105 | } | ||
106 | |||
107 | async runMuxing () { | ||
108 | this.createFiles() | ||
109 | |||
110 | const outPath = await this.prepareDirectories() | ||
111 | |||
112 | this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED | ||
113 | ? await getLiveTranscodingCommand({ | ||
114 | rtmpUrl: this.rtmpUrl, | ||
115 | outPath, | ||
116 | resolutions: this.allResolutions, | ||
117 | fps: this.fps, | ||
118 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
119 | profile: CONFIG.LIVE.TRANSCODING.PROFILE | ||
120 | }) | ||
121 | : getLiveMuxingCommand(this.rtmpUrl, outPath) | ||
122 | |||
123 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags) | ||
124 | |||
125 | this.watchTSFiles(outPath) | ||
126 | this.watchMasterFile(outPath) | ||
127 | |||
128 | this.ffmpegCommand.on('error', (err, stdout, stderr) => { | ||
129 | this.onFFmpegError(err, stdout, stderr, outPath) | ||
130 | }) | ||
131 | |||
132 | this.ffmpegCommand.on('end', () => this.onFFmpegEnded(outPath)) | ||
133 | |||
134 | this.ffmpegCommand.run() | ||
135 | } | ||
136 | |||
137 | abort () { | ||
138 | if (!this.ffmpegCommand) return | ||
139 | |||
140 | this.ffmpegCommand.kill('SIGINT') | ||
141 | } | ||
142 | |||
143 | destroy () { | ||
144 | this.removeAllListeners() | ||
145 | this.isAbleToUploadVideoWithCache.clear() | ||
146 | this.hasClientSocketInBadHealthWithCache.clear() | ||
147 | } | ||
148 | |||
149 | private onFFmpegError (err: any, stdout: string, stderr: string, outPath: string) { | ||
150 | this.onFFmpegEnded(outPath) | ||
151 | |||
152 | // Don't care that we killed the ffmpeg process | ||
153 | if (err?.message?.includes('Exiting normally')) return | ||
154 | |||
155 | logger.error('Live transcoding error.', { err, stdout, stderr, ...this.lTags }) | ||
156 | |||
157 | this.emit('ffmpeg-error', ({ sessionId: this.sessionId })) | ||
158 | } | ||
159 | |||
160 | private onFFmpegEnded (outPath: string) { | ||
161 | logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.rtmpUrl, this.lTags) | ||
162 | |||
163 | setTimeout(() => { | ||
164 | // Wait latest segments generation, and close watchers | ||
165 | |||
166 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ]) | ||
167 | .then(() => { | ||
168 | // Process remaining segments hash | ||
169 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | ||
170 | this.processSegments(outPath, this.segmentsToProcessPerPlaylist[key]) | ||
171 | } | ||
172 | }) | ||
173 | .catch(err => { | ||
174 | logger.error( | ||
175 | 'Cannot close watchers of %s or process remaining hash segments.', outPath, | ||
176 | { err, ...this.lTags } | ||
177 | ) | ||
178 | }) | ||
179 | |||
180 | this.emit('after-cleanup', { videoId: this.videoId }) | ||
181 | }, 1000) | ||
182 | } | ||
183 | |||
184 | private watchMasterFile (outPath: string) { | ||
185 | this.masterWatcher = chokidar.watch(outPath + '/master.m3u8') | ||
186 | |||
187 | this.masterWatcher.on('add', async () => { | ||
188 | this.emit('master-playlist-created', { videoId: this.videoId }) | ||
189 | |||
190 | this.masterWatcher.close() | ||
191 | .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags })) | ||
192 | }) | ||
193 | } | ||
194 | |||
195 | private watchTSFiles (outPath: string) { | ||
196 | const startStreamDateTime = new Date().getTime() | ||
197 | |||
198 | this.tsWatcher = chokidar.watch(outPath + '/*.ts') | ||
199 | |||
200 | const playlistIdMatcher = /^([\d+])-/ | ||
201 | |||
202 | const addHandler = async segmentPath => { | ||
203 | logger.debug('Live add handler of %s.', segmentPath, this.lTags) | ||
204 | |||
205 | const playlistId = basename(segmentPath).match(playlistIdMatcher)[0] | ||
206 | |||
207 | const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || [] | ||
208 | this.processSegments(outPath, segmentsToProcess) | ||
209 | |||
210 | this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] | ||
211 | |||
212 | if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { | ||
213 | this.emit('bad-socket-health', { videoId: this.videoId }) | ||
214 | return | ||
215 | } | ||
216 | |||
217 | // Duration constraint check | ||
218 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { | ||
219 | this.emit('duration-exceeded', { videoId: this.videoId }) | ||
220 | return | ||
221 | } | ||
222 | |||
223 | // Check user quota if the user enabled replay saving | ||
224 | if (await this.isQuotaExceeded(segmentPath) === true) { | ||
225 | this.emit('quota-exceeded', { videoId: this.videoId }) | ||
226 | } | ||
227 | } | ||
228 | |||
229 | const deleteHandler = segmentPath => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath) | ||
230 | |||
231 | this.tsWatcher.on('add', p => addHandler(p)) | ||
232 | this.tsWatcher.on('unlink', p => deleteHandler(p)) | ||
233 | } | ||
234 | |||
235 | private async isQuotaExceeded (segmentPath: string) { | ||
236 | if (this.saveReplay !== true) return false | ||
237 | |||
238 | try { | ||
239 | const segmentStat = await stat(segmentPath) | ||
240 | |||
241 | LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.videoLive.id, segmentStat.size) | ||
242 | |||
243 | const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id) | ||
244 | |||
245 | return canUpload !== true | ||
246 | } catch (err) { | ||
247 | logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags }) | ||
248 | } | ||
249 | } | ||
250 | |||
251 | private createFiles () { | ||
252 | for (let i = 0; i < this.allResolutions.length; i++) { | ||
253 | const resolution = this.allResolutions[i] | ||
254 | |||
255 | const file = new VideoFileModel({ | ||
256 | resolution, | ||
257 | size: -1, | ||
258 | extname: '.ts', | ||
259 | infoHash: null, | ||
260 | fps: this.fps, | ||
261 | videoStreamingPlaylistId: this.streamingPlaylist.id | ||
262 | }) | ||
263 | |||
264 | VideoFileModel.customUpsert(file, 'streaming-playlist', null) | ||
265 | .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags })) | ||
266 | } | ||
267 | } | ||
268 | |||
269 | private async prepareDirectories () { | ||
270 | const outPath = getHLSDirectory(this.videoLive.Video) | ||
271 | await ensureDir(outPath) | ||
272 | |||
273 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) | ||
274 | |||
275 | if (this.videoLive.saveReplay === true) { | ||
276 | await ensureDir(replayDirectory) | ||
277 | } | ||
278 | |||
279 | return outPath | ||
280 | } | ||
281 | |||
282 | private isDurationConstraintValid (streamingStartTime: number) { | ||
283 | const maxDuration = CONFIG.LIVE.MAX_DURATION | ||
284 | // No limit | ||
285 | if (maxDuration < 0) return true | ||
286 | |||
287 | const now = new Date().getTime() | ||
288 | const max = streamingStartTime + maxDuration | ||
289 | |||
290 | return now <= max | ||
291 | } | ||
292 | |||
293 | private processSegments (hlsVideoPath: string, segmentPaths: string[]) { | ||
294 | Bluebird.mapSeries(segmentPaths, async previousSegment => { | ||
295 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | ||
296 | await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment) | ||
297 | |||
298 | if (this.saveReplay) { | ||
299 | await this.addSegmentToReplay(hlsVideoPath, previousSegment) | ||
300 | } | ||
301 | }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags })) | ||
302 | } | ||
303 | |||
304 | private hasClientSocketInBadHealth (sessionId: string) { | ||
305 | const rtmpSession = this.context.sessions.get(sessionId) | ||
306 | |||
307 | if (!rtmpSession) { | ||
308 | logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags) | ||
309 | return | ||
310 | } | ||
311 | |||
312 | for (const playerSessionId of rtmpSession.players) { | ||
313 | const playerSession = this.context.sessions.get(playerSessionId) | ||
314 | |||
315 | if (!playerSession) { | ||
316 | logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags) | ||
317 | continue | ||
318 | } | ||
319 | |||
320 | if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { | ||
321 | return true | ||
322 | } | ||
323 | } | ||
324 | |||
325 | return false | ||
326 | } | ||
327 | |||
328 | private async addSegmentToReplay (hlsVideoPath: string, segmentPath: string) { | ||
329 | const segmentName = basename(segmentPath) | ||
330 | const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, buildConcatenatedName(segmentName)) | ||
331 | |||
332 | try { | ||
333 | const data = await readFile(segmentPath) | ||
334 | |||
335 | await appendFile(dest, data) | ||
336 | } catch (err) { | ||
337 | logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags }) | ||
338 | } | ||
339 | } | ||
340 | } | ||
341 | |||
342 | // --------------------------------------------------------------------------- | ||
343 | |||
344 | export { | ||
345 | MuxingSession | ||
346 | } | ||
diff --git a/server/lib/actor-image.ts b/server/lib/local-actor.ts index f271f0b5b..77667f6b0 100644 --- a/server/lib/actor-image.ts +++ b/server/lib/local-actor.ts | |||
@@ -1,19 +1,38 @@ | |||
1 | import 'multer' | 1 | import 'multer' |
2 | import { queue } from 'async' | 2 | import { queue } from 'async' |
3 | import * as LRUCache from 'lru-cache' | 3 | import * as LRUCache from 'lru-cache' |
4 | import { extname, join } from 'path' | 4 | import { join } from 'path' |
5 | import { v4 as uuidv4 } from 'uuid' | 5 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
6 | import { ActorImageType } from '@shared/models' | 6 | import { buildUUID } from '@server/helpers/uuid' |
7 | import { ActorModel } from '@server/models/actor/actor' | ||
8 | import { ActivityPubActorType, ActorImageType } from '@shared/models' | ||
7 | import { retryTransactionWrapper } from '../helpers/database-utils' | 9 | import { retryTransactionWrapper } from '../helpers/database-utils' |
8 | import { processImage } from '../helpers/image-utils' | 10 | import { processImage } from '../helpers/image-utils' |
9 | import { downloadImage } from '../helpers/requests' | 11 | import { downloadImage } from '../helpers/requests' |
10 | import { CONFIG } from '../initializers/config' | 12 | import { CONFIG } from '../initializers/config' |
11 | import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants' | 13 | import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY, WEBSERVER } from '../initializers/constants' |
12 | import { sequelizeTypescript } from '../initializers/database' | 14 | import { sequelizeTypescript } from '../initializers/database' |
13 | import { MAccountDefault, MChannelDefault } from '../types/models' | 15 | import { MAccountDefault, MActor, MChannelDefault } from '../types/models' |
14 | import { deleteActorImageInstance, updateActorImageInstance } from './activitypub/actor' | 16 | import { deleteActorImageInstance, updateActorImageInstance } from './activitypub/actors' |
15 | import { sendUpdateActor } from './activitypub/send' | 17 | import { sendUpdateActor } from './activitypub/send' |
16 | 18 | ||
19 | function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { | ||
20 | return new ActorModel({ | ||
21 | type, | ||
22 | url, | ||
23 | preferredUsername, | ||
24 | publicKey: null, | ||
25 | privateKey: null, | ||
26 | followersCount: 0, | ||
27 | followingCount: 0, | ||
28 | inboxUrl: url + '/inbox', | ||
29 | outboxUrl: url + '/outbox', | ||
30 | sharedInboxUrl: WEBSERVER.URL + '/inbox', | ||
31 | followersUrl: url + '/followers', | ||
32 | followingUrl: url + '/following' | ||
33 | }) as MActor | ||
34 | } | ||
35 | |||
17 | async function updateLocalActorImageFile ( | 36 | async function updateLocalActorImageFile ( |
18 | accountOrChannel: MAccountDefault | MChannelDefault, | 37 | accountOrChannel: MAccountDefault | MChannelDefault, |
19 | imagePhysicalFile: Express.Multer.File, | 38 | imagePhysicalFile: Express.Multer.File, |
@@ -23,9 +42,9 @@ async function updateLocalActorImageFile ( | |||
23 | ? ACTOR_IMAGES_SIZE.AVATARS | 42 | ? ACTOR_IMAGES_SIZE.AVATARS |
24 | : ACTOR_IMAGES_SIZE.BANNERS | 43 | : ACTOR_IMAGES_SIZE.BANNERS |
25 | 44 | ||
26 | const extension = extname(imagePhysicalFile.filename) | 45 | const extension = getLowercaseExtension(imagePhysicalFile.filename) |
27 | 46 | ||
28 | const imageName = uuidv4() + extension | 47 | const imageName = buildUUID() + extension |
29 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) | 48 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) |
30 | await processImage(imagePhysicalFile.path, destination, imageSize) | 49 | await processImage(imagePhysicalFile.path, destination, imageSize) |
31 | 50 | ||
@@ -93,5 +112,6 @@ export { | |||
93 | actorImagePathUnsafeCache, | 112 | actorImagePathUnsafeCache, |
94 | updateLocalActorImageFile, | 113 | updateLocalActorImageFile, |
95 | deleteLocalActorImageFile, | 114 | deleteLocalActorImageFile, |
96 | pushActorImageProcessInQueue | 115 | pushActorImageProcessInQueue, |
116 | buildActorInstance | ||
97 | } | 117 | } |
diff --git a/server/lib/model-loaders/actor.ts b/server/lib/model-loaders/actor.ts new file mode 100644 index 000000000..1355d8ee2 --- /dev/null +++ b/server/lib/model-loaders/actor.ts | |||
@@ -0,0 +1,17 @@ | |||
1 | |||
2 | import { ActorModel } from '../../models/actor/actor' | ||
3 | import { MActorAccountChannelId, MActorFull } from '../../types/models' | ||
4 | |||
5 | type ActorLoadByUrlType = 'all' | 'association-ids' | ||
6 | |||
7 | function loadActorByUrl (url: string, fetchType: ActorLoadByUrlType): Promise<MActorFull | MActorAccountChannelId> { | ||
8 | if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url) | ||
9 | |||
10 | if (fetchType === 'association-ids') return ActorModel.loadByUrl(url) | ||
11 | } | ||
12 | |||
13 | export { | ||
14 | ActorLoadByUrlType, | ||
15 | |||
16 | loadActorByUrl | ||
17 | } | ||
diff --git a/server/lib/model-loaders/index.ts b/server/lib/model-loaders/index.ts new file mode 100644 index 000000000..9e5152cb2 --- /dev/null +++ b/server/lib/model-loaders/index.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | export * from './actor' | ||
2 | export * from './video' | ||
diff --git a/server/lib/model-loaders/video.ts b/server/lib/model-loaders/video.ts new file mode 100644 index 000000000..0a3c15ad8 --- /dev/null +++ b/server/lib/model-loaders/video.ts | |||
@@ -0,0 +1,73 @@ | |||
1 | import { VideoModel } from '@server/models/video/video' | ||
2 | import { | ||
3 | MVideoAccountLightBlacklistAllFiles, | ||
4 | MVideoFormattableDetails, | ||
5 | MVideoFullLight, | ||
6 | MVideoId, | ||
7 | MVideoImmutable, | ||
8 | MVideoThumbnail | ||
9 | } from '@server/types/models' | ||
10 | import { Hooks } from '../plugins/hooks' | ||
11 | |||
12 | type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes' | ||
13 | |||
14 | function loadVideo (id: number | string, fetchType: 'for-api', userId?: number): Promise<MVideoFormattableDetails> | ||
15 | function loadVideo (id: number | string, fetchType: 'all', userId?: number): Promise<MVideoFullLight> | ||
16 | function loadVideo (id: number | string, fetchType: 'only-immutable-attributes'): Promise<MVideoImmutable> | ||
17 | function loadVideo (id: number | string, fetchType: 'only-video', userId?: number): Promise<MVideoThumbnail> | ||
18 | function loadVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Promise<MVideoId> | ||
19 | function loadVideo ( | ||
20 | id: number | string, | ||
21 | fetchType: VideoLoadType, | ||
22 | userId?: number | ||
23 | ): Promise<MVideoFullLight | MVideoThumbnail | MVideoId | MVideoImmutable> | ||
24 | function loadVideo ( | ||
25 | id: number | string, | ||
26 | fetchType: VideoLoadType, | ||
27 | userId?: number | ||
28 | ): Promise<MVideoFullLight | MVideoThumbnail | MVideoId | MVideoImmutable> { | ||
29 | |||
30 | if (fetchType === 'for-api') { | ||
31 | return Hooks.wrapPromiseFun( | ||
32 | VideoModel.loadForGetAPI, | ||
33 | { id, userId }, | ||
34 | 'filter:api.video.get.result' | ||
35 | ) | ||
36 | } | ||
37 | |||
38 | if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) | ||
39 | |||
40 | if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id) | ||
41 | |||
42 | if (fetchType === 'only-video') return VideoModel.load(id) | ||
43 | |||
44 | if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) | ||
45 | } | ||
46 | |||
47 | type VideoLoadByUrlType = 'all' | 'only-video' | 'only-immutable-attributes' | ||
48 | |||
49 | function loadVideoByUrl (url: string, fetchType: 'all'): Promise<MVideoAccountLightBlacklistAllFiles> | ||
50 | function loadVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Promise<MVideoImmutable> | ||
51 | function loadVideoByUrl (url: string, fetchType: 'only-video'): Promise<MVideoThumbnail> | ||
52 | function loadVideoByUrl ( | ||
53 | url: string, | ||
54 | fetchType: VideoLoadByUrlType | ||
55 | ): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> | ||
56 | function loadVideoByUrl ( | ||
57 | url: string, | ||
58 | fetchType: VideoLoadByUrlType | ||
59 | ): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
60 | if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) | ||
61 | |||
62 | if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url) | ||
63 | |||
64 | if (fetchType === 'only-video') return VideoModel.loadByUrl(url) | ||
65 | } | ||
66 | |||
67 | export { | ||
68 | VideoLoadType, | ||
69 | VideoLoadByUrlType, | ||
70 | |||
71 | loadVideo, | ||
72 | loadVideoByUrl | ||
73 | } | ||
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts index 925d64902..14e00518e 100644 --- a/server/lib/moderation.ts +++ b/server/lib/moderation.ts | |||
@@ -23,9 +23,9 @@ import { ActivityCreate } from '../../shared/models/activitypub' | |||
23 | import { VideoObject } from '../../shared/models/activitypub/objects' | 23 | import { VideoObject } from '../../shared/models/activitypub/objects' |
24 | import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object' | 24 | import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object' |
25 | import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' | 25 | import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' |
26 | import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model' | 26 | import { VideoCommentCreate } from '../../shared/models/videos/comment/video-comment.model' |
27 | import { UserModel } from '../models/account/user' | 27 | import { ActorModel } from '../models/actor/actor' |
28 | import { ActorModel } from '../models/activitypub/actor' | 28 | import { UserModel } from '../models/user/user' |
29 | import { VideoModel } from '../models/video/video' | 29 | import { VideoModel } from '../models/video/video' |
30 | import { VideoCommentModel } from '../models/video/video-comment' | 30 | import { VideoCommentModel } from '../models/video/video-comment' |
31 | import { sendAbuse } from './activitypub/send/send-flag' | 31 | import { sendAbuse } from './activitypub/send/send-flag' |
@@ -221,7 +221,7 @@ async function createAbuse (options: { | |||
221 | const { isOwned } = await associateFun(abuseInstance) | 221 | const { isOwned } = await associateFun(abuseInstance) |
222 | 222 | ||
223 | if (isOwned === false) { | 223 | if (isOwned === false) { |
224 | await sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction) | 224 | sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction) |
225 | } | 225 | } |
226 | 226 | ||
227 | const abuseJSON = abuseInstance.toFormattedAdminJSON() | 227 | const abuseJSON = abuseInstance.toFormattedAdminJSON() |
diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts index da7f7cc05..1f9ff16df 100644 --- a/server/lib/notifier.ts +++ b/server/lib/notifier.ts | |||
@@ -17,8 +17,8 @@ import { VideoPrivacy, VideoState } from '../../shared/models/videos' | |||
17 | import { logger } from '../helpers/logger' | 17 | import { logger } from '../helpers/logger' |
18 | import { CONFIG } from '../initializers/config' | 18 | import { CONFIG } from '../initializers/config' |
19 | import { AccountBlocklistModel } from '../models/account/account-blocklist' | 19 | import { AccountBlocklistModel } from '../models/account/account-blocklist' |
20 | import { UserModel } from '../models/account/user' | 20 | import { UserModel } from '../models/user/user' |
21 | import { UserNotificationModel } from '../models/account/user-notification' | 21 | import { UserNotificationModel } from '../models/user/user-notification' |
22 | import { MAbuseFull, MAbuseMessage, MAccountServer, MActorFollowFull, MApplication, MPlugin } from '../types/models' | 22 | import { MAbuseFull, MAbuseMessage, MAccountServer, MActorFollowFull, MApplication, MPlugin } from '../types/models' |
23 | import { MCommentOwnerVideo, MVideoAccountLight, MVideoFullLight } from '../types/models/video' | 23 | import { MCommentOwnerVideo, MVideoAccountLight, MVideoFullLight } from '../types/models/video' |
24 | import { isBlockedByServerOrAccount } from './blocklist' | 24 | import { isBlockedByServerOrAccount } from './blocklist' |
diff --git a/server/lib/plugins/hooks.ts b/server/lib/plugins/hooks.ts index aa92f03cc..5e97b52a0 100644 --- a/server/lib/plugins/hooks.ts +++ b/server/lib/plugins/hooks.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { ServerActionHookName, ServerFilterHookName } from '../../../shared/models/plugins/server-hook.model' | ||
2 | import { PluginManager } from './plugin-manager' | ||
3 | import { logger } from '../../helpers/logger' | ||
4 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
2 | import { ServerActionHookName, ServerFilterHookName } from '../../../shared/models' | ||
3 | import { logger } from '../../helpers/logger' | ||
4 | import { PluginManager } from './plugin-manager' | ||
5 | 5 | ||
6 | type PromiseFunction <U, T> = (params: U) => Promise<T> | Bluebird<T> | 6 | type PromiseFunction <U, T> = (params: U) => Promise<T> | Bluebird<T> |
7 | type RawFunction <U, T> = (params: U) => T | 7 | type RawFunction <U, T> = (params: U) => T |
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts index f1bc24d8b..8487672ba 100644 --- a/server/lib/plugins/plugin-helpers-builder.ts +++ b/server/lib/plugins/plugin-helpers-builder.ts | |||
@@ -15,9 +15,9 @@ import { MPlugin } from '@server/types/models' | |||
15 | import { PeerTubeHelpers } from '@server/types/plugins' | 15 | import { PeerTubeHelpers } from '@server/types/plugins' |
16 | import { VideoBlacklistCreate } from '@shared/models' | 16 | import { VideoBlacklistCreate } from '@shared/models' |
17 | import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' | 17 | import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' |
18 | import { getServerConfig } from '../config' | 18 | import { ServerConfigManager } from '../server-config-manager' |
19 | import { blacklistVideo, unblacklistVideo } from '../video-blacklist' | 19 | import { blacklistVideo, unblacklistVideo } from '../video-blacklist' |
20 | import { UserModel } from '@server/models/account/user' | 20 | import { UserModel } from '@server/models/user/user' |
21 | 21 | ||
22 | function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers { | 22 | function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers { |
23 | const logger = buildPluginLogger(npmName) | 23 | const logger = buildPluginLogger(npmName) |
@@ -147,7 +147,7 @@ function buildConfigHelpers () { | |||
147 | }, | 147 | }, |
148 | 148 | ||
149 | getServerConfig () { | 149 | getServerConfig () { |
150 | return getServerConfig() | 150 | return ServerConfigManager.Instance.getServerConfig() |
151 | } | 151 | } |
152 | } | 152 | } |
153 | } | 153 | } |
diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts index 165bc91b3..119cee8e0 100644 --- a/server/lib/plugins/plugin-index.ts +++ b/server/lib/plugins/plugin-index.ts | |||
@@ -1,16 +1,16 @@ | |||
1 | import { sanitizeUrl } from '@server/helpers/core-utils' | 1 | import { sanitizeUrl } from '@server/helpers/core-utils' |
2 | import { ResultList } from '../../../shared/models' | 2 | import { logger } from '@server/helpers/logger' |
3 | import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model' | 3 | import { doJSONRequest } from '@server/helpers/requests' |
4 | import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model' | 4 | import { CONFIG } from '@server/initializers/config' |
5 | import { PEERTUBE_VERSION } from '@server/initializers/constants' | ||
6 | import { PluginModel } from '@server/models/server/plugin' | ||
5 | import { | 7 | import { |
8 | PeerTubePluginIndex, | ||
9 | PeertubePluginIndexList, | ||
6 | PeertubePluginLatestVersionRequest, | 10 | PeertubePluginLatestVersionRequest, |
7 | PeertubePluginLatestVersionResponse | 11 | PeertubePluginLatestVersionResponse, |
8 | } from '../../../shared/models/plugins/peertube-plugin-latest-version.model' | 12 | ResultList |
9 | import { logger } from '../../helpers/logger' | 13 | } from '@shared/models' |
10 | import { doJSONRequest } from '../../helpers/requests' | ||
11 | import { CONFIG } from '../../initializers/config' | ||
12 | import { PEERTUBE_VERSION } from '../../initializers/constants' | ||
13 | import { PluginModel } from '../../models/server/plugin' | ||
14 | import { PluginManager } from './plugin-manager' | 14 | import { PluginManager } from './plugin-manager' |
15 | 15 | ||
16 | async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { | 16 | async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { |
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts index ba9814383..6599bccca 100644 --- a/server/lib/plugins/plugin-manager.ts +++ b/server/lib/plugins/plugin-manager.ts | |||
@@ -4,16 +4,11 @@ import { createReadStream, createWriteStream } from 'fs' | |||
4 | import { ensureDir, outputFile, readJSON } from 'fs-extra' | 4 | import { ensureDir, outputFile, readJSON } from 'fs-extra' |
5 | import { basename, join } from 'path' | 5 | import { basename, join } from 'path' |
6 | import { MOAuthTokenUser, MUser } from '@server/types/models' | 6 | import { MOAuthTokenUser, MUser } from '@server/types/models' |
7 | import { RegisterServerHookOptions } from '@shared/models/plugins/register-server-hook.model' | 7 | import { getCompleteLocale } from '@shared/core-utils' |
8 | import { ClientScript, PluginPackageJson, PluginTranslation, PluginTranslationPaths, RegisterServerHookOptions } from '@shared/models' | ||
8 | import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' | 9 | import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' |
9 | import { | ||
10 | ClientScript, | ||
11 | PluginPackageJson, | ||
12 | PluginTranslationPaths as PackagePluginTranslations | ||
13 | } from '../../../shared/models/plugins/plugin-package-json.model' | ||
14 | import { PluginTranslation } from '../../../shared/models/plugins/plugin-translation.model' | ||
15 | import { PluginType } from '../../../shared/models/plugins/plugin.type' | 10 | import { PluginType } from '../../../shared/models/plugins/plugin.type' |
16 | import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server-hook.model' | 11 | import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model' |
17 | import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins' | 12 | import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins' |
18 | import { logger } from '../../helpers/logger' | 13 | import { logger } from '../../helpers/logger' |
19 | import { CONFIG } from '../../initializers/config' | 14 | import { CONFIG } from '../../initializers/config' |
@@ -23,7 +18,6 @@ import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPas | |||
23 | import { ClientHtml } from '../client-html' | 18 | import { ClientHtml } from '../client-html' |
24 | import { RegisterHelpers } from './register-helpers' | 19 | import { RegisterHelpers } from './register-helpers' |
25 | import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn' | 20 | import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn' |
26 | import { getCompleteLocale } from '@shared/core-utils' | ||
27 | 21 | ||
28 | export interface RegisteredPlugin { | 22 | export interface RegisteredPlugin { |
29 | npmName: string | 23 | npmName: string |
@@ -310,22 +304,28 @@ export class PluginManager implements ServerHook { | |||
310 | uninstalled: false, | 304 | uninstalled: false, |
311 | peertubeEngine: packageJSON.engine.peertube | 305 | peertubeEngine: packageJSON.engine.peertube |
312 | }, { returning: true }) | 306 | }, { returning: true }) |
313 | } catch (err) { | 307 | |
314 | logger.error('Cannot install plugin %s, removing it...', toInstall, { err }) | 308 | logger.info('Successful installation of plugin %s.', toInstall) |
309 | |||
310 | await this.registerPluginOrTheme(plugin) | ||
311 | } catch (rootErr) { | ||
312 | logger.error('Cannot install plugin %s, removing it...', toInstall, { err: rootErr }) | ||
315 | 313 | ||
316 | try { | 314 | try { |
317 | await removeNpmPlugin(npmName) | 315 | await this.uninstall(npmName) |
318 | } catch (err) { | 316 | } catch (err) { |
319 | logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err }) | 317 | logger.error('Cannot uninstall plugin %s after failed installation.', toInstall, { err }) |
318 | |||
319 | try { | ||
320 | await removeNpmPlugin(npmName) | ||
321 | } catch (err) { | ||
322 | logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err }) | ||
323 | } | ||
320 | } | 324 | } |
321 | 325 | ||
322 | throw err | 326 | throw rootErr |
323 | } | 327 | } |
324 | 328 | ||
325 | logger.info('Successful installation of plugin %s.', toInstall) | ||
326 | |||
327 | await this.registerPluginOrTheme(plugin) | ||
328 | |||
329 | return plugin | 329 | return plugin |
330 | } | 330 | } |
331 | 331 | ||
@@ -431,8 +431,7 @@ export class PluginManager implements ServerHook { | |||
431 | 431 | ||
432 | await ensureDir(registerOptions.peertubeHelpers.plugin.getDataDirectoryPath()) | 432 | await ensureDir(registerOptions.peertubeHelpers.plugin.getDataDirectoryPath()) |
433 | 433 | ||
434 | library.register(registerOptions) | 434 | await library.register(registerOptions) |
435 | .catch(err => logger.error('Cannot register plugin %s.', npmName, { err })) | ||
436 | 435 | ||
437 | logger.info('Add plugin %s CSS to global file.', npmName) | 436 | logger.info('Add plugin %s CSS to global file.', npmName) |
438 | 437 | ||
@@ -443,7 +442,7 @@ export class PluginManager implements ServerHook { | |||
443 | 442 | ||
444 | // ###################### Translations ###################### | 443 | // ###################### Translations ###################### |
445 | 444 | ||
446 | private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PackagePluginTranslations) { | 445 | private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPaths) { |
447 | for (const locale of Object.keys(translationPaths)) { | 446 | for (const locale of Object.keys(translationPaths)) { |
448 | const path = translationPaths[locale] | 447 | const path = translationPaths[locale] |
449 | const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) | 448 | const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) |
diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts index aa69ca2a2..09275f9ba 100644 --- a/server/lib/plugins/register-helpers.ts +++ b/server/lib/plugins/register-helpers.ts | |||
@@ -26,10 +26,10 @@ import { | |||
26 | PluginVideoLicenceManager, | 26 | PluginVideoLicenceManager, |
27 | PluginVideoPrivacyManager, | 27 | PluginVideoPrivacyManager, |
28 | RegisterServerHookOptions, | 28 | RegisterServerHookOptions, |
29 | RegisterServerSettingOptions | 29 | RegisterServerSettingOptions, |
30 | serverHookObject | ||
30 | } from '@shared/models' | 31 | } from '@shared/models' |
31 | import { serverHookObject } from '@shared/models/plugins/server-hook.model' | 32 | import { VideoTranscodingProfilesManager } from '../transcoding/video-transcoding-profiles' |
32 | import { VideoTranscodingProfilesManager } from '../video-transcoding-profiles' | ||
33 | import { buildPluginHelpers } from './plugin-helpers-builder' | 33 | import { buildPluginHelpers } from './plugin-helpers-builder' |
34 | 34 | ||
35 | type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy' | 35 | type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy' |
@@ -37,18 +37,20 @@ type VideoConstant = { [key in number | string]: string } | |||
37 | 37 | ||
38 | type UpdatedVideoConstant = { | 38 | type UpdatedVideoConstant = { |
39 | [name in AlterableVideoConstant]: { | 39 | [name in AlterableVideoConstant]: { |
40 | added: { key: number | string, label: string }[] | 40 | [ npmName: string]: { |
41 | deleted: { key: number | string, label: string }[] | 41 | added: { key: number | string, label: string }[] |
42 | deleted: { key: number | string, label: string }[] | ||
43 | } | ||
42 | } | 44 | } |
43 | } | 45 | } |
44 | 46 | ||
45 | export class RegisterHelpers { | 47 | export class RegisterHelpers { |
46 | private readonly updatedVideoConstants: UpdatedVideoConstant = { | 48 | private readonly updatedVideoConstants: UpdatedVideoConstant = { |
47 | playlistPrivacy: { added: [], deleted: [] }, | 49 | playlistPrivacy: { }, |
48 | privacy: { added: [], deleted: [] }, | 50 | privacy: { }, |
49 | language: { added: [], deleted: [] }, | 51 | language: { }, |
50 | licence: { added: [], deleted: [] }, | 52 | licence: { }, |
51 | category: { added: [], deleted: [] } | 53 | category: { } |
52 | } | 54 | } |
53 | 55 | ||
54 | private readonly transcodingProfiles: { | 56 | private readonly transcodingProfiles: { |
@@ -377,7 +379,7 @@ export class RegisterHelpers { | |||
377 | const { npmName, type, obj, key } = parameters | 379 | const { npmName, type, obj, key } = parameters |
378 | 380 | ||
379 | if (!obj[key]) { | 381 | if (!obj[key]) { |
380 | logger.warn('Cannot delete %s %s by plugin %s: key does not exist.', type, npmName, key) | 382 | logger.warn('Cannot delete %s by plugin %s: key %s does not exist.', type, npmName, key) |
381 | return false | 383 | return false |
382 | } | 384 | } |
383 | 385 | ||
@@ -388,7 +390,15 @@ export class RegisterHelpers { | |||
388 | } | 390 | } |
389 | } | 391 | } |
390 | 392 | ||
391 | this.updatedVideoConstants[type][npmName].deleted.push({ key, label: obj[key] }) | 393 | const updatedConstants = this.updatedVideoConstants[type][npmName] |
394 | |||
395 | const alreadyAdded = updatedConstants.added.find(a => a.key === key) | ||
396 | if (alreadyAdded) { | ||
397 | updatedConstants.added.filter(a => a.key !== key) | ||
398 | } else if (obj[key]) { | ||
399 | updatedConstants.deleted.push({ key, label: obj[key] }) | ||
400 | } | ||
401 | |||
392 | delete obj[key] | 402 | delete obj[key] |
393 | 403 | ||
394 | return true | 404 | return true |
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts index da620b607..2a9241249 100644 --- a/server/lib/redundancy.ts +++ b/server/lib/redundancy.ts | |||
@@ -1,12 +1,12 @@ | |||
1 | import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' | ||
2 | import { sendUndoCacheFile } from './activitypub/send' | ||
3 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
4 | import { MActorSignature, MVideoRedundancyVideo } from '@server/types/models' | ||
5 | import { CONFIG } from '@server/initializers/config' | ||
6 | import { logger } from '@server/helpers/logger' | 2 | import { logger } from '@server/helpers/logger' |
7 | import { ActorFollowModel } from '@server/models/activitypub/actor-follow' | 3 | import { CONFIG } from '@server/initializers/config' |
8 | import { Activity } from '@shared/models' | 4 | import { ActorFollowModel } from '@server/models/actor/actor-follow' |
9 | import { getServerActor } from '@server/models/application/application' | 5 | import { getServerActor } from '@server/models/application/application' |
6 | import { MActorSignature, MVideoRedundancyVideo } from '@server/types/models' | ||
7 | import { Activity } from '@shared/models' | ||
8 | import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' | ||
9 | import { sendUndoCacheFile } from './activitypub/send' | ||
10 | 10 | ||
11 | async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { | 11 | async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { |
12 | const serverActor = await getServerActor() | 12 | const serverActor = await getServerActor() |
diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts index 598c0211f..1b80316e9 100644 --- a/server/lib/schedulers/actor-follow-scheduler.ts +++ b/server/lib/schedulers/actor-follow-scheduler.ts | |||
@@ -1,9 +1,9 @@ | |||
1 | import { isTestInstance } from '../../helpers/core-utils' | 1 | import { isTestInstance } from '../../helpers/core-utils' |
2 | import { logger } from '../../helpers/logger' | 2 | import { logger } from '../../helpers/logger' |
3 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' | ||
4 | import { AbstractScheduler } from './abstract-scheduler' | ||
5 | import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | 3 | import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
4 | import { ActorFollowModel } from '../../models/actor/actor-follow' | ||
6 | import { ActorFollowScoreCache } from '../files-cache' | 5 | import { ActorFollowScoreCache } from '../files-cache' |
6 | import { AbstractScheduler } from './abstract-scheduler' | ||
7 | 7 | ||
8 | export class ActorFollowScheduler extends AbstractScheduler { | 8 | export class ActorFollowScheduler extends AbstractScheduler { |
9 | 9 | ||
diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts index 0b8cd1389..aaa5feed5 100644 --- a/server/lib/schedulers/auto-follow-index-instances.ts +++ b/server/lib/schedulers/auto-follow-index-instances.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { chunk } from 'lodash' | 1 | import { chunk } from 'lodash' |
2 | import { doJSONRequest } from '@server/helpers/requests' | 2 | import { doJSONRequest } from '@server/helpers/requests' |
3 | import { JobQueue } from '@server/lib/job-queue' | 3 | import { JobQueue } from '@server/lib/job-queue' |
4 | import { ActorFollowModel } from '@server/models/activitypub/actor-follow' | 4 | import { ActorFollowModel } from '@server/models/actor/actor-follow' |
5 | import { getServerActor } from '@server/models/application/application' | 5 | import { getServerActor } from '@server/models/application/application' |
6 | import { logger } from '../../helpers/logger' | 6 | import { logger } from '../../helpers/logger' |
7 | import { CONFIG } from '../../initializers/config' | 7 | import { CONFIG } from '../../initializers/config' |
diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts index 17a42b2c4..225669ea2 100644 --- a/server/lib/schedulers/remove-old-history-scheduler.ts +++ b/server/lib/schedulers/remove-old-history-scheduler.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { logger } from '../../helpers/logger' | 1 | import { logger } from '../../helpers/logger' |
2 | import { AbstractScheduler } from './abstract-scheduler' | 2 | import { AbstractScheduler } from './abstract-scheduler' |
3 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | 3 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
4 | import { UserVideoHistoryModel } from '../../models/account/user-video-history' | 4 | import { UserVideoHistoryModel } from '../../models/user/user-video-history' |
5 | import { CONFIG } from '../../initializers/config' | 5 | import { CONFIG } from '../../initializers/config' |
6 | 6 | ||
7 | export class RemoveOldHistoryScheduler extends AbstractScheduler { | 7 | export class RemoveOldHistoryScheduler extends AbstractScheduler { |
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 3e75babcb..af69bda89 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts | |||
@@ -1,12 +1,12 @@ | |||
1 | import { VideoModel } from '@server/models/video/video' | ||
2 | import { MVideoFullLight } from '@server/types/models' | ||
1 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
2 | import { AbstractScheduler } from './abstract-scheduler' | 4 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
5 | import { sequelizeTypescript } from '../../initializers/database' | ||
3 | import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' | 6 | import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' |
4 | import { retryTransactionWrapper } from '../../helpers/database-utils' | ||
5 | import { federateVideoIfNeeded } from '../activitypub/videos' | 7 | import { federateVideoIfNeeded } from '../activitypub/videos' |
6 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
7 | import { Notifier } from '../notifier' | 8 | import { Notifier } from '../notifier' |
8 | import { sequelizeTypescript } from '../../initializers/database' | 9 | import { AbstractScheduler } from './abstract-scheduler' |
9 | import { MVideoFullLight } from '@server/types/models' | ||
10 | 10 | ||
11 | export class UpdateVideosScheduler extends AbstractScheduler { | 11 | export class UpdateVideosScheduler extends AbstractScheduler { |
12 | 12 | ||
@@ -19,18 +19,19 @@ export class UpdateVideosScheduler extends AbstractScheduler { | |||
19 | } | 19 | } |
20 | 20 | ||
21 | protected async internalExecute () { | 21 | protected async internalExecute () { |
22 | return retryTransactionWrapper(this.updateVideos.bind(this)) | 22 | return this.updateVideos() |
23 | } | 23 | } |
24 | 24 | ||
25 | private async updateVideos () { | 25 | private async updateVideos () { |
26 | if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined | 26 | if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined |
27 | 27 | ||
28 | const publishedVideos = await sequelizeTypescript.transaction(async t => { | 28 | const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() |
29 | const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t) | 29 | const publishedVideos: MVideoFullLight[] = [] |
30 | const publishedVideos: MVideoFullLight[] = [] | 30 | |
31 | for (const schedule of schedules) { | ||
32 | await sequelizeTypescript.transaction(async t => { | ||
33 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(schedule.videoId, t) | ||
31 | 34 | ||
32 | for (const schedule of schedules) { | ||
33 | const video = schedule.Video | ||
34 | logger.info('Executing scheduled video update on %s.', video.uuid) | 35 | logger.info('Executing scheduled video update on %s.', video.uuid) |
35 | 36 | ||
36 | if (schedule.privacy) { | 37 | if (schedule.privacy) { |
@@ -42,16 +43,13 @@ export class UpdateVideosScheduler extends AbstractScheduler { | |||
42 | await federateVideoIfNeeded(video, isNewVideo, t) | 43 | await federateVideoIfNeeded(video, isNewVideo, t) |
43 | 44 | ||
44 | if (wasConfidentialVideo) { | 45 | if (wasConfidentialVideo) { |
45 | const videoToPublish: MVideoFullLight = Object.assign(video, { ScheduleVideoUpdate: schedule, UserVideoHistories: [] }) | 46 | publishedVideos.push(video) |
46 | publishedVideos.push(videoToPublish) | ||
47 | } | 47 | } |
48 | } | 48 | } |
49 | 49 | ||
50 | await schedule.destroy({ transaction: t }) | 50 | await schedule.destroy({ transaction: t }) |
51 | } | 51 | }) |
52 | 52 | } | |
53 | return publishedVideos | ||
54 | }) | ||
55 | 53 | ||
56 | for (const v of publishedVideos) { | 54 | for (const v of publishedVideos) { |
57 | Notifier.Instance.notifyOnNewVideoIfNeeded(v) | 55 | Notifier.Instance.notifyOnNewVideoIfNeeded(v) |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 59b55cccc..b5a5eb697 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -23,7 +23,7 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../. | |||
23 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 23 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
24 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 24 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
25 | import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' | 25 | import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
26 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' | 26 | import { getOrCreateAPVideo } from '../activitypub/videos' |
27 | import { downloadPlaylistSegments } from '../hls' | 27 | import { downloadPlaylistSegments } from '../hls' |
28 | import { removeVideoRedundancy } from '../redundancy' | 28 | import { removeVideoRedundancy } from '../redundancy' |
29 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' | 29 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' |
@@ -351,7 +351,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
351 | syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true }, | 351 | syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true }, |
352 | fetchType: 'all' as 'all' | 352 | fetchType: 'all' as 'all' |
353 | } | 353 | } |
354 | const { video } = await getOrCreateVideoAndAccountAndChannel(getVideoOptions) | 354 | const { video } = await getOrCreateAPVideo(getVideoOptions) |
355 | 355 | ||
356 | return video | 356 | return video |
357 | } | 357 | } |
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index aefe6aba4..898691c13 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { YoutubeDL } from '@server/helpers/youtube-dl' |
2 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | 2 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
3 | import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' | 3 | import { AbstractScheduler } from './abstract-scheduler' |
4 | 4 | ||
5 | export class YoutubeDlUpdateScheduler extends AbstractScheduler { | 5 | export class YoutubeDlUpdateScheduler extends AbstractScheduler { |
6 | 6 | ||
@@ -13,7 +13,7 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler { | |||
13 | } | 13 | } |
14 | 14 | ||
15 | protected internalExecute () { | 15 | protected internalExecute () { |
16 | return updateYoutubeDLBinary() | 16 | return YoutubeDL.updateYoutubeDLBinary() |
17 | } | 17 | } |
18 | 18 | ||
19 | static get Instance () { | 19 | static get Instance () { |
diff --git a/server/lib/search.ts b/server/lib/search.ts new file mode 100644 index 000000000..b643a4055 --- /dev/null +++ b/server/lib/search.ts | |||
@@ -0,0 +1,50 @@ | |||
1 | import * as express from 'express' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { AccountBlocklistModel } from '@server/models/account/account-blocklist' | ||
4 | import { getServerActor } from '@server/models/application/application' | ||
5 | import { ServerBlocklistModel } from '@server/models/server/server-blocklist' | ||
6 | import { SearchTargetQuery } from '@shared/models' | ||
7 | |||
8 | function isSearchIndexSearch (query: SearchTargetQuery) { | ||
9 | if (query.searchTarget === 'search-index') return true | ||
10 | |||
11 | const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX | ||
12 | |||
13 | if (searchIndexConfig.ENABLED !== true) return false | ||
14 | |||
15 | if (searchIndexConfig.DISABLE_LOCAL_SEARCH) return true | ||
16 | if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true | ||
17 | |||
18 | return false | ||
19 | } | ||
20 | |||
21 | async function buildMutedForSearchIndex (res: express.Response) { | ||
22 | const serverActor = await getServerActor() | ||
23 | const accountIds = [ serverActor.Account.id ] | ||
24 | |||
25 | if (res.locals.oauth) { | ||
26 | accountIds.push(res.locals.oauth.token.User.Account.id) | ||
27 | } | ||
28 | |||
29 | const [ blockedHosts, blockedAccounts ] = await Promise.all([ | ||
30 | ServerBlocklistModel.listHostsBlockedBy(accountIds), | ||
31 | AccountBlocklistModel.listHandlesBlockedBy(accountIds) | ||
32 | ]) | ||
33 | |||
34 | return { | ||
35 | blockedHosts, | ||
36 | blockedAccounts | ||
37 | } | ||
38 | } | ||
39 | |||
40 | function isURISearch (search: string) { | ||
41 | if (!search) return false | ||
42 | |||
43 | return search.startsWith('http://') || search.startsWith('https://') | ||
44 | } | ||
45 | |||
46 | export { | ||
47 | isSearchIndexSearch, | ||
48 | buildMutedForSearchIndex, | ||
49 | isURISearch | ||
50 | } | ||
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts new file mode 100644 index 000000000..80d87a9d3 --- /dev/null +++ b/server/lib/server-config-manager.ts | |||
@@ -0,0 +1,304 @@ | |||
1 | import { getServerCommit } from '@server/helpers/utils' | ||
2 | import { CONFIG, isEmailEnabled } from '@server/initializers/config' | ||
3 | import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants' | ||
4 | import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup' | ||
5 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' | ||
6 | import { HTMLServerConfig, RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models' | ||
7 | import { Hooks } from './plugins/hooks' | ||
8 | import { PluginManager } from './plugins/plugin-manager' | ||
9 | import { getThemeOrDefault } from './plugins/theme-utils' | ||
10 | import { VideoTranscodingProfilesManager } from './transcoding/video-transcoding-profiles' | ||
11 | |||
12 | /** | ||
13 | * | ||
14 | * Used to send the server config to clients (using REST/API or plugins API) | ||
15 | * We need a singleton class to manage config state depending on external events (to build menu entries etc) | ||
16 | * | ||
17 | */ | ||
18 | |||
19 | class ServerConfigManager { | ||
20 | |||
21 | private static instance: ServerConfigManager | ||
22 | |||
23 | private serverCommit: string | ||
24 | |||
25 | private homepageEnabled = false | ||
26 | |||
27 | private constructor () {} | ||
28 | |||
29 | async init () { | ||
30 | const instanceHomepage = await ActorCustomPageModel.loadInstanceHomepage() | ||
31 | |||
32 | this.updateHomepageState(instanceHomepage?.content) | ||
33 | } | ||
34 | |||
35 | updateHomepageState (content: string) { | ||
36 | this.homepageEnabled = !!content | ||
37 | } | ||
38 | |||
39 | async getHTMLServerConfig (): Promise<HTMLServerConfig> { | ||
40 | if (this.serverCommit === undefined) this.serverCommit = await getServerCommit() | ||
41 | |||
42 | const defaultTheme = getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) | ||
43 | |||
44 | return { | ||
45 | instance: { | ||
46 | name: CONFIG.INSTANCE.NAME, | ||
47 | shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, | ||
48 | isNSFW: CONFIG.INSTANCE.IS_NSFW, | ||
49 | defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, | ||
50 | defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE, | ||
51 | customizations: { | ||
52 | javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT, | ||
53 | css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS | ||
54 | } | ||
55 | }, | ||
56 | search: { | ||
57 | remoteUri: { | ||
58 | users: CONFIG.SEARCH.REMOTE_URI.USERS, | ||
59 | anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS | ||
60 | }, | ||
61 | searchIndex: { | ||
62 | enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, | ||
63 | url: CONFIG.SEARCH.SEARCH_INDEX.URL, | ||
64 | disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, | ||
65 | isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH | ||
66 | } | ||
67 | }, | ||
68 | plugin: { | ||
69 | registered: this.getRegisteredPlugins(), | ||
70 | registeredExternalAuths: this.getExternalAuthsPlugins(), | ||
71 | registeredIdAndPassAuths: this.getIdAndPassAuthPlugins() | ||
72 | }, | ||
73 | theme: { | ||
74 | registered: this.getRegisteredThemes(), | ||
75 | default: defaultTheme | ||
76 | }, | ||
77 | email: { | ||
78 | enabled: isEmailEnabled() | ||
79 | }, | ||
80 | contactForm: { | ||
81 | enabled: CONFIG.CONTACT_FORM.ENABLED | ||
82 | }, | ||
83 | serverVersion: PEERTUBE_VERSION, | ||
84 | serverCommit: this.serverCommit, | ||
85 | transcoding: { | ||
86 | hls: { | ||
87 | enabled: CONFIG.TRANSCODING.HLS.ENABLED | ||
88 | }, | ||
89 | webtorrent: { | ||
90 | enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED | ||
91 | }, | ||
92 | enabledResolutions: this.getEnabledResolutions('vod'), | ||
93 | profile: CONFIG.TRANSCODING.PROFILE, | ||
94 | availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod') | ||
95 | }, | ||
96 | live: { | ||
97 | enabled: CONFIG.LIVE.ENABLED, | ||
98 | |||
99 | allowReplay: CONFIG.LIVE.ALLOW_REPLAY, | ||
100 | maxDuration: CONFIG.LIVE.MAX_DURATION, | ||
101 | maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES, | ||
102 | maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, | ||
103 | |||
104 | transcoding: { | ||
105 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, | ||
106 | enabledResolutions: this.getEnabledResolutions('live'), | ||
107 | profile: CONFIG.LIVE.TRANSCODING.PROFILE, | ||
108 | availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') | ||
109 | }, | ||
110 | |||
111 | rtmp: { | ||
112 | port: CONFIG.LIVE.RTMP.PORT | ||
113 | } | ||
114 | }, | ||
115 | import: { | ||
116 | videos: { | ||
117 | http: { | ||
118 | enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED | ||
119 | }, | ||
120 | torrent: { | ||
121 | enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED | ||
122 | } | ||
123 | } | ||
124 | }, | ||
125 | autoBlacklist: { | ||
126 | videos: { | ||
127 | ofUsers: { | ||
128 | enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED | ||
129 | } | ||
130 | } | ||
131 | }, | ||
132 | avatar: { | ||
133 | file: { | ||
134 | size: { | ||
135 | max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max | ||
136 | }, | ||
137 | extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME | ||
138 | } | ||
139 | }, | ||
140 | banner: { | ||
141 | file: { | ||
142 | size: { | ||
143 | max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max | ||
144 | }, | ||
145 | extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME | ||
146 | } | ||
147 | }, | ||
148 | video: { | ||
149 | image: { | ||
150 | extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME, | ||
151 | size: { | ||
152 | max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max | ||
153 | } | ||
154 | }, | ||
155 | file: { | ||
156 | extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME | ||
157 | } | ||
158 | }, | ||
159 | videoCaption: { | ||
160 | file: { | ||
161 | size: { | ||
162 | max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max | ||
163 | }, | ||
164 | extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME | ||
165 | } | ||
166 | }, | ||
167 | user: { | ||
168 | videoQuota: CONFIG.USER.VIDEO_QUOTA, | ||
169 | videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY | ||
170 | }, | ||
171 | trending: { | ||
172 | videos: { | ||
173 | intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS, | ||
174 | algorithms: { | ||
175 | enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED, | ||
176 | default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT | ||
177 | } | ||
178 | } | ||
179 | }, | ||
180 | tracker: { | ||
181 | enabled: CONFIG.TRACKER.ENABLED | ||
182 | }, | ||
183 | |||
184 | followings: { | ||
185 | instance: { | ||
186 | autoFollowIndex: { | ||
187 | indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL | ||
188 | } | ||
189 | } | ||
190 | }, | ||
191 | |||
192 | broadcastMessage: { | ||
193 | enabled: CONFIG.BROADCAST_MESSAGE.ENABLED, | ||
194 | message: CONFIG.BROADCAST_MESSAGE.MESSAGE, | ||
195 | level: CONFIG.BROADCAST_MESSAGE.LEVEL, | ||
196 | dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE | ||
197 | }, | ||
198 | |||
199 | homepage: { | ||
200 | enabled: this.homepageEnabled | ||
201 | } | ||
202 | } | ||
203 | } | ||
204 | |||
205 | async getServerConfig (ip?: string): Promise<ServerConfig> { | ||
206 | const { allowed } = await Hooks.wrapPromiseFun( | ||
207 | isSignupAllowed, | ||
208 | { | ||
209 | ip | ||
210 | }, | ||
211 | 'filter:api.user.signup.allowed.result' | ||
212 | ) | ||
213 | |||
214 | const allowedForCurrentIP = isSignupAllowedForCurrentIP(ip) | ||
215 | |||
216 | const signup = { | ||
217 | allowed, | ||
218 | allowedForCurrentIP, | ||
219 | minimumAge: CONFIG.SIGNUP.MINIMUM_AGE, | ||
220 | requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION | ||
221 | } | ||
222 | |||
223 | const htmlConfig = await this.getHTMLServerConfig() | ||
224 | |||
225 | return { ...htmlConfig, signup } | ||
226 | } | ||
227 | |||
228 | getRegisteredThemes () { | ||
229 | return PluginManager.Instance.getRegisteredThemes() | ||
230 | .map(t => ({ | ||
231 | name: t.name, | ||
232 | version: t.version, | ||
233 | description: t.description, | ||
234 | css: t.css, | ||
235 | clientScripts: t.clientScripts | ||
236 | })) | ||
237 | } | ||
238 | |||
239 | getRegisteredPlugins () { | ||
240 | return PluginManager.Instance.getRegisteredPlugins() | ||
241 | .map(p => ({ | ||
242 | name: p.name, | ||
243 | version: p.version, | ||
244 | description: p.description, | ||
245 | clientScripts: p.clientScripts | ||
246 | })) | ||
247 | } | ||
248 | |||
249 | getEnabledResolutions (type: 'vod' | 'live') { | ||
250 | const transcoding = type === 'vod' | ||
251 | ? CONFIG.TRANSCODING | ||
252 | : CONFIG.LIVE.TRANSCODING | ||
253 | |||
254 | return Object.keys(transcoding.RESOLUTIONS) | ||
255 | .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true) | ||
256 | .map(r => parseInt(r, 10)) | ||
257 | } | ||
258 | |||
259 | private getIdAndPassAuthPlugins () { | ||
260 | const result: RegisteredIdAndPassAuthConfig[] = [] | ||
261 | |||
262 | for (const p of PluginManager.Instance.getIdAndPassAuths()) { | ||
263 | for (const auth of p.idAndPassAuths) { | ||
264 | result.push({ | ||
265 | npmName: p.npmName, | ||
266 | name: p.name, | ||
267 | version: p.version, | ||
268 | authName: auth.authName, | ||
269 | weight: auth.getWeight() | ||
270 | }) | ||
271 | } | ||
272 | } | ||
273 | |||
274 | return result | ||
275 | } | ||
276 | |||
277 | private getExternalAuthsPlugins () { | ||
278 | const result: RegisteredExternalAuthConfig[] = [] | ||
279 | |||
280 | for (const p of PluginManager.Instance.getExternalAuths()) { | ||
281 | for (const auth of p.externalAuths) { | ||
282 | result.push({ | ||
283 | npmName: p.npmName, | ||
284 | name: p.name, | ||
285 | version: p.version, | ||
286 | authName: auth.authName, | ||
287 | authDisplayName: auth.authDisplayName() | ||
288 | }) | ||
289 | } | ||
290 | } | ||
291 | |||
292 | return result | ||
293 | } | ||
294 | |||
295 | static get Instance () { | ||
296 | return this.instance || (this.instance = new this()) | ||
297 | } | ||
298 | } | ||
299 | |||
300 | // --------------------------------------------------------------------------- | ||
301 | |||
302 | export { | ||
303 | ServerConfigManager | ||
304 | } | ||
diff --git a/server/lib/signup.ts b/server/lib/signup.ts new file mode 100644 index 000000000..8fa81e601 --- /dev/null +++ b/server/lib/signup.ts | |||
@@ -0,0 +1,62 @@ | |||
1 | import { UserModel } from '../models/user/user' | ||
2 | import * as ipaddr from 'ipaddr.js' | ||
3 | import { CONFIG } from '../initializers/config' | ||
4 | |||
5 | const isCidr = require('is-cidr') | ||
6 | |||
7 | async function isSignupAllowed (): Promise<{ allowed: boolean, errorMessage?: string }> { | ||
8 | if (CONFIG.SIGNUP.ENABLED === false) { | ||
9 | return { allowed: false } | ||
10 | } | ||
11 | |||
12 | // No limit and signup is enabled | ||
13 | if (CONFIG.SIGNUP.LIMIT === -1) { | ||
14 | return { allowed: true } | ||
15 | } | ||
16 | |||
17 | const totalUsers = await UserModel.countTotal() | ||
18 | |||
19 | return { allowed: totalUsers < CONFIG.SIGNUP.LIMIT } | ||
20 | } | ||
21 | |||
22 | function isSignupAllowedForCurrentIP (ip: string) { | ||
23 | if (!ip) return false | ||
24 | |||
25 | const addr = ipaddr.parse(ip) | ||
26 | const excludeList = [ 'blacklist' ] | ||
27 | let matched = '' | ||
28 | |||
29 | // if there is a valid, non-empty whitelist, we exclude all unknown adresses too | ||
30 | if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) { | ||
31 | excludeList.push('unknown') | ||
32 | } | ||
33 | |||
34 | if (addr.kind() === 'ipv4') { | ||
35 | const addrV4 = ipaddr.IPv4.parse(ip) | ||
36 | const rangeList = { | ||
37 | whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr)) | ||
38 | .map(cidr => ipaddr.IPv4.parseCIDR(cidr)), | ||
39 | blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr)) | ||
40 | .map(cidr => ipaddr.IPv4.parseCIDR(cidr)) | ||
41 | } | ||
42 | matched = ipaddr.subnetMatch(addrV4, rangeList, 'unknown') | ||
43 | } else if (addr.kind() === 'ipv6') { | ||
44 | const addrV6 = ipaddr.IPv6.parse(ip) | ||
45 | const rangeList = { | ||
46 | whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr)) | ||
47 | .map(cidr => ipaddr.IPv6.parseCIDR(cidr)), | ||
48 | blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr)) | ||
49 | .map(cidr => ipaddr.IPv6.parseCIDR(cidr)) | ||
50 | } | ||
51 | matched = ipaddr.subnetMatch(addrV6, rangeList, 'unknown') | ||
52 | } | ||
53 | |||
54 | return !excludeList.includes(matched) | ||
55 | } | ||
56 | |||
57 | // --------------------------------------------------------------------------- | ||
58 | |||
59 | export { | ||
60 | isSignupAllowed, | ||
61 | isSignupAllowedForCurrentIP | ||
62 | } | ||
diff --git a/server/lib/stat-manager.ts b/server/lib/stat-manager.ts index 5d703f610..3c5e0a93e 100644 --- a/server/lib/stat-manager.ts +++ b/server/lib/stat-manager.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | 1 | import { CONFIG } from '@server/initializers/config' |
2 | import { UserModel } from '@server/models/account/user' | 2 | import { UserModel } from '@server/models/user/user' |
3 | import { ActorFollowModel } from '@server/models/activitypub/actor-follow' | 3 | import { ActorFollowModel } from '@server/models/actor/actor-follow' |
4 | import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' | 4 | import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' |
5 | import { VideoModel } from '@server/models/video/video' | 5 | import { VideoModel } from '@server/models/video/video' |
6 | import { VideoChannelModel } from '@server/models/video/video-channel' | 6 | import { VideoChannelModel } from '@server/models/video/video-channel' |
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index cfee69cfc..c08523988 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts | |||
@@ -14,7 +14,7 @@ import { getVideoFilePath } from './video-paths' | |||
14 | 14 | ||
15 | type ImageSize = { height?: number, width?: number } | 15 | type ImageSize = { height?: number, width?: number } |
16 | 16 | ||
17 | function createPlaylistMiniatureFromExisting (options: { | 17 | function updatePlaylistMiniatureFromExisting (options: { |
18 | inputPath: string | 18 | inputPath: string |
19 | playlist: MVideoPlaylistThumbnail | 19 | playlist: MVideoPlaylistThumbnail |
20 | automaticallyGenerated: boolean | 20 | automaticallyGenerated: boolean |
@@ -26,7 +26,7 @@ function createPlaylistMiniatureFromExisting (options: { | |||
26 | const type = ThumbnailType.MINIATURE | 26 | const type = ThumbnailType.MINIATURE |
27 | 27 | ||
28 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) | 28 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) |
29 | return createThumbnailFromFunction({ | 29 | return updateThumbnailFromFunction({ |
30 | thumbnailCreator, | 30 | thumbnailCreator, |
31 | filename, | 31 | filename, |
32 | height, | 32 | height, |
@@ -37,7 +37,7 @@ function createPlaylistMiniatureFromExisting (options: { | |||
37 | }) | 37 | }) |
38 | } | 38 | } |
39 | 39 | ||
40 | function createPlaylistMiniatureFromUrl (options: { | 40 | function updatePlaylistMiniatureFromUrl (options: { |
41 | downloadUrl: string | 41 | downloadUrl: string |
42 | playlist: MVideoPlaylistThumbnail | 42 | playlist: MVideoPlaylistThumbnail |
43 | size?: ImageSize | 43 | size?: ImageSize |
@@ -52,10 +52,10 @@ function createPlaylistMiniatureFromUrl (options: { | |||
52 | : downloadUrl | 52 | : downloadUrl |
53 | 53 | ||
54 | const thumbnailCreator = () => downloadImage(downloadUrl, basePath, filename, { width, height }) | 54 | const thumbnailCreator = () => downloadImage(downloadUrl, basePath, filename, { width, height }) |
55 | return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) | 55 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) |
56 | } | 56 | } |
57 | 57 | ||
58 | function createVideoMiniatureFromUrl (options: { | 58 | function updateVideoMiniatureFromUrl (options: { |
59 | downloadUrl: string | 59 | downloadUrl: string |
60 | video: MVideoThumbnail | 60 | video: MVideoThumbnail |
61 | type: ThumbnailType | 61 | type: ThumbnailType |
@@ -82,10 +82,10 @@ function createVideoMiniatureFromUrl (options: { | |||
82 | return Promise.resolve() | 82 | return Promise.resolve() |
83 | } | 83 | } |
84 | 84 | ||
85 | return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) | 85 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) |
86 | } | 86 | } |
87 | 87 | ||
88 | function createVideoMiniatureFromExisting (options: { | 88 | function updateVideoMiniatureFromExisting (options: { |
89 | inputPath: string | 89 | inputPath: string |
90 | video: MVideoThumbnail | 90 | video: MVideoThumbnail |
91 | type: ThumbnailType | 91 | type: ThumbnailType |
@@ -98,7 +98,7 @@ function createVideoMiniatureFromExisting (options: { | |||
98 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | 98 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) |
99 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) | 99 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) |
100 | 100 | ||
101 | return createThumbnailFromFunction({ | 101 | return updateThumbnailFromFunction({ |
102 | thumbnailCreator, | 102 | thumbnailCreator, |
103 | filename, | 103 | filename, |
104 | height, | 104 | height, |
@@ -123,7 +123,7 @@ function generateVideoMiniature (options: { | |||
123 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) | 123 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) |
124 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) | 124 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) |
125 | 125 | ||
126 | return createThumbnailFromFunction({ | 126 | return updateThumbnailFromFunction({ |
127 | thumbnailCreator, | 127 | thumbnailCreator, |
128 | filename, | 128 | filename, |
129 | height, | 129 | height, |
@@ -134,7 +134,7 @@ function generateVideoMiniature (options: { | |||
134 | }) | 134 | }) |
135 | } | 135 | } |
136 | 136 | ||
137 | function createPlaceholderThumbnail (options: { | 137 | function updatePlaceholderThumbnail (options: { |
138 | fileUrl: string | 138 | fileUrl: string |
139 | video: MVideoThumbnail | 139 | video: MVideoThumbnail |
140 | type: ThumbnailType | 140 | type: ThumbnailType |
@@ -165,11 +165,11 @@ function createPlaceholderThumbnail (options: { | |||
165 | 165 | ||
166 | export { | 166 | export { |
167 | generateVideoMiniature, | 167 | generateVideoMiniature, |
168 | createVideoMiniatureFromUrl, | 168 | updateVideoMiniatureFromUrl, |
169 | createVideoMiniatureFromExisting, | 169 | updateVideoMiniatureFromExisting, |
170 | createPlaceholderThumbnail, | 170 | updatePlaceholderThumbnail, |
171 | createPlaylistMiniatureFromUrl, | 171 | updatePlaylistMiniatureFromUrl, |
172 | createPlaylistMiniatureFromExisting | 172 | updatePlaylistMiniatureFromExisting |
173 | } | 173 | } |
174 | 174 | ||
175 | function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { | 175 | function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { |
@@ -231,7 +231,7 @@ function buildMetadataFromVideo (video: MVideoThumbnail, type: ThumbnailType, si | |||
231 | return undefined | 231 | return undefined |
232 | } | 232 | } |
233 | 233 | ||
234 | async function createThumbnailFromFunction (parameters: { | 234 | async function updateThumbnailFromFunction (parameters: { |
235 | thumbnailCreator: () => Promise<any> | 235 | thumbnailCreator: () => Promise<any> |
236 | filename: string | 236 | filename: string |
237 | height: number | 237 | height: number |
diff --git a/server/lib/video-transcoding-profiles.ts b/server/lib/transcoding/video-transcoding-profiles.ts index 81f5e1962..c5ea72a5f 100644 --- a/server/lib/video-transcoding-profiles.ts +++ b/server/lib/transcoding/video-transcoding-profiles.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { logger } from '@server/helpers/logger' | 1 | import { logger } from '@server/helpers/logger' |
2 | import { AvailableEncoders, EncoderOptionsBuilder, getTargetBitrate, VideoResolution } from '../../shared/models/videos' | 2 | import { AvailableEncoders, EncoderOptionsBuilder, getTargetBitrate, VideoResolution } from '../../../shared/models/videos' |
3 | import { buildStreamSuffix, resetSupportedEncoders } from '../helpers/ffmpeg-utils' | 3 | import { buildStreamSuffix, resetSupportedEncoders } from '../../helpers/ffmpeg-utils' |
4 | import { | 4 | import { |
5 | canDoQuickAudioTranscode, | 5 | canDoQuickAudioTranscode, |
6 | ffprobePromise, | 6 | ffprobePromise, |
@@ -8,8 +8,8 @@ import { | |||
8 | getMaxAudioBitrate, | 8 | getMaxAudioBitrate, |
9 | getVideoFileBitrate, | 9 | getVideoFileBitrate, |
10 | getVideoStreamFromFile | 10 | getVideoStreamFromFile |
11 | } from '../helpers/ffprobe-utils' | 11 | } from '../../helpers/ffprobe-utils' |
12 | import { VIDEO_TRANSCODING_FPS } from '../initializers/constants' | 12 | import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants' |
13 | 13 | ||
14 | /** | 14 | /** |
15 | * | 15 | * |
diff --git a/server/lib/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts index c949dca2e..1ad63baf3 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/transcoding/video-transcoding.ts | |||
@@ -1,19 +1,20 @@ | |||
1 | import { Job } from 'bull' | 1 | import { Job } from 'bull' |
2 | import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' | 2 | import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' |
3 | import { basename, extname as extnameUtil, join } from 'path' | 3 | import { basename, extname as extnameUtil, join } from 'path' |
4 | import { toEven } from '@server/helpers/core-utils' | ||
4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
5 | import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 6 | import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
6 | import { VideoResolution } from '../../shared/models/videos' | 7 | import { VideoResolution } from '../../../shared/models/videos' |
7 | import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' | 8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
8 | import { transcode, TranscodeOptions, TranscodeOptionsType } from '../helpers/ffmpeg-utils' | 9 | import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' |
9 | import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../helpers/ffprobe-utils' | 10 | import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' |
10 | import { logger } from '../helpers/logger' | 11 | import { logger } from '../../helpers/logger' |
11 | import { CONFIG } from '../initializers/config' | 12 | import { CONFIG } from '../../initializers/config' |
12 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../initializers/constants' | 13 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../../initializers/constants' |
13 | import { VideoFileModel } from '../models/video/video-file' | 14 | import { VideoFileModel } from '../../models/video/video-file' |
14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' |
15 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from './hls' | 16 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' |
16 | import { generateVideoFilename, generateVideoStreamingPlaylistName, getVideoFilePath } from './video-paths' | 17 | import { generateVideoFilename, generateVideoStreamingPlaylistName, getVideoFilePath } from '../video-paths' |
17 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | 18 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' |
18 | 19 | ||
19 | /** | 20 | /** |
@@ -35,6 +36,8 @@ async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile | |||
35 | ? 'quick-transcode' | 36 | ? 'quick-transcode' |
36 | : 'video' | 37 | : 'video' |
37 | 38 | ||
39 | const resolution = toEven(inputVideoFile.resolution) | ||
40 | |||
38 | const transcodeOptions: TranscodeOptions = { | 41 | const transcodeOptions: TranscodeOptions = { |
39 | type: transcodeType, | 42 | type: transcodeType, |
40 | 43 | ||
@@ -44,7 +47,7 @@ async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile | |||
44 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 47 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
45 | profile: CONFIG.TRANSCODING.PROFILE, | 48 | profile: CONFIG.TRANSCODING.PROFILE, |
46 | 49 | ||
47 | resolution: inputVideoFile.resolution, | 50 | resolution, |
48 | 51 | ||
49 | job | 52 | job |
50 | } | 53 | } |
@@ -57,7 +60,7 @@ async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile | |||
57 | 60 | ||
58 | // Important to do this before getVideoFilename() to take in account the new filename | 61 | // Important to do this before getVideoFilename() to take in account the new filename |
59 | inputVideoFile.extname = newExtname | 62 | inputVideoFile.extname = newExtname |
60 | inputVideoFile.filename = generateVideoFilename(video, false, inputVideoFile.resolution, newExtname) | 63 | inputVideoFile.filename = generateVideoFilename(video, false, resolution, newExtname) |
61 | 64 | ||
62 | const videoOutputPath = getVideoFilePath(video, inputVideoFile) | 65 | const videoOutputPath = getVideoFilePath(video, inputVideoFile) |
63 | 66 | ||
@@ -215,16 +218,6 @@ function generateHlsPlaylistResolution (options: { | |||
215 | }) | 218 | }) |
216 | } | 219 | } |
217 | 220 | ||
218 | function getEnabledResolutions (type: 'vod' | 'live') { | ||
219 | const transcoding = type === 'vod' | ||
220 | ? CONFIG.TRANSCODING | ||
221 | : CONFIG.LIVE.TRANSCODING | ||
222 | |||
223 | return Object.keys(transcoding.RESOLUTIONS) | ||
224 | .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true) | ||
225 | .map(r => parseInt(r, 10)) | ||
226 | } | ||
227 | |||
228 | // --------------------------------------------------------------------------- | 221 | // --------------------------------------------------------------------------- |
229 | 222 | ||
230 | export { | 223 | export { |
@@ -232,8 +225,7 @@ export { | |||
232 | generateHlsPlaylistResolutionFromTS, | 225 | generateHlsPlaylistResolutionFromTS, |
233 | optimizeOriginalVideofile, | 226 | optimizeOriginalVideofile, |
234 | transcodeNewWebTorrentResolution, | 227 | transcodeNewWebTorrentResolution, |
235 | mergeAudioVideofile, | 228 | mergeAudioVideofile |
236 | getEnabledResolutions | ||
237 | } | 229 | } |
238 | 230 | ||
239 | // --------------------------------------------------------------------------- | 231 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/user.ts b/server/lib/user.ts index 9b0a0a2f1..936403692 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts | |||
@@ -1,19 +1,21 @@ | |||
1 | import { Transaction } from 'sequelize/types' | 1 | import { Transaction } from 'sequelize/types' |
2 | import { v4 as uuidv4 } from 'uuid' | 2 | import { buildUUID } from '@server/helpers/uuid' |
3 | import { UserModel } from '@server/models/account/user' | 3 | import { UserModel } from '@server/models/user/user' |
4 | import { MActorDefault } from '@server/types/models/actor' | ||
4 | import { ActivityPubActorType } from '../../shared/models/activitypub' | 5 | import { ActivityPubActorType } from '../../shared/models/activitypub' |
5 | import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' | 6 | import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' |
6 | import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' | 7 | import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' |
7 | import { sequelizeTypescript } from '../initializers/database' | 8 | import { sequelizeTypescript } from '../initializers/database' |
8 | import { AccountModel } from '../models/account/account' | 9 | import { AccountModel } from '../models/account/account' |
9 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' | 10 | import { ActorModel } from '../models/actor/actor' |
10 | import { ActorModel } from '../models/activitypub/actor' | 11 | import { UserNotificationSettingModel } from '../models/user/user-notification-setting' |
11 | import { MAccountDefault, MActorDefault, MChannelActor } from '../types/models' | 12 | import { MAccountDefault, MChannelActor } from '../types/models' |
12 | import { MUser, MUserDefault, MUserId } from '../types/models/user' | 13 | import { MUser, MUserDefault, MUserId } from '../types/models/user' |
13 | import { buildActorInstance, generateAndSaveActorKeys } from './activitypub/actor' | 14 | import { generateAndSaveActorKeys } from './activitypub/actors' |
14 | import { getLocalAccountActivityPubUrl } from './activitypub/url' | 15 | import { getLocalAccountActivityPubUrl } from './activitypub/url' |
15 | import { Emailer } from './emailer' | 16 | import { Emailer } from './emailer' |
16 | import { LiveManager } from './live-manager' | 17 | import { LiveQuotaStore } from './live/live-quota-store' |
18 | import { buildActorInstance } from './local-actor' | ||
17 | import { Redis } from './redis' | 19 | import { Redis } from './redis' |
18 | import { createLocalVideoChannel } from './video-channel' | 20 | import { createLocalVideoChannel } from './video-channel' |
19 | import { createWatchLaterPlaylist } from './video-playlist' | 21 | import { createWatchLaterPlaylist } from './video-playlist' |
@@ -42,11 +44,11 @@ async function createUserAccountAndChannelAndPlaylist (parameters: { | |||
42 | displayName: userDisplayName, | 44 | displayName: userDisplayName, |
43 | userId: userCreated.id, | 45 | userId: userCreated.id, |
44 | applicationId: null, | 46 | applicationId: null, |
45 | t: t | 47 | t |
46 | }) | 48 | }) |
47 | userCreated.Account = accountCreated | 49 | userCreated.Account = accountCreated |
48 | 50 | ||
49 | const channelAttributes = await buildChannelAttributes(userCreated, channelNames) | 51 | const channelAttributes = await buildChannelAttributes(userCreated, t, channelNames) |
50 | const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t) | 52 | const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t) |
51 | 53 | ||
52 | const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t) | 54 | const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t) |
@@ -127,7 +129,7 @@ async function getOriginalVideoFileTotalFromUser (user: MUserId) { | |||
127 | 129 | ||
128 | const base = await UserModel.getTotalRawQuery(query, user.id) | 130 | const base = await UserModel.getTotalRawQuery(query, user.id) |
129 | 131 | ||
130 | return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) | 132 | return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) |
131 | } | 133 | } |
132 | 134 | ||
133 | // Returns cumulative size of all video files uploaded in the last 24 hours. | 135 | // Returns cumulative size of all video files uploaded in the last 24 hours. |
@@ -141,10 +143,10 @@ async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) { | |||
141 | 143 | ||
142 | const base = await UserModel.getTotalRawQuery(query, user.id) | 144 | const base = await UserModel.getTotalRawQuery(query, user.id) |
143 | 145 | ||
144 | return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) | 146 | return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) |
145 | } | 147 | } |
146 | 148 | ||
147 | async function isAbleToUploadVideo (userId: number, size: number) { | 149 | async function isAbleToUploadVideo (userId: number, newVideoSize: number) { |
148 | const user = await UserModel.loadById(userId) | 150 | const user = await UserModel.loadById(userId) |
149 | 151 | ||
150 | if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) | 152 | if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) |
@@ -154,8 +156,8 @@ async function isAbleToUploadVideo (userId: number, size: number) { | |||
154 | getOriginalVideoFileTotalDailyFromUser(user) | 156 | getOriginalVideoFileTotalDailyFromUser(user) |
155 | ]) | 157 | ]) |
156 | 158 | ||
157 | const uploadedTotal = size + totalBytes | 159 | const uploadedTotal = newVideoSize + totalBytes |
158 | const uploadedDaily = size + totalBytesDaily | 160 | const uploadedDaily = newVideoSize + totalBytesDaily |
159 | 161 | ||
160 | if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota | 162 | if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota |
161 | if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily | 163 | if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily |
@@ -201,14 +203,14 @@ function createDefaultUserNotificationSettings (user: MUserId, t: Transaction | | |||
201 | return UserNotificationSettingModel.create(values, { transaction: t }) | 203 | return UserNotificationSettingModel.create(values, { transaction: t }) |
202 | } | 204 | } |
203 | 205 | ||
204 | async function buildChannelAttributes (user: MUser, channelNames?: ChannelNames) { | 206 | async function buildChannelAttributes (user: MUser, transaction?: Transaction, channelNames?: ChannelNames) { |
205 | if (channelNames) return channelNames | 207 | if (channelNames) return channelNames |
206 | 208 | ||
207 | let channelName = user.username + '_channel' | 209 | let channelName = user.username + '_channel' |
208 | 210 | ||
209 | // Conflict, generate uuid instead | 211 | // Conflict, generate uuid instead |
210 | const actor = await ActorModel.loadLocalByName(channelName) | 212 | const actor = await ActorModel.loadLocalByName(channelName, transaction) |
211 | if (actor) channelName = uuidv4() | 213 | if (actor) channelName = buildUUID() |
212 | 214 | ||
213 | const videoChannelDisplayName = `Main ${user.username} channel` | 215 | const videoChannelDisplayName = `Main ${user.username} channel` |
214 | 216 | ||
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts index 37c43c3b0..0984c0d7a 100644 --- a/server/lib/video-blacklist.ts +++ b/server/lib/video-blacklist.ts | |||
@@ -16,7 +16,7 @@ import { CONFIG } from '../initializers/config' | |||
16 | import { VideoBlacklistModel } from '../models/video/video-blacklist' | 16 | import { VideoBlacklistModel } from '../models/video/video-blacklist' |
17 | import { sendDeleteVideo } from './activitypub/send' | 17 | import { sendDeleteVideo } from './activitypub/send' |
18 | import { federateVideoIfNeeded } from './activitypub/videos' | 18 | import { federateVideoIfNeeded } from './activitypub/videos' |
19 | import { LiveManager } from './live-manager' | 19 | import { LiveManager } from './live/live-manager' |
20 | import { Notifier } from './notifier' | 20 | import { Notifier } from './notifier' |
21 | import { Hooks } from './plugins/hooks' | 21 | import { Hooks } from './plugins/hooks' |
22 | 22 | ||
diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts index 0476cb2d5..2fd63a8c4 100644 --- a/server/lib/video-channel.ts +++ b/server/lib/video-channel.ts | |||
@@ -1,17 +1,15 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | import { v4 as uuidv4 } from 'uuid' | ||
3 | import { VideoChannelCreate } from '../../shared/models' | 2 | import { VideoChannelCreate } from '../../shared/models' |
4 | import { VideoModel } from '../models/video/video' | 3 | import { VideoModel } from '../models/video/video' |
5 | import { VideoChannelModel } from '../models/video/video-channel' | 4 | import { VideoChannelModel } from '../models/video/video-channel' |
6 | import { MAccountId, MChannelId } from '../types/models' | 5 | import { MAccountId, MChannelId } from '../types/models' |
7 | import { buildActorInstance } from './activitypub/actor' | ||
8 | import { getLocalVideoChannelActivityPubUrl } from './activitypub/url' | 6 | import { getLocalVideoChannelActivityPubUrl } from './activitypub/url' |
9 | import { federateVideoIfNeeded } from './activitypub/videos' | 7 | import { federateVideoIfNeeded } from './activitypub/videos' |
8 | import { buildActorInstance } from './local-actor' | ||
10 | 9 | ||
11 | async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) { | 10 | async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) { |
12 | const uuid = uuidv4() | ||
13 | const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name) | 11 | const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name) |
14 | const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name, uuid) | 12 | const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name) |
15 | 13 | ||
16 | const actorInstanceCreated = await actorInstance.save({ transaction: t }) | 14 | const actorInstanceCreated = await actorInstance.save({ transaction: t }) |
17 | 15 | ||
diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts index 736ebb2f8..c76570a5d 100644 --- a/server/lib/video-comment.ts +++ b/server/lib/video-comment.ts | |||
@@ -3,7 +3,7 @@ import * as Sequelize from 'sequelize' | |||
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
4 | import { sequelizeTypescript } from '@server/initializers/database' | 4 | import { sequelizeTypescript } from '@server/initializers/database' |
5 | import { ResultList } from '../../shared/models' | 5 | import { ResultList } from '../../shared/models' |
6 | import { VideoCommentThreadTree } from '../../shared/models/videos/video-comment.model' | 6 | import { VideoCommentThreadTree } from '../../shared/models/videos/comment/video-comment.model' |
7 | import { VideoCommentModel } from '../models/video/video-comment' | 7 | import { VideoCommentModel } from '../models/video/video-comment' |
8 | import { MAccountDefault, MComment, MCommentOwnerVideo, MCommentOwnerVideoReply, MVideoFullLight } from '../types/models' | 8 | import { MAccountDefault, MComment, MCommentOwnerVideo, MCommentOwnerVideoReply, MVideoFullLight } from '../types/models' |
9 | import { sendCreateVideoComment, sendDeleteVideoComment } from './activitypub/send' | 9 | import { sendCreateVideoComment, sendDeleteVideoComment } from './activitypub/send' |
@@ -18,9 +18,9 @@ async function removeComment (videoCommentInstance: MCommentOwnerVideo) { | |||
18 | await sendDeleteVideoComment(videoCommentInstance, t) | 18 | await sendDeleteVideoComment(videoCommentInstance, t) |
19 | } | 19 | } |
20 | 20 | ||
21 | markCommentAsDeleted(videoCommentInstance) | 21 | videoCommentInstance.markAsDeleted() |
22 | 22 | ||
23 | await videoCommentInstance.save() | 23 | await videoCommentInstance.save({ transaction: t }) |
24 | }) | 24 | }) |
25 | 25 | ||
26 | logger.info('Video comment %d deleted.', videoCommentInstance.id) | 26 | logger.info('Video comment %d deleted.', videoCommentInstance.id) |
@@ -95,17 +95,10 @@ function buildFormattedCommentTree (resultList: ResultList<VideoCommentModel>): | |||
95 | return thread | 95 | return thread |
96 | } | 96 | } |
97 | 97 | ||
98 | function markCommentAsDeleted (comment: MComment): void { | ||
99 | comment.text = '' | ||
100 | comment.deletedAt = new Date() | ||
101 | comment.accountId = null | ||
102 | } | ||
103 | |||
104 | // --------------------------------------------------------------------------- | 98 | // --------------------------------------------------------------------------- |
105 | 99 | ||
106 | export { | 100 | export { |
107 | removeComment, | 101 | removeComment, |
108 | createVideoComment, | 102 | createVideoComment, |
109 | buildFormattedCommentTree, | 103 | buildFormattedCommentTree |
110 | markCommentAsDeleted | ||
111 | } | 104 | } |
diff --git a/server/lib/video.ts b/server/lib/video.ts index 21e4b7ff2..daf998704 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts | |||
@@ -10,7 +10,7 @@ import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } fro | |||
10 | import { federateVideoIfNeeded } from './activitypub/videos' | 10 | import { federateVideoIfNeeded } from './activitypub/videos' |
11 | import { JobQueue } from './job-queue/job-queue' | 11 | import { JobQueue } from './job-queue/job-queue' |
12 | import { Notifier } from './notifier' | 12 | import { Notifier } from './notifier' |
13 | import { createVideoMiniatureFromExisting } from './thumbnail' | 13 | import { updateVideoMiniatureFromExisting } from './thumbnail' |
14 | 14 | ||
15 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { | 15 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { |
16 | return { | 16 | return { |
@@ -28,6 +28,8 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil | |||
28 | privacy: videoInfo.privacy || VideoPrivacy.PRIVATE, | 28 | privacy: videoInfo.privacy || VideoPrivacy.PRIVATE, |
29 | channelId: channelId, | 29 | channelId: channelId, |
30 | originallyPublishedAt: videoInfo.originallyPublishedAt | 30 | originallyPublishedAt: videoInfo.originallyPublishedAt |
31 | ? new Date(videoInfo.originallyPublishedAt) | ||
32 | : null | ||
31 | } | 33 | } |
32 | } | 34 | } |
33 | 35 | ||
@@ -52,7 +54,7 @@ async function buildVideoThumbnailsFromReq (options: { | |||
52 | const fields = files?.[p.fieldName] | 54 | const fields = files?.[p.fieldName] |
53 | 55 | ||
54 | if (fields) { | 56 | if (fields) { |
55 | return createVideoMiniatureFromExisting({ | 57 | return updateVideoMiniatureFromExisting({ |
56 | inputPath: fields[0].path, | 58 | inputPath: fields[0].path, |
57 | video, | 59 | video, |
58 | type: p.type, | 60 | type: p.type, |