aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/activitypub
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib/activitypub')
-rw-r--r--server/lib/activitypub/actor.ts594
-rw-r--r--server/lib/activitypub/actors/get.ts122
-rw-r--r--server/lib/activitypub/actors/image.ts94
-rw-r--r--server/lib/activitypub/actors/index.ts6
-rw-r--r--server/lib/activitypub/actors/keys.ts16
-rw-r--r--server/lib/activitypub/actors/refresh.ts81
-rw-r--r--server/lib/activitypub/actors/shared/creator.ts149
-rw-r--r--server/lib/activitypub/actors/shared/index.ts3
-rw-r--r--server/lib/activitypub/actors/shared/object-to-model-attributes.ts70
-rw-r--r--server/lib/activitypub/actors/shared/url-to-object.ts54
-rw-r--r--server/lib/activitypub/actors/updater.ts90
-rw-r--r--server/lib/activitypub/actors/webfinger.ts67
-rw-r--r--server/lib/activitypub/audience.ts2
-rw-r--r--server/lib/activitypub/cache-file.ts91
-rw-r--r--server/lib/activitypub/crawl.ts7
-rw-r--r--server/lib/activitypub/follow.ts13
-rw-r--r--server/lib/activitypub/outbox.ts24
-rw-r--r--server/lib/activitypub/playlist.ts204
-rw-r--r--server/lib/activitypub/playlists/create-update.ts156
-rw-r--r--server/lib/activitypub/playlists/get.ts35
-rw-r--r--server/lib/activitypub/playlists/index.ts3
-rw-r--r--server/lib/activitypub/playlists/refresh.ts53
-rw-r--r--server/lib/activitypub/playlists/shared/index.ts2
-rw-r--r--server/lib/activitypub/playlists/shared/object-to-model-attributes.ts40
-rw-r--r--server/lib/activitypub/playlists/shared/url-to-object.ts47
-rw-r--r--server/lib/activitypub/process/process-accept.ts4
-rw-r--r--server/lib/activitypub/process/process-announce.ts4
-rw-r--r--server/lib/activitypub/process/process-create.ts12
-rw-r--r--server/lib/activitypub/process/process-delete.ts11
-rw-r--r--server/lib/activitypub/process/process-dislike.ts4
-rw-r--r--server/lib/activitypub/process/process-follow.ts21
-rw-r--r--server/lib/activitypub/process/process-like.ts4
-rw-r--r--server/lib/activitypub/process/process-reject.ts2
-rw-r--r--server/lib/activitypub/process/process-undo.ts16
-rw-r--r--server/lib/activitypub/process/process-update.ts104
-rw-r--r--server/lib/activitypub/process/process-view.ts13
-rw-r--r--server/lib/activitypub/process/process.ts14
-rw-r--r--server/lib/activitypub/send/send-delete.ts2
-rw-r--r--server/lib/activitypub/send/send-view.ts2
-rw-r--r--server/lib/activitypub/send/utils.ts16
-rw-r--r--server/lib/activitypub/share.ts40
-rw-r--r--server/lib/activitypub/video-comments.ts41
-rw-r--r--server/lib/activitypub/video-rates.ts64
-rw-r--r--server/lib/activitypub/videos.ts931
-rw-r--r--server/lib/activitypub/videos/federate.ts36
-rw-r--r--server/lib/activitypub/videos/get.ts113
-rw-r--r--server/lib/activitypub/videos/index.ts4
-rw-r--r--server/lib/activitypub/videos/refresh.ts68
-rw-r--r--server/lib/activitypub/videos/shared/abstract-builder.ts173
-rw-r--r--server/lib/activitypub/videos/shared/creator.ts88
-rw-r--r--server/lib/activitypub/videos/shared/index.ts6
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts256
-rw-r--r--server/lib/activitypub/videos/shared/trackers.ts43
-rw-r--r--server/lib/activitypub/videos/shared/url-to-object.ts25
-rw-r--r--server/lib/activitypub/videos/shared/video-sync-attributes.ts94
-rw-r--r--server/lib/activitypub/videos/updater.ts166
56 files changed, 2404 insertions, 1996 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
deleted file mode 100644
index 5fe7381c9..000000000
--- a/server/lib/activitypub/actor.ts
+++ /dev/null
@@ -1,594 +0,0 @@
1import * as Bluebird from 'bluebird'
2import { extname } from 'path'
3import { Op, Transaction } from 'sequelize'
4import { URL } from 'url'
5import { v4 as uuidv4 } from 'uuid'
6import { getServerActor } from '@server/models/application/application'
7import { ActorImageType } from '@shared/models'
8import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
9import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
10import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
11import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
12import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
13import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor'
14import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
15import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
16import { logger } from '../../helpers/logger'
17import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
18import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
19import { getUrlFromWebfinger } from '../../helpers/webfinger'
20import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
21import { sequelizeTypescript } from '../../initializers/database'
22import { AccountModel } from '../../models/account/account'
23import { ActorImageModel } from '../../models/account/actor-image'
24import { ActorModel } from '../../models/activitypub/actor'
25import { ServerModel } from '../../models/server/server'
26import { VideoChannelModel } from '../../models/video/video-channel'
27import {
28 MAccount,
29 MAccountDefault,
30 MActor,
31 MActorAccountChannelId,
32 MActorAccountChannelIdActor,
33 MActorAccountId,
34 MActorFull,
35 MActorFullActor,
36 MActorId,
37 MActorImage,
38 MActorImages,
39 MChannel
40} from '../../types/models'
41import { JobQueue } from '../job-queue'
42
43// Set account keys, this could be long so process after the account creation and do not block the client
44async function generateAndSaveActorKeys <T extends MActor> (actor: T) {
45 const { publicKey, privateKey } = await createPrivateAndPublicKeys()
46
47 actor.publicKey = publicKey
48 actor.privateKey = privateKey
49
50 return actor.save()
51}
52
53function getOrCreateActorAndServerAndModel (
54 activityActor: string | ActivityPubActor,
55 fetchType: 'all',
56 recurseIfNeeded?: boolean,
57 updateCollections?: boolean
58): Promise<MActorFullActor>
59
60function getOrCreateActorAndServerAndModel (
61 activityActor: string | ActivityPubActor,
62 fetchType?: 'association-ids',
63 recurseIfNeeded?: boolean,
64 updateCollections?: boolean
65): Promise<MActorAccountChannelId>
66
67async function getOrCreateActorAndServerAndModel (
68 activityActor: string | ActivityPubActor,
69 fetchType: ActorFetchByUrlType = 'association-ids',
70 recurseIfNeeded = true,
71 updateCollections = false
72): Promise<MActorFullActor | MActorAccountChannelId> {
73 const actorUrl = getAPId(activityActor)
74 let created = false
75 let accountPlaylistsUrl: string
76
77 let actor = await fetchActorByUrl(actorUrl, fetchType)
78 // Orphan actor (not associated to an account of channel) so recreate it
79 if (actor && (!actor.Account && !actor.VideoChannel)) {
80 await actor.destroy()
81 actor = null
82 }
83
84 // We don't have this actor in our database, fetch it on remote
85 if (!actor) {
86 const { result } = await fetchRemoteActor(actorUrl)
87 if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
88
89 // Create the attributed to actor
90 // In PeerTube a video channel is owned by an account
91 let ownerActor: MActorFullActor
92 if (recurseIfNeeded === true && result.actor.type === 'Group') {
93 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
94 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
95
96 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
97 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
98 }
99
100 try {
101 // Don't recurse another time
102 const recurseIfNeeded = false
103 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
104 } catch (err) {
105 logger.error('Cannot get or create account attributed to video channel ' + actorUrl)
106 throw new Error(err)
107 }
108 }
109
110 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
111 created = true
112 accountPlaylistsUrl = result.playlists
113 }
114
115 if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor
116 if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
117
118 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
119 if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.')
120
121 if ((created === true || refreshed === true) && updateCollections === true) {
122 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
123 await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
124 }
125
126 // We created a new account: fetch the playlists
127 if (created === true && actor.Account && accountPlaylistsUrl) {
128 const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
129 await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
130 }
131
132 return actorRefreshed
133}
134
135function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
136 return new ActorModel({
137 type,
138 url,
139 preferredUsername,
140 uuid,
141 publicKey: null,
142 privateKey: null,
143 followersCount: 0,
144 followingCount: 0,
145 inboxUrl: url + '/inbox',
146 outboxUrl: url + '/outbox',
147 sharedInboxUrl: WEBSERVER.URL + '/inbox',
148 followersUrl: url + '/followers',
149 followingUrl: url + '/following'
150 }) as MActor
151}
152
153async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
154 const followersCount = await fetchActorTotalItems(attributes.followers)
155 const followingCount = await fetchActorTotalItems(attributes.following)
156
157 actorInstance.type = attributes.type
158 actorInstance.preferredUsername = attributes.preferredUsername
159 actorInstance.url = attributes.id
160 actorInstance.publicKey = attributes.publicKey.publicKeyPem
161 actorInstance.followersCount = followersCount
162 actorInstance.followingCount = followingCount
163 actorInstance.inboxUrl = attributes.inbox
164 actorInstance.outboxUrl = attributes.outbox
165 actorInstance.followersUrl = attributes.followers
166 actorInstance.followingUrl = attributes.following
167
168 if (attributes.published) actorInstance.remoteCreatedAt = new Date(attributes.published)
169
170 if (attributes.endpoints?.sharedInbox) {
171 actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox
172 }
173}
174
175type ImageInfo = {
176 name: string
177 fileUrl: string
178 height: number
179 width: number
180 onDisk?: boolean
181}
182async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) {
183 const oldImageModel = type === ActorImageType.AVATAR
184 ? actor.Avatar
185 : actor.Banner
186
187 if (oldImageModel) {
188 // Don't update the avatar if the file URL did not change
189 if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor
190
191 try {
192 await oldImageModel.destroy({ transaction: t })
193
194 setActorImage(actor, type, null)
195 } catch (err) {
196 logger.error('Cannot remove old actor image of actor %s.', actor.url, { err })
197 }
198 }
199
200 if (imageInfo) {
201 const imageModel = await ActorImageModel.create({
202 filename: imageInfo.name,
203 onDisk: imageInfo.onDisk ?? false,
204 fileUrl: imageInfo.fileUrl,
205 height: imageInfo.height,
206 width: imageInfo.width,
207 type
208 }, { transaction: t })
209
210 setActorImage(actor, type, imageModel)
211 }
212
213 return actor
214}
215
216async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) {
217 try {
218 if (type === ActorImageType.AVATAR) {
219 await actor.Avatar.destroy({ transaction: t })
220
221 actor.avatarId = null
222 actor.Avatar = null
223 } else {
224 await actor.Banner.destroy({ transaction: t })
225
226 actor.bannerId = null
227 actor.Banner = null
228 }
229 } catch (err) {
230 logger.error('Cannot remove old image of actor %s.', actor.url, { err })
231 }
232
233 return actor
234}
235
236async function fetchActorTotalItems (url: string) {
237 try {
238 const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true })
239
240 return body.totalItems || 0
241 } catch (err) {
242 logger.warn('Cannot fetch remote actor count %s.', url, { err })
243 return 0
244 }
245}
246
247function getImageInfoIfExists (actorJSON: ActivityPubActor, type: ActorImageType) {
248 const mimetypes = MIMETYPES.IMAGE
249 const icon = type === ActorImageType.AVATAR
250 ? actorJSON.icon
251 : actorJSON.image
252
253 if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined
254
255 let extension: string
256
257 if (icon.mediaType) {
258 extension = mimetypes.MIMETYPE_EXT[icon.mediaType]
259 } else {
260 const tmp = extname(icon.url)
261
262 if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp
263 }
264
265 if (!extension) return undefined
266
267 return {
268 name: uuidv4() + extension,
269 fileUrl: icon.url,
270 height: icon.height,
271 width: icon.width,
272 type
273 }
274}
275
276async function addFetchOutboxJob (actor: Pick<ActorModel, 'id' | 'outboxUrl'>) {
277 // Don't fetch ourselves
278 const serverActor = await getServerActor()
279 if (serverActor.id === actor.id) {
280 logger.error('Cannot fetch our own outbox!')
281 return undefined
282 }
283
284 const payload = {
285 uri: actor.outboxUrl,
286 type: 'activity' as 'activity'
287 }
288
289 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
290}
291
292async function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (
293 actorArg: T,
294 fetchedType: ActorFetchByUrlType
295): Promise<{ actor: T | MActorFull, refreshed: boolean }> {
296 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
297
298 // We need more attributes
299 const actor = fetchedType === 'all'
300 ? actorArg as MActorFull
301 : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
302
303 try {
304 let actorUrl: string
305 try {
306 actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
307 } catch (err) {
308 logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
309 actorUrl = actor.url
310 }
311
312 const { result } = await fetchRemoteActor(actorUrl)
313
314 if (result === undefined) {
315 logger.warn('Cannot fetch remote actor in refresh actor.')
316 return { actor, refreshed: false }
317 }
318
319 return sequelizeTypescript.transaction(async t => {
320 updateInstanceWithAnother(actor, result.actor)
321
322 await updateActorImageInstance(actor, ActorImageType.AVATAR, result.avatar, t)
323 await updateActorImageInstance(actor, ActorImageType.BANNER, result.banner, t)
324
325 // Force update
326 actor.setDataValue('updatedAt', new Date())
327 await actor.save({ transaction: t })
328
329 if (actor.Account) {
330 actor.Account.name = result.name
331 actor.Account.description = result.summary
332
333 await actor.Account.save({ transaction: t })
334 } else if (actor.VideoChannel) {
335 actor.VideoChannel.name = result.name
336 actor.VideoChannel.description = result.summary
337 actor.VideoChannel.support = result.support
338
339 await actor.VideoChannel.save({ transaction: t })
340 }
341
342 return { refreshed: true, actor }
343 })
344 } catch (err) {
345 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
346 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
347 actor.Account
348 ? await actor.Account.destroy()
349 : await actor.VideoChannel.destroy()
350
351 return { actor: undefined, refreshed: false }
352 }
353
354 logger.warn('Cannot refresh actor %s.', actor.url, { err })
355 return { actor, refreshed: false }
356 }
357}
358
359export {
360 getOrCreateActorAndServerAndModel,
361 buildActorInstance,
362 generateAndSaveActorKeys,
363 fetchActorTotalItems,
364 getImageInfoIfExists,
365 updateActorInstance,
366 deleteActorImageInstance,
367 refreshActorIfNeeded,
368 updateActorImageInstance,
369 addFetchOutboxJob
370}
371
372// ---------------------------------------------------------------------------
373
374function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) {
375 const id = imageModel
376 ? imageModel.id
377 : null
378
379 if (type === ActorImageType.AVATAR) {
380 actorModel.avatarId = id
381 actorModel.Avatar = imageModel
382 } else {
383 actorModel.bannerId = id
384 actorModel.Banner = imageModel
385 }
386
387 return actorModel
388}
389
390function saveActorAndServerAndModelIfNotExist (
391 result: FetchRemoteActorResult,
392 ownerActor?: MActorFullActor,
393 t?: Transaction
394): Bluebird<MActorFullActor> | Promise<MActorFullActor> {
395 const actor = result.actor
396
397 if (t !== undefined) return save(t)
398
399 return sequelizeTypescript.transaction(t => save(t))
400
401 async function save (t: Transaction) {
402 const actorHost = new URL(actor.url).host
403
404 const serverOptions = {
405 where: {
406 host: actorHost
407 },
408 defaults: {
409 host: actorHost
410 },
411 transaction: t
412 }
413 const [ server ] = await ServerModel.findOrCreate(serverOptions)
414
415 // Save our new account in database
416 actor.serverId = server.id
417
418 // Avatar?
419 if (result.avatar) {
420 const avatar = await ActorImageModel.create({
421 filename: result.avatar.name,
422 fileUrl: result.avatar.fileUrl,
423 width: result.avatar.width,
424 height: result.avatar.height,
425 onDisk: false,
426 type: ActorImageType.AVATAR
427 }, { transaction: t })
428
429 actor.avatarId = avatar.id
430 }
431
432 // Banner?
433 if (result.banner) {
434 const banner = await ActorImageModel.create({
435 filename: result.banner.name,
436 fileUrl: result.banner.fileUrl,
437 width: result.banner.width,
438 height: result.banner.height,
439 onDisk: false,
440 type: ActorImageType.BANNER
441 }, { transaction: t })
442
443 actor.bannerId = banner.id
444 }
445
446 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
447 // (which could be false in a retried query)
448 const [ actorCreated, created ] = await ActorModel.findOrCreate<MActorFullActor>({
449 defaults: actor.toJSON(),
450 where: {
451 [Op.or]: [
452 {
453 url: actor.url
454 },
455 {
456 serverId: actor.serverId,
457 preferredUsername: actor.preferredUsername
458 }
459 ]
460 },
461 transaction: t
462 })
463
464 // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards
465 if (created !== true && actorCreated.url !== actor.url) {
466 // Only fix http://example.com/account/djidane to https://example.com/account/djidane
467 if (actorCreated.url.replace(/^http:\/\//, '') !== actor.url.replace(/^https:\/\//, '')) {
468 throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${actor.url}`)
469 }
470
471 actorCreated.url = actor.url
472 await actorCreated.save({ transaction: t })
473 }
474
475 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
476 actorCreated.Account = await saveAccount(actorCreated, result, t) as MAccountDefault
477 actorCreated.Account.Actor = actorCreated
478 } else if (actorCreated.type === 'Group') { // Video channel
479 const channel = await saveVideoChannel(actorCreated, result, ownerActor, t)
480 actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: ownerActor.Account })
481 }
482
483 actorCreated.Server = server
484
485 return actorCreated
486 }
487}
488
489type ImageResult = {
490 name: string
491 fileUrl: string
492 height: number
493 width: number
494}
495
496type FetchRemoteActorResult = {
497 actor: MActor
498 name: string
499 summary: string
500 support?: string
501 playlists?: string
502 avatar?: ImageResult
503 banner?: ImageResult
504 attributedTo: ActivityPubAttributedTo[]
505}
506async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
507 logger.info('Fetching remote actor %s.', actorUrl)
508
509 const requestResult = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true })
510 const actorJSON = requestResult.body
511
512 if (sanitizeAndCheckActorObject(actorJSON) === false) {
513 logger.debug('Remote actor JSON is not valid.', { actorJSON })
514 return { result: undefined, statusCode: requestResult.statusCode }
515 }
516
517 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
518 logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
519 return { result: undefined, statusCode: requestResult.statusCode }
520 }
521
522 const followersCount = await fetchActorTotalItems(actorJSON.followers)
523 const followingCount = await fetchActorTotalItems(actorJSON.following)
524
525 const actor = new ActorModel({
526 type: actorJSON.type,
527 preferredUsername: actorJSON.preferredUsername,
528 url: actorJSON.id,
529 publicKey: actorJSON.publicKey.publicKeyPem,
530 privateKey: null,
531 followersCount: followersCount,
532 followingCount: followingCount,
533 inboxUrl: actorJSON.inbox,
534 outboxUrl: actorJSON.outbox,
535 followersUrl: actorJSON.followers,
536 followingUrl: actorJSON.following,
537
538 sharedInboxUrl: actorJSON.endpoints?.sharedInbox
539 ? actorJSON.endpoints.sharedInbox
540 : null
541 })
542
543 const avatarInfo = getImageInfoIfExists(actorJSON, ActorImageType.AVATAR)
544 const bannerInfo = getImageInfoIfExists(actorJSON, ActorImageType.BANNER)
545
546 const name = actorJSON.name || actorJSON.preferredUsername
547 return {
548 statusCode: requestResult.statusCode,
549 result: {
550 actor,
551 name,
552 avatar: avatarInfo,
553 banner: bannerInfo,
554 summary: actorJSON.summary,
555 support: actorJSON.support,
556 playlists: actorJSON.playlists,
557 attributedTo: actorJSON.attributedTo
558 }
559 }
560}
561
562async function saveAccount (actor: MActorId, result: FetchRemoteActorResult, t: Transaction) {
563 const [ accountCreated ] = await AccountModel.findOrCreate({
564 defaults: {
565 name: result.name,
566 description: result.summary,
567 actorId: actor.id
568 },
569 where: {
570 actorId: actor.id
571 },
572 transaction: t
573 })
574
575 return accountCreated as MAccount
576}
577
578async function saveVideoChannel (actor: MActorId, result: FetchRemoteActorResult, ownerActor: MActorAccountId, t: Transaction) {
579 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
580 defaults: {
581 name: result.name,
582 description: result.summary,
583 support: result.support,
584 actorId: actor.id,
585 accountId: ownerActor.Account.id
586 },
587 where: {
588 actorId: actor.id
589 },
590 transaction: t
591 })
592
593 return videoChannelCreated as MChannel
594}
diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts
new file mode 100644
index 000000000..8681ea02a
--- /dev/null
+++ b/server/lib/activitypub/actors/get.ts
@@ -0,0 +1,122 @@
1
2import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub'
3import { retryTransactionWrapper } from '@server/helpers/database-utils'
4import { logger } from '@server/helpers/logger'
5import { JobQueue } from '@server/lib/job-queue'
6import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders'
7import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models'
8import { ActivityPubActor } from '@shared/models'
9import { refreshActorIfNeeded } from './refresh'
10import { APActorCreator, fetchRemoteActor } from './shared'
11
12function getOrCreateAPActor (
13 activityActor: string | ActivityPubActor,
14 fetchType: 'all',
15 recurseIfNeeded?: boolean,
16 updateCollections?: boolean
17): Promise<MActorFullActor>
18
19function getOrCreateAPActor (
20 activityActor: string | ActivityPubActor,
21 fetchType?: 'association-ids',
22 recurseIfNeeded?: boolean,
23 updateCollections?: boolean
24): Promise<MActorAccountChannelId>
25
26async function getOrCreateAPActor (
27 activityActor: string | ActivityPubActor,
28 fetchType: ActorLoadByUrlType = 'association-ids',
29 recurseIfNeeded = true,
30 updateCollections = false
31): Promise<MActorFullActor | MActorAccountChannelId> {
32 const actorUrl = getAPId(activityActor)
33 let actor = await loadActorFromDB(actorUrl, fetchType)
34
35 let created = false
36 let accountPlaylistsUrl: string
37
38 // We don't have this actor in our database, fetch it on remote
39 if (!actor) {
40 const { actorObject } = await fetchRemoteActor(actorUrl)
41 if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
42
43 // actorUrl is just an alias/rediraction, so process object id instead
44 if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections)
45
46 // Create the attributed to actor
47 // In PeerTube a video channel is owned by an account
48 let ownerActor: MActorFullActor
49 if (recurseIfNeeded === true && actorObject.type === 'Group') {
50 ownerActor = await getOrCreateAPOwner(actorObject, actorUrl)
51 }
52
53 const creator = new APActorCreator(actorObject, ownerActor)
54 actor = await retryTransactionWrapper(creator.create.bind(creator))
55 created = true
56 accountPlaylistsUrl = actorObject.playlists
57 }
58
59 if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor
60 if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
61
62 const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType })
63 if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.')
64
65 await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections)
66 await schedulePlaylistFetchIfNeeded(actor, created, accountPlaylistsUrl)
67
68 return actorRefreshed
69}
70
71// ---------------------------------------------------------------------------
72
73export {
74 getOrCreateAPActor
75}
76
77// ---------------------------------------------------------------------------
78
79async function loadActorFromDB (actorUrl: string, fetchType: ActorLoadByUrlType) {
80 let actor = await loadActorByUrl(actorUrl, fetchType)
81
82 // Orphan actor (not associated to an account of channel) so recreate it
83 if (actor && (!actor.Account && !actor.VideoChannel)) {
84 await actor.destroy()
85 actor = null
86 }
87
88 return actor
89}
90
91function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) {
92 const accountAttributedTo = actorObject.attributedTo.find(a => a.type === 'Person')
93 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actorUrl)
94
95 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
96 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
97 }
98
99 try {
100 // Don't recurse another time
101 const recurseIfNeeded = false
102 return getOrCreateAPActor(accountAttributedTo.id, 'all', recurseIfNeeded)
103 } catch (err) {
104 logger.error('Cannot get or create account attributed to video channel ' + actorUrl)
105 throw new Error(err)
106 }
107}
108
109async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) {
110 if ((created === true || refreshed === true) && updateCollections === true) {
111 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
112 await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
113 }
114}
115
116async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) {
117 // We created a new account: fetch the playlists
118 if (created === true && actor.Account && accountPlaylistsUrl) {
119 const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' }
120 await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
121 }
122}
diff --git a/server/lib/activitypub/actors/image.ts b/server/lib/activitypub/actors/image.ts
new file mode 100644
index 000000000..443ad0a63
--- /dev/null
+++ b/server/lib/activitypub/actors/image.ts
@@ -0,0 +1,94 @@
1import { Transaction } from 'sequelize/types'
2import { logger } from '@server/helpers/logger'
3import { ActorImageModel } from '@server/models/actor/actor-image'
4import { MActorImage, MActorImages } from '@server/types/models'
5import { ActorImageType } from '@shared/models'
6
7type ImageInfo = {
8 name: string
9 fileUrl: string
10 height: number
11 width: number
12 onDisk?: boolean
13}
14
15async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) {
16 const oldImageModel = type === ActorImageType.AVATAR
17 ? actor.Avatar
18 : actor.Banner
19
20 if (oldImageModel) {
21 // Don't update the avatar if the file URL did not change
22 if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor
23
24 try {
25 await oldImageModel.destroy({ transaction: t })
26
27 setActorImage(actor, type, null)
28 } catch (err) {
29 logger.error('Cannot remove old actor image of actor %s.', actor.url, { err })
30 }
31 }
32
33 if (imageInfo) {
34 const imageModel = await ActorImageModel.create({
35 filename: imageInfo.name,
36 onDisk: imageInfo.onDisk ?? false,
37 fileUrl: imageInfo.fileUrl,
38 height: imageInfo.height,
39 width: imageInfo.width,
40 type
41 }, { transaction: t })
42
43 setActorImage(actor, type, imageModel)
44 }
45
46 return actor
47}
48
49async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) {
50 try {
51 if (type === ActorImageType.AVATAR) {
52 await actor.Avatar.destroy({ transaction: t })
53
54 actor.avatarId = null
55 actor.Avatar = null
56 } else {
57 await actor.Banner.destroy({ transaction: t })
58
59 actor.bannerId = null
60 actor.Banner = null
61 }
62 } catch (err) {
63 logger.error('Cannot remove old image of actor %s.', actor.url, { err })
64 }
65
66 return actor
67}
68
69// ---------------------------------------------------------------------------
70
71export {
72 ImageInfo,
73
74 updateActorImageInstance,
75 deleteActorImageInstance
76}
77
78// ---------------------------------------------------------------------------
79
80function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) {
81 const id = imageModel
82 ? imageModel.id
83 : null
84
85 if (type === ActorImageType.AVATAR) {
86 actorModel.avatarId = id
87 actorModel.Avatar = imageModel
88 } else {
89 actorModel.bannerId = id
90 actorModel.Banner = imageModel
91 }
92
93 return actorModel
94}
diff --git a/server/lib/activitypub/actors/index.ts b/server/lib/activitypub/actors/index.ts
new file mode 100644
index 000000000..5ee2a6f1a
--- /dev/null
+++ b/server/lib/activitypub/actors/index.ts
@@ -0,0 +1,6 @@
1export * from './get'
2export * from './image'
3export * from './keys'
4export * from './refresh'
5export * from './updater'
6export * from './webfinger'
diff --git a/server/lib/activitypub/actors/keys.ts b/server/lib/activitypub/actors/keys.ts
new file mode 100644
index 000000000..c3d18abd8
--- /dev/null
+++ b/server/lib/activitypub/actors/keys.ts
@@ -0,0 +1,16 @@
1import { createPrivateAndPublicKeys } from '@server/helpers/peertube-crypto'
2import { MActor } from '@server/types/models'
3
4// Set account keys, this could be long so process after the account creation and do not block the client
5async function generateAndSaveActorKeys <T extends MActor> (actor: T) {
6 const { publicKey, privateKey } = await createPrivateAndPublicKeys()
7
8 actor.publicKey = publicKey
9 actor.privateKey = privateKey
10
11 return actor.save()
12}
13
14export {
15 generateAndSaveActorKeys
16}
diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts
new file mode 100644
index 000000000..b2fe3932f
--- /dev/null
+++ b/server/lib/activitypub/actors/refresh.ts
@@ -0,0 +1,81 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { PromiseCache } from '@server/helpers/promise-cache'
3import { PeerTubeRequestError } from '@server/helpers/requests'
4import { ActorLoadByUrlType } from '@server/lib/model-loaders'
5import { ActorModel } from '@server/models/actor/actor'
6import { MActorAccountChannelId, MActorFull } from '@server/types/models'
7import { HttpStatusCode } from '@shared/core-utils'
8import { fetchRemoteActor } from './shared'
9import { APActorUpdater } from './updater'
10import { getUrlFromWebfinger } from './webfinger'
11
12type RefreshResult <T> = Promise<{ actor: T | MActorFull, refreshed: boolean }>
13
14type RefreshOptions <T> = {
15 actor: T
16 fetchedType: ActorLoadByUrlType
17}
18
19const promiseCache = new PromiseCache(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url)
20
21function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <T> {
22 const actorArg = options.actor
23 if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false })
24
25 return promiseCache.run(options)
26}
27
28export {
29 refreshActorIfNeeded
30}
31
32// ---------------------------------------------------------------------------
33
34async function doRefresh <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <MActorFull> {
35 const { actor: actorArg, fetchedType } = options
36
37 // We need more attributes
38 const actor = fetchedType === 'all'
39 ? actorArg as MActorFull
40 : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
41
42 const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url)
43
44 logger.info('Refreshing actor %s.', actor.url, lTags())
45
46 try {
47 const actorUrl = await getActorUrl(actor)
48 const { actorObject } = await fetchRemoteActor(actorUrl)
49
50 if (actorObject === undefined) {
51 logger.warn('Cannot fetch remote actor in refresh actor.')
52 return { actor, refreshed: false }
53 }
54
55 const updater = new APActorUpdater(actorObject, actor)
56 await updater.update()
57
58 return { refreshed: true, actor }
59 } catch (err) {
60 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
61 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url, lTags())
62
63 actor.Account
64 ? await actor.Account.destroy()
65 : await actor.VideoChannel.destroy()
66
67 return { actor: undefined, refreshed: false }
68 }
69
70 logger.warn('Cannot refresh actor %s.', actor.url, { err, ...lTags() })
71 return { actor, refreshed: false }
72 }
73}
74
75function getActorUrl (actor: MActorFull) {
76 return getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
77 .catch(err => {
78 logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
79 return actor.url
80 })
81}
diff --git a/server/lib/activitypub/actors/shared/creator.ts b/server/lib/activitypub/actors/shared/creator.ts
new file mode 100644
index 000000000..999aed97d
--- /dev/null
+++ b/server/lib/activitypub/actors/shared/creator.ts
@@ -0,0 +1,149 @@
1import { Op, Transaction } from 'sequelize'
2import { sequelizeTypescript } from '@server/initializers/database'
3import { AccountModel } from '@server/models/account/account'
4import { ActorModel } from '@server/models/actor/actor'
5import { ServerModel } from '@server/models/server/server'
6import { VideoChannelModel } from '@server/models/video/video-channel'
7import { MAccount, MAccountDefault, MActor, MActorFullActor, MActorId, MActorImages, MChannel, MServer } from '@server/types/models'
8import { ActivityPubActor, ActorImageType } from '@shared/models'
9import { updateActorImageInstance } from '../image'
10import { getActorAttributesFromObject, getActorDisplayNameFromObject, getImageInfoFromObject } from './object-to-model-attributes'
11import { fetchActorFollowsCount } from './url-to-object'
12
13export class APActorCreator {
14
15 constructor (
16 private readonly actorObject: ActivityPubActor,
17 private readonly ownerActor?: MActorFullActor
18 ) {
19
20 }
21
22 async create (): Promise<MActorFullActor> {
23 const { followersCount, followingCount } = await fetchActorFollowsCount(this.actorObject)
24
25 const actorInstance = new ActorModel(getActorAttributesFromObject(this.actorObject, followersCount, followingCount))
26
27 return sequelizeTypescript.transaction(async t => {
28 const server = await this.setServer(actorInstance, t)
29
30 await this.setImageIfNeeded(actorInstance, ActorImageType.AVATAR, t)
31 await this.setImageIfNeeded(actorInstance, ActorImageType.BANNER, t)
32
33 const { actorCreated, created } = await this.saveActor(actorInstance, t)
34
35 await this.tryToFixActorUrlIfNeeded(actorCreated, actorInstance, created, t)
36
37 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { // Account or PeerTube instance
38 actorCreated.Account = await this.saveAccount(actorCreated, t) as MAccountDefault
39 actorCreated.Account.Actor = actorCreated
40 }
41
42 if (actorCreated.type === 'Group') { // Video channel
43 const channel = await this.saveVideoChannel(actorCreated, t)
44 actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: this.ownerActor.Account })
45 }
46
47 actorCreated.Server = server
48
49 return actorCreated
50 })
51 }
52
53 private async setServer (actor: MActor, t: Transaction) {
54 const actorHost = new URL(actor.url).host
55
56 const serverOptions = {
57 where: {
58 host: actorHost
59 },
60 defaults: {
61 host: actorHost
62 },
63 transaction: t
64 }
65 const [ server ] = await ServerModel.findOrCreate(serverOptions)
66
67 // Save our new account in database
68 actor.serverId = server.id
69
70 return server as MServer
71 }
72
73 private async setImageIfNeeded (actor: MActor, type: ActorImageType, t: Transaction) {
74 const imageInfo = getImageInfoFromObject(this.actorObject, type)
75 if (!imageInfo) return
76
77 return updateActorImageInstance(actor as MActorImages, type, imageInfo, t)
78 }
79
80 private async saveActor (actor: MActor, t: Transaction) {
81 // Force the actor creation using findOrCreate() instead of save()
82 // Sometimes Sequelize skips the save() when it thinks the instance already exists
83 // (which could be false in a retried query)
84 const [ actorCreated, created ] = await ActorModel.findOrCreate<MActorFullActor>({
85 defaults: actor.toJSON(),
86 where: {
87 [Op.or]: [
88 {
89 url: actor.url
90 },
91 {
92 serverId: actor.serverId,
93 preferredUsername: actor.preferredUsername
94 }
95 ]
96 },
97 transaction: t
98 })
99
100 return { actorCreated, created }
101 }
102
103 private async tryToFixActorUrlIfNeeded (actorCreated: MActor, newActor: MActor, created: boolean, t: Transaction) {
104 // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards
105 if (created !== true && actorCreated.url !== newActor.url) {
106 // Only fix http://example.com/account/djidane to https://example.com/account/djidane
107 if (actorCreated.url.replace(/^http:\/\//, '') !== newActor.url.replace(/^https:\/\//, '')) {
108 throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${newActor.url}`)
109 }
110
111 actorCreated.url = newActor.url
112 await actorCreated.save({ transaction: t })
113 }
114 }
115
116 private async saveAccount (actor: MActorId, t: Transaction) {
117 const [ accountCreated ] = await AccountModel.findOrCreate({
118 defaults: {
119 name: getActorDisplayNameFromObject(this.actorObject),
120 description: this.actorObject.summary,
121 actorId: actor.id
122 },
123 where: {
124 actorId: actor.id
125 },
126 transaction: t
127 })
128
129 return accountCreated as MAccount
130 }
131
132 private async saveVideoChannel (actor: MActorId, t: Transaction) {
133 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
134 defaults: {
135 name: getActorDisplayNameFromObject(this.actorObject),
136 description: this.actorObject.summary,
137 support: this.actorObject.support,
138 actorId: actor.id,
139 accountId: this.ownerActor.Account.id
140 },
141 where: {
142 actorId: actor.id
143 },
144 transaction: t
145 })
146
147 return videoChannelCreated as MChannel
148 }
149}
diff --git a/server/lib/activitypub/actors/shared/index.ts b/server/lib/activitypub/actors/shared/index.ts
new file mode 100644
index 000000000..52af1a8e1
--- /dev/null
+++ b/server/lib/activitypub/actors/shared/index.ts
@@ -0,0 +1,3 @@
1export * from './creator'
2export * from './object-to-model-attributes'
3export * from './url-to-object'
diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts
new file mode 100644
index 000000000..1612b3ad0
--- /dev/null
+++ b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts
@@ -0,0 +1,70 @@
1import { getLowercaseExtension } from '@server/helpers/core-utils'
2import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
3import { buildUUID } from '@server/helpers/uuid'
4import { MIMETYPES } from '@server/initializers/constants'
5import { ActorModel } from '@server/models/actor/actor'
6import { FilteredModelAttributes } from '@server/types'
7import { ActivityPubActor, ActorImageType } from '@shared/models'
8
9function getActorAttributesFromObject (
10 actorObject: ActivityPubActor,
11 followersCount: number,
12 followingCount: number
13): FilteredModelAttributes<ActorModel> {
14 return {
15 type: actorObject.type,
16 preferredUsername: actorObject.preferredUsername,
17 url: actorObject.id,
18 publicKey: actorObject.publicKey.publicKeyPem,
19 privateKey: null,
20 followersCount,
21 followingCount,
22 inboxUrl: actorObject.inbox,
23 outboxUrl: actorObject.outbox,
24 followersUrl: actorObject.followers,
25 followingUrl: actorObject.following,
26
27 sharedInboxUrl: actorObject.endpoints?.sharedInbox
28 ? actorObject.endpoints.sharedInbox
29 : null
30 }
31}
32
33function getImageInfoFromObject (actorObject: ActivityPubActor, type: ActorImageType) {
34 const mimetypes = MIMETYPES.IMAGE
35 const icon = type === ActorImageType.AVATAR
36 ? actorObject.icon
37 : actorObject.image
38
39 if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined
40
41 let extension: string
42
43 if (icon.mediaType) {
44 extension = mimetypes.MIMETYPE_EXT[icon.mediaType]
45 } else {
46 const tmp = getLowercaseExtension(icon.url)
47
48 if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp
49 }
50
51 if (!extension) return undefined
52
53 return {
54 name: buildUUID() + extension,
55 fileUrl: icon.url,
56 height: icon.height,
57 width: icon.width,
58 type
59 }
60}
61
62function getActorDisplayNameFromObject (actorObject: ActivityPubActor) {
63 return actorObject.name || actorObject.preferredUsername
64}
65
66export {
67 getActorAttributesFromObject,
68 getImageInfoFromObject,
69 getActorDisplayNameFromObject
70}
diff --git a/server/lib/activitypub/actors/shared/url-to-object.ts b/server/lib/activitypub/actors/shared/url-to-object.ts
new file mode 100644
index 000000000..f4f16b044
--- /dev/null
+++ b/server/lib/activitypub/actors/shared/url-to-object.ts
@@ -0,0 +1,54 @@
1
2import { checkUrlsSameHost } from '@server/helpers/activitypub'
3import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor'
4import { logger } from '@server/helpers/logger'
5import { doJSONRequest } from '@server/helpers/requests'
6import { ActivityPubActor, ActivityPubOrderedCollection } from '@shared/models'
7
8async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> {
9 logger.info('Fetching remote actor %s.', actorUrl)
10
11 const { body, statusCode } = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true })
12
13 if (sanitizeAndCheckActorObject(body) === false) {
14 logger.debug('Remote actor JSON is not valid.', { actorJSON: body })
15 return { actorObject: undefined, statusCode: statusCode }
16 }
17
18 if (checkUrlsSameHost(body.id, actorUrl) !== true) {
19 logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, body.id)
20 return { actorObject: undefined, statusCode: statusCode }
21 }
22
23 return {
24 statusCode,
25
26 actorObject: body
27 }
28}
29
30async function fetchActorFollowsCount (actorObject: ActivityPubActor) {
31 const followersCount = await fetchActorTotalItems(actorObject.followers)
32 const followingCount = await fetchActorTotalItems(actorObject.following)
33
34 return { followersCount, followingCount }
35}
36
37// ---------------------------------------------------------------------------
38export {
39 fetchActorFollowsCount,
40 fetchRemoteActor
41}
42
43// ---------------------------------------------------------------------------
44
45async function fetchActorTotalItems (url: string) {
46 try {
47 const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true })
48
49 return body.totalItems || 0
50 } catch (err) {
51 logger.warn('Cannot fetch remote actor count %s.', url, { err })
52 return 0
53 }
54}
diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts
new file mode 100644
index 000000000..de5e03eee
--- /dev/null
+++ b/server/lib/activitypub/actors/updater.ts
@@ -0,0 +1,90 @@
1import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils'
2import { logger } from '@server/helpers/logger'
3import { VideoChannelModel } from '@server/models/video/video-channel'
4import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models'
5import { ActivityPubActor, ActorImageType } from '@shared/models'
6import { updateActorImageInstance } from './image'
7import { fetchActorFollowsCount } from './shared'
8import { getImageInfoFromObject } from './shared/object-to-model-attributes'
9
10export class APActorUpdater {
11
12 private accountOrChannel: MAccount | MChannel
13
14 private readonly actorFieldsSave: object
15 private readonly accountOrChannelFieldsSave: object
16
17 constructor (
18 private readonly actorObject: ActivityPubActor,
19 private readonly actor: MActorFull
20 ) {
21 this.actorFieldsSave = this.actor.toJSON()
22
23 if (this.actorObject.type === 'Group') this.accountOrChannel = this.actor.VideoChannel
24 else this.accountOrChannel = this.actor.Account
25
26 this.accountOrChannelFieldsSave = this.accountOrChannel.toJSON()
27 }
28
29 async update () {
30 const avatarInfo = getImageInfoFromObject(this.actorObject, ActorImageType.AVATAR)
31 const bannerInfo = getImageInfoFromObject(this.actorObject, ActorImageType.BANNER)
32
33 try {
34 await this.updateActorInstance(this.actor, this.actorObject)
35
36 this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername
37 this.accountOrChannel.description = this.actorObject.summary
38
39 if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support
40
41 await runInReadCommittedTransaction(async t => {
42 await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t)
43 await updateActorImageInstance(this.actor, ActorImageType.BANNER, bannerInfo, t)
44 })
45
46 await runInReadCommittedTransaction(async t => {
47 await this.actor.save({ transaction: t })
48 await this.accountOrChannel.save({ transaction: t })
49 })
50
51 logger.info('Remote account %s updated', this.actorObject.url)
52 } catch (err) {
53 if (this.actor !== undefined && this.actorFieldsSave !== undefined) {
54 resetSequelizeInstance(this.actor, this.actorFieldsSave)
55 }
56
57 if (this.accountOrChannel !== undefined && this.accountOrChannelFieldsSave !== undefined) {
58 resetSequelizeInstance(this.accountOrChannel, this.accountOrChannelFieldsSave)
59 }
60
61 // This is just a debug because we will retry the insert
62 logger.debug('Cannot update the remote account.', { err })
63 throw err
64 }
65 }
66
67 private async updateActorInstance (actorInstance: MActor, actorObject: ActivityPubActor) {
68 const { followersCount, followingCount } = await fetchActorFollowsCount(actorObject)
69
70 actorInstance.type = actorObject.type
71 actorInstance.preferredUsername = actorObject.preferredUsername
72 actorInstance.url = actorObject.id
73 actorInstance.publicKey = actorObject.publicKey.publicKeyPem
74 actorInstance.followersCount = followersCount
75 actorInstance.followingCount = followingCount
76 actorInstance.inboxUrl = actorObject.inbox
77 actorInstance.outboxUrl = actorObject.outbox
78 actorInstance.followersUrl = actorObject.followers
79 actorInstance.followingUrl = actorObject.following
80
81 if (actorObject.published) actorInstance.remoteCreatedAt = new Date(actorObject.published)
82
83 if (actorObject.endpoints?.sharedInbox) {
84 actorInstance.sharedInboxUrl = actorObject.endpoints.sharedInbox
85 }
86
87 // Force actor update
88 actorInstance.changed('updatedAt', true)
89 }
90}
diff --git a/server/lib/activitypub/actors/webfinger.ts b/server/lib/activitypub/actors/webfinger.ts
new file mode 100644
index 000000000..1c7ec4717
--- /dev/null
+++ b/server/lib/activitypub/actors/webfinger.ts
@@ -0,0 +1,67 @@
1import * as WebFinger from 'webfinger.js'
2import { isProdInstance } from '@server/helpers/core-utils'
3import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
4import { REQUEST_TIMEOUT, WEBSERVER } from '@server/initializers/constants'
5import { ActorModel } from '@server/models/actor/actor'
6import { MActorFull } from '@server/types/models'
7import { WebFingerData } from '@shared/models'
8
9const webfinger = new WebFinger({
10 webfist_fallback: false,
11 tls_only: isProdInstance(),
12 uri_fallback: false,
13 request_timeout: REQUEST_TIMEOUT
14})
15
16async function loadActorUrlOrGetFromWebfinger (uriArg: string) {
17 // Handle strings like @toto@example.com
18 const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg
19
20 const [ name, host ] = uri.split('@')
21 let actor: MActorFull
22
23 if (!host || host === WEBSERVER.HOST) {
24 actor = await ActorModel.loadLocalByName(name)
25 } else {
26 actor = await ActorModel.loadByNameAndHost(name, host)
27 }
28
29 if (actor) return actor.url
30
31 return getUrlFromWebfinger(uri)
32}
33
34async function getUrlFromWebfinger (uri: string) {
35 const webfingerData: WebFingerData = await webfingerLookup(uri)
36 return getLinkOrThrow(webfingerData)
37}
38
39// ---------------------------------------------------------------------------
40
41export {
42 getUrlFromWebfinger,
43 loadActorUrlOrGetFromWebfinger
44}
45
46// ---------------------------------------------------------------------------
47
48function getLinkOrThrow (webfingerData: WebFingerData) {
49 if (Array.isArray(webfingerData.links) === false) throw new Error('WebFinger links is not an array.')
50
51 const selfLink = webfingerData.links.find(l => l.rel === 'self')
52 if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) {
53 throw new Error('Cannot find self link or href is not a valid URL.')
54 }
55
56 return selfLink.href
57}
58
59function webfingerLookup (nameWithHost: string) {
60 return new Promise<WebFingerData>((res, rej) => {
61 webfinger.lookup(nameWithHost, (err, p) => {
62 if (err) return rej(err)
63
64 return res(p.object)
65 })
66 })
67}
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts
index 2986714d3..d0558f191 100644
--- a/server/lib/activitypub/audience.ts
+++ b/server/lib/activitypub/audience.ts
@@ -1,7 +1,7 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { ActivityAudience } from '../../../shared/models/activitypub' 2import { ActivityAudience } from '../../../shared/models/activitypub'
3import { ACTIVITY_PUB } from '../../initializers/constants' 3import { ACTIVITY_PUB } from '../../initializers/constants'
4import { ActorModel } from '../../models/activitypub/actor' 4import { ActorModel } from '../../models/actor/actor'
5import { VideoModel } from '../../models/video/video' 5import { VideoModel } from '../../models/video/video'
6import { VideoShareModel } from '../../models/video/video-share' 6import { VideoShareModel } from '../../models/video/video-share'
7import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '../../types/models' 7import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '../../types/models'
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 2e6dd34e0..a16d2cd93 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,54 +1,27 @@
1import { CacheFileObject } from '../../../shared/index'
2import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
3import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
4import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' 2import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models'
3import { CacheFileObject } from '../../../shared/index'
4import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6 6
7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { 7async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
8 8 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
9 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
10 const url = cacheFileObject.url
11
12 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14 9
15 return { 10 if (redundancyModel) {
16 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, 11 return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
17 url: cacheFileObject.id,
18 fileUrl: url.href,
19 strategy: null,
20 videoStreamingPlaylistId: playlist.id,
21 actorId: byActor.id
22 }
23 } 12 }
24 13
25 const url = cacheFileObject.url 14 return createCacheFile(cacheFileObject, video, byActor, t)
26 const videoFile = video.VideoFiles.find(f => {
27 return f.resolution === url.height && f.fps === url.fps
28 })
29
30 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
31
32 return {
33 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
34 url: cacheFileObject.id,
35 fileUrl: url.href,
36 strategy: null,
37 videoFileId: videoFile.id,
38 actorId: byActor.id
39 }
40} 15}
41 16
42async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { 17// ---------------------------------------------------------------------------
43 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
44 18
45 if (!redundancyModel) { 19export {
46 await createCacheFile(cacheFileObject, video, byActor, t) 20 createOrUpdateCacheFile
47 } else {
48 await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
49 }
50} 21}
51 22
23// ---------------------------------------------------------------------------
24
52function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { 25function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
53 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) 26 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
54 27
@@ -74,9 +47,37 @@ function updateCacheFile (
74 return redundancyModel.save({ transaction: t }) 47 return redundancyModel.save({ transaction: t })
75} 48}
76 49
77export { 50function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) {
78 createOrUpdateCacheFile, 51
79 createCacheFile, 52 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
80 updateCacheFile, 53 const url = cacheFileObject.url
81 cacheFileActivityObjectToDBAttributes 54
55 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
56 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
57
58 return {
59 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
60 url: cacheFileObject.id,
61 fileUrl: url.href,
62 strategy: null,
63 videoStreamingPlaylistId: playlist.id,
64 actorId: byActor.id
65 }
66 }
67
68 const url = cacheFileObject.url
69 const videoFile = video.VideoFiles.find(f => {
70 return f.resolution === url.height && f.fps === url.fps
71 })
72
73 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
74
75 return {
76 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
77 url: cacheFileObject.id,
78 fileUrl: url.href,
79 strategy: null,
80 videoFileId: videoFile.id,
81 actorId: byActor.id
82 }
82} 83}
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index 278abf7de..cd117f571 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -3,7 +3,7 @@ import { URL } from 'url'
3import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' 3import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
4import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
5import { doJSONRequest } from '../../helpers/requests' 5import { doJSONRequest } from '../../helpers/requests'
6import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' 6import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants'
7 7
8type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) 8type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
9type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) 9type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
@@ -13,10 +13,7 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction
13 13
14 logger.info('Crawling ActivityPub data on %s.', url) 14 logger.info('Crawling ActivityPub data on %s.', url)
15 15
16 const options = { 16 const options = { activityPub: true }
17 activityPub: true,
18 timeout: REQUEST_TIMEOUT
19 }
20 17
21 const startDate = new Date() 18 const startDate = new Date()
22 19
diff --git a/server/lib/activitypub/follow.ts b/server/lib/activitypub/follow.ts
index 351499bd1..c1bd667e0 100644
--- a/server/lib/activitypub/follow.ts
+++ b/server/lib/activitypub/follow.ts
@@ -1,12 +1,13 @@
1import { MActorFollowActors } from '../../types/models' 1import { Transaction } from 'sequelize'
2import { getServerActor } from '@server/models/application/application'
3import { logger } from '../../helpers/logger'
2import { CONFIG } from '../../initializers/config' 4import { CONFIG } from '../../initializers/config'
3import { SERVER_ACTOR_NAME } from '../../initializers/constants' 5import { SERVER_ACTOR_NAME } from '../../initializers/constants'
4import { JobQueue } from '../job-queue'
5import { logger } from '../../helpers/logger'
6import { ServerModel } from '../../models/server/server' 6import { ServerModel } from '../../models/server/server'
7import { getServerActor } from '@server/models/application/application' 7import { MActorFollowActors } from '../../types/models'
8import { JobQueue } from '../job-queue'
8 9
9async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors) { 10async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors, transaction?: Transaction) {
10 if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return 11 if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return
11 12
12 const follower = actorFollow.ActorFollower 13 const follower = actorFollow.ActorFollower
@@ -16,7 +17,7 @@ async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors) {
16 17
17 const me = await getServerActor() 18 const me = await getServerActor()
18 19
19 const server = await ServerModel.load(follower.serverId) 20 const server = await ServerModel.load(follower.serverId, transaction)
20 const host = server.host 21 const host = server.host
21 22
22 const payload = { 23 const payload = {
diff --git a/server/lib/activitypub/outbox.ts b/server/lib/activitypub/outbox.ts
new file mode 100644
index 000000000..ecdc33a77
--- /dev/null
+++ b/server/lib/activitypub/outbox.ts
@@ -0,0 +1,24 @@
1import { logger } from '@server/helpers/logger'
2import { ActorModel } from '@server/models/actor/actor'
3import { getServerActor } from '@server/models/application/application'
4import { JobQueue } from '../job-queue'
5
6async function addFetchOutboxJob (actor: Pick<ActorModel, 'id' | 'outboxUrl'>) {
7 // Don't fetch ourselves
8 const serverActor = await getServerActor()
9 if (serverActor.id === actor.id) {
10 logger.error('Cannot fetch our own outbox!')
11 return undefined
12 }
13
14 const payload = {
15 uri: actor.outboxUrl,
16 type: 'activity' as 'activity'
17 }
18
19 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
20}
21
22export {
23 addFetchOutboxJob
24}
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
deleted file mode 100644
index 7166c68a6..000000000
--- a/server/lib/activitypub/playlist.ts
+++ /dev/null
@@ -1,204 +0,0 @@
1import * as Bluebird from 'bluebird'
2import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
3import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
4import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
5import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
6import { checkUrlsSameHost } from '../../helpers/activitypub'
7import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
8import { isArray } from '../../helpers/custom-validators/misc'
9import { logger } from '../../helpers/logger'
10import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
11import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
12import { sequelizeTypescript } from '../../initializers/database'
13import { VideoPlaylistModel } from '../../models/video/video-playlist'
14import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
15import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
16import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
17import { FilteredModelAttributes } from '../../types/sequelize'
18import { createPlaylistMiniatureFromUrl } from '../thumbnail'
19import { getOrCreateActorAndServerAndModel } from './actor'
20import { crawlCollectionPage } from './crawl'
21import { getOrCreateVideoAndAccountAndChannel } from './videos'
22
23function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
24 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
25 ? VideoPlaylistPrivacy.PUBLIC
26 : VideoPlaylistPrivacy.UNLISTED
27
28 return {
29 name: playlistObject.name,
30 description: playlistObject.content,
31 privacy,
32 url: playlistObject.id,
33 uuid: playlistObject.uuid,
34 ownerAccountId: byAccount.id,
35 videoChannelId: null,
36 createdAt: new Date(playlistObject.published),
37 updatedAt: new Date(playlistObject.updated)
38 }
39}
40
41function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
42 return {
43 position: elementObject.position,
44 url: elementObject.id,
45 startTimestamp: elementObject.startTimestamp || null,
46 stopTimestamp: elementObject.stopTimestamp || null,
47 videoPlaylistId: videoPlaylist.id,
48 videoId: video.id
49 }
50}
51
52async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
53 await Bluebird.map(playlistUrls, async playlistUrl => {
54 try {
55 const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
56 if (exists === true) return
57
58 // Fetch url
59 const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
60
61 if (!isPlaylistObjectValid(body)) {
62 throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
63 }
64
65 if (!isArray(body.to)) {
66 throw new Error('Playlist does not have an audience.')
67 }
68
69 return createOrUpdateVideoPlaylist(body, account, body.to)
70 } catch (err) {
71 logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
72 }
73 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
74}
75
76async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
77 const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
78
79 if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
80 const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
81
82 if (actor.VideoChannel) {
83 playlistAttributes.videoChannelId = actor.VideoChannel.id
84 } else {
85 logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
86 }
87 }
88
89 const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
90
91 let accItems: string[] = []
92 await crawlCollectionPage<string>(playlistObject.id, items => {
93 accItems = accItems.concat(items)
94
95 return Promise.resolve()
96 })
97
98 const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
99
100 if (playlistObject.icon) {
101 try {
102 const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
103 await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
104 } catch (err) {
105 logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
106 }
107 } else if (refreshedPlaylist.hasThumbnail()) {
108 await refreshedPlaylist.Thumbnail.destroy()
109 refreshedPlaylist.Thumbnail = null
110 }
111
112 return resetVideoPlaylistElements(accItems, refreshedPlaylist)
113}
114
115async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
116 if (!videoPlaylist.isOutdated()) return videoPlaylist
117
118 try {
119 const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
120
121 if (playlistObject === undefined) {
122 logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
123
124 await videoPlaylist.setAsRefreshed()
125 return videoPlaylist
126 }
127
128 const byAccount = videoPlaylist.OwnerAccount
129 await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
130
131 return videoPlaylist
132 } catch (err) {
133 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
134 logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
135
136 await videoPlaylist.destroy()
137 return undefined
138 }
139
140 logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
141
142 await videoPlaylist.setAsRefreshed()
143 return videoPlaylist
144 }
145}
146
147// ---------------------------------------------------------------------------
148
149export {
150 createAccountPlaylists,
151 playlistObjectToDBAttributes,
152 playlistElementObjectToDBAttributes,
153 createOrUpdateVideoPlaylist,
154 refreshVideoPlaylistIfNeeded
155}
156
157// ---------------------------------------------------------------------------
158
159async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
160 const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
161
162 await Bluebird.map(elementUrls, async elementUrl => {
163 try {
164 const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
165
166 if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
167
168 if (checkUrlsSameHost(body.id, elementUrl) !== true) {
169 throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
170 }
171
172 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
173
174 elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
175 } catch (err) {
176 logger.warn('Cannot add playlist element %s.', elementUrl, { err })
177 }
178 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
179
180 await sequelizeTypescript.transaction(async t => {
181 await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
182
183 for (const element of elementsToCreate) {
184 await VideoPlaylistElementModel.create(element, { transaction: t })
185 }
186 })
187
188 logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
189
190 return undefined
191}
192
193async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
194 logger.info('Fetching remote playlist %s.', playlistUrl)
195
196 const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
197
198 if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
199 logger.debug('Remote video playlist JSON is not valid.', { body })
200 return { statusCode, playlistObject: undefined }
201 }
202
203 return { statusCode, playlistObject: body }
204}
diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts
new file mode 100644
index 000000000..ea3e61ac5
--- /dev/null
+++ b/server/lib/activitypub/playlists/create-update.ts
@@ -0,0 +1,156 @@
1import * as Bluebird from 'bluebird'
2import { getAPId } from '@server/helpers/activitypub'
3import { isArray } from '@server/helpers/custom-validators/misc'
4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
6import { sequelizeTypescript } from '@server/initializers/database'
7import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
8import { VideoPlaylistModel } from '@server/models/video/video-playlist'
9import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
10import { FilteredModelAttributes } from '@server/types'
11import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models'
12import { AttributesOnly } from '@shared/core-utils'
13import { PlaylistObject } from '@shared/models'
14import { getOrCreateAPActor } from '../actors'
15import { crawlCollectionPage } from '../crawl'
16import { getOrCreateAPVideo } from '../videos'
17import {
18 fetchRemotePlaylistElement,
19 fetchRemoteVideoPlaylist,
20 playlistElementObjectToDBAttributes,
21 playlistObjectToDBAttributes
22} from './shared'
23
24const lTags = loggerTagsFactory('ap', 'video-playlist')
25
26async function createAccountPlaylists (playlistUrls: string[]) {
27 await Bluebird.map(playlistUrls, async playlistUrl => {
28 try {
29 const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
30 if (exists === true) return
31
32 const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
33
34 if (playlistObject === undefined) {
35 throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
36 }
37
38 return createOrUpdateVideoPlaylist(playlistObject)
39 } catch (err) {
40 logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
41 }
42 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
43}
44
45async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) {
46 const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to)
47
48 await setVideoChannel(playlistObject, playlistAttributes)
49
50 const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true })
51
52 const playlistElementUrls = await fetchElementUrls(playlistObject)
53
54 // Refetch playlist from DB since elements fetching could be long in time
55 const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
56
57 await updatePlaylistThumbnail(playlistObject, playlist)
58
59 const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist)
60 playlist.setVideosLength(elementsLength)
61
62 return playlist
63}
64
65// ---------------------------------------------------------------------------
66
67export {
68 createAccountPlaylists,
69 createOrUpdateVideoPlaylist
70}
71
72// ---------------------------------------------------------------------------
73
74async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
75 if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) {
76 throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject))
77 }
78
79 const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all')
80
81 if (!actor.VideoChannel) {
82 logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
83 return
84 }
85
86 playlistAttributes.videoChannelId = actor.VideoChannel.id
87 playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id
88}
89
90async function fetchElementUrls (playlistObject: PlaylistObject) {
91 let accItems: string[] = []
92 await crawlCollectionPage<string>(playlistObject.id, items => {
93 accItems = accItems.concat(items)
94
95 return Promise.resolve()
96 })
97
98 return accItems
99}
100
101async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
102 if (playlistObject.icon) {
103 let thumbnailModel: MThumbnail
104
105 try {
106 thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
107 await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
108 } catch (err) {
109 logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
110
111 if (thumbnailModel) await thumbnailModel.removeThumbnail()
112 }
113
114 return
115 }
116
117 // Playlist does not have an icon, destroy existing one
118 if (playlist.hasThumbnail()) {
119 await playlist.Thumbnail.destroy()
120 playlist.Thumbnail = null
121 }
122}
123
124async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
125 const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
126
127 await sequelizeTypescript.transaction(async t => {
128 await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
129
130 for (const element of elementsToCreate) {
131 await VideoPlaylistElementModel.create(element, { transaction: t })
132 }
133 })
134
135 logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
136
137 return elementsToCreate.length
138}
139
140async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
141 const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
142
143 await Bluebird.map(elementUrls, async elementUrl => {
144 try {
145 const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
146
147 const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
148
149 elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
150 } catch (err) {
151 logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
152 }
153 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
154
155 return elementsToCreate
156}
diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts
new file mode 100644
index 000000000..2c19c503a
--- /dev/null
+++ b/server/lib/activitypub/playlists/get.ts
@@ -0,0 +1,35 @@
1import { getAPId } from '@server/helpers/activitypub'
2import { VideoPlaylistModel } from '@server/models/video/video-playlist'
3import { MVideoPlaylistFullSummary } from '@server/types/models'
4import { APObject } from '@shared/models'
5import { createOrUpdateVideoPlaylist } from './create-update'
6import { scheduleRefreshIfNeeded } from './refresh'
7import { fetchRemoteVideoPlaylist } from './shared'
8
9async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObject): Promise<MVideoPlaylistFullSummary> {
10 const playlistUrl = getAPId(playlistObjectArg)
11
12 const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl)
13
14 if (playlistFromDatabase) {
15 scheduleRefreshIfNeeded(playlistFromDatabase)
16
17 return playlistFromDatabase
18 }
19
20 const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
21 if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl)
22
23 // playlistUrl is just an alias/rediraction, so process object id instead
24 if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject)
25
26 const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject)
27
28 return playlistCreated
29}
30
31// ---------------------------------------------------------------------------
32
33export {
34 getOrCreateAPVideoPlaylist
35}
diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts
new file mode 100644
index 000000000..e2470a674
--- /dev/null
+++ b/server/lib/activitypub/playlists/index.ts
@@ -0,0 +1,3 @@
1export * from './get'
2export * from './create-update'
3export * from './refresh'
diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts
new file mode 100644
index 000000000..ef3cb3fe4
--- /dev/null
+++ b/server/lib/activitypub/playlists/refresh.ts
@@ -0,0 +1,53 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { PeerTubeRequestError } from '@server/helpers/requests'
3import { JobQueue } from '@server/lib/job-queue'
4import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models'
5import { HttpStatusCode } from '@shared/core-utils'
6import { createOrUpdateVideoPlaylist } from './create-update'
7import { fetchRemoteVideoPlaylist } from './shared'
8
9function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) {
10 if (!playlist.isOutdated()) return
11
12 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } })
13}
14
15async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
16 if (!videoPlaylist.isOutdated()) return videoPlaylist
17
18 const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url)
19
20 logger.info('Refreshing playlist %s.', videoPlaylist.url, lTags())
21
22 try {
23 const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
24
25 if (playlistObject === undefined) {
26 logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags())
27
28 await videoPlaylist.setAsRefreshed()
29 return videoPlaylist
30 }
31
32 await createOrUpdateVideoPlaylist(playlistObject)
33
34 return videoPlaylist
35 } catch (err) {
36 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
37 logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags())
38
39 await videoPlaylist.destroy()
40 return undefined
41 }
42
43 logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() })
44
45 await videoPlaylist.setAsRefreshed()
46 return videoPlaylist
47 }
48}
49
50export {
51 scheduleRefreshIfNeeded,
52 refreshVideoPlaylistIfNeeded
53}
diff --git a/server/lib/activitypub/playlists/shared/index.ts b/server/lib/activitypub/playlists/shared/index.ts
new file mode 100644
index 000000000..a217f2291
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/index.ts
@@ -0,0 +1,2 @@
1export * from './object-to-model-attributes'
2export * from './url-to-object'
diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
new file mode 100644
index 000000000..70fd335bc
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
@@ -0,0 +1,40 @@
1import { ACTIVITY_PUB } from '@server/initializers/constants'
2import { VideoPlaylistModel } from '@server/models/video/video-playlist'
3import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
4import { MVideoId, MVideoPlaylistId } from '@server/types/models'
5import { AttributesOnly } from '@shared/core-utils'
6import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models'
7
8function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) {
9 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
10 ? VideoPlaylistPrivacy.PUBLIC
11 : VideoPlaylistPrivacy.UNLISTED
12
13 return {
14 name: playlistObject.name,
15 description: playlistObject.content,
16 privacy,
17 url: playlistObject.id,
18 uuid: playlistObject.uuid,
19 ownerAccountId: null,
20 videoChannelId: null,
21 createdAt: new Date(playlistObject.published),
22 updatedAt: new Date(playlistObject.updated)
23 } as AttributesOnly<VideoPlaylistModel>
24}
25
26function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
27 return {
28 position: elementObject.position,
29 url: elementObject.id,
30 startTimestamp: elementObject.startTimestamp || null,
31 stopTimestamp: elementObject.stopTimestamp || null,
32 videoPlaylistId: videoPlaylist.id,
33 videoId: video.id
34 } as AttributesOnly<VideoPlaylistElementModel>
35}
36
37export {
38 playlistObjectToDBAttributes,
39 playlistElementObjectToDBAttributes
40}
diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts
new file mode 100644
index 000000000..ec8c01255
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/url-to-object.ts
@@ -0,0 +1,47 @@
1import { isArray } from 'lodash'
2import { checkUrlsSameHost } from '@server/helpers/activitypub'
3import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist'
4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { doJSONRequest } from '@server/helpers/requests'
6import { PlaylistElementObject, PlaylistObject } from '@shared/models'
7
8async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
9 const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl)
10
11 logger.info('Fetching remote playlist %s.', playlistUrl, lTags())
12
13 const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
14
15 if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
16 logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() })
17 return { statusCode, playlistObject: undefined }
18 }
19
20 if (!isArray(body.to)) {
21 logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() })
22 return { statusCode, playlistObject: undefined }
23 }
24
25 return { statusCode, playlistObject: body }
26}
27
28async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> {
29 const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl)
30
31 logger.debug('Fetching remote playlist element %s.', elementUrl, lTags())
32
33 const { body, statusCode } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
34
35 if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`)
36
37 if (checkUrlsSameHost(body.id, elementUrl) !== true) {
38 throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
39 }
40
41 return { statusCode, elementObject: body }
42}
43
44export {
45 fetchRemoteVideoPlaylist,
46 fetchRemotePlaylistElement
47}
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts
index 1799829f8..077b01eda 100644
--- a/server/lib/activitypub/process/process-accept.ts
+++ b/server/lib/activitypub/process/process-accept.ts
@@ -1,8 +1,8 @@
1import { ActivityAccept } from '../../../../shared/models/activitypub' 1import { ActivityAccept } from '../../../../shared/models/activitypub'
2import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 2import { ActorFollowModel } from '../../../models/actor/actor-follow'
3import { addFetchOutboxJob } from '../actor'
4import { APProcessorOptions } from '../../../types/activitypub-processor.model' 3import { APProcessorOptions } from '../../../types/activitypub-processor.model'
5import { MActorDefault, MActorSignature } from '../../../types/models' 4import { MActorDefault, MActorSignature } from '../../../types/models'
5import { addFetchOutboxJob } from '../outbox'
6 6
7async function processAcceptActivity (options: APProcessorOptions<ActivityAccept>) { 7async function processAcceptActivity (options: APProcessorOptions<ActivityAccept>) {
8 const { byActor: targetActor, inboxActor } = options 8 const { byActor: targetActor, inboxActor } = options
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index 63082466e..ec23c705e 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -3,7 +3,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { sequelizeTypescript } from '../../../initializers/database' 3import { sequelizeTypescript } from '../../../initializers/database'
4import { VideoShareModel } from '../../../models/video/video-share' 4import { VideoShareModel } from '../../../models/video/video-share'
5import { forwardVideoRelatedActivity } from '../send/utils' 5import { forwardVideoRelatedActivity } from '../send/utils'
6import { getOrCreateVideoAndAccountAndChannel } from '../videos' 6import { getOrCreateAPVideo } from '../videos'
7import { Notifier } from '../../notifier' 7import { Notifier } from '../../notifier'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9import { APProcessorOptions } from '../../../types/activitypub-processor.model' 9import { APProcessorOptions } from '../../../types/activitypub-processor.model'
@@ -32,7 +32,7 @@ async function processVideoShare (actorAnnouncer: MActorSignature, activity: Act
32 let videoCreated: boolean 32 let videoCreated: boolean
33 33
34 try { 34 try {
35 const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) 35 const result = await getOrCreateAPVideo({ videoObject: objectUri })
36 video = result.video 36 video = result.video
37 videoCreated = result.created 37 videoCreated = result.created
38 } catch (err) { 38 } catch (err) {
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 9cded4dec..70e048d6e 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -1,3 +1,4 @@
1import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
1import { isRedundancyAccepted } from '@server/lib/redundancy' 2import { isRedundancyAccepted } from '@server/lib/redundancy'
2import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared' 3import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared'
3import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' 4import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
@@ -9,11 +10,10 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model'
9import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' 10import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models'
10import { Notifier } from '../../notifier' 11import { Notifier } from '../../notifier'
11import { createOrUpdateCacheFile } from '../cache-file' 12import { createOrUpdateCacheFile } from '../cache-file'
12import { createOrUpdateVideoPlaylist } from '../playlist' 13import { createOrUpdateVideoPlaylist } from '../playlists'
13import { forwardVideoRelatedActivity } from '../send/utils' 14import { forwardVideoRelatedActivity } from '../send/utils'
14import { resolveThread } from '../video-comments' 15import { resolveThread } from '../video-comments'
15import { getOrCreateVideoAndAccountAndChannel } from '../videos' 16import { getOrCreateAPVideo } from '../videos'
16import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
17 17
18async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { 18async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
19 const { activity, byActor } = options 19 const { activity, byActor } = options
@@ -55,7 +55,7 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) {
55 const videoToCreateData = activity.object as VideoObject 55 const videoToCreateData = activity.object as VideoObject
56 56
57 const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } 57 const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false }
58 const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData, syncParam }) 58 const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam })
59 59
60 if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) 60 if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video)
61 61
@@ -67,7 +67,7 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor
67 67
68 const cacheFile = activity.object as CacheFileObject 68 const cacheFile = activity.object as CacheFileObject
69 69
70 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) 70 const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object })
71 71
72 await sequelizeTypescript.transaction(async t => { 72 await sequelizeTypescript.transaction(async t => {
73 return createOrUpdateCacheFile(cacheFile, video, byActor, t) 73 return createOrUpdateCacheFile(cacheFile, video, byActor, t)
@@ -128,5 +128,5 @@ async function processCreatePlaylist (activity: ActivityCreate, byActor: MActorS
128 128
129 if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) 129 if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url)
130 130
131 await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) 131 await createOrUpdateVideoPlaylist(playlistObject, activity.to)
132} 132}
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts
index 88a968318..1d2279df5 100644
--- a/server/lib/activitypub/process/process-delete.ts
+++ b/server/lib/activitypub/process/process-delete.ts
@@ -2,7 +2,7 @@ import { ActivityDelete } from '../../../../shared/models/activitypub'
2import { retryTransactionWrapper } from '../../../helpers/database-utils' 2import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { sequelizeTypescript } from '../../../initializers/database' 4import { sequelizeTypescript } from '../../../initializers/database'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/actor/actor'
6import { VideoModel } from '../../../models/video/video' 6import { VideoModel } from '../../../models/video/video'
7import { VideoCommentModel } from '../../../models/video/video-comment' 7import { VideoCommentModel } from '../../../models/video/video-comment'
8import { VideoPlaylistModel } from '../../../models/video/video-playlist' 8import { VideoPlaylistModel } from '../../../models/video/video-playlist'
@@ -16,7 +16,6 @@ import {
16 MChannelActor, 16 MChannelActor,
17 MCommentOwnerVideo 17 MCommentOwnerVideo
18} from '../../../types/models' 18} from '../../../types/models'
19import { markCommentAsDeleted } from '../../video-comment'
20import { forwardVideoRelatedActivity } from '../send/utils' 19import { forwardVideoRelatedActivity } from '../send/utils'
21 20
22async function processDeleteActivity (options: APProcessorOptions<ActivityDelete>) { 21async function processDeleteActivity (options: APProcessorOptions<ActivityDelete>) {
@@ -130,7 +129,7 @@ async function processDeleteVideoChannel (videoChannelToRemove: MChannelActor) {
130 129
131function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) { 130function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) {
132 // Already deleted 131 // Already deleted
133 if (videoComment.isDeleted()) return 132 if (videoComment.isDeleted()) return Promise.resolve()
134 133
135 logger.debug('Removing remote video comment "%s".', videoComment.url) 134 logger.debug('Removing remote video comment "%s".', videoComment.url)
136 135
@@ -139,11 +138,9 @@ function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCom
139 throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`) 138 throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`)
140 } 139 }
141 140
142 await sequelizeTypescript.transaction(async t => { 141 videoComment.markAsDeleted()
143 markCommentAsDeleted(videoComment)
144 142
145 await videoComment.save() 143 await videoComment.save({ transaction: t })
146 })
147 144
148 if (videoComment.Video.isOwned()) { 145 if (videoComment.Video.isOwned()) {
149 // Don't resend the activity to the sender 146 // Don't resend the activity to the sender
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts
index 089c7b881..ecc57cd10 100644
--- a/server/lib/activitypub/process/process-dislike.ts
+++ b/server/lib/activitypub/process/process-dislike.ts
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
6import { APProcessorOptions } from '../../../types/activitypub-processor.model' 6import { APProcessorOptions } from '../../../types/activitypub-processor.model'
7import { MActorSignature } from '../../../types/models' 7import { MActorSignature } from '../../../types/models'
8import { forwardVideoRelatedActivity } from '../send/utils' 8import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateVideoAndAccountAndChannel } from '../videos' 9import { getOrCreateAPVideo } from '../videos'
10 10
11async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) { 11async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) {
12 const { activity, byActor } = options 12 const { activity, byActor } = options
@@ -30,7 +30,7 @@ async function processDislike (activity: ActivityCreate | ActivityDislike, byAct
30 30
31 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) 31 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
32 32
33 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) 33 const { video } = await getOrCreateAPVideo({ videoObject: dislikeObject })
34 34
35 return sequelizeTypescript.transaction(async t => { 35 return sequelizeTypescript.transaction(async t => {
36 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) 36 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t)
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts
index 38d684512..f85238f8e 100644
--- a/server/lib/activitypub/process/process-follow.ts
+++ b/server/lib/activitypub/process/process-follow.ts
@@ -1,17 +1,17 @@
1import { getServerActor } from '@server/models/application/application'
1import { ActivityFollow } from '../../../../shared/models/activitypub' 2import { ActivityFollow } from '../../../../shared/models/activitypub'
3import { getAPId } from '../../../helpers/activitypub'
2import { retryTransactionWrapper } from '../../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { logger } from '../../../helpers/logger' 5import { logger } from '../../../helpers/logger'
4import { sequelizeTypescript } from '../../../initializers/database'
5import { ActorModel } from '../../../models/activitypub/actor'
6import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
7import { sendAccept, sendReject } from '../send'
8import { Notifier } from '../../notifier'
9import { getAPId } from '../../../helpers/activitypub'
10import { CONFIG } from '../../../initializers/config' 6import { CONFIG } from '../../../initializers/config'
7import { sequelizeTypescript } from '../../../initializers/database'
8import { ActorModel } from '../../../models/actor/actor'
9import { ActorFollowModel } from '../../../models/actor/actor-follow'
11import { APProcessorOptions } from '../../../types/activitypub-processor.model' 10import { APProcessorOptions } from '../../../types/activitypub-processor.model'
12import { MActorFollowActors, MActorSignature } from '../../../types/models' 11import { MActorFollowActors, MActorSignature } from '../../../types/models'
12import { Notifier } from '../../notifier'
13import { autoFollowBackIfNeeded } from '../follow' 13import { autoFollowBackIfNeeded } from '../follow'
14import { getServerActor } from '@server/models/application/application' 14import { sendAccept, sendReject } from '../send'
15 15
16async function processFollowActivity (options: APProcessorOptions<ActivityFollow>) { 16async function processFollowActivity (options: APProcessorOptions<ActivityFollow>) {
17 const { activity, byActor } = options 17 const { activity, byActor } = options
@@ -43,7 +43,7 @@ async function processFollow (byActor: MActorSignature, activityId: string, targ
43 if (isFollowingInstance && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { 43 if (isFollowingInstance && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) {
44 logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) 44 logger.info('Rejecting %s because instance followers are disabled.', targetActor.url)
45 45
46 await sendReject(activityId, byActor, targetActor) 46 sendReject(activityId, byActor, targetActor)
47 47
48 return { actorFollow: undefined as MActorFollowActors } 48 return { actorFollow: undefined as MActorFollowActors }
49 } 49 }
@@ -84,8 +84,9 @@ async function processFollow (byActor: MActorSignature, activityId: string, targ
84 84
85 // Target sends to actor he accepted the follow request 85 // Target sends to actor he accepted the follow request
86 if (actorFollow.state === 'accepted') { 86 if (actorFollow.state === 'accepted') {
87 await sendAccept(actorFollow) 87 sendAccept(actorFollow)
88 await autoFollowBackIfNeeded(actorFollow) 88
89 await autoFollowBackIfNeeded(actorFollow, t)
89 } 90 }
90 91
91 return { actorFollow, created, isFollowingInstance, targetActor } 92 return { actorFollow, created, isFollowingInstance, targetActor }
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index 8688b3b47..cd4e86cbb 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
6import { APProcessorOptions } from '../../../types/activitypub-processor.model' 6import { APProcessorOptions } from '../../../types/activitypub-processor.model'
7import { MActorSignature } from '../../../types/models' 7import { MActorSignature } from '../../../types/models'
8import { forwardVideoRelatedActivity } from '../send/utils' 8import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateVideoAndAccountAndChannel } from '../videos' 9import { getOrCreateAPVideo } from '../videos'
10 10
11async function processLikeActivity (options: APProcessorOptions<ActivityLike>) { 11async function processLikeActivity (options: APProcessorOptions<ActivityLike>) {
12 const { activity, byActor } = options 12 const { activity, byActor } = options
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: MActorSignature, activity: ActivityLik
27 const byAccount = byActor.Account 27 const byAccount = byActor.Account
28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
29 29
30 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) 30 const { video } = await getOrCreateAPVideo({ videoObject: videoUrl })
31 31
32 return sequelizeTypescript.transaction(async t => { 32 return sequelizeTypescript.transaction(async t => {
33 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) 33 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t)
diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts
index 03b669fd9..7f7ab305f 100644
--- a/server/lib/activitypub/process/process-reject.ts
+++ b/server/lib/activitypub/process/process-reject.ts
@@ -1,6 +1,6 @@
1import { ActivityReject } from '../../../../shared/models/activitypub/activity' 1import { ActivityReject } from '../../../../shared/models/activitypub/activity'
2import { sequelizeTypescript } from '../../../initializers/database' 2import { sequelizeTypescript } from '../../../initializers/database'
3import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../../models/actor/actor-follow'
4import { APProcessorOptions } from '../../../types/activitypub-processor.model' 4import { APProcessorOptions } from '../../../types/activitypub-processor.model'
5import { MActor } from '../../../types/models' 5import { MActor } from '../../../types/models'
6 6
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index e520c2f0d..d4b2a795f 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -4,14 +4,14 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { sequelizeTypescript } from '../../../initializers/database' 5import { sequelizeTypescript } from '../../../initializers/database'
6import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 6import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
7import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/actor/actor'
8import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 8import { ActorFollowModel } from '../../../models/actor/actor-follow'
9import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 9import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
10import { VideoShareModel } from '../../../models/video/video-share' 10import { VideoShareModel } from '../../../models/video/video-share'
11import { APProcessorOptions } from '../../../types/activitypub-processor.model' 11import { APProcessorOptions } from '../../../types/activitypub-processor.model'
12import { MActorSignature } from '../../../types/models' 12import { MActorSignature } from '../../../types/models'
13import { forwardVideoRelatedActivity } from '../send/utils' 13import { forwardVideoRelatedActivity } from '../send/utils'
14import { getOrCreateVideoAndAccountAndChannel } from '../videos' 14import { getOrCreateAPVideo } from '../videos'
15 15
16async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { 16async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) {
17 const { activity, byActor } = options 17 const { activity, byActor } = options
@@ -55,7 +55,7 @@ export {
55async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { 55async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) {
56 const likeActivity = activity.object as ActivityLike 56 const likeActivity = activity.object as ActivityLike
57 57
58 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) 58 const { video } = await getOrCreateAPVideo({ videoObject: likeActivity.object })
59 59
60 return sequelizeTypescript.transaction(async t => { 60 return sequelizeTypescript.transaction(async t => {
61 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 61 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
@@ -80,7 +80,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU
80 ? activity.object 80 ? activity.object
81 : activity.object.object as DislikeObject 81 : activity.object.object as DislikeObject
82 82
83 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) 83 const { video } = await getOrCreateAPVideo({ videoObject: dislike.object })
84 84
85 return sequelizeTypescript.transaction(async t => { 85 return sequelizeTypescript.transaction(async t => {
86 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 86 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
@@ -103,10 +103,10 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU
103async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { 103async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) {
104 const cacheFileObject = activity.object.object as CacheFileObject 104 const cacheFileObject = activity.object.object as CacheFileObject
105 105
106 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) 106 const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object })
107 107
108 return sequelizeTypescript.transaction(async t => { 108 return sequelizeTypescript.transaction(async t => {
109 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 109 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
110 if (!cacheFile) { 110 if (!cacheFile) {
111 logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) 111 logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id)
112 return 112 return
@@ -114,7 +114,7 @@ async function processUndoCacheFile (byActor: MActorSignature, activity: Activit
114 114
115 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') 115 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.')
116 116
117 await cacheFile.destroy() 117 await cacheFile.destroy({ transaction: t })
118 118
119 if (video.isOwned()) { 119 if (video.isOwned()) {
120 // Don't resend the activity to the sender 120 // Don't resend the activity to the sender
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index 6df9b93b2..f40008a6b 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -1,23 +1,20 @@
1import { isRedundancyAccepted } from '@server/lib/redundancy'
1import { ActivityUpdate, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' 2import { ActivityUpdate, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub'
2import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' 3import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
3import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' 4import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
5import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
6import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
7import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
5import { sequelizeTypescript } from '../../../initializers/database' 9import { sequelizeTypescript } from '../../../initializers/database'
6import { AccountModel } from '../../../models/account/account' 10import { ActorModel } from '../../../models/actor/actor'
7import { ActorModel } from '../../../models/activitypub/actor' 11import { APProcessorOptions } from '../../../types/activitypub-processor.model'
8import { VideoChannelModel } from '../../../models/video/video-channel' 12import { MActorFull, MActorSignature } from '../../../types/models'
9import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } from '../actor' 13import { APActorUpdater } from '../actors/updater'
10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
13import { createOrUpdateCacheFile } from '../cache-file' 14import { createOrUpdateCacheFile } from '../cache-file'
15import { createOrUpdateVideoPlaylist } from '../playlists'
14import { forwardVideoRelatedActivity } from '../send/utils' 16import { forwardVideoRelatedActivity } from '../send/utils'
15import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' 17import { APVideoUpdater, getOrCreateAPVideo } from '../videos'
16import { createOrUpdateVideoPlaylist } from '../playlist'
17import { APProcessorOptions } from '../../../types/activitypub-processor.model'
18import { MActorSignature, MAccountIdActor } from '../../../types/models'
19import { isRedundancyAccepted } from '@server/lib/redundancy'
20import { ActorImageType } from '@shared/models'
21 18
22async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { 19async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) {
23 const { activity, byActor } = options 20 const { activity, byActor } = options
@@ -25,7 +22,7 @@ async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate
25 const objectType = activity.object.type 22 const objectType = activity.object.type
26 23
27 if (objectType === 'Video') { 24 if (objectType === 'Video') {
28 return retryTransactionWrapper(processUpdateVideo, byActor, activity) 25 return retryTransactionWrapper(processUpdateVideo, activity)
29 } 26 }
30 27
31 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { 28 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
@@ -55,7 +52,7 @@ export {
55 52
56// --------------------------------------------------------------------------- 53// ---------------------------------------------------------------------------
57 54
58async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpdate) { 55async function processUpdateVideo (activity: ActivityUpdate) {
59 const videoObject = activity.object as VideoObject 56 const videoObject = activity.object as VideoObject
60 57
61 if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { 58 if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) {
@@ -63,7 +60,7 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd
63 return undefined 60 return undefined
64 } 61 }
65 62
66 const { video, created } = await getOrCreateVideoAndAccountAndChannel({ 63 const { video, created } = await getOrCreateAPVideo({
67 videoObject: videoObject.id, 64 videoObject: videoObject.id,
68 allowRefresh: false, 65 allowRefresh: false,
69 fetchType: 'all' 66 fetchType: 'all'
@@ -71,20 +68,8 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd
71 // We did not have this video, it has been created so no need to update 68 // We did not have this video, it has been created so no need to update
72 if (created) return 69 if (created) return
73 70
74 // Load new channel 71 const updater = new APVideoUpdater(videoObject, video)
75 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 72 return updater.update(activity.to)
76
77 const account = actor.Account as MAccountIdActor
78 account.Actor = actor
79
80 const updateOptions = {
81 video,
82 videoObject,
83 account,
84 channel: channelActor.VideoChannel,
85 overrideTo: activity.to
86 }
87 return updateVideoFromAP(updateOptions)
88} 73}
89 74
90async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) { 75async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) {
@@ -97,7 +82,7 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ
97 return undefined 82 return undefined
98 } 83 }
99 84
100 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) 85 const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object })
101 86
102 await sequelizeTypescript.transaction(async t => { 87 await sequelizeTypescript.transaction(async t => {
103 await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) 88 await createOrUpdateCacheFile(cacheFileObject, video, byActor, t)
@@ -111,56 +96,13 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ
111 } 96 }
112} 97}
113 98
114async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { 99async function processUpdateActor (actor: MActorFull, activity: ActivityUpdate) {
115 const actorAttributesToUpdate = activity.object as ActivityPubActor 100 const actorObject = activity.object as ActivityPubActor
116 101
117 logger.debug('Updating remote account "%s".', actorAttributesToUpdate.url) 102 logger.debug('Updating remote account "%s".', actorObject.url)
118 let accountOrChannelInstance: AccountModel | VideoChannelModel
119 let actorFieldsSave: object
120 let accountOrChannelFieldsSave: object
121 103
122 // Fetch icon? 104 const updater = new APActorUpdater(actorObject, actor)
123 const avatarInfo = getImageInfoIfExists(actorAttributesToUpdate, ActorImageType.AVATAR) 105 return updater.update()
124 const bannerInfo = getImageInfoIfExists(actorAttributesToUpdate, ActorImageType.BANNER)
125
126 try {
127 await sequelizeTypescript.transaction(async t => {
128 actorFieldsSave = actor.toJSON()
129
130 if (actorAttributesToUpdate.type === 'Group') accountOrChannelInstance = actor.VideoChannel
131 else accountOrChannelInstance = actor.Account
132
133 accountOrChannelFieldsSave = accountOrChannelInstance.toJSON()
134
135 await updateActorInstance(actor, actorAttributesToUpdate)
136
137 await updateActorImageInstance(actor, ActorImageType.AVATAR, avatarInfo, t)
138 await updateActorImageInstance(actor, ActorImageType.BANNER, bannerInfo, t)
139
140 await actor.save({ transaction: t })
141
142 accountOrChannelInstance.name = actorAttributesToUpdate.name || actorAttributesToUpdate.preferredUsername
143 accountOrChannelInstance.description = actorAttributesToUpdate.summary
144
145 if (accountOrChannelInstance instanceof VideoChannelModel) accountOrChannelInstance.support = actorAttributesToUpdate.support
146
147 await accountOrChannelInstance.save({ transaction: t })
148 })
149
150 logger.info('Remote account %s updated', actorAttributesToUpdate.url)
151 } catch (err) {
152 if (actor !== undefined && actorFieldsSave !== undefined) {
153 resetSequelizeInstance(actor, actorFieldsSave)
154 }
155
156 if (accountOrChannelInstance !== undefined && accountOrChannelFieldsSave !== undefined) {
157 resetSequelizeInstance(accountOrChannelInstance, accountOrChannelFieldsSave)
158 }
159
160 // This is just a debug because we will retry the insert
161 logger.debug('Cannot update the remote account.', { err })
162 throw err
163 }
164} 106}
165 107
166async function processUpdatePlaylist (byActor: MActorSignature, activity: ActivityUpdate) { 108async function processUpdatePlaylist (byActor: MActorSignature, activity: ActivityUpdate) {
@@ -169,5 +111,5 @@ async function processUpdatePlaylist (byActor: MActorSignature, activity: Activi
169 111
170 if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) 112 if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url)
171 113
172 await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) 114 await createOrUpdateVideoPlaylist(playlistObject, activity.to)
173} 115}
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
index 84697673b..5593ee257 100644
--- a/server/lib/activitypub/process/process-view.ts
+++ b/server/lib/activitypub/process/process-view.ts
@@ -1,10 +1,10 @@
1import { getOrCreateVideoAndAccountAndChannel } from '../videos' 1import { getOrCreateAPVideo } from '../videos'
2import { forwardVideoRelatedActivity } from '../send/utils' 2import { forwardVideoRelatedActivity } from '../send/utils'
3import { Redis } from '../../redis' 3import { Redis } from '../../redis'
4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' 4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub'
5import { APProcessorOptions } from '../../../types/activitypub-processor.model' 5import { APProcessorOptions } from '../../../types/activitypub-processor.model'
6import { MActorSignature } from '../../../types/models' 6import { MActorSignature } from '../../../types/models'
7import { LiveManager } from '@server/lib/live-manager' 7import { LiveManager } from '@server/lib/live/live-manager'
8 8
9async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) { 9async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) {
10 const { activity, byActor } = options 10 const { activity, byActor } = options
@@ -24,12 +24,11 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct
24 ? activity.object 24 ? activity.object
25 : (activity.object as ViewObject).object 25 : (activity.object as ViewObject).object
26 26
27 const options = { 27 const { video } = await getOrCreateAPVideo({
28 videoObject, 28 videoObject,
29 fetchType: 'only-video' as 'only-video', 29 fetchType: 'only-video',
30 allowRefresh: false as false 30 allowRefresh: false
31 } 31 })
32 const { video } = await getOrCreateVideoAndAccountAndChannel(options)
33 32
34 if (!video.isLive) { 33 if (!video.isLive) {
35 await Redis.Instance.addVideoView(video.id) 34 await Redis.Instance.addVideoView(video.id)
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts
index 5cef75665..02a23d098 100644
--- a/server/lib/activitypub/process/process.ts
+++ b/server/lib/activitypub/process/process.ts
@@ -1,22 +1,22 @@
1import { StatsManager } from '@server/lib/stat-manager'
1import { Activity, ActivityType } from '../../../../shared/models/activitypub' 2import { Activity, ActivityType } from '../../../../shared/models/activitypub'
2import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' 3import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub'
3import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { APProcessorOptions } from '../../../types/activitypub-processor.model'
6import { MActorDefault, MActorSignature } from '../../../types/models'
7import { getOrCreateAPActor } from '../actors'
4import { processAcceptActivity } from './process-accept' 8import { processAcceptActivity } from './process-accept'
5import { processAnnounceActivity } from './process-announce' 9import { processAnnounceActivity } from './process-announce'
6import { processCreateActivity } from './process-create' 10import { processCreateActivity } from './process-create'
7import { processDeleteActivity } from './process-delete' 11import { processDeleteActivity } from './process-delete'
12import { processDislikeActivity } from './process-dislike'
13import { processFlagActivity } from './process-flag'
8import { processFollowActivity } from './process-follow' 14import { processFollowActivity } from './process-follow'
9import { processLikeActivity } from './process-like' 15import { processLikeActivity } from './process-like'
10import { processRejectActivity } from './process-reject' 16import { processRejectActivity } from './process-reject'
11import { processUndoActivity } from './process-undo' 17import { processUndoActivity } from './process-undo'
12import { processUpdateActivity } from './process-update' 18import { processUpdateActivity } from './process-update'
13import { getOrCreateActorAndServerAndModel } from '../actor'
14import { processDislikeActivity } from './process-dislike'
15import { processFlagActivity } from './process-flag'
16import { processViewActivity } from './process-view' 19import { processViewActivity } from './process-view'
17import { APProcessorOptions } from '../../../types/activitypub-processor.model'
18import { MActorDefault, MActorSignature } from '../../../types/models'
19import { StatsManager } from '@server/lib/stat-manager'
20 20
21const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions<Activity>) => Promise<any> } = { 21const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions<Activity>) => Promise<any> } = {
22 Create: processCreateActivity, 22 Create: processCreateActivity,
@@ -65,7 +65,7 @@ async function processActivities (
65 continue 65 continue
66 } 66 }
67 67
68 const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) 68 const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateAPActor(actorUrl)
69 actorsCache[actorUrl] = byActor 69 actorsCache[actorUrl] = byActor
70 70
71 const activityProcessor = processActivity[activity.type] 71 const activityProcessor = processActivity[activity.type]
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index e0acced18..d31f8c10b 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize'
2import { getServerActor } from '@server/models/application/application' 2import { getServerActor } from '@server/models/application/application'
3import { ActivityAudience, ActivityDelete } from '../../../../shared/models/activitypub' 3import { ActivityAudience, ActivityDelete } from '../../../../shared/models/activitypub'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/actor/actor'
6import { VideoCommentModel } from '../../../models/video/video-comment' 6import { VideoCommentModel } from '../../../models/video/video-comment'
7import { VideoShareModel } from '../../../models/video/video-share' 7import { VideoShareModel } from '../../../models/video/video-share'
8import { MActorUrl } from '../../../types/models' 8import { MActorUrl } from '../../../types/models'
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts
index 9254dc7c5..153e94295 100644
--- a/server/lib/activitypub/send/send-view.ts
+++ b/server/lib/activitypub/send/send-view.ts
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize'
2import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models' 2import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models'
3import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' 3import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/actor/actor'
6import { audiencify, getAudience } from '../audience' 6import { audiencify, getAudience } from '../audience'
7import { getLocalVideoViewActivityPubUrl } from '../url' 7import { getLocalVideoViewActivityPubUrl } from '../url'
8import { sendVideoRelatedActivity } from './utils' 8import { sendVideoRelatedActivity } from './utils'
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts
index 85a9f009d..7cd8030e1 100644
--- a/server/lib/activitypub/send/utils.ts
+++ b/server/lib/activitypub/send/utils.ts
@@ -1,14 +1,14 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { getServerActor } from '@server/models/application/application'
3import { ContextType } from '@shared/models/activitypub/context'
2import { Activity, ActivityAudience } from '../../../../shared/models/activitypub' 4import { Activity, ActivityAudience } from '../../../../shared/models/activitypub'
5import { afterCommitIfTransaction } from '../../../helpers/database-utils'
3import { logger } from '../../../helpers/logger' 6import { logger } from '../../../helpers/logger'
4import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/actor/actor'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 8import { ActorFollowModel } from '../../../models/actor/actor-follow'
9import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../types/models'
6import { JobQueue } from '../../job-queue' 10import { JobQueue } from '../../job-queue'
7import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience' 11import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
8import { afterCommitIfTransaction } from '../../../helpers/database-utils'
9import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../types/models'
10import { getServerActor } from '@server/models/application/application'
11import { ContextType } from '@shared/models/activitypub/context'
12 12
13async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { 13async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
14 byActor: MActorLight 14 byActor: MActorLight
@@ -22,7 +22,9 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
22 22
23 // Send to origin 23 // Send to origin
24 if (video.isOwned() === false) { 24 if (video.isOwned() === false) {
25 const accountActor = (video as MVideoAccountLight).VideoChannel?.Account?.Actor || await ActorModel.loadAccountActorByVideoId(video.id) 25 let accountActor: MActorLight = (video as MVideoAccountLight).VideoChannel?.Account?.Actor
26
27 if (!accountActor) accountActor = await ActorModel.loadAccountActorByVideoId(video.id, transaction)
26 28
27 const audience = getRemoteVideoAudience(accountActor, actorsInvolvedInVideo) 29 const audience = getRemoteVideoAudience(accountActor, actorsInvolvedInVideo)
28 const activity = activityBuilder(audience) 30 const activity = activityBuilder(audience)
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index c22fa0893..1ff01a175 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -7,7 +7,7 @@ import { doJSONRequest } from '../../helpers/requests'
7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
8import { VideoShareModel } from '../../models/video/video-share' 8import { VideoShareModel } from '../../models/video/video-share'
9import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' 9import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video'
10import { getOrCreateActorAndServerAndModel } from './actor' 10import { getOrCreateAPActor } from './actors'
11import { sendUndoAnnounce, sendVideoAnnounce } from './send' 11import { sendUndoAnnounce, sendVideoAnnounce } from './send'
12import { getLocalVideoAnnounceActivityPubUrl } from './url' 12import { getLocalVideoAnnounceActivityPubUrl } from './url'
13 13
@@ -40,23 +40,7 @@ async function changeVideoChannelShare (
40async function addVideoShares (shareUrls: string[], video: MVideoId) { 40async function addVideoShares (shareUrls: string[], video: MVideoId) {
41 await Bluebird.map(shareUrls, async shareUrl => { 41 await Bluebird.map(shareUrls, async shareUrl => {
42 try { 42 try {
43 const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true }) 43 await addVideoShare(shareUrl, video)
44 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
45
46 const actorUrl = getAPId(body.actor)
47 if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
48 throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
49 }
50
51 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
52
53 const entry = {
54 actorId: actor.id,
55 videoId: video.id,
56 url: shareUrl
57 }
58
59 await VideoShareModel.upsert(entry)
60 } catch (err) { 44 } catch (err) {
61 logger.warn('Cannot add share %s.', shareUrl, { err }) 45 logger.warn('Cannot add share %s.', shareUrl, { err })
62 } 46 }
@@ -71,6 +55,26 @@ export {
71 55
72// --------------------------------------------------------------------------- 56// ---------------------------------------------------------------------------
73 57
58async function addVideoShare (shareUrl: string, video: MVideoId) {
59 const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
60 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
61
62 const actorUrl = getAPId(body.actor)
63 if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
64 throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
65 }
66
67 const actor = await getOrCreateAPActor(actorUrl)
68
69 const entry = {
70 actorId: actor.id,
71 videoId: video.id,
72 url: shareUrl
73 }
74
75 await VideoShareModel.upsert(entry)
76}
77
74async function shareByServer (video: MVideo, t: Transaction) { 78async function shareByServer (video: MVideo, t: Transaction) {
75 const serverActor = await getServerActor() 79 const serverActor = await getServerActor()
76 80
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index e23e0c0e7..6b7f9504f 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -6,8 +6,8 @@ import { doJSONRequest } from '../../helpers/requests'
6import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 6import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
7import { VideoCommentModel } from '../../models/video/video-comment' 7import { VideoCommentModel } from '../../models/video/video-comment'
8import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' 8import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
9import { getOrCreateActorAndServerAndModel } from './actor' 9import { getOrCreateAPActor } from './actors'
10import { getOrCreateVideoAndAccountAndChannel } from './videos' 10import { getOrCreateAPVideo } from './videos'
11 11
12type ResolveThreadParams = { 12type ResolveThreadParams = {
13 url: string 13 url: string
@@ -29,10 +29,11 @@ async function addVideoComments (commentUrls: string[]) {
29 29
30async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { 30async function resolveThread (params: ResolveThreadParams): ResolveThreadResult {
31 const { url, isVideo } = params 31 const { url, isVideo } = params
32
32 if (params.commentCreated === undefined) params.commentCreated = false 33 if (params.commentCreated === undefined) params.commentCreated = false
33 if (params.comments === undefined) params.comments = [] 34 if (params.comments === undefined) params.comments = []
34 35
35 // If it is not a video, or if we don't know if it's a video 36 // If it is not a video, or if we don't know if it's a video, try to get the thread from DB
36 if (isVideo === false || isVideo === undefined) { 37 if (isVideo === false || isVideo === undefined) {
37 const result = await resolveCommentFromDB(params) 38 const result = await resolveCommentFromDB(params)
38 if (result) return result 39 if (result) return result
@@ -42,7 +43,7 @@ async function resolveThread (params: ResolveThreadParams): ResolveThreadResult
42 // If it is a video, or if we don't know if it's a video 43 // If it is a video, or if we don't know if it's a video
43 if (isVideo === true || isVideo === undefined) { 44 if (isVideo === true || isVideo === undefined) {
44 // Keep await so we catch the exception 45 // Keep await so we catch the exception
45 return await tryResolveThreadFromVideo(params) 46 return await tryToResolveThreadFromVideo(params)
46 } 47 }
47 } catch (err) { 48 } catch (err) {
48 logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) 49 logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err })
@@ -62,34 +63,32 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {
62 const { url, comments, commentCreated } = params 63 const { url, comments, commentCreated } = params
63 64
64 const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) 65 const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url)
65 if (commentFromDatabase) { 66 if (!commentFromDatabase) return undefined
66 let parentComments = comments.concat([ commentFromDatabase ])
67 67
68 // Speed up things and resolve directly the thread 68 let parentComments = comments.concat([ commentFromDatabase ])
69 if (commentFromDatabase.InReplyToVideoComment) {
70 const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
71 69
72 parentComments = parentComments.concat(data) 70 // Speed up things and resolve directly the thread
73 } 71 if (commentFromDatabase.InReplyToVideoComment) {
72 const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
74 73
75 return resolveThread({ 74 parentComments = parentComments.concat(data)
76 url: commentFromDatabase.Video.url,
77 comments: parentComments,
78 isVideo: true,
79 commentCreated
80 })
81 } 75 }
82 76
83 return undefined 77 return resolveThread({
78 url: commentFromDatabase.Video.url,
79 comments: parentComments,
80 isVideo: true,
81 commentCreated
82 })
84} 83}
85 84
86async function tryResolveThreadFromVideo (params: ResolveThreadParams) { 85async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
87 const { url, comments, commentCreated } = params 86 const { url, comments, commentCreated } = params
88 87
89 // Maybe it's a reply to a video? 88 // Maybe it's a reply to a video?
90 // If yes, it's done: we resolved all the thread 89 // If yes, it's done: we resolved all the thread
91 const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false } 90 const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false }
92 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) 91 const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam })
93 92
94 if (video.isOwned() && !video.hasPrivacyForFederation()) { 93 if (video.isOwned() && !video.hasPrivacyForFederation()) {
95 throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') 94 throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation')
@@ -148,7 +147,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
148 } 147 }
149 148
150 const actor = actorUrl 149 const actor = actorUrl
151 ? await getOrCreateActorAndServerAndModel(actorUrl, 'all') 150 ? await getOrCreateAPActor(actorUrl, 'all')
152 : null 151 : null
153 152
154 const comment = new VideoCommentModel({ 153 const comment = new VideoCommentModel({
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index f40c07fea..9fb97ef84 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -3,44 +3,23 @@ import { Transaction } from 'sequelize'
3import { doJSONRequest } from '@server/helpers/requests' 3import { doJSONRequest } from '@server/helpers/requests'
4import { VideoRateType } from '../../../shared/models/videos' 4import { VideoRateType } from '../../../shared/models/videos'
5import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 5import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
6import { logger } from '../../helpers/logger' 6import { logger, loggerTagsFactory } from '../../helpers/logger'
7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
8import { AccountVideoRateModel } from '../../models/account/account-video-rate' 8import { AccountVideoRateModel } from '../../models/account/account-video-rate'
9import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' 9import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models'
10import { getOrCreateActorAndServerAndModel } from './actor' 10import { getOrCreateAPActor } from './actors'
11import { sendLike, sendUndoDislike, sendUndoLike } from './send' 11import { sendLike, sendUndoDislike, sendUndoLike } from './send'
12import { sendDislike } from './send/send-dislike' 12import { sendDislike } from './send/send-dislike'
13import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' 13import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url'
14 14
15const lTags = loggerTagsFactory('ap', 'video-rate', 'create')
16
15async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { 17async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) {
16 await Bluebird.map(ratesUrl, async rateUrl => { 18 await Bluebird.map(ratesUrl, async rateUrl => {
17 try { 19 try {
18 // Fetch url 20 await createRate(rateUrl, video, rate)
19 const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
20 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
21
22 const actorUrl = getAPId(body.actor)
23 if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
24 throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
25 }
26
27 if (checkUrlsSameHost(body.id, rateUrl) !== true) {
28 throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
29 }
30
31 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
32
33 const entry = {
34 videoId: video.id,
35 accountId: actor.Account.id,
36 type: rate,
37 url: body.id
38 }
39
40 // Video "likes"/"dislikes" will be updated by the caller
41 await AccountVideoRateModel.upsert(entry)
42 } catch (err) { 21 } catch (err) {
43 logger.warn('Cannot add rate %s.', rateUrl, { err }) 22 logger.info('Cannot add rate %s.', rateUrl, { err, ...lTags(rateUrl, video.uuid, video.url) })
44 } 23 }
45 }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) 24 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
46} 25}
@@ -73,8 +52,39 @@ function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVid
73 : getVideoDislikeActivityPubUrlByLocalActor(actor, video) 52 : getVideoDislikeActivityPubUrlByLocalActor(actor, video)
74} 53}
75 54
55// ---------------------------------------------------------------------------
56
76export { 57export {
77 getLocalRateUrl, 58 getLocalRateUrl,
78 createRates, 59 createRates,
79 sendVideoRateChange 60 sendVideoRateChange
80} 61}
62
63// ---------------------------------------------------------------------------
64
65async function createRate (rateUrl: string, video: MVideo, rate: VideoRateType) {
66 // Fetch url
67 const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
68 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
69
70 const actorUrl = getAPId(body.actor)
71 if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
72 throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
73 }
74
75 if (checkUrlsSameHost(body.id, rateUrl) !== true) {
76 throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
77 }
78
79 const actor = await getOrCreateAPActor(actorUrl)
80
81 const entry = {
82 videoId: video.id,
83 accountId: actor.Account.id,
84 type: rate,
85 url: body.id
86 }
87
88 // Video "likes"/"dislikes" will be updated by the caller
89 await AccountVideoRateModel.upsert(entry)
90}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
deleted file mode 100644
index 127a0dd8a..000000000
--- a/server/lib/activitypub/videos.ts
+++ /dev/null
@@ -1,931 +0,0 @@
1import * as Bluebird from 'bluebird'
2import { maxBy, minBy } from 'lodash'
3import * as magnetUtil from 'magnet-uri'
4import { basename } from 'path'
5import { Transaction } from 'sequelize/types'
6import { TrackerModel } from '@server/models/server/tracker'
7import { VideoLiveModel } from '@server/models/video/video-live'
8import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
9import {
10 ActivityHashTagObject,
11 ActivityMagnetUrlObject,
12 ActivityPlaylistSegmentHashesObject,
13 ActivityPlaylistUrlObject,
14 ActivitypubHttpFetcherPayload,
15 ActivityTagObject,
16 ActivityUrlObject,
17 ActivityVideoUrlObject
18} from '../../../shared/index'
19import { ActivityTrackerUrlObject, VideoObject } from '../../../shared/models/activitypub/objects'
20import { VideoPrivacy } from '../../../shared/models/videos'
21import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
22import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
23import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
24import {
25 isAPVideoFileUrlMetadataObject,
26 isAPVideoTrackerUrlObject,
27 sanitizeAndCheckVideoTorrentObject
28} from '../../helpers/custom-validators/activitypub/videos'
29import { isArray } from '../../helpers/custom-validators/misc'
30import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
31import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
32import { logger } from '../../helpers/logger'
33import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
34import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
35import {
36 ACTIVITY_PUB,
37 MIMETYPES,
38 P2P_MEDIA_LOADER_PEER_VERSION,
39 PREVIEWS_SIZE,
40 REMOTE_SCHEME,
41 THUMBNAILS_SIZE
42} from '../../initializers/constants'
43import { sequelizeTypescript } from '../../initializers/database'
44import { AccountVideoRateModel } from '../../models/account/account-video-rate'
45import { VideoModel } from '../../models/video/video'
46import { VideoCaptionModel } from '../../models/video/video-caption'
47import { VideoCommentModel } from '../../models/video/video-comment'
48import { VideoFileModel } from '../../models/video/video-file'
49import { VideoShareModel } from '../../models/video/video-share'
50import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
51import {
52 MAccountIdActor,
53 MChannelAccountLight,
54 MChannelDefault,
55 MChannelId,
56 MStreamingPlaylist,
57 MStreamingPlaylistFilesVideo,
58 MStreamingPlaylistVideo,
59 MVideo,
60 MVideoAccountLight,
61 MVideoAccountLightBlacklistAllFiles,
62 MVideoAP,
63 MVideoAPWithoutCaption,
64 MVideoCaption,
65 MVideoFile,
66 MVideoFullLight,
67 MVideoId,
68 MVideoImmutable,
69 MVideoThumbnail,
70 MVideoWithHost
71} from '../../types/models'
72import { MThumbnail } from '../../types/models/video/thumbnail'
73import { FilteredModelAttributes } from '../../types/sequelize'
74import { ActorFollowScoreCache } from '../files-cache'
75import { JobQueue } from '../job-queue'
76import { Notifier } from '../notifier'
77import { PeerTubeSocket } from '../peertube-socket'
78import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
79import { setVideoTags } from '../video'
80import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
81import { generateTorrentFileName } from '../video-paths'
82import { getOrCreateActorAndServerAndModel } from './actor'
83import { crawlCollectionPage } from './crawl'
84import { sendCreateVideo, sendUpdateVideo } from './send'
85import { addVideoShares, shareVideoByServerAndChannel } from './share'
86import { addVideoComments } from './video-comments'
87import { createRates } from './video-rates'
88
89async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
90 const video = videoArg as MVideoAP
91
92 if (
93 // Check this is not a blacklisted video, or unfederated blacklisted video
94 (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
95 // Check the video is public/unlisted and published
96 video.hasPrivacyForFederation() && video.hasStateForFederation()
97 ) {
98 // Fetch more attributes that we will need to serialize in AP object
99 if (isArray(video.VideoCaptions) === false) {
100 video.VideoCaptions = await video.$get('VideoCaptions', {
101 attributes: [ 'filename', 'language' ],
102 transaction
103 })
104 }
105
106 if (isNewVideo) {
107 // Now we'll add the video's meta data to our followers
108 await sendCreateVideo(video, transaction)
109 await shareVideoByServerAndChannel(video, transaction)
110 } else {
111 await sendUpdateVideo(video, transaction)
112 }
113 }
114}
115
116async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
117 logger.info('Fetching remote video %s.', videoUrl)
118
119 const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
120
121 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
122 logger.debug('Remote video JSON is not valid.', { body })
123 return { statusCode, videoObject: undefined }
124 }
125
126 return { statusCode, videoObject: body }
127}
128
129async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
130 const host = video.VideoChannel.Account.Actor.Server.host
131 const path = video.getDescriptionAPIPath()
132 const url = REMOTE_SCHEME.HTTP + '://' + host + path
133
134 const { body } = await doJSONRequest<any>(url)
135 return body.description || ''
136}
137
138function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) {
139 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
140 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
141
142 if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
143 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
144 }
145
146 return getOrCreateActorAndServerAndModel(channel.id, 'all')
147}
148
149type SyncParam = {
150 likes: boolean
151 dislikes: boolean
152 shares: boolean
153 comments: boolean
154 thumbnail: boolean
155 refreshVideo?: boolean
156}
157async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoObject, syncParam: SyncParam) {
158 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
159
160 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
161
162 if (syncParam.likes === true) {
163 const handler = items => createRates(items, video, 'like')
164 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
165
166 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
167 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes }))
168 } else {
169 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
170 }
171
172 if (syncParam.dislikes === true) {
173 const handler = items => createRates(items, video, 'dislike')
174 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
175
176 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
177 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes }))
178 } else {
179 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
180 }
181
182 if (syncParam.shares === true) {
183 const handler = items => addVideoShares(items, video)
184 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
185
186 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
187 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
188 } else {
189 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
190 }
191
192 if (syncParam.comments === true) {
193 const handler = items => addVideoComments(items)
194 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
195
196 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
197 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
198 } else {
199 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
200 }
201
202 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
203}
204
205type GetVideoResult <T> = Promise<{
206 video: T
207 created: boolean
208 autoBlacklisted?: boolean
209}>
210
211type GetVideoParamAll = {
212 videoObject: { id: string } | string
213 syncParam?: SyncParam
214 fetchType?: 'all'
215 allowRefresh?: boolean
216}
217
218type GetVideoParamImmutable = {
219 videoObject: { id: string } | string
220 syncParam?: SyncParam
221 fetchType: 'only-immutable-attributes'
222 allowRefresh: false
223}
224
225type GetVideoParamOther = {
226 videoObject: { id: string } | string
227 syncParam?: SyncParam
228 fetchType?: 'all' | 'only-video'
229 allowRefresh?: boolean
230}
231
232function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
233function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
234function getOrCreateVideoAndAccountAndChannel (
235 options: GetVideoParamOther
236): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
237async function getOrCreateVideoAndAccountAndChannel (
238 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
239): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
240 // Default params
241 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
242 const fetchType = options.fetchType || 'all'
243 const allowRefresh = options.allowRefresh !== false
244
245 // Get video url
246 const videoUrl = getAPId(options.videoObject)
247 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
248
249 if (videoFromDatabase) {
250 // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
251 if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
252 const refreshOptions = {
253 video: videoFromDatabase as MVideoThumbnail,
254 fetchedType: fetchType,
255 syncParam
256 }
257
258 if (syncParam.refreshVideo === true) {
259 videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
260 } else {
261 await JobQueue.Instance.createJobWithPromise({
262 type: 'activitypub-refresher',
263 payload: { type: 'video', url: videoFromDatabase.url }
264 })
265 }
266 }
267
268 return { video: videoFromDatabase, created: false }
269 }
270
271 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
272 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
273
274 const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
275 const videoChannel = actor.VideoChannel
276
277 try {
278 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
279
280 await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
281
282 return { video: videoCreated, created: true, autoBlacklisted }
283 } catch (err) {
284 // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video
285 if (err.name === 'SequelizeUniqueConstraintError') {
286 const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType)
287 if (fallbackVideo) return { video: fallbackVideo, created: false }
288 }
289
290 throw err
291 }
292}
293
294async function updateVideoFromAP (options: {
295 video: MVideoAccountLightBlacklistAllFiles
296 videoObject: VideoObject
297 account: MAccountIdActor
298 channel: MChannelDefault
299 overrideTo?: string[]
300}) {
301 const { video, videoObject, account, channel, overrideTo } = options
302
303 logger.debug('Updating remote video "%s".', options.videoObject.uuid, { videoObject: options.videoObject, account, channel })
304
305 let videoFieldsSave: any
306 const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE
307 const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED
308
309 try {
310 let thumbnailModel: MThumbnail
311
312 try {
313 thumbnailModel = await createVideoMiniatureFromUrl({
314 downloadUrl: getThumbnailFromIcons(videoObject).url,
315 video,
316 type: ThumbnailType.MINIATURE
317 })
318 } catch (err) {
319 logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
320 }
321
322 const videoUpdated = await sequelizeTypescript.transaction(async t => {
323 const sequelizeOptions = { transaction: t }
324
325 videoFieldsSave = video.toJSON()
326
327 // Check we can update the channel: we trust the remote server
328 const oldVideoChannel = video.VideoChannel
329
330 if (!oldVideoChannel.Actor.serverId || !channel.Actor.serverId) {
331 throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
332 }
333
334 if (oldVideoChannel.Actor.serverId !== channel.Actor.serverId) {
335 throw new Error('New channel ' + channel.Actor.url + ' is not on the same server than new channel ' + oldVideoChannel.Actor.url)
336 }
337
338 const to = overrideTo || videoObject.to
339 const videoData = videoActivityObjectToDBAttributes(channel, videoObject, to)
340 video.name = videoData.name
341 video.uuid = videoData.uuid
342 video.url = videoData.url
343 video.category = videoData.category
344 video.licence = videoData.licence
345 video.language = videoData.language
346 video.description = videoData.description
347 video.support = videoData.support
348 video.nsfw = videoData.nsfw
349 video.commentsEnabled = videoData.commentsEnabled
350 video.downloadEnabled = videoData.downloadEnabled
351 video.waitTranscoding = videoData.waitTranscoding
352 video.state = videoData.state
353 video.duration = videoData.duration
354 video.createdAt = videoData.createdAt
355 video.publishedAt = videoData.publishedAt
356 video.originallyPublishedAt = videoData.originallyPublishedAt
357 video.privacy = videoData.privacy
358 video.channelId = videoData.channelId
359 video.views = videoData.views
360 video.isLive = videoData.isLive
361
362 // Ensures we update the updated video attribute
363 video.changed('updatedAt', true)
364
365 const videoUpdated = await video.save(sequelizeOptions) as MVideoFullLight
366
367 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
368
369 const previewIcon = getPreviewFromIcons(videoObject)
370 if (videoUpdated.getPreview() && previewIcon) {
371 const previewModel = createPlaceholderThumbnail({
372 fileUrl: previewIcon.url,
373 video,
374 type: ThumbnailType.PREVIEW,
375 size: previewIcon
376 })
377 await videoUpdated.addAndSaveThumbnail(previewModel, t)
378 }
379
380 {
381 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url)
382 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
383
384 // Remove video files that do not exist anymore
385 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
386 await Promise.all(destroyTasks)
387
388 // Update or add other one
389 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
390 videoUpdated.VideoFiles = await Promise.all(upsertTasks)
391 }
392
393 {
394 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles)
395 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
396
397 // Remove video playlists that do not exist anymore
398 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
399 await Promise.all(destroyTasks)
400
401 let oldStreamingPlaylistFiles: MVideoFile[] = []
402 for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
403 oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
404 }
405
406 videoUpdated.VideoStreamingPlaylists = []
407
408 for (const playlistAttributes of streamingPlaylistAttributes) {
409 const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
410 .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo)
411 streamingPlaylistModel.Video = videoUpdated
412
413 const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
414 .map(a => new VideoFileModel(a))
415 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
416 await Promise.all(destroyTasks)
417
418 // Update or add other one
419 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
420 streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
421
422 videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
423 }
424 }
425
426 {
427 // Update Tags
428 const tags = videoObject.tag
429 .filter(isAPHashTagObject)
430 .map(tag => tag.name)
431 await setVideoTags({ video: videoUpdated, tags, transaction: t })
432 }
433
434 // Update trackers
435 {
436 const trackers = getTrackerUrls(videoObject, videoUpdated)
437 await setVideoTrackers({ video: videoUpdated, trackers, transaction: t })
438 }
439
440 {
441 // Update captions
442 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
443
444 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
445 const caption = new VideoCaptionModel({
446 videoId: videoUpdated.id,
447 filename: VideoCaptionModel.generateCaptionName(c.identifier),
448 language: c.identifier,
449 fileUrl: c.url
450 }) as MVideoCaption
451
452 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
453 })
454 await Promise.all(videoCaptionsPromises)
455 }
456
457 {
458 // Create or update existing live
459 if (video.isLive) {
460 const [ videoLive ] = await VideoLiveModel.upsert({
461 saveReplay: videoObject.liveSaveReplay,
462 permanentLive: videoObject.permanentLive,
463 videoId: video.id
464 }, { transaction: t, returning: true })
465
466 videoUpdated.VideoLive = videoLive
467 } else { // Delete existing live if it exists
468 await VideoLiveModel.destroy({
469 where: {
470 videoId: video.id
471 },
472 transaction: t
473 })
474
475 videoUpdated.VideoLive = null
476 }
477 }
478
479 return videoUpdated
480 })
481
482 await autoBlacklistVideoIfNeeded({
483 video: videoUpdated,
484 user: undefined,
485 isRemote: true,
486 isNew: false,
487 transaction: undefined
488 })
489
490 // Notify our users?
491 if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated)
492
493 if (videoUpdated.isLive) {
494 PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated)
495 PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
496 }
497
498 logger.info('Remote video with uuid %s updated', videoObject.uuid)
499
500 return videoUpdated
501 } catch (err) {
502 if (video !== undefined && videoFieldsSave !== undefined) {
503 resetSequelizeInstance(video, videoFieldsSave)
504 }
505
506 // This is just a debug because we will retry the insert
507 logger.debug('Cannot update the remote video.', { err })
508 throw err
509 }
510}
511
512async function refreshVideoIfNeeded (options: {
513 video: MVideoThumbnail
514 fetchedType: VideoFetchByUrlType
515 syncParam: SyncParam
516}): Promise<MVideoThumbnail> {
517 if (!options.video.isOutdated()) return options.video
518
519 // We need more attributes if the argument video was fetched with not enough joints
520 const video = options.fetchedType === 'all'
521 ? options.video as MVideoAccountLightBlacklistAllFiles
522 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
523
524 try {
525 const { videoObject } = await fetchRemoteVideo(video.url)
526
527 if (videoObject === undefined) {
528 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
529
530 await video.setAsRefreshed()
531 return video
532 }
533
534 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
535
536 const updateOptions = {
537 video,
538 videoObject,
539 account: channelActor.VideoChannel.Account,
540 channel: channelActor.VideoChannel
541 }
542 await updateVideoFromAP(updateOptions)
543 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
544
545 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
546
547 return video
548 } catch (err) {
549 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
550 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
551
552 // Video does not exist anymore
553 await video.destroy()
554 return undefined
555 }
556
557 logger.warn('Cannot refresh video %s.', options.video.url, { err })
558
559 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
560
561 // Don't refresh in loop
562 await video.setAsRefreshed()
563 return video
564 }
565}
566
567export {
568 updateVideoFromAP,
569 refreshVideoIfNeeded,
570 federateVideoIfNeeded,
571 fetchRemoteVideo,
572 getOrCreateVideoAndAccountAndChannel,
573 fetchRemoteVideoDescription,
574 getOrCreateVideoChannelFromVideoObject
575}
576
577// ---------------------------------------------------------------------------
578
579function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
580 const urlMediaType = url.mediaType
581
582 return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
583}
584
585function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject {
586 return url && url.mediaType === 'application/x-mpegURL'
587}
588
589function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
590 return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
591}
592
593function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
594 return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
595}
596
597function isAPHashTagObject (url: any): url is ActivityHashTagObject {
598 return url && url.type === 'Hashtag'
599}
600
601async function createVideo (videoObject: VideoObject, channel: MChannelAccountLight, waitThumbnail = false) {
602 logger.debug('Adding remote video %s.', videoObject.id)
603
604 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
605 const video = VideoModel.build(videoData) as MVideoThumbnail
606
607 const promiseThumbnail = createVideoMiniatureFromUrl({
608 downloadUrl: getThumbnailFromIcons(videoObject).url,
609 video,
610 type: ThumbnailType.MINIATURE
611 }).catch(err => {
612 logger.error('Cannot create miniature from url.', { err })
613 return undefined
614 })
615
616 let thumbnailModel: MThumbnail
617 if (waitThumbnail === true) {
618 thumbnailModel = await promiseThumbnail
619 }
620
621 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
622 try {
623 const sequelizeOptions = { transaction: t }
624
625 const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
626 videoCreated.VideoChannel = channel
627
628 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
629
630 const previewIcon = getPreviewFromIcons(videoObject)
631 if (previewIcon) {
632 const previewModel = createPlaceholderThumbnail({
633 fileUrl: previewIcon.url,
634 video: videoCreated,
635 type: ThumbnailType.PREVIEW,
636 size: previewIcon
637 })
638
639 await videoCreated.addAndSaveThumbnail(previewModel, t)
640 }
641
642 // Process files
643 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
644
645 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
646 const videoFiles = await Promise.all(videoFilePromises)
647
648 const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
649 videoCreated.VideoStreamingPlaylists = []
650
651 for (const playlistAttributes of streamingPlaylistsAttributes) {
652 const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo
653 playlist.Video = videoCreated
654
655 const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject)
656 const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
657 playlist.VideoFiles = await Promise.all(videoFilePromises)
658
659 videoCreated.VideoStreamingPlaylists.push(playlist)
660 }
661
662 // Process tags
663 const tags = videoObject.tag
664 .filter(isAPHashTagObject)
665 .map(t => t.name)
666 await setVideoTags({ video: videoCreated, tags, transaction: t })
667
668 // Process captions
669 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
670 const caption = new VideoCaptionModel({
671 videoId: videoCreated.id,
672 filename: VideoCaptionModel.generateCaptionName(c.identifier),
673 language: c.identifier,
674 fileUrl: c.url
675 }) as MVideoCaption
676
677 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
678 })
679 await Promise.all(videoCaptionsPromises)
680
681 // Process trackers
682 {
683 const trackers = getTrackerUrls(videoObject, videoCreated)
684 await setVideoTrackers({ video: videoCreated, trackers, transaction: t })
685 }
686
687 videoCreated.VideoFiles = videoFiles
688
689 if (videoCreated.isLive) {
690 const videoLive = new VideoLiveModel({
691 streamKey: null,
692 saveReplay: videoObject.liveSaveReplay,
693 permanentLive: videoObject.permanentLive,
694 videoId: videoCreated.id
695 })
696
697 videoCreated.VideoLive = await videoLive.save({ transaction: t })
698 }
699
700 // We added a video in this channel, set it as updated
701 await channel.setAsUpdated(t)
702
703 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
704 video: videoCreated,
705 user: undefined,
706 isRemote: true,
707 isNew: true,
708 transaction: t
709 })
710
711 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
712
713 return { autoBlacklisted, videoCreated }
714 } catch (err) {
715 // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
716 // Remove thumbnail
717 if (thumbnailModel) await thumbnailModel.removeThumbnail()
718
719 throw err
720 }
721 })
722
723 if (waitThumbnail === false) {
724 // Error is already caught above
725 // eslint-disable-next-line @typescript-eslint/no-floating-promises
726 promiseThumbnail.then(thumbnailModel => {
727 if (!thumbnailModel) return
728
729 thumbnailModel = videoCreated.id
730
731 return thumbnailModel.save()
732 })
733 }
734
735 return { autoBlacklisted, videoCreated }
736}
737
738function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
739 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
740 ? VideoPrivacy.PUBLIC
741 : VideoPrivacy.UNLISTED
742
743 const duration = videoObject.duration.replace(/[^\d]+/, '')
744 const language = videoObject.language?.identifier
745
746 const category = videoObject.category
747 ? parseInt(videoObject.category.identifier, 10)
748 : undefined
749
750 const licence = videoObject.licence
751 ? parseInt(videoObject.licence.identifier, 10)
752 : undefined
753
754 const description = videoObject.content || null
755 const support = videoObject.support || null
756
757 return {
758 name: videoObject.name,
759 uuid: videoObject.uuid,
760 url: videoObject.id,
761 category,
762 licence,
763 language,
764 description,
765 support,
766 nsfw: videoObject.sensitive,
767 commentsEnabled: videoObject.commentsEnabled,
768 downloadEnabled: videoObject.downloadEnabled,
769 waitTranscoding: videoObject.waitTranscoding,
770 isLive: videoObject.isLiveBroadcast,
771 state: videoObject.state,
772 channelId: videoChannel.id,
773 duration: parseInt(duration, 10),
774 createdAt: new Date(videoObject.published),
775 publishedAt: new Date(videoObject.published),
776
777 originallyPublishedAt: videoObject.originallyPublishedAt
778 ? new Date(videoObject.originallyPublishedAt)
779 : null,
780
781 updatedAt: new Date(videoObject.updated),
782 views: videoObject.views,
783 likes: 0,
784 dislikes: 0,
785 remote: true,
786 privacy
787 }
788}
789
790function videoFileActivityUrlToDBAttributes (
791 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
792 urls: (ActivityTagObject | ActivityUrlObject)[]
793) {
794 const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
795
796 if (fileUrls.length === 0) return []
797
798 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
799 for (const fileUrl of fileUrls) {
800 // Fetch associated magnet uri
801 const magnet = urls.filter(isAPMagnetUrlObject)
802 .find(u => u.height === fileUrl.height)
803
804 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
805
806 const parsed = magnetUtil.decode(magnet.href)
807 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
808 throw new Error('Cannot parse magnet URI ' + magnet.href)
809 }
810
811 const torrentUrl = Array.isArray(parsed.xs)
812 ? parsed.xs[0]
813 : parsed.xs
814
815 // Fetch associated metadata url, if any
816 const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
817 .find(u => {
818 return u.height === fileUrl.height &&
819 u.fps === fileUrl.fps &&
820 u.rel.includes(fileUrl.mediaType)
821 })
822
823 const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
824 const resolution = fileUrl.height
825 const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
826 const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
827
828 const attribute = {
829 extname,
830 infoHash: parsed.infoHash,
831 resolution,
832 size: fileUrl.size,
833 fps: fileUrl.fps || -1,
834 metadataUrl: metadata?.href,
835
836 // Use the name of the remote file because we don't proxify video file requests
837 filename: basename(fileUrl.href),
838 fileUrl: fileUrl.href,
839
840 torrentUrl,
841 // Use our own torrent name since we proxify torrent requests
842 torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution),
843
844 // This is a video file owned by a video or by a streaming playlist
845 videoId,
846 videoStreamingPlaylistId
847 }
848
849 attributes.push(attribute)
850 }
851
852 return attributes
853}
854
855function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) {
856 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
857 if (playlistUrls.length === 0) return []
858
859 const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
860 for (const playlistUrlObject of playlistUrls) {
861 const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
862
863 let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
864
865 // FIXME: backward compatibility introduced in v2.1.0
866 if (files.length === 0) files = videoFiles
867
868 if (!segmentsSha256UrlObject) {
869 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
870 continue
871 }
872
873 const attribute = {
874 type: VideoStreamingPlaylistType.HLS,
875 playlistUrl: playlistUrlObject.href,
876 segmentsSha256Url: segmentsSha256UrlObject.href,
877 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
878 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
879 videoId: video.id,
880 tagAPObject: playlistUrlObject.tag
881 }
882
883 attributes.push(attribute)
884 }
885
886 return attributes
887}
888
889function getThumbnailFromIcons (videoObject: VideoObject) {
890 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
891 // Fallback if there are not valid icons
892 if (validIcons.length === 0) validIcons = videoObject.icon
893
894 return minBy(validIcons, 'width')
895}
896
897function getPreviewFromIcons (videoObject: VideoObject) {
898 const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
899
900 return maxBy(validIcons, 'width')
901}
902
903function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
904 let wsFound = false
905
906 const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u))
907 .map((u: ActivityTrackerUrlObject) => {
908 if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true
909
910 return u.href
911 })
912
913 if (wsFound) return trackers
914
915 return [
916 buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS),
917 buildRemoteVideoBaseUrl(video, '/tracker/announce')
918 ]
919}
920
921async function setVideoTrackers (options: {
922 video: MVideo
923 trackers: string[]
924 transaction?: Transaction
925}) {
926 const { video, trackers, transaction } = options
927
928 const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction)
929
930 await video.$set('Trackers', trackerInstances, { transaction })
931}
diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts
new file mode 100644
index 000000000..bd0c54b0c
--- /dev/null
+++ b/server/lib/activitypub/videos/federate.ts
@@ -0,0 +1,36 @@
1import { Transaction } from 'sequelize/types'
2import { isArray } from '@server/helpers/custom-validators/misc'
3import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models'
4import { sendCreateVideo, sendUpdateVideo } from '../send'
5import { shareVideoByServerAndChannel } from '../share'
6
7async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
8 const video = videoArg as MVideoAP
9
10 if (
11 // Check this is not a blacklisted video, or unfederated blacklisted video
12 (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
13 // Check the video is public/unlisted and published
14 video.hasPrivacyForFederation() && video.hasStateForFederation()
15 ) {
16 // Fetch more attributes that we will need to serialize in AP object
17 if (isArray(video.VideoCaptions) === false) {
18 video.VideoCaptions = await video.$get('VideoCaptions', {
19 attributes: [ 'filename', 'language' ],
20 transaction
21 })
22 }
23
24 if (isNewVideo) {
25 // Now we'll add the video's meta data to our followers
26 await sendCreateVideo(video, transaction)
27 await shareVideoByServerAndChannel(video, transaction)
28 } else {
29 await sendUpdateVideo(video, transaction)
30 }
31 }
32}
33
34export {
35 federateVideoIfNeeded
36}
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts
new file mode 100644
index 000000000..f3e2f0625
--- /dev/null
+++ b/server/lib/activitypub/videos/get.ts
@@ -0,0 +1,113 @@
1import { getAPId } from '@server/helpers/activitypub'
2import { retryTransactionWrapper } from '@server/helpers/database-utils'
3import { JobQueue } from '@server/lib/job-queue'
4import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders'
5import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models'
6import { APObject } from '@shared/models'
7import { refreshVideoIfNeeded } from './refresh'
8import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
9
10type GetVideoResult <T> = Promise<{
11 video: T
12 created: boolean
13 autoBlacklisted?: boolean
14}>
15
16type GetVideoParamAll = {
17 videoObject: APObject
18 syncParam?: SyncParam
19 fetchType?: 'all'
20 allowRefresh?: boolean
21}
22
23type GetVideoParamImmutable = {
24 videoObject: APObject
25 syncParam?: SyncParam
26 fetchType: 'only-immutable-attributes'
27 allowRefresh: false
28}
29
30type GetVideoParamOther = {
31 videoObject: APObject
32 syncParam?: SyncParam
33 fetchType?: 'all' | 'only-video'
34 allowRefresh?: boolean
35}
36
37function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
38function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
39function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
40
41async function getOrCreateAPVideo (
42 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
43): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
44 // Default params
45 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
46 const fetchType = options.fetchType || 'all'
47 const allowRefresh = options.allowRefresh !== false
48
49 // Get video url
50 const videoUrl = getAPId(options.videoObject)
51 let videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType)
52
53 if (videoFromDatabase) {
54 if (allowRefresh === true) {
55 // Typings ensure allowRefresh === false in only-immutable-attributes fetch type
56 videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam)
57 }
58
59 return { video: videoFromDatabase, created: false }
60 }
61
62 const { videoObject } = await fetchRemoteVideo(videoUrl)
63 if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
64
65 // videoUrl is just an alias/rediraction, so process object id instead
66 if (videoObject.id !== videoUrl) return getOrCreateAPVideo({ ...options, fetchType: 'all', videoObject })
67
68 try {
69 const creator = new APVideoCreator(videoObject)
70 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail)
71
72 await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
73
74 return { video: videoCreated, created: true, autoBlacklisted }
75 } catch (err) {
76 // Maybe a concurrent getOrCreateAPVideo call created this video
77 if (err.name === 'SequelizeUniqueConstraintError') {
78 const alreadyCreatedVideo = await loadVideoByUrl(videoUrl, fetchType)
79 if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false }
80 }
81
82 throw err
83 }
84}
85
86// ---------------------------------------------------------------------------
87
88export {
89 getOrCreateAPVideo
90}
91
92// ---------------------------------------------------------------------------
93
94async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoLoadByUrlType, syncParam: SyncParam) {
95 if (!video.isOutdated()) return video
96
97 const refreshOptions = {
98 video,
99 fetchedType: fetchType,
100 syncParam
101 }
102
103 if (syncParam.refreshVideo === true) {
104 return refreshVideoIfNeeded(refreshOptions)
105 }
106
107 await JobQueue.Instance.createJobWithPromise({
108 type: 'activitypub-refresher',
109 payload: { type: 'video', url: video.url }
110 })
111
112 return video
113}
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts
new file mode 100644
index 000000000..b22062598
--- /dev/null
+++ b/server/lib/activitypub/videos/index.ts
@@ -0,0 +1,4 @@
1export * from './federate'
2export * from './get'
3export * from './refresh'
4export * from './updater'
diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts
new file mode 100644
index 000000000..a7b82f286
--- /dev/null
+++ b/server/lib/activitypub/videos/refresh.ts
@@ -0,0 +1,68 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { PeerTubeRequestError } from '@server/helpers/requests'
3import { ActorFollowScoreCache } from '@server/lib/files-cache'
4import { VideoLoadByUrlType } from '@server/lib/model-loaders'
5import { VideoModel } from '@server/models/video/video'
6import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models'
7import { HttpStatusCode } from '@shared/core-utils'
8import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
9import { APVideoUpdater } from './updater'
10
11async function refreshVideoIfNeeded (options: {
12 video: MVideoThumbnail
13 fetchedType: VideoLoadByUrlType
14 syncParam: SyncParam
15}): Promise<MVideoThumbnail> {
16 if (!options.video.isOutdated()) return options.video
17
18 // We need more attributes if the argument video was fetched with not enough joints
19 const video = options.fetchedType === 'all'
20 ? options.video as MVideoAccountLightBlacklistAllFiles
21 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
22
23 const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url)
24
25 logger.info('Refreshing video %s.', video.url, lTags())
26
27 try {
28 const { videoObject } = await fetchRemoteVideo(video.url)
29
30 if (videoObject === undefined) {
31 logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags())
32
33 await video.setAsRefreshed()
34 return video
35 }
36
37 const videoUpdater = new APVideoUpdater(videoObject, video)
38 await videoUpdater.update()
39
40 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
41
42 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
43
44 return video
45 } catch (err) {
46 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
47 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags())
48
49 // Video does not exist anymore
50 await video.destroy()
51 return undefined
52 }
53
54 logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags() })
55
56 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
57
58 // Don't refresh in loop
59 await video.setAsRefreshed()
60 return video
61 }
62}
63
64// ---------------------------------------------------------------------------
65
66export {
67 refreshVideoIfNeeded
68}
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
new file mode 100644
index 000000000..e89c94bcd
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -0,0 +1,173 @@
1import { Transaction } from 'sequelize/types'
2import { checkUrlsSameHost } from '@server/helpers/activitypub'
3import { deleteNonExistingModels } from '@server/helpers/database-utils'
4import { logger, LoggerTagsFn } from '@server/helpers/logger'
5import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
6import { setVideoTags } from '@server/lib/video'
7import { VideoCaptionModel } from '@server/models/video/video-caption'
8import { VideoFileModel } from '@server/models/video/video-file'
9import { VideoLiveModel } from '@server/models/video/video-live'
10import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
11import { MStreamingPlaylistFilesVideo, MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models'
12import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models'
13import { getOrCreateAPActor } from '../../actors'
14import {
15 getCaptionAttributesFromObject,
16 getFileAttributesFromUrl,
17 getLiveAttributesFromObject,
18 getPreviewFromIcons,
19 getStreamingPlaylistAttributesFromObject,
20 getTagsFromObject,
21 getThumbnailFromIcons
22} from './object-to-model-attributes'
23import { getTrackerUrls, setVideoTrackers } from './trackers'
24
25export abstract class APVideoAbstractBuilder {
26 protected abstract videoObject: VideoObject
27 protected abstract lTags: LoggerTagsFn
28
29 protected async getOrCreateVideoChannelFromVideoObject () {
30 const channel = this.videoObject.attributedTo.find(a => a.type === 'Group')
31 if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url)
32
33 if (checkUrlsSameHost(channel.id, this.videoObject.id) !== true) {
34 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${this.videoObject.id}`)
35 }
36
37 return getOrCreateAPActor(channel.id, 'all')
38 }
39
40 protected tryToGenerateThumbnail (video: MVideoThumbnail): Promise<MThumbnail> {
41 return updateVideoMiniatureFromUrl({
42 downloadUrl: getThumbnailFromIcons(this.videoObject).url,
43 video,
44 type: ThumbnailType.MINIATURE
45 }).catch(err => {
46 logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err, ...this.lTags() })
47
48 return undefined
49 })
50 }
51
52 protected async setPreview (video: MVideoFullLight, t?: Transaction) {
53 // Don't fetch the preview that could be big, create a placeholder instead
54 const previewIcon = getPreviewFromIcons(this.videoObject)
55 if (!previewIcon) return
56
57 const previewModel = updatePlaceholderThumbnail({
58 fileUrl: previewIcon.url,
59 video,
60 type: ThumbnailType.PREVIEW,
61 size: previewIcon
62 })
63
64 await video.addAndSaveThumbnail(previewModel, t)
65 }
66
67 protected async setTags (video: MVideoFullLight, t: Transaction) {
68 const tags = getTagsFromObject(this.videoObject)
69 await setVideoTags({ video, tags, transaction: t })
70 }
71
72 protected async setTrackers (video: MVideoFullLight, t: Transaction) {
73 const trackers = getTrackerUrls(this.videoObject, video)
74 await setVideoTrackers({ video, trackers, transaction: t })
75 }
76
77 protected async insertOrReplaceCaptions (video: MVideoFullLight, t: Transaction) {
78 const existingCaptions = await VideoCaptionModel.listVideoCaptions(video.id, t)
79
80 let captionsToCreate = getCaptionAttributesFromObject(video, this.videoObject)
81 .map(a => new VideoCaptionModel(a) as MVideoCaption)
82
83 for (const existingCaption of existingCaptions) {
84 // Only keep captions that do not already exist
85 const filtered = captionsToCreate.filter(c => !c.isEqual(existingCaption))
86
87 // This caption already exists, we don't need to destroy and create it
88 if (filtered.length !== captionsToCreate.length) {
89 captionsToCreate = filtered
90 continue
91 }
92
93 // Destroy this caption that does not exist anymore
94 await existingCaption.destroy({ transaction: t })
95 }
96
97 for (const captionToCreate of captionsToCreate) {
98 await captionToCreate.save({ transaction: t })
99 }
100 }
101
102 protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) {
103 const attributes = getLiveAttributesFromObject(video, this.videoObject)
104 const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true })
105
106 video.VideoLive = videoLive
107 }
108
109 protected async setWebTorrentFiles (video: MVideoFullLight, t: Transaction) {
110 const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url)
111 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
112
113 // Remove video files that do not exist anymore
114 const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t)
115 await Promise.all(destroyTasks)
116
117 // Update or add other one
118 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
119 video.VideoFiles = await Promise.all(upsertTasks)
120 }
121
122 protected async setStreamingPlaylists (video: MVideoFullLight, t: Transaction) {
123 const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject, video.VideoFiles || [])
124 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
125
126 // Remove video playlists that do not exist anymore
127 const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t)
128 await Promise.all(destroyTasks)
129
130 video.VideoStreamingPlaylists = []
131
132 for (const playlistAttributes of streamingPlaylistAttributes) {
133
134 const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
135 streamingPlaylistModel.Video = video
136
137 await this.setStreamingPlaylistFiles(video, streamingPlaylistModel, playlistAttributes.tagAPObject, t)
138
139 video.VideoStreamingPlaylists.push(streamingPlaylistModel)
140 }
141 }
142
143 private async insertOrReplaceStreamingPlaylist (attributes: VideoStreamingPlaylistModel['_creationAttributes'], t: Transaction) {
144 const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t })
145
146 return streamingPlaylist as MStreamingPlaylistFilesVideo
147 }
148
149 private getStreamingPlaylistFiles (video: MVideoFullLight, type: VideoStreamingPlaylistType) {
150 const playlist = video.VideoStreamingPlaylists.find(s => s.type === type)
151 if (!playlist) return []
152
153 return playlist.VideoFiles
154 }
155
156 private async setStreamingPlaylistFiles (
157 video: MVideoFullLight,
158 playlistModel: MStreamingPlaylistFilesVideo,
159 tagObjects: ActivityTagObject[],
160 t: Transaction
161 ) {
162 const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(video, playlistModel.type)
163
164 const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a))
165
166 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
167 await Promise.all(destroyTasks)
168
169 // Update or add other one
170 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
171 playlistModel.VideoFiles = await Promise.all(upsertTasks)
172 }
173}
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts
new file mode 100644
index 000000000..ad3b88936
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/creator.ts
@@ -0,0 +1,88 @@
1
2import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
3import { sequelizeTypescript } from '@server/initializers/database'
4import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
5import { VideoModel } from '@server/models/video/video'
6import { MThumbnail, MVideoFullLight, MVideoThumbnail } from '@server/types/models'
7import { VideoObject } from '@shared/models'
8import { APVideoAbstractBuilder } from './abstract-builder'
9import { getVideoAttributesFromObject } from './object-to-model-attributes'
10
11export class APVideoCreator extends APVideoAbstractBuilder {
12 protected lTags: LoggerTagsFn
13
14 constructor (protected readonly videoObject: VideoObject) {
15 super()
16
17 this.lTags = loggerTagsFactory('ap', 'video', 'create', this.videoObject.uuid, this.videoObject.id)
18 }
19
20 async create (waitThumbnail = false) {
21 logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags())
22
23 const channelActor = await this.getOrCreateVideoChannelFromVideoObject()
24 const channel = channelActor.VideoChannel
25
26 const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to)
27 const video = VideoModel.build(videoData) as MVideoThumbnail
28
29 const promiseThumbnail = this.tryToGenerateThumbnail(video)
30
31 let thumbnailModel: MThumbnail
32 if (waitThumbnail === true) {
33 thumbnailModel = await promiseThumbnail
34 }
35
36 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
37 try {
38 const videoCreated = await video.save({ transaction: t }) as MVideoFullLight
39 videoCreated.VideoChannel = channel
40
41 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
42
43 await this.setPreview(videoCreated, t)
44 await this.setWebTorrentFiles(videoCreated, t)
45 await this.setStreamingPlaylists(videoCreated, t)
46 await this.setTags(videoCreated, t)
47 await this.setTrackers(videoCreated, t)
48 await this.insertOrReplaceCaptions(videoCreated, t)
49 await this.insertOrReplaceLive(videoCreated, t)
50
51 // We added a video in this channel, set it as updated
52 await channel.setAsUpdated(t)
53
54 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
55 video: videoCreated,
56 user: undefined,
57 isRemote: true,
58 isNew: true,
59 transaction: t
60 })
61
62 logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags())
63
64 return { autoBlacklisted, videoCreated }
65 } catch (err) {
66 // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
67 // Remove thumbnail
68 if (thumbnailModel) await thumbnailModel.removeThumbnail()
69
70 throw err
71 }
72 })
73
74 if (waitThumbnail === false) {
75 // Error is already caught above
76 // eslint-disable-next-line @typescript-eslint/no-floating-promises
77 promiseThumbnail.then(thumbnailModel => {
78 if (!thumbnailModel) return
79
80 thumbnailModel = videoCreated.id
81
82 return thumbnailModel.save()
83 })
84 }
85
86 return { autoBlacklisted, videoCreated }
87 }
88}
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts
new file mode 100644
index 000000000..951403493
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/index.ts
@@ -0,0 +1,6 @@
1export * from './abstract-builder'
2export * from './creator'
3export * from './object-to-model-attributes'
4export * from './trackers'
5export * from './url-to-object'
6export * from './video-sync-attributes'
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
new file mode 100644
index 000000000..85548428c
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -0,0 +1,256 @@
1import { maxBy, minBy } from 'lodash'
2import * as magnetUtil from 'magnet-uri'
3import { basename } from 'path'
4import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos'
5import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos'
6import { logger } from '@server/helpers/logger'
7import { getExtFromMimetype } from '@server/helpers/video'
8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { generateTorrentFileName } from '@server/lib/video-paths'
10import { VideoFileModel } from '@server/models/video/video-file'
11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
12import { FilteredModelAttributes } from '@server/types'
13import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
14import {
15 ActivityHashTagObject,
16 ActivityMagnetUrlObject,
17 ActivityPlaylistSegmentHashesObject,
18 ActivityPlaylistUrlObject,
19 ActivityTagObject,
20 ActivityUrlObject,
21 ActivityVideoUrlObject,
22 VideoObject,
23 VideoPrivacy,
24 VideoStreamingPlaylistType
25} from '@shared/models'
26import { VideoCaptionModel } from '@server/models/video/video-caption'
27
28function getThumbnailFromIcons (videoObject: VideoObject) {
29 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
30 // Fallback if there are not valid icons
31 if (validIcons.length === 0) validIcons = videoObject.icon
32
33 return minBy(validIcons, 'width')
34}
35
36function getPreviewFromIcons (videoObject: VideoObject) {
37 const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
38
39 return maxBy(validIcons, 'width')
40}
41
42function getTagsFromObject (videoObject: VideoObject) {
43 return videoObject.tag
44 .filter(isAPHashTagObject)
45 .map(t => t.name)
46}
47
48function getFileAttributesFromUrl (
49 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
50 urls: (ActivityTagObject | ActivityUrlObject)[]
51) {
52 const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
53
54 if (fileUrls.length === 0) return []
55
56 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
57 for (const fileUrl of fileUrls) {
58 // Fetch associated magnet uri
59 const magnet = urls.filter(isAPMagnetUrlObject)
60 .find(u => u.height === fileUrl.height)
61
62 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
63
64 const parsed = magnetUtil.decode(magnet.href)
65 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
66 throw new Error('Cannot parse magnet URI ' + magnet.href)
67 }
68
69 const torrentUrl = Array.isArray(parsed.xs)
70 ? parsed.xs[0]
71 : parsed.xs
72
73 // Fetch associated metadata url, if any
74 const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
75 .find(u => {
76 return u.height === fileUrl.height &&
77 u.fps === fileUrl.fps &&
78 u.rel.includes(fileUrl.mediaType)
79 })
80
81 const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
82 const resolution = fileUrl.height
83 const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
84 const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
85
86 const attribute = {
87 extname,
88 infoHash: parsed.infoHash,
89 resolution,
90 size: fileUrl.size,
91 fps: fileUrl.fps || -1,
92 metadataUrl: metadata?.href,
93
94 // Use the name of the remote file because we don't proxify video file requests
95 filename: basename(fileUrl.href),
96 fileUrl: fileUrl.href,
97
98 torrentUrl,
99 // Use our own torrent name since we proxify torrent requests
100 torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution),
101
102 // This is a video file owned by a video or by a streaming playlist
103 videoId,
104 videoStreamingPlaylistId
105 }
106
107 attributes.push(attribute)
108 }
109
110 return attributes
111}
112
113function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) {
114 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
115 if (playlistUrls.length === 0) return []
116
117 const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
118 for (const playlistUrlObject of playlistUrls) {
119 const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
120
121 let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
122
123 // FIXME: backward compatibility introduced in v2.1.0
124 if (files.length === 0) files = videoFiles
125
126 if (!segmentsSha256UrlObject) {
127 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
128 continue
129 }
130
131 const attribute = {
132 type: VideoStreamingPlaylistType.HLS,
133 playlistUrl: playlistUrlObject.href,
134 segmentsSha256Url: segmentsSha256UrlObject.href,
135 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
136 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
137 videoId: video.id,
138
139 tagAPObject: playlistUrlObject.tag
140 }
141
142 attributes.push(attribute)
143 }
144
145 return attributes
146}
147
148function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) {
149 return {
150 saveReplay: videoObject.liveSaveReplay,
151 permanentLive: videoObject.permanentLive,
152 videoId: video.id
153 }
154}
155
156function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObject) {
157 return videoObject.subtitleLanguage.map(c => ({
158 videoId: video.id,
159 filename: VideoCaptionModel.generateCaptionName(c.identifier),
160 language: c.identifier,
161 fileUrl: c.url
162 }))
163}
164
165function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
166 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
167 ? VideoPrivacy.PUBLIC
168 : VideoPrivacy.UNLISTED
169
170 const duration = videoObject.duration.replace(/[^\d]+/, '')
171 const language = videoObject.language?.identifier
172
173 const category = videoObject.category
174 ? parseInt(videoObject.category.identifier, 10)
175 : undefined
176
177 const licence = videoObject.licence
178 ? parseInt(videoObject.licence.identifier, 10)
179 : undefined
180
181 const description = videoObject.content || null
182 const support = videoObject.support || null
183
184 return {
185 name: videoObject.name,
186 uuid: videoObject.uuid,
187 url: videoObject.id,
188 category,
189 licence,
190 language,
191 description,
192 support,
193 nsfw: videoObject.sensitive,
194 commentsEnabled: videoObject.commentsEnabled,
195 downloadEnabled: videoObject.downloadEnabled,
196 waitTranscoding: videoObject.waitTranscoding,
197 isLive: videoObject.isLiveBroadcast,
198 state: videoObject.state,
199 channelId: videoChannel.id,
200 duration: parseInt(duration, 10),
201 createdAt: new Date(videoObject.published),
202 publishedAt: new Date(videoObject.published),
203
204 originallyPublishedAt: videoObject.originallyPublishedAt
205 ? new Date(videoObject.originallyPublishedAt)
206 : null,
207
208 updatedAt: new Date(videoObject.updated),
209 views: videoObject.views,
210 likes: 0,
211 dislikes: 0,
212 remote: true,
213 privacy
214 }
215}
216
217// ---------------------------------------------------------------------------
218
219export {
220 getThumbnailFromIcons,
221 getPreviewFromIcons,
222
223 getTagsFromObject,
224
225 getFileAttributesFromUrl,
226 getStreamingPlaylistAttributesFromObject,
227
228 getLiveAttributesFromObject,
229 getCaptionAttributesFromObject,
230
231 getVideoAttributesFromObject
232}
233
234// ---------------------------------------------------------------------------
235
236function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
237 const urlMediaType = url.mediaType
238
239 return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
240}
241
242function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject {
243 return url && url.mediaType === 'application/x-mpegURL'
244}
245
246function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
247 return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
248}
249
250function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
251 return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
252}
253
254function isAPHashTagObject (url: any): url is ActivityHashTagObject {
255 return url && url.type === 'Hashtag'
256}
diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts
new file mode 100644
index 000000000..1c5fc4f84
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/trackers.ts
@@ -0,0 +1,43 @@
1import { Transaction } from 'sequelize/types'
2import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
3import { isAPVideoTrackerUrlObject } from '@server/helpers/custom-validators/activitypub/videos'
4import { isArray } from '@server/helpers/custom-validators/misc'
5import { REMOTE_SCHEME } from '@server/initializers/constants'
6import { TrackerModel } from '@server/models/server/tracker'
7import { MVideo, MVideoWithHost } from '@server/types/models'
8import { ActivityTrackerUrlObject, VideoObject } from '@shared/models'
9
10function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
11 let wsFound = false
12
13 const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u))
14 .map((u: ActivityTrackerUrlObject) => {
15 if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true
16
17 return u.href
18 })
19
20 if (wsFound) return trackers
21
22 return [
23 buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS),
24 buildRemoteVideoBaseUrl(video, '/tracker/announce')
25 ]
26}
27
28async function setVideoTrackers (options: {
29 video: MVideo
30 trackers: string[]
31 transaction: Transaction
32}) {
33 const { video, trackers, transaction } = options
34
35 const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction)
36
37 await video.$set('Trackers', trackerInstances, { transaction })
38}
39
40export {
41 getTrackerUrls,
42 setVideoTrackers
43}
diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts
new file mode 100644
index 000000000..dba3e9480
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/url-to-object.ts
@@ -0,0 +1,25 @@
1import { checkUrlsSameHost } from '@server/helpers/activitypub'
2import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos'
3import { logger, loggerTagsFactory } from '@server/helpers/logger'
4import { doJSONRequest } from '@server/helpers/requests'
5import { VideoObject } from '@shared/models'
6
7const lTags = loggerTagsFactory('ap', 'video')
8
9async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
10 logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl))
11
12 const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
13
14 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
15 logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) })
16
17 return { statusCode, videoObject: undefined }
18 }
19
20 return { statusCode, videoObject: body }
21}
22
23export {
24 fetchRemoteVideo
25}
diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts
new file mode 100644
index 000000000..c4e101005
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/video-sync-attributes.ts
@@ -0,0 +1,94 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { JobQueue } from '@server/lib/job-queue'
3import { AccountVideoRateModel } from '@server/models/account/account-video-rate'
4import { VideoCommentModel } from '@server/models/video/video-comment'
5import { VideoShareModel } from '@server/models/video/video-share'
6import { MVideo } from '@server/types/models'
7import { ActivitypubHttpFetcherPayload, VideoObject } from '@shared/models'
8import { crawlCollectionPage } from '../../crawl'
9import { addVideoShares } from '../../share'
10import { addVideoComments } from '../../video-comments'
11import { createRates } from '../../video-rates'
12
13const lTags = loggerTagsFactory('ap', 'video')
14
15type SyncParam = {
16 likes: boolean
17 dislikes: boolean
18 shares: boolean
19 comments: boolean
20 thumbnail: boolean
21 refreshVideo?: boolean
22}
23
24async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoObject, syncParam: SyncParam) {
25 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
26
27 await syncRates('like', video, fetchedVideo, syncParam.likes)
28 await syncRates('dislike', video, fetchedVideo, syncParam.dislikes)
29
30 await syncShares(video, fetchedVideo, syncParam.shares)
31
32 await syncComments(video, fetchedVideo, syncParam.comments)
33}
34
35// ---------------------------------------------------------------------------
36
37export {
38 SyncParam,
39 syncVideoExternalAttributes
40}
41
42// ---------------------------------------------------------------------------
43
44function createJob (payload: ActivitypubHttpFetcherPayload) {
45 return JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
46}
47
48function syncRates (type: 'like' | 'dislike', video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
49 const uri = type === 'like'
50 ? fetchedVideo.likes
51 : fetchedVideo.dislikes
52
53 if (!isSync) {
54 const jobType = type === 'like'
55 ? 'video-likes'
56 : 'video-dislikes'
57
58 return createJob({ uri, videoId: video.id, type: jobType })
59 }
60
61 const handler = items => createRates(items, video, type)
62 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, type, crawlStartDate)
63
64 return crawlCollectionPage<string>(uri, handler, cleaner)
65 .catch(err => logger.error('Cannot add rate of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) }))
66}
67
68function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
69 const uri = fetchedVideo.shares
70
71 if (!isSync) {
72 return createJob({ uri, videoId: video.id, type: 'video-shares' })
73 }
74
75 const handler = items => addVideoShares(items, video)
76 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
77
78 return crawlCollectionPage<string>(uri, handler, cleaner)
79 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) }))
80}
81
82function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
83 const uri = fetchedVideo.comments
84
85 if (!isSync) {
86 return createJob({ uri, videoId: video.id, type: 'video-comments' })
87 }
88
89 const handler = items => addVideoComments(items)
90 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
91
92 return crawlCollectionPage<string>(uri, handler, cleaner)
93 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) }))
94}
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts
new file mode 100644
index 000000000..157569414
--- /dev/null
+++ b/server/lib/activitypub/videos/updater.ts
@@ -0,0 +1,166 @@
1import { Transaction } from 'sequelize/types'
2import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils'
3import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
4import { Notifier } from '@server/lib/notifier'
5import { PeerTubeSocket } from '@server/lib/peertube-socket'
6import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
7import { VideoLiveModel } from '@server/models/video/video-live'
8import { MActor, MChannelAccountLight, MChannelId, MVideoAccountLightBlacklistAllFiles, MVideoFullLight } from '@server/types/models'
9import { VideoObject, VideoPrivacy } from '@shared/models'
10import { APVideoAbstractBuilder, getVideoAttributesFromObject } from './shared'
11
12export class APVideoUpdater extends APVideoAbstractBuilder {
13 private readonly wasPrivateVideo: boolean
14 private readonly wasUnlistedVideo: boolean
15
16 private readonly videoFieldsSave: any
17
18 private readonly oldVideoChannel: MChannelAccountLight
19
20 protected lTags: LoggerTagsFn
21
22 constructor (
23 protected readonly videoObject: VideoObject,
24 private readonly video: MVideoAccountLightBlacklistAllFiles
25 ) {
26 super()
27
28 this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE
29 this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED
30
31 this.oldVideoChannel = this.video.VideoChannel
32
33 this.videoFieldsSave = this.video.toJSON()
34
35 this.lTags = loggerTagsFactory('ap', 'video', 'update', video.uuid, video.url)
36 }
37
38 async update (overrideTo?: string[]) {
39 logger.debug(
40 'Updating remote video "%s".', this.videoObject.uuid,
41 { videoObject: this.videoObject, ...this.lTags() }
42 )
43
44 try {
45 const channelActor = await this.getOrCreateVideoChannelFromVideoObject()
46
47 const thumbnailModel = await this.tryToGenerateThumbnail(this.video)
48
49 this.checkChannelUpdateOrThrow(channelActor)
50
51 const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo)
52
53 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel)
54
55 await runInReadCommittedTransaction(async t => {
56 await this.setWebTorrentFiles(videoUpdated, t)
57 await this.setStreamingPlaylists(videoUpdated, t)
58 })
59
60 await Promise.all([
61 runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
62 runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
63 this.setOrDeleteLive(videoUpdated),
64 this.setPreview(videoUpdated)
65 ])
66
67 await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t))
68
69 await autoBlacklistVideoIfNeeded({
70 video: videoUpdated,
71 user: undefined,
72 isRemote: true,
73 isNew: false,
74 transaction: undefined
75 })
76
77 // Notify our users?
78 if (this.wasPrivateVideo || this.wasUnlistedVideo) {
79 Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated)
80 }
81
82 if (videoUpdated.isLive) {
83 PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated)
84 PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
85 }
86
87 logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags())
88
89 return videoUpdated
90 } catch (err) {
91 this.catchUpdateError(err)
92 }
93 }
94
95 // Check we can update the channel: we trust the remote server
96 private checkChannelUpdateOrThrow (newChannelActor: MActor) {
97 if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) {
98 throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
99 }
100
101 if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) {
102 throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`)
103 }
104 }
105
106 private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) {
107 const to = overrideTo || this.videoObject.to
108 const videoData = getVideoAttributesFromObject(channel, this.videoObject, to)
109 this.video.name = videoData.name
110 this.video.uuid = videoData.uuid
111 this.video.url = videoData.url
112 this.video.category = videoData.category
113 this.video.licence = videoData.licence
114 this.video.language = videoData.language
115 this.video.description = videoData.description
116 this.video.support = videoData.support
117 this.video.nsfw = videoData.nsfw
118 this.video.commentsEnabled = videoData.commentsEnabled
119 this.video.downloadEnabled = videoData.downloadEnabled
120 this.video.waitTranscoding = videoData.waitTranscoding
121 this.video.state = videoData.state
122 this.video.duration = videoData.duration
123 this.video.createdAt = videoData.createdAt
124 this.video.publishedAt = videoData.publishedAt
125 this.video.originallyPublishedAt = videoData.originallyPublishedAt
126 this.video.privacy = videoData.privacy
127 this.video.channelId = videoData.channelId
128 this.video.views = videoData.views
129 this.video.isLive = videoData.isLive
130
131 // Ensures we update the updatedAt attribute, even if main attributes did not change
132 this.video.changed('updatedAt', true)
133
134 return this.video.save({ transaction }) as Promise<MVideoFullLight>
135 }
136
137 private async setCaptions (videoUpdated: MVideoFullLight, t: Transaction) {
138 await this.insertOrReplaceCaptions(videoUpdated, t)
139 }
140
141 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
142 if (!this.video.isLive) return
143
144 if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction)
145
146 // Delete existing live if it exists
147 await VideoLiveModel.destroy({
148 where: {
149 videoId: this.video.id
150 },
151 transaction
152 })
153
154 videoUpdated.VideoLive = null
155 }
156
157 private catchUpdateError (err: Error) {
158 if (this.video !== undefined && this.videoFieldsSave !== undefined) {
159 resetSequelizeInstance(this.video, this.videoFieldsSave)
160 }
161
162 // This is just a debug because we will retry the insert
163 logger.debug('Cannot update the remote video.', { err, ...this.lTags() })
164 throw err
165 }
166}
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-02 14:38:28 +0000