aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-06-02 09:35:01 +0200
committerChocobozzz <me@florianbigard.com>2021-06-02 16:57:53 +0200
commit69290ab37b8aead01477b9b98fdfad0e69b08582 (patch)
tree4b93d349dfce5014925e7f060b3b158ac9b3bbc2 /server/lib
parent81628e5069e0168b11857f276fe8e03b93102dde (diff)
downloadPeerTube-69290ab37b8aead01477b9b98fdfad0e69b08582.tar.gz
PeerTube-69290ab37b8aead01477b9b98fdfad0e69b08582.tar.zst
PeerTube-69290ab37b8aead01477b9b98fdfad0e69b08582.zip
Refactor AP video update
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/process/process-update.ts9
-rw-r--r--server/lib/activitypub/videos.ts931
-rw-r--r--server/lib/activitypub/videos/federate.ts36
-rw-r--r--server/lib/activitypub/videos/fetch.ts202
-rw-r--r--server/lib/activitypub/videos/index.ts3
-rw-r--r--server/lib/activitypub/videos/shared/index.ts4
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts234
-rw-r--r--server/lib/activitypub/videos/shared/trackers.ts43
-rw-r--r--server/lib/activitypub/videos/shared/video-create.ts167
-rw-r--r--server/lib/activitypub/videos/shared/video-sync-attributes.ts75
-rw-r--r--server/lib/activitypub/videos/update.ts293
-rw-r--r--server/lib/job-queue/handlers/activitypub-refresher.ts2
12 files changed, 1062 insertions, 937 deletions
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index 6cd9d0fba..de1ff5d90 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -17,7 +17,7 @@ import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } f
17import { createOrUpdateCacheFile } from '../cache-file' 17import { createOrUpdateCacheFile } from '../cache-file'
18import { createOrUpdateVideoPlaylist } from '../playlist' 18import { createOrUpdateVideoPlaylist } from '../playlist'
19import { forwardVideoRelatedActivity } from '../send/utils' 19import { forwardVideoRelatedActivity } from '../send/utils'
20import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' 20import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, APVideoUpdater } from '../videos'
21 21
22async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { 22async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) {
23 const { activity, byActor } = options 23 const { activity, byActor } = options
@@ -77,14 +77,13 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd
77 const account = actor.Account as MAccountIdActor 77 const account = actor.Account as MAccountIdActor
78 account.Actor = actor 78 account.Actor = actor
79 79
80 const updateOptions = { 80 const updater = new APVideoUpdater({
81 video, 81 video,
82 videoObject, 82 videoObject,
83 account,
84 channel: channelActor.VideoChannel, 83 channel: channelActor.VideoChannel,
85 overrideTo: activity.to 84 overrideTo: activity.to
86 } 85 })
87 return updateVideoFromAP(updateOptions) 86 return updater.update()
88} 87}
89 88
90async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) { 89async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) {
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
deleted file mode 100644
index 127a0dd8a..000000000
--- a/server/lib/activitypub/videos.ts
+++ /dev/null
@@ -1,931 +0,0 @@
1import * as Bluebird from 'bluebird'
2import { maxBy, minBy } from 'lodash'
3import * as magnetUtil from 'magnet-uri'
4import { basename } from 'path'
5import { Transaction } from 'sequelize/types'
6import { TrackerModel } from '@server/models/server/tracker'
7import { VideoLiveModel } from '@server/models/video/video-live'
8import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
9import {
10 ActivityHashTagObject,
11 ActivityMagnetUrlObject,
12 ActivityPlaylistSegmentHashesObject,
13 ActivityPlaylistUrlObject,
14 ActivitypubHttpFetcherPayload,
15 ActivityTagObject,
16 ActivityUrlObject,
17 ActivityVideoUrlObject
18} from '../../../shared/index'
19import { ActivityTrackerUrlObject, VideoObject } from '../../../shared/models/activitypub/objects'
20import { VideoPrivacy } from '../../../shared/models/videos'
21import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
22import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
23import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
24import {
25 isAPVideoFileUrlMetadataObject,
26 isAPVideoTrackerUrlObject,
27 sanitizeAndCheckVideoTorrentObject
28} from '../../helpers/custom-validators/activitypub/videos'
29import { isArray } from '../../helpers/custom-validators/misc'
30import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
31import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
32import { logger } from '../../helpers/logger'
33import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
34import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
35import {
36 ACTIVITY_PUB,
37 MIMETYPES,
38 P2P_MEDIA_LOADER_PEER_VERSION,
39 PREVIEWS_SIZE,
40 REMOTE_SCHEME,
41 THUMBNAILS_SIZE
42} from '../../initializers/constants'
43import { sequelizeTypescript } from '../../initializers/database'
44import { AccountVideoRateModel } from '../../models/account/account-video-rate'
45import { VideoModel } from '../../models/video/video'
46import { VideoCaptionModel } from '../../models/video/video-caption'
47import { VideoCommentModel } from '../../models/video/video-comment'
48import { VideoFileModel } from '../../models/video/video-file'
49import { VideoShareModel } from '../../models/video/video-share'
50import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
51import {
52 MAccountIdActor,
53 MChannelAccountLight,
54 MChannelDefault,
55 MChannelId,
56 MStreamingPlaylist,
57 MStreamingPlaylistFilesVideo,
58 MStreamingPlaylistVideo,
59 MVideo,
60 MVideoAccountLight,
61 MVideoAccountLightBlacklistAllFiles,
62 MVideoAP,
63 MVideoAPWithoutCaption,
64 MVideoCaption,
65 MVideoFile,
66 MVideoFullLight,
67 MVideoId,
68 MVideoImmutable,
69 MVideoThumbnail,
70 MVideoWithHost
71} from '../../types/models'
72import { MThumbnail } from '../../types/models/video/thumbnail'
73import { FilteredModelAttributes } from '../../types/sequelize'
74import { ActorFollowScoreCache } from '../files-cache'
75import { JobQueue } from '../job-queue'
76import { Notifier } from '../notifier'
77import { PeerTubeSocket } from '../peertube-socket'
78import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
79import { setVideoTags } from '../video'
80import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
81import { generateTorrentFileName } from '../video-paths'
82import { getOrCreateActorAndServerAndModel } from './actor'
83import { crawlCollectionPage } from './crawl'
84import { sendCreateVideo, sendUpdateVideo } from './send'
85import { addVideoShares, shareVideoByServerAndChannel } from './share'
86import { addVideoComments } from './video-comments'
87import { createRates } from './video-rates'
88
89async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
90 const video = videoArg as MVideoAP
91
92 if (
93 // Check this is not a blacklisted video, or unfederated blacklisted video
94 (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
95 // Check the video is public/unlisted and published
96 video.hasPrivacyForFederation() && video.hasStateForFederation()
97 ) {
98 // Fetch more attributes that we will need to serialize in AP object
99 if (isArray(video.VideoCaptions) === false) {
100 video.VideoCaptions = await video.$get('VideoCaptions', {
101 attributes: [ 'filename', 'language' ],
102 transaction
103 })
104 }
105
106 if (isNewVideo) {
107 // Now we'll add the video's meta data to our followers
108 await sendCreateVideo(video, transaction)
109 await shareVideoByServerAndChannel(video, transaction)
110 } else {
111 await sendUpdateVideo(video, transaction)
112 }
113 }
114}
115
116async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
117 logger.info('Fetching remote video %s.', videoUrl)
118
119 const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
120
121 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
122 logger.debug('Remote video JSON is not valid.', { body })
123 return { statusCode, videoObject: undefined }
124 }
125
126 return { statusCode, videoObject: body }
127}
128
129async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
130 const host = video.VideoChannel.Account.Actor.Server.host
131 const path = video.getDescriptionAPIPath()
132 const url = REMOTE_SCHEME.HTTP + '://' + host + path
133
134 const { body } = await doJSONRequest<any>(url)
135 return body.description || ''
136}
137
138function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) {
139 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
140 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
141
142 if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
143 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
144 }
145
146 return getOrCreateActorAndServerAndModel(channel.id, 'all')
147}
148
149type SyncParam = {
150 likes: boolean
151 dislikes: boolean
152 shares: boolean
153 comments: boolean
154 thumbnail: boolean
155 refreshVideo?: boolean
156}
157async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoObject, syncParam: SyncParam) {
158 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
159
160 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
161
162 if (syncParam.likes === true) {
163 const handler = items => createRates(items, video, 'like')
164 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
165
166 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
167 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes }))
168 } else {
169 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
170 }
171
172 if (syncParam.dislikes === true) {
173 const handler = items => createRates(items, video, 'dislike')
174 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
175
176 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
177 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes }))
178 } else {
179 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
180 }
181
182 if (syncParam.shares === true) {
183 const handler = items => addVideoShares(items, video)
184 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
185
186 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
187 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
188 } else {
189 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
190 }
191
192 if (syncParam.comments === true) {
193 const handler = items => addVideoComments(items)
194 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
195
196 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
197 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
198 } else {
199 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
200 }
201
202 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
203}
204
205type GetVideoResult <T> = Promise<{
206 video: T
207 created: boolean
208 autoBlacklisted?: boolean
209}>
210
211type GetVideoParamAll = {
212 videoObject: { id: string } | string
213 syncParam?: SyncParam
214 fetchType?: 'all'
215 allowRefresh?: boolean
216}
217
218type GetVideoParamImmutable = {
219 videoObject: { id: string } | string
220 syncParam?: SyncParam
221 fetchType: 'only-immutable-attributes'
222 allowRefresh: false
223}
224
225type GetVideoParamOther = {
226 videoObject: { id: string } | string
227 syncParam?: SyncParam
228 fetchType?: 'all' | 'only-video'
229 allowRefresh?: boolean
230}
231
232function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
233function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
234function getOrCreateVideoAndAccountAndChannel (
235 options: GetVideoParamOther
236): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
237async function getOrCreateVideoAndAccountAndChannel (
238 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
239): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
240 // Default params
241 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
242 const fetchType = options.fetchType || 'all'
243 const allowRefresh = options.allowRefresh !== false
244
245 // Get video url
246 const videoUrl = getAPId(options.videoObject)
247 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
248
249 if (videoFromDatabase) {
250 // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
251 if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
252 const refreshOptions = {
253 video: videoFromDatabase as MVideoThumbnail,
254 fetchedType: fetchType,
255 syncParam
256 }
257
258 if (syncParam.refreshVideo === true) {
259 videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
260 } else {
261 await JobQueue.Instance.createJobWithPromise({
262 type: 'activitypub-refresher',
263 payload: { type: 'video', url: videoFromDatabase.url }
264 })
265 }
266 }
267
268 return { video: videoFromDatabase, created: false }
269 }
270
271 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
272 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
273
274 const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
275 const videoChannel = actor.VideoChannel
276
277 try {
278 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
279
280 await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
281
282 return { video: videoCreated, created: true, autoBlacklisted }
283 } catch (err) {
284 // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video
285 if (err.name === 'SequelizeUniqueConstraintError') {
286 const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType)
287 if (fallbackVideo) return { video: fallbackVideo, created: false }
288 }
289
290 throw err
291 }
292}
293
294async function updateVideoFromAP (options: {
295 video: MVideoAccountLightBlacklistAllFiles
296 videoObject: VideoObject
297 account: MAccountIdActor
298 channel: MChannelDefault
299 overrideTo?: string[]
300}) {
301 const { video, videoObject, account, channel, overrideTo } = options
302
303 logger.debug('Updating remote video "%s".', options.videoObject.uuid, { videoObject: options.videoObject, account, channel })
304
305 let videoFieldsSave: any
306 const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE
307 const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED
308
309 try {
310 let thumbnailModel: MThumbnail
311
312 try {
313 thumbnailModel = await createVideoMiniatureFromUrl({
314 downloadUrl: getThumbnailFromIcons(videoObject).url,
315 video,
316 type: ThumbnailType.MINIATURE
317 })
318 } catch (err) {
319 logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
320 }
321
322 const videoUpdated = await sequelizeTypescript.transaction(async t => {
323 const sequelizeOptions = { transaction: t }
324
325 videoFieldsSave = video.toJSON()
326
327 // Check we can update the channel: we trust the remote server
328 const oldVideoChannel = video.VideoChannel
329
330 if (!oldVideoChannel.Actor.serverId || !channel.Actor.serverId) {
331 throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
332 }
333
334 if (oldVideoChannel.Actor.serverId !== channel.Actor.serverId) {
335 throw new Error('New channel ' + channel.Actor.url + ' is not on the same server than new channel ' + oldVideoChannel.Actor.url)
336 }
337
338 const to = overrideTo || videoObject.to
339 const videoData = videoActivityObjectToDBAttributes(channel, videoObject, to)
340 video.name = videoData.name
341 video.uuid = videoData.uuid
342 video.url = videoData.url
343 video.category = videoData.category
344 video.licence = videoData.licence
345 video.language = videoData.language
346 video.description = videoData.description
347 video.support = videoData.support
348 video.nsfw = videoData.nsfw
349 video.commentsEnabled = videoData.commentsEnabled
350 video.downloadEnabled = videoData.downloadEnabled
351 video.waitTranscoding = videoData.waitTranscoding
352 video.state = videoData.state
353 video.duration = videoData.duration
354 video.createdAt = videoData.createdAt
355 video.publishedAt = videoData.publishedAt
356 video.originallyPublishedAt = videoData.originallyPublishedAt
357 video.privacy = videoData.privacy
358 video.channelId = videoData.channelId
359 video.views = videoData.views
360 video.isLive = videoData.isLive
361
362 // Ensures we update the updated video attribute
363 video.changed('updatedAt', true)
364
365 const videoUpdated = await video.save(sequelizeOptions) as MVideoFullLight
366
367 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
368
369 const previewIcon = getPreviewFromIcons(videoObject)
370 if (videoUpdated.getPreview() && previewIcon) {
371 const previewModel = createPlaceholderThumbnail({
372 fileUrl: previewIcon.url,
373 video,
374 type: ThumbnailType.PREVIEW,
375 size: previewIcon
376 })
377 await videoUpdated.addAndSaveThumbnail(previewModel, t)
378 }
379
380 {
381 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url)
382 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
383
384 // Remove video files that do not exist anymore
385 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
386 await Promise.all(destroyTasks)
387
388 // Update or add other one
389 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
390 videoUpdated.VideoFiles = await Promise.all(upsertTasks)
391 }
392
393 {
394 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles)
395 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
396
397 // Remove video playlists that do not exist anymore
398 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
399 await Promise.all(destroyTasks)
400
401 let oldStreamingPlaylistFiles: MVideoFile[] = []
402 for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
403 oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
404 }
405
406 videoUpdated.VideoStreamingPlaylists = []
407
408 for (const playlistAttributes of streamingPlaylistAttributes) {
409 const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
410 .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo)
411 streamingPlaylistModel.Video = videoUpdated
412
413 const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
414 .map(a => new VideoFileModel(a))
415 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
416 await Promise.all(destroyTasks)
417
418 // Update or add other one
419 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
420 streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
421
422 videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
423 }
424 }
425
426 {
427 // Update Tags
428 const tags = videoObject.tag
429 .filter(isAPHashTagObject)
430 .map(tag => tag.name)
431 await setVideoTags({ video: videoUpdated, tags, transaction: t })
432 }
433
434 // Update trackers
435 {
436 const trackers = getTrackerUrls(videoObject, videoUpdated)
437 await setVideoTrackers({ video: videoUpdated, trackers, transaction: t })
438 }
439
440 {
441 // Update captions
442 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
443
444 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
445 const caption = new VideoCaptionModel({
446 videoId: videoUpdated.id,
447 filename: VideoCaptionModel.generateCaptionName(c.identifier),
448 language: c.identifier,
449 fileUrl: c.url
450 }) as MVideoCaption
451
452 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
453 })
454 await Promise.all(videoCaptionsPromises)
455 }
456
457 {
458 // Create or update existing live
459 if (video.isLive) {
460 const [ videoLive ] = await VideoLiveModel.upsert({
461 saveReplay: videoObject.liveSaveReplay,
462 permanentLive: videoObject.permanentLive,
463 videoId: video.id
464 }, { transaction: t, returning: true })
465
466 videoUpdated.VideoLive = videoLive
467 } else { // Delete existing live if it exists
468 await VideoLiveModel.destroy({
469 where: {
470 videoId: video.id
471 },
472 transaction: t
473 })
474
475 videoUpdated.VideoLive = null
476 }
477 }
478
479 return videoUpdated
480 })
481
482 await autoBlacklistVideoIfNeeded({
483 video: videoUpdated,
484 user: undefined,
485 isRemote: true,
486 isNew: false,
487 transaction: undefined
488 })
489
490 // Notify our users?
491 if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated)
492
493 if (videoUpdated.isLive) {
494 PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated)
495 PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
496 }
497
498 logger.info('Remote video with uuid %s updated', videoObject.uuid)
499
500 return videoUpdated
501 } catch (err) {
502 if (video !== undefined && videoFieldsSave !== undefined) {
503 resetSequelizeInstance(video, videoFieldsSave)
504 }
505
506 // This is just a debug because we will retry the insert
507 logger.debug('Cannot update the remote video.', { err })
508 throw err
509 }
510}
511
512async function refreshVideoIfNeeded (options: {
513 video: MVideoThumbnail
514 fetchedType: VideoFetchByUrlType
515 syncParam: SyncParam
516}): Promise<MVideoThumbnail> {
517 if (!options.video.isOutdated()) return options.video
518
519 // We need more attributes if the argument video was fetched with not enough joints
520 const video = options.fetchedType === 'all'
521 ? options.video as MVideoAccountLightBlacklistAllFiles
522 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
523
524 try {
525 const { videoObject } = await fetchRemoteVideo(video.url)
526
527 if (videoObject === undefined) {
528 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
529
530 await video.setAsRefreshed()
531 return video
532 }
533
534 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
535
536 const updateOptions = {
537 video,
538 videoObject,
539 account: channelActor.VideoChannel.Account,
540 channel: channelActor.VideoChannel
541 }
542 await updateVideoFromAP(updateOptions)
543 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
544
545 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
546
547 return video
548 } catch (err) {
549 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
550 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
551
552 // Video does not exist anymore
553 await video.destroy()
554 return undefined
555 }
556
557 logger.warn('Cannot refresh video %s.', options.video.url, { err })
558
559 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
560
561 // Don't refresh in loop
562 await video.setAsRefreshed()
563 return video
564 }
565}
566
567export {
568 updateVideoFromAP,
569 refreshVideoIfNeeded,
570 federateVideoIfNeeded,
571 fetchRemoteVideo,
572 getOrCreateVideoAndAccountAndChannel,
573 fetchRemoteVideoDescription,
574 getOrCreateVideoChannelFromVideoObject
575}
576
577// ---------------------------------------------------------------------------
578
579function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
580 const urlMediaType = url.mediaType
581
582 return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
583}
584
585function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject {
586 return url && url.mediaType === 'application/x-mpegURL'
587}
588
589function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
590 return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
591}
592
593function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
594 return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
595}
596
597function isAPHashTagObject (url: any): url is ActivityHashTagObject {
598 return url && url.type === 'Hashtag'
599}
600
601async function createVideo (videoObject: VideoObject, channel: MChannelAccountLight, waitThumbnail = false) {
602 logger.debug('Adding remote video %s.', videoObject.id)
603
604 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
605 const video = VideoModel.build(videoData) as MVideoThumbnail
606
607 const promiseThumbnail = createVideoMiniatureFromUrl({
608 downloadUrl: getThumbnailFromIcons(videoObject).url,
609 video,
610 type: ThumbnailType.MINIATURE
611 }).catch(err => {
612 logger.error('Cannot create miniature from url.', { err })
613 return undefined
614 })
615
616 let thumbnailModel: MThumbnail
617 if (waitThumbnail === true) {
618 thumbnailModel = await promiseThumbnail
619 }
620
621 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
622 try {
623 const sequelizeOptions = { transaction: t }
624
625 const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
626 videoCreated.VideoChannel = channel
627
628 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
629
630 const previewIcon = getPreviewFromIcons(videoObject)
631 if (previewIcon) {
632 const previewModel = createPlaceholderThumbnail({
633 fileUrl: previewIcon.url,
634 video: videoCreated,
635 type: ThumbnailType.PREVIEW,
636 size: previewIcon
637 })
638
639 await videoCreated.addAndSaveThumbnail(previewModel, t)
640 }
641
642 // Process files
643 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
644
645 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
646 const videoFiles = await Promise.all(videoFilePromises)
647
648 const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
649 videoCreated.VideoStreamingPlaylists = []
650
651 for (const playlistAttributes of streamingPlaylistsAttributes) {
652 const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo
653 playlist.Video = videoCreated
654
655 const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject)
656 const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
657 playlist.VideoFiles = await Promise.all(videoFilePromises)
658
659 videoCreated.VideoStreamingPlaylists.push(playlist)
660 }
661
662 // Process tags
663 const tags = videoObject.tag
664 .filter(isAPHashTagObject)
665 .map(t => t.name)
666 await setVideoTags({ video: videoCreated, tags, transaction: t })
667
668 // Process captions
669 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
670 const caption = new VideoCaptionModel({
671 videoId: videoCreated.id,
672 filename: VideoCaptionModel.generateCaptionName(c.identifier),
673 language: c.identifier,
674 fileUrl: c.url
675 }) as MVideoCaption
676
677 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
678 })
679 await Promise.all(videoCaptionsPromises)
680
681 // Process trackers
682 {
683 const trackers = getTrackerUrls(videoObject, videoCreated)
684 await setVideoTrackers({ video: videoCreated, trackers, transaction: t })
685 }
686
687 videoCreated.VideoFiles = videoFiles
688
689 if (videoCreated.isLive) {
690 const videoLive = new VideoLiveModel({
691 streamKey: null,
692 saveReplay: videoObject.liveSaveReplay,
693 permanentLive: videoObject.permanentLive,
694 videoId: videoCreated.id
695 })
696
697 videoCreated.VideoLive = await videoLive.save({ transaction: t })
698 }
699
700 // We added a video in this channel, set it as updated
701 await channel.setAsUpdated(t)
702
703 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
704 video: videoCreated,
705 user: undefined,
706 isRemote: true,
707 isNew: true,
708 transaction: t
709 })
710
711 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
712
713 return { autoBlacklisted, videoCreated }
714 } catch (err) {
715 // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
716 // Remove thumbnail
717 if (thumbnailModel) await thumbnailModel.removeThumbnail()
718
719 throw err
720 }
721 })
722
723 if (waitThumbnail === false) {
724 // Error is already caught above
725 // eslint-disable-next-line @typescript-eslint/no-floating-promises
726 promiseThumbnail.then(thumbnailModel => {
727 if (!thumbnailModel) return
728
729 thumbnailModel = videoCreated.id
730
731 return thumbnailModel.save()
732 })
733 }
734
735 return { autoBlacklisted, videoCreated }
736}
737
738function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
739 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
740 ? VideoPrivacy.PUBLIC
741 : VideoPrivacy.UNLISTED
742
743 const duration = videoObject.duration.replace(/[^\d]+/, '')
744 const language = videoObject.language?.identifier
745
746 const category = videoObject.category
747 ? parseInt(videoObject.category.identifier, 10)
748 : undefined
749
750 const licence = videoObject.licence
751 ? parseInt(videoObject.licence.identifier, 10)
752 : undefined
753
754 const description = videoObject.content || null
755 const support = videoObject.support || null
756
757 return {
758 name: videoObject.name,
759 uuid: videoObject.uuid,
760 url: videoObject.id,
761 category,
762 licence,
763 language,
764 description,
765 support,
766 nsfw: videoObject.sensitive,
767 commentsEnabled: videoObject.commentsEnabled,
768 downloadEnabled: videoObject.downloadEnabled,
769 waitTranscoding: videoObject.waitTranscoding,
770 isLive: videoObject.isLiveBroadcast,
771 state: videoObject.state,
772 channelId: videoChannel.id,
773 duration: parseInt(duration, 10),
774 createdAt: new Date(videoObject.published),
775 publishedAt: new Date(videoObject.published),
776
777 originallyPublishedAt: videoObject.originallyPublishedAt
778 ? new Date(videoObject.originallyPublishedAt)
779 : null,
780
781 updatedAt: new Date(videoObject.updated),
782 views: videoObject.views,
783 likes: 0,
784 dislikes: 0,
785 remote: true,
786 privacy
787 }
788}
789
790function videoFileActivityUrlToDBAttributes (
791 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
792 urls: (ActivityTagObject | ActivityUrlObject)[]
793) {
794 const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
795
796 if (fileUrls.length === 0) return []
797
798 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
799 for (const fileUrl of fileUrls) {
800 // Fetch associated magnet uri
801 const magnet = urls.filter(isAPMagnetUrlObject)
802 .find(u => u.height === fileUrl.height)
803
804 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
805
806 const parsed = magnetUtil.decode(magnet.href)
807 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
808 throw new Error('Cannot parse magnet URI ' + magnet.href)
809 }
810
811 const torrentUrl = Array.isArray(parsed.xs)
812 ? parsed.xs[0]
813 : parsed.xs
814
815 // Fetch associated metadata url, if any
816 const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
817 .find(u => {
818 return u.height === fileUrl.height &&
819 u.fps === fileUrl.fps &&
820 u.rel.includes(fileUrl.mediaType)
821 })
822
823 const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
824 const resolution = fileUrl.height
825 const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
826 const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
827
828 const attribute = {
829 extname,
830 infoHash: parsed.infoHash,
831 resolution,
832 size: fileUrl.size,
833 fps: fileUrl.fps || -1,
834 metadataUrl: metadata?.href,
835
836 // Use the name of the remote file because we don't proxify video file requests
837 filename: basename(fileUrl.href),
838 fileUrl: fileUrl.href,
839
840 torrentUrl,
841 // Use our own torrent name since we proxify torrent requests
842 torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution),
843
844 // This is a video file owned by a video or by a streaming playlist
845 videoId,
846 videoStreamingPlaylistId
847 }
848
849 attributes.push(attribute)
850 }
851
852 return attributes
853}
854
855function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) {
856 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
857 if (playlistUrls.length === 0) return []
858
859 const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
860 for (const playlistUrlObject of playlistUrls) {
861 const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
862
863 let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
864
865 // FIXME: backward compatibility introduced in v2.1.0
866 if (files.length === 0) files = videoFiles
867
868 if (!segmentsSha256UrlObject) {
869 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
870 continue
871 }
872
873 const attribute = {
874 type: VideoStreamingPlaylistType.HLS,
875 playlistUrl: playlistUrlObject.href,
876 segmentsSha256Url: segmentsSha256UrlObject.href,
877 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
878 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
879 videoId: video.id,
880 tagAPObject: playlistUrlObject.tag
881 }
882
883 attributes.push(attribute)
884 }
885
886 return attributes
887}
888
889function getThumbnailFromIcons (videoObject: VideoObject) {
890 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
891 // Fallback if there are not valid icons
892 if (validIcons.length === 0) validIcons = videoObject.icon
893
894 return minBy(validIcons, 'width')
895}
896
897function getPreviewFromIcons (videoObject: VideoObject) {
898 const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
899
900 return maxBy(validIcons, 'width')
901}
902
903function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
904 let wsFound = false
905
906 const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u))
907 .map((u: ActivityTrackerUrlObject) => {
908 if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true
909
910 return u.href
911 })
912
913 if (wsFound) return trackers
914
915 return [
916 buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS),
917 buildRemoteVideoBaseUrl(video, '/tracker/announce')
918 ]
919}
920
921async function setVideoTrackers (options: {
922 video: MVideo
923 trackers: string[]
924 transaction?: Transaction
925}) {
926 const { video, trackers, transaction } = options
927
928 const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction)
929
930 await video.$set('Trackers', trackerInstances, { transaction })
931}
diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts
new file mode 100644
index 000000000..bd0c54b0c
--- /dev/null
+++ b/server/lib/activitypub/videos/federate.ts
@@ -0,0 +1,36 @@
1import { Transaction } from 'sequelize/types'
2import { isArray } from '@server/helpers/custom-validators/misc'
3import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models'
4import { sendCreateVideo, sendUpdateVideo } from '../send'
5import { shareVideoByServerAndChannel } from '../share'
6
7async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
8 const video = videoArg as MVideoAP
9
10 if (
11 // Check this is not a blacklisted video, or unfederated blacklisted video
12 (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
13 // Check the video is public/unlisted and published
14 video.hasPrivacyForFederation() && video.hasStateForFederation()
15 ) {
16 // Fetch more attributes that we will need to serialize in AP object
17 if (isArray(video.VideoCaptions) === false) {
18 video.VideoCaptions = await video.$get('VideoCaptions', {
19 attributes: [ 'filename', 'language' ],
20 transaction
21 })
22 }
23
24 if (isNewVideo) {
25 // Now we'll add the video's meta data to our followers
26 await sendCreateVideo(video, transaction)
27 await shareVideoByServerAndChannel(video, transaction)
28 } else {
29 await sendUpdateVideo(video, transaction)
30 }
31 }
32}
33
34export {
35 federateVideoIfNeeded
36}
diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts
new file mode 100644
index 000000000..fdcf4ee5c
--- /dev/null
+++ b/server/lib/activitypub/videos/fetch.ts
@@ -0,0 +1,202 @@
1import { checkUrlsSameHost, getAPId } from "@server/helpers/activitypub"
2import { sanitizeAndCheckVideoTorrentObject } from "@server/helpers/custom-validators/activitypub/videos"
3import { retryTransactionWrapper } from "@server/helpers/database-utils"
4import { logger } from "@server/helpers/logger"
5import { doJSONRequest, PeerTubeRequestError } from "@server/helpers/requests"
6import { fetchVideoByUrl, VideoFetchByUrlType } from "@server/helpers/video"
7import { REMOTE_SCHEME } from "@server/initializers/constants"
8import { ActorFollowScoreCache } from "@server/lib/files-cache"
9import { JobQueue } from "@server/lib/job-queue"
10import { VideoModel } from "@server/models/video/video"
11import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from "@server/types/models"
12import { HttpStatusCode } from "@shared/core-utils"
13import { VideoObject } from "@shared/models"
14import { getOrCreateActorAndServerAndModel } from "../actor"
15import { SyncParam, syncVideoExternalAttributes } from "./shared"
16import { createVideo } from "./shared/video-create"
17import { APVideoUpdater } from "./update"
18
19async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
20 logger.info('Fetching remote video %s.', videoUrl)
21
22 const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
23
24 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
25 logger.debug('Remote video JSON is not valid.', { body })
26 return { statusCode, videoObject: undefined }
27 }
28
29 return { statusCode, videoObject: body }
30}
31
32async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
33 const host = video.VideoChannel.Account.Actor.Server.host
34 const path = video.getDescriptionAPIPath()
35 const url = REMOTE_SCHEME.HTTP + '://' + host + path
36
37 const { body } = await doJSONRequest<any>(url)
38 return body.description || ''
39}
40
41function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) {
42 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
43 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
44
45 if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
46 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
47 }
48
49 return getOrCreateActorAndServerAndModel(channel.id, 'all')
50}
51
52type GetVideoResult <T> = Promise<{
53 video: T
54 created: boolean
55 autoBlacklisted?: boolean
56}>
57
58type GetVideoParamAll = {
59 videoObject: { id: string } | string
60 syncParam?: SyncParam
61 fetchType?: 'all'
62 allowRefresh?: boolean
63}
64
65type GetVideoParamImmutable = {
66 videoObject: { id: string } | string
67 syncParam?: SyncParam
68 fetchType: 'only-immutable-attributes'
69 allowRefresh: false
70}
71
72type GetVideoParamOther = {
73 videoObject: { id: string } | string
74 syncParam?: SyncParam
75 fetchType?: 'all' | 'only-video'
76 allowRefresh?: boolean
77}
78
79function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
80function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
81function getOrCreateVideoAndAccountAndChannel (
82 options: GetVideoParamOther
83): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
84async function getOrCreateVideoAndAccountAndChannel (
85 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
86): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
87 // Default params
88 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
89 const fetchType = options.fetchType || 'all'
90 const allowRefresh = options.allowRefresh !== false
91
92 // Get video url
93 const videoUrl = getAPId(options.videoObject)
94 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
95
96 if (videoFromDatabase) {
97 // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
98 if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
99 const refreshOptions = {
100 video: videoFromDatabase as MVideoThumbnail,
101 fetchedType: fetchType,
102 syncParam
103 }
104
105 if (syncParam.refreshVideo === true) {
106 videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
107 } else {
108 await JobQueue.Instance.createJobWithPromise({
109 type: 'activitypub-refresher',
110 payload: { type: 'video', url: videoFromDatabase.url }
111 })
112 }
113 }
114
115 return { video: videoFromDatabase, created: false }
116 }
117
118 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
119 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
120
121 const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
122 const videoChannel = actor.VideoChannel
123
124 try {
125 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
126
127 await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
128
129 return { video: videoCreated, created: true, autoBlacklisted }
130 } catch (err) {
131 // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video
132 if (err.name === 'SequelizeUniqueConstraintError') {
133 const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType)
134 if (fallbackVideo) return { video: fallbackVideo, created: false }
135 }
136
137 throw err
138 }
139}
140
141async function refreshVideoIfNeeded (options: {
142 video: MVideoThumbnail
143 fetchedType: VideoFetchByUrlType
144 syncParam: SyncParam
145}): Promise<MVideoThumbnail> {
146 if (!options.video.isOutdated()) return options.video
147
148 // We need more attributes if the argument video was fetched with not enough joints
149 const video = options.fetchedType === 'all'
150 ? options.video as MVideoAccountLightBlacklistAllFiles
151 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
152
153 try {
154 const { videoObject } = await fetchRemoteVideo(video.url)
155
156 if (videoObject === undefined) {
157 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
158
159 await video.setAsRefreshed()
160 return video
161 }
162
163 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
164
165 const videoUpdater = new APVideoUpdater({
166 video,
167 videoObject,
168 channel: channelActor.VideoChannel
169 })
170 await videoUpdater.update()
171
172 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
173
174 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
175
176 return video
177 } catch (err) {
178 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
179 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
180
181 // Video does not exist anymore
182 await video.destroy()
183 return undefined
184 }
185
186 logger.warn('Cannot refresh video %s.', options.video.url, { err })
187
188 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
189
190 // Don't refresh in loop
191 await video.setAsRefreshed()
192 return video
193 }
194}
195
196export {
197 fetchRemoteVideo,
198 fetchRemoteVideoDescription,
199 refreshVideoIfNeeded,
200 getOrCreateVideoChannelFromVideoObject,
201 getOrCreateVideoAndAccountAndChannel
202}
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts
new file mode 100644
index 000000000..0e126c85a
--- /dev/null
+++ b/server/lib/activitypub/videos/index.ts
@@ -0,0 +1,3 @@
1export * from './federate'
2export * from './fetch'
3export * from './update'
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts
new file mode 100644
index 000000000..4d24fbc6a
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/index.ts
@@ -0,0 +1,4 @@
1export * from './object-to-model-attributes'
2export * from './trackers'
3export * from './video-create'
4export * from './video-sync-attributes'
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
new file mode 100644
index 000000000..8a8105500
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -0,0 +1,234 @@
1import { maxBy, minBy } from 'lodash'
2import * as magnetUtil from 'magnet-uri'
3import { basename } from 'path'
4import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos'
5import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos'
6import { logger } from '@server/helpers/logger'
7import { getExtFromMimetype } from '@server/helpers/video'
8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { generateTorrentFileName } from '@server/lib/video-paths'
10import { VideoFileModel } from '@server/models/video/video-file'
11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
12import { FilteredModelAttributes } from '@server/types'
13import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
14import {
15 ActivityHashTagObject,
16 ActivityMagnetUrlObject,
17 ActivityPlaylistSegmentHashesObject,
18 ActivityPlaylistUrlObject,
19 ActivityTagObject,
20 ActivityUrlObject,
21 ActivityVideoUrlObject,
22 VideoObject,
23 VideoPrivacy,
24 VideoStreamingPlaylistType
25} from '@shared/models'
26
27function getThumbnailFromIcons (videoObject: VideoObject) {
28 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
29 // Fallback if there are not valid icons
30 if (validIcons.length === 0) validIcons = videoObject.icon
31
32 return minBy(validIcons, 'width')
33}
34
35function getPreviewFromIcons (videoObject: VideoObject) {
36 const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
37
38 return maxBy(validIcons, 'width')
39}
40
41function getTagsFromObject (videoObject: VideoObject) {
42 return videoObject.tag
43 .filter(isAPHashTagObject)
44 .map(t => t.name)
45}
46
47function videoFileActivityUrlToDBAttributes (
48 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
49 urls: (ActivityTagObject | ActivityUrlObject)[]
50) {
51 const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
52
53 if (fileUrls.length === 0) return []
54
55 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
56 for (const fileUrl of fileUrls) {
57 // Fetch associated magnet uri
58 const magnet = urls.filter(isAPMagnetUrlObject)
59 .find(u => u.height === fileUrl.height)
60
61 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
62
63 const parsed = magnetUtil.decode(magnet.href)
64 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
65 throw new Error('Cannot parse magnet URI ' + magnet.href)
66 }
67
68 const torrentUrl = Array.isArray(parsed.xs)
69 ? parsed.xs[0]
70 : parsed.xs
71
72 // Fetch associated metadata url, if any
73 const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
74 .find(u => {
75 return u.height === fileUrl.height &&
76 u.fps === fileUrl.fps &&
77 u.rel.includes(fileUrl.mediaType)
78 })
79
80 const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
81 const resolution = fileUrl.height
82 const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
83 const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
84
85 const attribute = {
86 extname,
87 infoHash: parsed.infoHash,
88 resolution,
89 size: fileUrl.size,
90 fps: fileUrl.fps || -1,
91 metadataUrl: metadata?.href,
92
93 // Use the name of the remote file because we don't proxify video file requests
94 filename: basename(fileUrl.href),
95 fileUrl: fileUrl.href,
96
97 torrentUrl,
98 // Use our own torrent name since we proxify torrent requests
99 torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution),
100
101 // This is a video file owned by a video or by a streaming playlist
102 videoId,
103 videoStreamingPlaylistId
104 }
105
106 attributes.push(attribute)
107 }
108
109 return attributes
110}
111
112function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) {
113 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
114 if (playlistUrls.length === 0) return []
115
116 const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
117 for (const playlistUrlObject of playlistUrls) {
118 const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
119
120 let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
121
122 // FIXME: backward compatibility introduced in v2.1.0
123 if (files.length === 0) files = videoFiles
124
125 if (!segmentsSha256UrlObject) {
126 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
127 continue
128 }
129
130 const attribute = {
131 type: VideoStreamingPlaylistType.HLS,
132 playlistUrl: playlistUrlObject.href,
133 segmentsSha256Url: segmentsSha256UrlObject.href,
134 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
135 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
136 videoId: video.id,
137 tagAPObject: playlistUrlObject.tag
138 }
139
140 attributes.push(attribute)
141 }
142
143 return attributes
144}
145
146function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
147 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
148 ? VideoPrivacy.PUBLIC
149 : VideoPrivacy.UNLISTED
150
151 const duration = videoObject.duration.replace(/[^\d]+/, '')
152 const language = videoObject.language?.identifier
153
154 const category = videoObject.category
155 ? parseInt(videoObject.category.identifier, 10)
156 : undefined
157
158 const licence = videoObject.licence
159 ? parseInt(videoObject.licence.identifier, 10)
160 : undefined
161
162 const description = videoObject.content || null
163 const support = videoObject.support || null
164
165 return {
166 name: videoObject.name,
167 uuid: videoObject.uuid,
168 url: videoObject.id,
169 category,
170 licence,
171 language,
172 description,
173 support,
174 nsfw: videoObject.sensitive,
175 commentsEnabled: videoObject.commentsEnabled,
176 downloadEnabled: videoObject.downloadEnabled,
177 waitTranscoding: videoObject.waitTranscoding,
178 isLive: videoObject.isLiveBroadcast,
179 state: videoObject.state,
180 channelId: videoChannel.id,
181 duration: parseInt(duration, 10),
182 createdAt: new Date(videoObject.published),
183 publishedAt: new Date(videoObject.published),
184
185 originallyPublishedAt: videoObject.originallyPublishedAt
186 ? new Date(videoObject.originallyPublishedAt)
187 : null,
188
189 updatedAt: new Date(videoObject.updated),
190 views: videoObject.views,
191 likes: 0,
192 dislikes: 0,
193 remote: true,
194 privacy
195 }
196}
197
198// ---------------------------------------------------------------------------
199
200export {
201 getThumbnailFromIcons,
202 getPreviewFromIcons,
203
204 getTagsFromObject,
205
206 videoActivityObjectToDBAttributes,
207
208 videoFileActivityUrlToDBAttributes,
209 streamingPlaylistActivityUrlToDBAttributes
210}
211
212// ---------------------------------------------------------------------------
213
214function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
215 const urlMediaType = url.mediaType
216
217 return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
218}
219
220function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject {
221 return url && url.mediaType === 'application/x-mpegURL'
222}
223
224function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
225 return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
226}
227
228function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
229 return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
230}
231
232function isAPHashTagObject (url: any): url is ActivityHashTagObject {
233 return url && url.type === 'Hashtag'
234}
diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts
new file mode 100644
index 000000000..fcb2a5091
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/trackers.ts
@@ -0,0 +1,43 @@
1import { Transaction } from 'sequelize/types'
2import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
3import { isAPVideoTrackerUrlObject } from '@server/helpers/custom-validators/activitypub/videos'
4import { isArray } from '@server/helpers/custom-validators/misc'
5import { REMOTE_SCHEME } from '@server/initializers/constants'
6import { TrackerModel } from '@server/models/server/tracker'
7import { MVideo, MVideoWithHost } from '@server/types/models'
8import { ActivityTrackerUrlObject, VideoObject } from '@shared/models'
9
10function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
11 let wsFound = false
12
13 const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u))
14 .map((u: ActivityTrackerUrlObject) => {
15 if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true
16
17 return u.href
18 })
19
20 if (wsFound) return trackers
21
22 return [
23 buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS),
24 buildRemoteVideoBaseUrl(video, '/tracker/announce')
25 ]
26}
27
28async function setVideoTrackers (options: {
29 video: MVideo
30 trackers: string[]
31 transaction?: Transaction
32}) {
33 const { video, trackers, transaction } = options
34
35 const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction)
36
37 await video.$set('Trackers', trackerInstances, { transaction })
38}
39
40export {
41 getTrackerUrls,
42 setVideoTrackers
43}
diff --git a/server/lib/activitypub/videos/shared/video-create.ts b/server/lib/activitypub/videos/shared/video-create.ts
new file mode 100644
index 000000000..80cc2ab37
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/video-create.ts
@@ -0,0 +1,167 @@
1import { logger } from '@server/helpers/logger'
2import { sequelizeTypescript } from '@server/initializers/database'
3import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '@server/lib/thumbnail'
4import { setVideoTags } from '@server/lib/video'
5import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
6import { VideoModel } from '@server/models/video/video'
7import { VideoCaptionModel } from '@server/models/video/video-caption'
8import { VideoFileModel } from '@server/models/video/video-file'
9import { VideoLiveModel } from '@server/models/video/video-live'
10import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
11import {
12 MChannelAccountLight,
13 MStreamingPlaylistFilesVideo,
14 MThumbnail,
15 MVideoCaption,
16 MVideoFullLight,
17 MVideoThumbnail
18} from '@server/types/models'
19import { ThumbnailType, VideoObject } from '@shared/models'
20import {
21 getPreviewFromIcons,
22 getTagsFromObject,
23 getThumbnailFromIcons,
24 streamingPlaylistActivityUrlToDBAttributes,
25 videoActivityObjectToDBAttributes,
26 videoFileActivityUrlToDBAttributes
27} from './object-to-model-attributes'
28import { getTrackerUrls, setVideoTrackers } from './trackers'
29
30async function createVideo (videoObject: VideoObject, channel: MChannelAccountLight, waitThumbnail = false) {
31 logger.debug('Adding remote video %s.', videoObject.id)
32
33 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
34 const video = VideoModel.build(videoData) as MVideoThumbnail
35
36 const promiseThumbnail = createVideoMiniatureFromUrl({
37 downloadUrl: getThumbnailFromIcons(videoObject).url,
38 video,
39 type: ThumbnailType.MINIATURE
40 }).catch(err => {
41 logger.error('Cannot create miniature from url.', { err })
42 return undefined
43 })
44
45 let thumbnailModel: MThumbnail
46 if (waitThumbnail === true) {
47 thumbnailModel = await promiseThumbnail
48 }
49
50 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
51 try {
52 const sequelizeOptions = { transaction: t }
53
54 const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
55 videoCreated.VideoChannel = channel
56
57 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
58
59 const previewIcon = getPreviewFromIcons(videoObject)
60 if (previewIcon) {
61 const previewModel = createPlaceholderThumbnail({
62 fileUrl: previewIcon.url,
63 video: videoCreated,
64 type: ThumbnailType.PREVIEW,
65 size: previewIcon
66 })
67
68 await videoCreated.addAndSaveThumbnail(previewModel, t)
69 }
70
71 // Process files
72 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
73
74 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
75 const videoFiles = await Promise.all(videoFilePromises)
76
77 const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
78 videoCreated.VideoStreamingPlaylists = []
79
80 for (const playlistAttributes of streamingPlaylistsAttributes) {
81 const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo
82 playlist.Video = videoCreated
83
84 const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject)
85 const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
86 playlist.VideoFiles = await Promise.all(videoFilePromises)
87
88 videoCreated.VideoStreamingPlaylists.push(playlist)
89 }
90
91 // Process tags
92 const tags = getTagsFromObject(videoObject)
93 await setVideoTags({ video: videoCreated, tags, transaction: t })
94
95 // Process captions
96 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
97 const caption = new VideoCaptionModel({
98 videoId: videoCreated.id,
99 filename: VideoCaptionModel.generateCaptionName(c.identifier),
100 language: c.identifier,
101 fileUrl: c.url
102 }) as MVideoCaption
103
104 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
105 })
106 await Promise.all(videoCaptionsPromises)
107
108 // Process trackers
109 {
110 const trackers = getTrackerUrls(videoObject, videoCreated)
111 await setVideoTrackers({ video: videoCreated, trackers, transaction: t })
112 }
113
114 videoCreated.VideoFiles = videoFiles
115
116 if (videoCreated.isLive) {
117 const videoLive = new VideoLiveModel({
118 streamKey: null,
119 saveReplay: videoObject.liveSaveReplay,
120 permanentLive: videoObject.permanentLive,
121 videoId: videoCreated.id
122 })
123
124 videoCreated.VideoLive = await videoLive.save({ transaction: t })
125 }
126
127 // We added a video in this channel, set it as updated
128 await channel.setAsUpdated(t)
129
130 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
131 video: videoCreated,
132 user: undefined,
133 isRemote: true,
134 isNew: true,
135 transaction: t
136 })
137
138 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
139
140 return { autoBlacklisted, videoCreated }
141 } catch (err) {
142 // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
143 // Remove thumbnail
144 if (thumbnailModel) await thumbnailModel.removeThumbnail()
145
146 throw err
147 }
148 })
149
150 if (waitThumbnail === false) {
151 // Error is already caught above
152 // eslint-disable-next-line @typescript-eslint/no-floating-promises
153 promiseThumbnail.then(thumbnailModel => {
154 if (!thumbnailModel) return
155
156 thumbnailModel = videoCreated.id
157
158 return thumbnailModel.save()
159 })
160 }
161
162 return { autoBlacklisted, videoCreated }
163}
164
165export {
166 createVideo
167}
diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts
new file mode 100644
index 000000000..181893c68
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/video-sync-attributes.ts
@@ -0,0 +1,75 @@
1import { logger } from '@server/helpers/logger'
2import { JobQueue } from '@server/lib/job-queue'
3import { AccountVideoRateModel } from '@server/models/account/account-video-rate'
4import { VideoCommentModel } from '@server/models/video/video-comment'
5import { VideoShareModel } from '@server/models/video/video-share'
6import { MVideo } from '@server/types/models'
7import { ActivitypubHttpFetcherPayload, VideoObject } from '@shared/models'
8import { crawlCollectionPage } from '../../crawl'
9import { addVideoShares } from '../../share'
10import { addVideoComments } from '../../video-comments'
11import { createRates } from '../../video-rates'
12
13import Bluebird = require('bluebird')
14
15type SyncParam = {
16 likes: boolean
17 dislikes: boolean
18 shares: boolean
19 comments: boolean
20 thumbnail: boolean
21 refreshVideo?: boolean
22}
23
24async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoObject, syncParam: SyncParam) {
25 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
26
27 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
28
29 if (syncParam.likes === true) {
30 const handler = items => createRates(items, video, 'like')
31 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
32
33 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
34 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes }))
35 } else {
36 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
37 }
38
39 if (syncParam.dislikes === true) {
40 const handler = items => createRates(items, video, 'dislike')
41 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
42
43 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
44 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes }))
45 } else {
46 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
47 }
48
49 if (syncParam.shares === true) {
50 const handler = items => addVideoShares(items, video)
51 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
52
53 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
54 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
55 } else {
56 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
57 }
58
59 if (syncParam.comments === true) {
60 const handler = items => addVideoComments(items)
61 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
62
63 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
64 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
65 } else {
66 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
67 }
68
69 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
70}
71
72export {
73 SyncParam,
74 syncVideoExternalAttributes
75}
diff --git a/server/lib/activitypub/videos/update.ts b/server/lib/activitypub/videos/update.ts
new file mode 100644
index 000000000..444b51628
--- /dev/null
+++ b/server/lib/activitypub/videos/update.ts
@@ -0,0 +1,293 @@
1import { Transaction } from 'sequelize/types'
2import { deleteNonExistingModels, resetSequelizeInstance } from '@server/helpers/database-utils'
3import { logger } from '@server/helpers/logger'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { Notifier } from '@server/lib/notifier'
6import { PeerTubeSocket } from '@server/lib/peertube-socket'
7import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '@server/lib/thumbnail'
8import { setVideoTags } from '@server/lib/video'
9import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
10import { VideoCaptionModel } from '@server/models/video/video-caption'
11import { VideoFileModel } from '@server/models/video/video-file'
12import { VideoLiveModel } from '@server/models/video/video-live'
13import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
14import {
15 MChannelAccountLight,
16 MChannelDefault,
17 MStreamingPlaylistFilesVideo,
18 MThumbnail,
19 MVideoAccountLightBlacklistAllFiles,
20 MVideoCaption,
21 MVideoFile,
22 MVideoFullLight
23} from '@server/types/models'
24import { ThumbnailType, VideoObject, VideoPrivacy } from '@shared/models'
25import {
26 getPreviewFromIcons,
27 getTagsFromObject,
28 getThumbnailFromIcons,
29 getTrackerUrls,
30 setVideoTrackers,
31 streamingPlaylistActivityUrlToDBAttributes,
32 videoActivityObjectToDBAttributes,
33 videoFileActivityUrlToDBAttributes
34} from './shared'
35
36export class APVideoUpdater {
37 private readonly video: MVideoAccountLightBlacklistAllFiles
38 private readonly videoObject: VideoObject
39 private readonly channel: MChannelDefault
40 private readonly overrideTo: string[]
41
42 private readonly wasPrivateVideo: boolean
43 private readonly wasUnlistedVideo: boolean
44
45 private readonly videoFieldsSave: any
46
47 private readonly oldVideoChannel: MChannelAccountLight
48
49 constructor (options: {
50 video: MVideoAccountLightBlacklistAllFiles
51 videoObject: VideoObject
52 channel: MChannelDefault
53 overrideTo?: string[]
54 }) {
55 this.video = options.video
56 this.videoObject = options.videoObject
57 this.channel = options.channel
58 this.overrideTo = options.overrideTo
59
60 this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE
61 this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED
62
63 this.oldVideoChannel = this.video.VideoChannel
64
65 this.videoFieldsSave = this.video.toJSON()
66 }
67
68 async update () {
69 logger.debug('Updating remote video "%s".', this.videoObject.uuid, { videoObject: this.videoObject, channel: this.channel })
70
71 try {
72 const thumbnailModel = await this.tryToGenerateThumbnail()
73
74 const videoUpdated = await sequelizeTypescript.transaction(async t => {
75 this.checkChannelUpdateOrThrow()
76
77 const videoUpdated = await this.updateVideo(t)
78
79 await this.processIcons(videoUpdated, thumbnailModel, t)
80 await this.processWebTorrentFiles(videoUpdated, t)
81 await this.processStreamingPlaylists(videoUpdated, t)
82 await this.processTags(videoUpdated, t)
83 await this.processTrackers(videoUpdated, t)
84 await this.processCaptions(videoUpdated, t)
85 await this.processLive(videoUpdated, t)
86
87 return videoUpdated
88 })
89
90 await autoBlacklistVideoIfNeeded({
91 video: videoUpdated,
92 user: undefined,
93 isRemote: true,
94 isNew: false,
95 transaction: undefined
96 })
97
98 // Notify our users?
99 if (this.wasPrivateVideo || this.wasUnlistedVideo) {
100 Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated)
101 }
102
103 if (videoUpdated.isLive) {
104 PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated)
105 PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
106 }
107
108 logger.info('Remote video with uuid %s updated', this.videoObject.uuid)
109
110 return videoUpdated
111 } catch (err) {
112 this.catchUpdateError(err)
113 }
114 }
115
116 private tryToGenerateThumbnail (): Promise<MThumbnail> {
117 return createVideoMiniatureFromUrl({
118 downloadUrl: getThumbnailFromIcons(this.videoObject).url,
119 video: this.video,
120 type: ThumbnailType.MINIATURE
121 }).catch(err => {
122 logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err })
123
124 return undefined
125 })
126 }
127
128 // Check we can update the channel: we trust the remote server
129 private checkChannelUpdateOrThrow () {
130 if (!this.oldVideoChannel.Actor.serverId || !this.channel.Actor.serverId) {
131 throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
132 }
133
134 if (this.oldVideoChannel.Actor.serverId !== this.channel.Actor.serverId) {
135 throw new Error(`New channel ${this.channel.Actor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`)
136 }
137 }
138
139 private updateVideo (transaction: Transaction) {
140 const to = this.overrideTo || this.videoObject.to
141 const videoData = videoActivityObjectToDBAttributes(this.channel, this.videoObject, to)
142 this.video.name = videoData.name
143 this.video.uuid = videoData.uuid
144 this.video.url = videoData.url
145 this.video.category = videoData.category
146 this.video.licence = videoData.licence
147 this.video.language = videoData.language
148 this.video.description = videoData.description
149 this.video.support = videoData.support
150 this.video.nsfw = videoData.nsfw
151 this.video.commentsEnabled = videoData.commentsEnabled
152 this.video.downloadEnabled = videoData.downloadEnabled
153 this.video.waitTranscoding = videoData.waitTranscoding
154 this.video.state = videoData.state
155 this.video.duration = videoData.duration
156 this.video.createdAt = videoData.createdAt
157 this.video.publishedAt = videoData.publishedAt
158 this.video.originallyPublishedAt = videoData.originallyPublishedAt
159 this.video.privacy = videoData.privacy
160 this.video.channelId = videoData.channelId
161 this.video.views = videoData.views
162 this.video.isLive = videoData.isLive
163
164 // Ensures we update the updated video attribute
165 this.video.changed('updatedAt', true)
166
167 return this.video.save({ transaction }) as Promise<MVideoFullLight>
168 }
169
170 private async processIcons (videoUpdated: MVideoFullLight, thumbnailModel: MThumbnail, t: Transaction) {
171 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
172
173 // Don't fetch the preview that could be big, create a placeholder instead
174 const previewIcon = getPreviewFromIcons(this.videoObject)
175 if (videoUpdated.getPreview() && previewIcon) {
176 const previewModel = createPlaceholderThumbnail({
177 fileUrl: previewIcon.url,
178 video: videoUpdated,
179 type: ThumbnailType.PREVIEW,
180 size: previewIcon
181 })
182 await videoUpdated.addAndSaveThumbnail(previewModel, t)
183 }
184 }
185
186 private async processWebTorrentFiles (videoUpdated: MVideoFullLight, t: Transaction) {
187 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, this.videoObject.url)
188 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
189
190 // Remove video files that do not exist anymore
191 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
192 await Promise.all(destroyTasks)
193
194 // Update or add other one
195 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
196 videoUpdated.VideoFiles = await Promise.all(upsertTasks)
197 }
198
199 private async processStreamingPlaylists (videoUpdated: MVideoFullLight, t: Transaction) {
200 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, this.videoObject, videoUpdated.VideoFiles)
201 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
202
203 // Remove video playlists that do not exist anymore
204 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
205 await Promise.all(destroyTasks)
206
207 let oldStreamingPlaylistFiles: MVideoFile[] = []
208 for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
209 oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
210 }
211
212 videoUpdated.VideoStreamingPlaylists = []
213
214 for (const playlistAttributes of streamingPlaylistAttributes) {
215 const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
216 .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo)
217 streamingPlaylistModel.Video = videoUpdated
218
219 const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
220 .map(a => new VideoFileModel(a))
221 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
222 await Promise.all(destroyTasks)
223
224 // Update or add other one
225 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
226 streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
227
228 videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
229 }
230 }
231
232 private async processTags (videoUpdated: MVideoFullLight, t: Transaction) {
233 const tags = getTagsFromObject(this.videoObject)
234 await setVideoTags({ video: videoUpdated, tags, transaction: t })
235 }
236
237 private async processTrackers (videoUpdated: MVideoFullLight, t: Transaction) {
238 const trackers = getTrackerUrls(this.videoObject, videoUpdated)
239 await setVideoTrackers({ video: videoUpdated, trackers, transaction: t })
240 }
241
242 private async processCaptions (videoUpdated: MVideoFullLight, t: Transaction) {
243 // Update captions
244 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
245
246 const videoCaptionsPromises = this.videoObject.subtitleLanguage.map(c => {
247 const caption = new VideoCaptionModel({
248 videoId: videoUpdated.id,
249 filename: VideoCaptionModel.generateCaptionName(c.identifier),
250 language: c.identifier,
251 fileUrl: c.url
252 }) as MVideoCaption
253
254 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
255 })
256
257 await Promise.all(videoCaptionsPromises)
258 }
259
260 private async processLive (videoUpdated: MVideoFullLight, t: Transaction) {
261 // Create or update existing live
262 if (this.video.isLive) {
263 const [ videoLive ] = await VideoLiveModel.upsert({
264 saveReplay: this.videoObject.liveSaveReplay,
265 permanentLive: this.videoObject.permanentLive,
266 videoId: this.video.id
267 }, { transaction: t, returning: true })
268
269 videoUpdated.VideoLive = videoLive
270 return
271 }
272
273 // Delete existing live if it exists
274 await VideoLiveModel.destroy({
275 where: {
276 videoId: this.video.id
277 },
278 transaction: t
279 })
280
281 videoUpdated.VideoLive = null
282 }
283
284 private catchUpdateError (err: Error) {
285 if (this.video !== undefined && this.videoFieldsSave !== undefined) {
286 resetSequelizeInstance(this.video, this.videoFieldsSave)
287 }
288
289 // This is just a debug because we will retry the insert
290 logger.debug('Cannot update the remote video.', { err })
291 throw err
292 }
293}
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts
index c09b1bcc8..a120e4ea8 100644
--- a/server/lib/job-queue/handlers/activitypub-refresher.ts
+++ b/server/lib/job-queue/handlers/activitypub-refresher.ts
@@ -1,12 +1,12 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist' 2import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist'
3import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos'
3import { RefreshPayload } from '@shared/models' 4import { RefreshPayload } from '@shared/models'
4import { logger } from '../../../helpers/logger' 5import { logger } from '../../../helpers/logger'
5import { fetchVideoByUrl } from '../../../helpers/video' 6import { fetchVideoByUrl } from '../../../helpers/video'
6import { ActorModel } from '../../../models/actor/actor' 7import { ActorModel } from '../../../models/actor/actor'
7import { VideoPlaylistModel } from '../../../models/video/video-playlist' 8import { VideoPlaylistModel } from '../../../models/video/video-playlist'
8import { refreshActorIfNeeded } from '../../activitypub/actor' 9import { refreshActorIfNeeded } from '../../activitypub/actor'
9import { refreshVideoIfNeeded } from '../../activitypub/videos'
10 10
11async function refreshAPObject (job: Bull.Job) { 11async function refreshAPObject (job: Bull.Job) {
12 const payload = job.data as RefreshPayload 12 const payload = job.data as RefreshPayload
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-01 08:41:42 +0000