aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/activitypub
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-06-02 10:41:46 +0200
committerChocobozzz <me@florianbigard.com>2021-06-02 16:57:53 +0200
commit08a47c75f992e7138dca5121f227909a8347d365 (patch)
treeff66fdaacef9b68a046a55b953fbf9dc6a97843a /server/lib/activitypub
parent69290ab37b8aead01477b9b98fdfad0e69b08582 (diff)
downloadPeerTube-08a47c75f992e7138dca5121f227909a8347d365.tar.gz
PeerTube-08a47c75f992e7138dca5121f227909a8347d365.tar.zst
PeerTube-08a47c75f992e7138dca5121f227909a8347d365.zip
Refactor AP video create/update
Diffstat (limited to 'server/lib/activitypub')
-rw-r--r--server/lib/activitypub/videos/fetch.ts44
-rw-r--r--server/lib/activitypub/videos/index.ts2
-rw-r--r--server/lib/activitypub/videos/shared/abstract-builder.ts142
-rw-r--r--server/lib/activitypub/videos/shared/creator.ts90
-rw-r--r--server/lib/activitypub/videos/shared/index.ts3
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts34
-rw-r--r--server/lib/activitypub/videos/shared/video-create.ts167
-rw-r--r--server/lib/activitypub/videos/update.ts293
-rw-r--r--server/lib/activitypub/videos/updater.ts170
9 files changed, 455 insertions, 490 deletions
diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts
index fdcf4ee5c..5e7f8552b 100644
--- a/server/lib/activitypub/videos/fetch.ts
+++ b/server/lib/activitypub/videos/fetch.ts
@@ -1,20 +1,19 @@
1import { checkUrlsSameHost, getAPId } from "@server/helpers/activitypub" 1import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub'
2import { sanitizeAndCheckVideoTorrentObject } from "@server/helpers/custom-validators/activitypub/videos" 2import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos'
3import { retryTransactionWrapper } from "@server/helpers/database-utils" 3import { retryTransactionWrapper } from '@server/helpers/database-utils'
4import { logger } from "@server/helpers/logger" 4import { logger } from '@server/helpers/logger'
5import { doJSONRequest, PeerTubeRequestError } from "@server/helpers/requests" 5import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests'
6import { fetchVideoByUrl, VideoFetchByUrlType } from "@server/helpers/video" 6import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video'
7import { REMOTE_SCHEME } from "@server/initializers/constants" 7import { REMOTE_SCHEME } from '@server/initializers/constants'
8import { ActorFollowScoreCache } from "@server/lib/files-cache" 8import { ActorFollowScoreCache } from '@server/lib/files-cache'
9import { JobQueue } from "@server/lib/job-queue" 9import { JobQueue } from '@server/lib/job-queue'
10import { VideoModel } from "@server/models/video/video" 10import { VideoModel } from '@server/models/video/video'
11import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from "@server/types/models" 11import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models'
12import { HttpStatusCode } from "@shared/core-utils" 12import { HttpStatusCode } from '@shared/core-utils'
13import { VideoObject } from "@shared/models" 13import { VideoObject } from '@shared/models'
14import { getOrCreateActorAndServerAndModel } from "../actor" 14import { getOrCreateActorAndServerAndModel } from '../actor'
15import { SyncParam, syncVideoExternalAttributes } from "./shared" 15import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared'
16import { createVideo } from "./shared/video-create" 16import { APVideoUpdater } from './updater'
17import { APVideoUpdater } from "./update"
18 17
19async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { 18async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
20 logger.info('Fetching remote video %s.', videoUrl) 19 logger.info('Fetching remote video %s.', videoUrl)
@@ -115,16 +114,17 @@ async function getOrCreateVideoAndAccountAndChannel (
115 return { video: videoFromDatabase, created: false } 114 return { video: videoFromDatabase, created: false }
116 } 115 }
117 116
118 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) 117 const { videoObject } = await fetchRemoteVideo(videoUrl)
119 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) 118 if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
120 119
121 const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) 120 const actor = await getOrCreateVideoChannelFromVideoObject(videoObject)
122 const videoChannel = actor.VideoChannel 121 const videoChannel = actor.VideoChannel
123 122
124 try { 123 try {
125 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) 124 const creator = new APVideoCreator({ videoObject, channel: videoChannel })
125 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail)
126 126
127 await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) 127 await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
128 128
129 return { video: videoCreated, created: true, autoBlacklisted } 129 return { video: videoCreated, created: true, autoBlacklisted }
130 } catch (err) { 130 } catch (err) {
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts
index 0e126c85a..b560acb76 100644
--- a/server/lib/activitypub/videos/index.ts
+++ b/server/lib/activitypub/videos/index.ts
@@ -1,3 +1,3 @@
1export * from './federate' 1export * from './federate'
2export * from './fetch' 2export * from './fetch'
3export * from './update' 3export * from './updater'
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
new file mode 100644
index 000000000..9d5f37e5f
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -0,0 +1,142 @@
1import { Transaction } from 'sequelize/types'
2import { deleteNonExistingModels } from '@server/helpers/database-utils'
3import { logger } from '@server/helpers/logger'
4import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '@server/lib/thumbnail'
5import { setVideoTags } from '@server/lib/video'
6import { VideoCaptionModel } from '@server/models/video/video-caption'
7import { VideoFileModel } from '@server/models/video/video-file'
8import { VideoLiveModel } from '@server/models/video/video-live'
9import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
10import { MStreamingPlaylistFilesVideo, MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models'
11import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models'
12import {
13 getCaptionAttributesFromObject,
14 getFileAttributesFromUrl,
15 getLiveAttributesFromObject,
16 getPreviewFromIcons,
17 getStreamingPlaylistAttributesFromObject,
18 getTagsFromObject,
19 getThumbnailFromIcons
20} from './object-to-model-attributes'
21import { getTrackerUrls, setVideoTrackers } from './trackers'
22
23export abstract class APVideoAbstractBuilder {
24 protected abstract videoObject: VideoObject
25
26 protected tryToGenerateThumbnail (video: MVideoThumbnail): Promise<MThumbnail> {
27 return createVideoMiniatureFromUrl({
28 downloadUrl: getThumbnailFromIcons(this.videoObject).url,
29 video,
30 type: ThumbnailType.MINIATURE
31 }).catch(err => {
32 logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err })
33
34 return undefined
35 })
36 }
37
38 protected async setPreview (video: MVideoFullLight, t: Transaction) {
39 // Don't fetch the preview that could be big, create a placeholder instead
40 const previewIcon = getPreviewFromIcons(this.videoObject)
41 if (!previewIcon) return
42
43 const previewModel = createPlaceholderThumbnail({
44 fileUrl: previewIcon.url,
45 video,
46 type: ThumbnailType.PREVIEW,
47 size: previewIcon
48 })
49
50 await video.addAndSaveThumbnail(previewModel, t)
51 }
52
53 protected async setTags (video: MVideoFullLight, t: Transaction) {
54 const tags = getTagsFromObject(this.videoObject)
55 await setVideoTags({ video, tags, transaction: t })
56 }
57
58 protected async setTrackers (video: MVideoFullLight, t: Transaction) {
59 const trackers = getTrackerUrls(this.videoObject, video)
60 await setVideoTrackers({ video, trackers, transaction: t })
61 }
62
63 protected async insertOrReplaceCaptions (video: MVideoFullLight, t: Transaction) {
64 const videoCaptionsPromises = getCaptionAttributesFromObject(video, this.videoObject)
65 .map(a => new VideoCaptionModel(a) as MVideoCaption)
66 .map(c => VideoCaptionModel.insertOrReplaceLanguage(c, t))
67
68 await Promise.all(videoCaptionsPromises)
69 }
70
71 protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) {
72 const attributes = getLiveAttributesFromObject(video, this.videoObject)
73 const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true })
74
75 video.VideoLive = videoLive
76 }
77
78 protected async setWebTorrentFiles (video: MVideoFullLight, t: Transaction) {
79 const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url)
80 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
81
82 // Remove video files that do not exist anymore
83 const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t)
84 await Promise.all(destroyTasks)
85
86 // Update or add other one
87 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
88 video.VideoFiles = await Promise.all(upsertTasks)
89 }
90
91 protected async setStreamingPlaylists (video: MVideoFullLight, t: Transaction) {
92 const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject, video.VideoFiles || [])
93 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
94
95 // Remove video playlists that do not exist anymore
96 const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t)
97 await Promise.all(destroyTasks)
98
99 video.VideoStreamingPlaylists = []
100
101 for (const playlistAttributes of streamingPlaylistAttributes) {
102
103 const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
104 streamingPlaylistModel.Video = video
105
106 await this.setStreamingPlaylistFiles(video, streamingPlaylistModel, playlistAttributes.tagAPObject, t)
107
108 video.VideoStreamingPlaylists.push(streamingPlaylistModel)
109 }
110 }
111
112 private async insertOrReplaceStreamingPlaylist (attributes: VideoStreamingPlaylistModel['_creationAttributes'], t: Transaction) {
113 const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t })
114
115 return streamingPlaylist as MStreamingPlaylistFilesVideo
116 }
117
118 private getStreamingPlaylistFiles (video: MVideoFullLight, type: VideoStreamingPlaylistType) {
119 const playlist = video.VideoStreamingPlaylists.find(s => s.type === type)
120 if (!playlist) return []
121
122 return playlist.VideoFiles
123 }
124
125 private async setStreamingPlaylistFiles (
126 video: MVideoFullLight,
127 playlistModel: MStreamingPlaylistFilesVideo,
128 tagObjects: ActivityTagObject[],
129 t: Transaction
130 ) {
131 const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(video, playlistModel.type)
132
133 const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a))
134
135 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
136 await Promise.all(destroyTasks)
137
138 // Update or add other one
139 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
140 playlistModel.VideoFiles = await Promise.all(upsertTasks)
141 }
142}
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts
new file mode 100644
index 000000000..4f2d79374
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/creator.ts
@@ -0,0 +1,90 @@
1
2import { logger } from '@server/helpers/logger'
3import { sequelizeTypescript } from '@server/initializers/database'
4import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
5import { VideoModel } from '@server/models/video/video'
6import { MChannelAccountLight, MThumbnail, MVideoFullLight, MVideoThumbnail } from '@server/types/models'
7import { VideoObject } from '@shared/models'
8import { APVideoAbstractBuilder } from './abstract-builder'
9import { getVideoAttributesFromObject } from './object-to-model-attributes'
10
11export class APVideoCreator extends APVideoAbstractBuilder {
12 protected readonly videoObject: VideoObject
13 private readonly channel: MChannelAccountLight
14
15 constructor (options: {
16 videoObject: VideoObject
17 channel: MChannelAccountLight
18 }) {
19 super()
20
21 this.videoObject = options.videoObject
22 this.channel = options.channel
23 }
24
25 async create (waitThumbnail = false) {
26 logger.debug('Adding remote video %s.', this.videoObject.id)
27
28 const videoData = await getVideoAttributesFromObject(this.channel, this.videoObject, this.videoObject.to)
29 const video = VideoModel.build(videoData) as MVideoThumbnail
30
31 const promiseThumbnail = this.tryToGenerateThumbnail(video)
32
33 let thumbnailModel: MThumbnail
34 if (waitThumbnail === true) {
35 thumbnailModel = await promiseThumbnail
36 }
37
38 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
39 try {
40 const videoCreated = await video.save({ transaction: t }) as MVideoFullLight
41 videoCreated.VideoChannel = this.channel
42
43 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
44
45 await this.setPreview(videoCreated, t)
46 await this.setWebTorrentFiles(videoCreated, t)
47 await this.setStreamingPlaylists(videoCreated, t)
48 await this.setTags(videoCreated, t)
49 await this.setTrackers(videoCreated, t)
50 await this.insertOrReplaceCaptions(videoCreated, t)
51 await this.insertOrReplaceLive(videoCreated, t)
52
53 // We added a video in this channel, set it as updated
54 await this.channel.setAsUpdated(t)
55
56 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
57 video: videoCreated,
58 user: undefined,
59 isRemote: true,
60 isNew: true,
61 transaction: t
62 })
63
64 logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid)
65
66 return { autoBlacklisted, videoCreated }
67 } catch (err) {
68 // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
69 // Remove thumbnail
70 if (thumbnailModel) await thumbnailModel.removeThumbnail()
71
72 throw err
73 }
74 })
75
76 if (waitThumbnail === false) {
77 // Error is already caught above
78 // eslint-disable-next-line @typescript-eslint/no-floating-promises
79 promiseThumbnail.then(thumbnailModel => {
80 if (!thumbnailModel) return
81
82 thumbnailModel = videoCreated.id
83
84 return thumbnailModel.save()
85 })
86 }
87
88 return { autoBlacklisted, videoCreated }
89 }
90}
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts
index 4d24fbc6a..208a43705 100644
--- a/server/lib/activitypub/videos/shared/index.ts
+++ b/server/lib/activitypub/videos/shared/index.ts
@@ -1,4 +1,5 @@
1export * from './abstract-builder'
2export * from './creator'
1export * from './object-to-model-attributes' 3export * from './object-to-model-attributes'
2export * from './trackers' 4export * from './trackers'
3export * from './video-create'
4export * from './video-sync-attributes' 5export * from './video-sync-attributes'
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index 8a8105500..85548428c 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -23,6 +23,7 @@ import {
23 VideoPrivacy, 23 VideoPrivacy,
24 VideoStreamingPlaylistType 24 VideoStreamingPlaylistType
25} from '@shared/models' 25} from '@shared/models'
26import { VideoCaptionModel } from '@server/models/video/video-caption'
26 27
27function getThumbnailFromIcons (videoObject: VideoObject) { 28function getThumbnailFromIcons (videoObject: VideoObject) {
28 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) 29 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@@ -44,7 +45,7 @@ function getTagsFromObject (videoObject: VideoObject) {
44 .map(t => t.name) 45 .map(t => t.name)
45} 46}
46 47
47function videoFileActivityUrlToDBAttributes ( 48function getFileAttributesFromUrl (
48 videoOrPlaylist: MVideo | MStreamingPlaylistVideo, 49 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
49 urls: (ActivityTagObject | ActivityUrlObject)[] 50 urls: (ActivityTagObject | ActivityUrlObject)[]
50) { 51) {
@@ -109,7 +110,7 @@ function videoFileActivityUrlToDBAttributes (
109 return attributes 110 return attributes
110} 111}
111 112
112function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) { 113function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) {
113 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] 114 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
114 if (playlistUrls.length === 0) return [] 115 if (playlistUrls.length === 0) return []
115 116
@@ -134,6 +135,7 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec
134 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), 135 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
135 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, 136 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
136 videoId: video.id, 137 videoId: video.id,
138
137 tagAPObject: playlistUrlObject.tag 139 tagAPObject: playlistUrlObject.tag
138 } 140 }
139 141
@@ -143,7 +145,24 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec
143 return attributes 145 return attributes
144} 146}
145 147
146function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { 148function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) {
149 return {
150 saveReplay: videoObject.liveSaveReplay,
151 permanentLive: videoObject.permanentLive,
152 videoId: video.id
153 }
154}
155
156function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObject) {
157 return videoObject.subtitleLanguage.map(c => ({
158 videoId: video.id,
159 filename: VideoCaptionModel.generateCaptionName(c.identifier),
160 language: c.identifier,
161 fileUrl: c.url
162 }))
163}
164
165function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
147 const privacy = to.includes(ACTIVITY_PUB.PUBLIC) 166 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
148 ? VideoPrivacy.PUBLIC 167 ? VideoPrivacy.PUBLIC
149 : VideoPrivacy.UNLISTED 168 : VideoPrivacy.UNLISTED
@@ -203,10 +222,13 @@ export {
203 222
204 getTagsFromObject, 223 getTagsFromObject,
205 224
206 videoActivityObjectToDBAttributes, 225 getFileAttributesFromUrl,
226 getStreamingPlaylistAttributesFromObject,
227
228 getLiveAttributesFromObject,
229 getCaptionAttributesFromObject,
207 230
208 videoFileActivityUrlToDBAttributes, 231 getVideoAttributesFromObject
209 streamingPlaylistActivityUrlToDBAttributes
210} 232}
211 233
212// --------------------------------------------------------------------------- 234// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/videos/shared/video-create.ts b/server/lib/activitypub/videos/shared/video-create.ts
deleted file mode 100644
index 80cc2ab37..000000000
--- a/server/lib/activitypub/videos/shared/video-create.ts
+++ /dev/null
@@ -1,167 +0,0 @@
1import { logger } from '@server/helpers/logger'
2import { sequelizeTypescript } from '@server/initializers/database'
3import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '@server/lib/thumbnail'
4import { setVideoTags } from '@server/lib/video'
5import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
6import { VideoModel } from '@server/models/video/video'
7import { VideoCaptionModel } from '@server/models/video/video-caption'
8import { VideoFileModel } from '@server/models/video/video-file'
9import { VideoLiveModel } from '@server/models/video/video-live'
10import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
11import {
12 MChannelAccountLight,
13 MStreamingPlaylistFilesVideo,
14 MThumbnail,
15 MVideoCaption,
16 MVideoFullLight,
17 MVideoThumbnail
18} from '@server/types/models'
19import { ThumbnailType, VideoObject } from '@shared/models'
20import {
21 getPreviewFromIcons,
22 getTagsFromObject,
23 getThumbnailFromIcons,
24 streamingPlaylistActivityUrlToDBAttributes,
25 videoActivityObjectToDBAttributes,
26 videoFileActivityUrlToDBAttributes
27} from './object-to-model-attributes'
28import { getTrackerUrls, setVideoTrackers } from './trackers'
29
30async function createVideo (videoObject: VideoObject, channel: MChannelAccountLight, waitThumbnail = false) {
31 logger.debug('Adding remote video %s.', videoObject.id)
32
33 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
34 const video = VideoModel.build(videoData) as MVideoThumbnail
35
36 const promiseThumbnail = createVideoMiniatureFromUrl({
37 downloadUrl: getThumbnailFromIcons(videoObject).url,
38 video,
39 type: ThumbnailType.MINIATURE
40 }).catch(err => {
41 logger.error('Cannot create miniature from url.', { err })
42 return undefined
43 })
44
45 let thumbnailModel: MThumbnail
46 if (waitThumbnail === true) {
47 thumbnailModel = await promiseThumbnail
48 }
49
50 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
51 try {
52 const sequelizeOptions = { transaction: t }
53
54 const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
55 videoCreated.VideoChannel = channel
56
57 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
58
59 const previewIcon = getPreviewFromIcons(videoObject)
60 if (previewIcon) {
61 const previewModel = createPlaceholderThumbnail({
62 fileUrl: previewIcon.url,
63 video: videoCreated,
64 type: ThumbnailType.PREVIEW,
65 size: previewIcon
66 })
67
68 await videoCreated.addAndSaveThumbnail(previewModel, t)
69 }
70
71 // Process files
72 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
73
74 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
75 const videoFiles = await Promise.all(videoFilePromises)
76
77 const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
78 videoCreated.VideoStreamingPlaylists = []
79
80 for (const playlistAttributes of streamingPlaylistsAttributes) {
81 const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo
82 playlist.Video = videoCreated
83
84 const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject)
85 const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
86 playlist.VideoFiles = await Promise.all(videoFilePromises)
87
88 videoCreated.VideoStreamingPlaylists.push(playlist)
89 }
90
91 // Process tags
92 const tags = getTagsFromObject(videoObject)
93 await setVideoTags({ video: videoCreated, tags, transaction: t })
94
95 // Process captions
96 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
97 const caption = new VideoCaptionModel({
98 videoId: videoCreated.id,
99 filename: VideoCaptionModel.generateCaptionName(c.identifier),
100 language: c.identifier,
101 fileUrl: c.url
102 }) as MVideoCaption
103
104 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
105 })
106 await Promise.all(videoCaptionsPromises)
107
108 // Process trackers
109 {
110 const trackers = getTrackerUrls(videoObject, videoCreated)
111 await setVideoTrackers({ video: videoCreated, trackers, transaction: t })
112 }
113
114 videoCreated.VideoFiles = videoFiles
115
116 if (videoCreated.isLive) {
117 const videoLive = new VideoLiveModel({
118 streamKey: null,
119 saveReplay: videoObject.liveSaveReplay,
120 permanentLive: videoObject.permanentLive,
121 videoId: videoCreated.id
122 })
123
124 videoCreated.VideoLive = await videoLive.save({ transaction: t })
125 }
126
127 // We added a video in this channel, set it as updated
128 await channel.setAsUpdated(t)
129
130 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
131 video: videoCreated,
132 user: undefined,
133 isRemote: true,
134 isNew: true,
135 transaction: t
136 })
137
138 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
139
140 return { autoBlacklisted, videoCreated }
141 } catch (err) {
142 // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
143 // Remove thumbnail
144 if (thumbnailModel) await thumbnailModel.removeThumbnail()
145
146 throw err
147 }
148 })
149
150 if (waitThumbnail === false) {
151 // Error is already caught above
152 // eslint-disable-next-line @typescript-eslint/no-floating-promises
153 promiseThumbnail.then(thumbnailModel => {
154 if (!thumbnailModel) return
155
156 thumbnailModel = videoCreated.id
157
158 return thumbnailModel.save()
159 })
160 }
161
162 return { autoBlacklisted, videoCreated }
163}
164
165export {
166 createVideo
167}
diff --git a/server/lib/activitypub/videos/update.ts b/server/lib/activitypub/videos/update.ts
deleted file mode 100644
index 444b51628..000000000
--- a/server/lib/activitypub/videos/update.ts
+++ /dev/null
@@ -1,293 +0,0 @@
1import { Transaction } from 'sequelize/types'
2import { deleteNonExistingModels, resetSequelizeInstance } from '@server/helpers/database-utils'
3import { logger } from '@server/helpers/logger'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { Notifier } from '@server/lib/notifier'
6import { PeerTubeSocket } from '@server/lib/peertube-socket'
7import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '@server/lib/thumbnail'
8import { setVideoTags } from '@server/lib/video'
9import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
10import { VideoCaptionModel } from '@server/models/video/video-caption'
11import { VideoFileModel } from '@server/models/video/video-file'
12import { VideoLiveModel } from '@server/models/video/video-live'
13import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
14import {
15 MChannelAccountLight,
16 MChannelDefault,
17 MStreamingPlaylistFilesVideo,
18 MThumbnail,
19 MVideoAccountLightBlacklistAllFiles,
20 MVideoCaption,
21 MVideoFile,
22 MVideoFullLight
23} from '@server/types/models'
24import { ThumbnailType, VideoObject, VideoPrivacy } from '@shared/models'
25import {
26 getPreviewFromIcons,
27 getTagsFromObject,
28 getThumbnailFromIcons,
29 getTrackerUrls,
30 setVideoTrackers,
31 streamingPlaylistActivityUrlToDBAttributes,
32 videoActivityObjectToDBAttributes,
33 videoFileActivityUrlToDBAttributes
34} from './shared'
35
36export class APVideoUpdater {
37 private readonly video: MVideoAccountLightBlacklistAllFiles
38 private readonly videoObject: VideoObject
39 private readonly channel: MChannelDefault
40 private readonly overrideTo: string[]
41
42 private readonly wasPrivateVideo: boolean
43 private readonly wasUnlistedVideo: boolean
44
45 private readonly videoFieldsSave: any
46
47 private readonly oldVideoChannel: MChannelAccountLight
48
49 constructor (options: {
50 video: MVideoAccountLightBlacklistAllFiles
51 videoObject: VideoObject
52 channel: MChannelDefault
53 overrideTo?: string[]
54 }) {
55 this.video = options.video
56 this.videoObject = options.videoObject
57 this.channel = options.channel
58 this.overrideTo = options.overrideTo
59
60 this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE
61 this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED
62
63 this.oldVideoChannel = this.video.VideoChannel
64
65 this.videoFieldsSave = this.video.toJSON()
66 }
67
68 async update () {
69 logger.debug('Updating remote video "%s".', this.videoObject.uuid, { videoObject: this.videoObject, channel: this.channel })
70
71 try {
72 const thumbnailModel = await this.tryToGenerateThumbnail()
73
74 const videoUpdated = await sequelizeTypescript.transaction(async t => {
75 this.checkChannelUpdateOrThrow()
76
77 const videoUpdated = await this.updateVideo(t)
78
79 await this.processIcons(videoUpdated, thumbnailModel, t)
80 await this.processWebTorrentFiles(videoUpdated, t)
81 await this.processStreamingPlaylists(videoUpdated, t)
82 await this.processTags(videoUpdated, t)
83 await this.processTrackers(videoUpdated, t)
84 await this.processCaptions(videoUpdated, t)
85 await this.processLive(videoUpdated, t)
86
87 return videoUpdated
88 })
89
90 await autoBlacklistVideoIfNeeded({
91 video: videoUpdated,
92 user: undefined,
93 isRemote: true,
94 isNew: false,
95 transaction: undefined
96 })
97
98 // Notify our users?
99 if (this.wasPrivateVideo || this.wasUnlistedVideo) {
100 Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated)
101 }
102
103 if (videoUpdated.isLive) {
104 PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated)
105 PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
106 }
107
108 logger.info('Remote video with uuid %s updated', this.videoObject.uuid)
109
110 return videoUpdated
111 } catch (err) {
112 this.catchUpdateError(err)
113 }
114 }
115
116 private tryToGenerateThumbnail (): Promise<MThumbnail> {
117 return createVideoMiniatureFromUrl({
118 downloadUrl: getThumbnailFromIcons(this.videoObject).url,
119 video: this.video,
120 type: ThumbnailType.MINIATURE
121 }).catch(err => {
122 logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err })
123
124 return undefined
125 })
126 }
127
128 // Check we can update the channel: we trust the remote server
129 private checkChannelUpdateOrThrow () {
130 if (!this.oldVideoChannel.Actor.serverId || !this.channel.Actor.serverId) {
131 throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
132 }
133
134 if (this.oldVideoChannel.Actor.serverId !== this.channel.Actor.serverId) {
135 throw new Error(`New channel ${this.channel.Actor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`)
136 }
137 }
138
139 private updateVideo (transaction: Transaction) {
140 const to = this.overrideTo || this.videoObject.to
141 const videoData = videoActivityObjectToDBAttributes(this.channel, this.videoObject, to)
142 this.video.name = videoData.name
143 this.video.uuid = videoData.uuid
144 this.video.url = videoData.url
145 this.video.category = videoData.category
146 this.video.licence = videoData.licence
147 this.video.language = videoData.language
148 this.video.description = videoData.description
149 this.video.support = videoData.support
150 this.video.nsfw = videoData.nsfw
151 this.video.commentsEnabled = videoData.commentsEnabled
152 this.video.downloadEnabled = videoData.downloadEnabled
153 this.video.waitTranscoding = videoData.waitTranscoding
154 this.video.state = videoData.state
155 this.video.duration = videoData.duration
156 this.video.createdAt = videoData.createdAt
157 this.video.publishedAt = videoData.publishedAt
158 this.video.originallyPublishedAt = videoData.originallyPublishedAt
159 this.video.privacy = videoData.privacy
160 this.video.channelId = videoData.channelId
161 this.video.views = videoData.views
162 this.video.isLive = videoData.isLive
163
164 // Ensures we update the updated video attribute
165 this.video.changed('updatedAt', true)
166
167 return this.video.save({ transaction }) as Promise<MVideoFullLight>
168 }
169
170 private async processIcons (videoUpdated: MVideoFullLight, thumbnailModel: MThumbnail, t: Transaction) {
171 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
172
173 // Don't fetch the preview that could be big, create a placeholder instead
174 const previewIcon = getPreviewFromIcons(this.videoObject)
175 if (videoUpdated.getPreview() && previewIcon) {
176 const previewModel = createPlaceholderThumbnail({
177 fileUrl: previewIcon.url,
178 video: videoUpdated,
179 type: ThumbnailType.PREVIEW,
180 size: previewIcon
181 })
182 await videoUpdated.addAndSaveThumbnail(previewModel, t)
183 }
184 }
185
186 private async processWebTorrentFiles (videoUpdated: MVideoFullLight, t: Transaction) {
187 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, this.videoObject.url)
188 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
189
190 // Remove video files that do not exist anymore
191 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
192 await Promise.all(destroyTasks)
193
194 // Update or add other one
195 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
196 videoUpdated.VideoFiles = await Promise.all(upsertTasks)
197 }
198
199 private async processStreamingPlaylists (videoUpdated: MVideoFullLight, t: Transaction) {
200 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, this.videoObject, videoUpdated.VideoFiles)
201 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
202
203 // Remove video playlists that do not exist anymore
204 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
205 await Promise.all(destroyTasks)
206
207 let oldStreamingPlaylistFiles: MVideoFile[] = []
208 for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
209 oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
210 }
211
212 videoUpdated.VideoStreamingPlaylists = []
213
214 for (const playlistAttributes of streamingPlaylistAttributes) {
215 const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
216 .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo)
217 streamingPlaylistModel.Video = videoUpdated
218
219 const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
220 .map(a => new VideoFileModel(a))
221 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
222 await Promise.all(destroyTasks)
223
224 // Update or add other one
225 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
226 streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
227
228 videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
229 }
230 }
231
232 private async processTags (videoUpdated: MVideoFullLight, t: Transaction) {
233 const tags = getTagsFromObject(this.videoObject)
234 await setVideoTags({ video: videoUpdated, tags, transaction: t })
235 }
236
237 private async processTrackers (videoUpdated: MVideoFullLight, t: Transaction) {
238 const trackers = getTrackerUrls(this.videoObject, videoUpdated)
239 await setVideoTrackers({ video: videoUpdated, trackers, transaction: t })
240 }
241
242 private async processCaptions (videoUpdated: MVideoFullLight, t: Transaction) {
243 // Update captions
244 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
245
246 const videoCaptionsPromises = this.videoObject.subtitleLanguage.map(c => {
247 const caption = new VideoCaptionModel({
248 videoId: videoUpdated.id,
249 filename: VideoCaptionModel.generateCaptionName(c.identifier),
250 language: c.identifier,
251 fileUrl: c.url
252 }) as MVideoCaption
253
254 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
255 })
256
257 await Promise.all(videoCaptionsPromises)
258 }
259
260 private async processLive (videoUpdated: MVideoFullLight, t: Transaction) {
261 // Create or update existing live
262 if (this.video.isLive) {
263 const [ videoLive ] = await VideoLiveModel.upsert({
264 saveReplay: this.videoObject.liveSaveReplay,
265 permanentLive: this.videoObject.permanentLive,
266 videoId: this.video.id
267 }, { transaction: t, returning: true })
268
269 videoUpdated.VideoLive = videoLive
270 return
271 }
272
273 // Delete existing live if it exists
274 await VideoLiveModel.destroy({
275 where: {
276 videoId: this.video.id
277 },
278 transaction: t
279 })
280
281 videoUpdated.VideoLive = null
282 }
283
284 private catchUpdateError (err: Error) {
285 if (this.video !== undefined && this.videoFieldsSave !== undefined) {
286 resetSequelizeInstance(this.video, this.videoFieldsSave)
287 }
288
289 // This is just a debug because we will retry the insert
290 logger.debug('Cannot update the remote video.', { err })
291 throw err
292 }
293}
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts
new file mode 100644
index 000000000..4338d1e22
--- /dev/null
+++ b/server/lib/activitypub/videos/updater.ts
@@ -0,0 +1,170 @@
1import { Transaction } from 'sequelize/types'
2import { resetSequelizeInstance } from '@server/helpers/database-utils'
3import { logger } from '@server/helpers/logger'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { Notifier } from '@server/lib/notifier'
6import { PeerTubeSocket } from '@server/lib/peertube-socket'
7import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
8import { VideoCaptionModel } from '@server/models/video/video-caption'
9import { VideoLiveModel } from '@server/models/video/video-live'
10import { MChannelAccountLight, MChannelDefault, MVideoAccountLightBlacklistAllFiles, MVideoFullLight } from '@server/types/models'
11import { VideoObject, VideoPrivacy } from '@shared/models'
12import { APVideoAbstractBuilder, getVideoAttributesFromObject } from './shared'
13
14export class APVideoUpdater extends APVideoAbstractBuilder {
15 protected readonly videoObject: VideoObject
16
17 private readonly video: MVideoAccountLightBlacklistAllFiles
18 private readonly channel: MChannelDefault
19 private readonly overrideTo: string[]
20
21 private readonly wasPrivateVideo: boolean
22 private readonly wasUnlistedVideo: boolean
23
24 private readonly videoFieldsSave: any
25
26 private readonly oldVideoChannel: MChannelAccountLight
27
28 constructor (options: {
29 video: MVideoAccountLightBlacklistAllFiles
30 videoObject: VideoObject
31 channel: MChannelDefault
32 overrideTo?: string[]
33 }) {
34 super()
35
36 this.video = options.video
37 this.videoObject = options.videoObject
38 this.channel = options.channel
39 this.overrideTo = options.overrideTo
40
41 this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE
42 this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED
43
44 this.oldVideoChannel = this.video.VideoChannel
45
46 this.videoFieldsSave = this.video.toJSON()
47 }
48
49 async update () {
50 logger.debug('Updating remote video "%s".', this.videoObject.uuid, { videoObject: this.videoObject, channel: this.channel })
51
52 try {
53 const thumbnailModel = await this.tryToGenerateThumbnail(this.video)
54
55 const videoUpdated = await sequelizeTypescript.transaction(async t => {
56 this.checkChannelUpdateOrThrow()
57
58 const videoUpdated = await this.updateVideo(t)
59
60 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
61
62 await this.setPreview(videoUpdated, t)
63 await this.setWebTorrentFiles(videoUpdated, t)
64 await this.setStreamingPlaylists(videoUpdated, t)
65 await this.setTags(videoUpdated, t)
66 await this.setTrackers(videoUpdated, t)
67 await this.setCaptions(videoUpdated, t)
68 await this.setOrDeleteLive(videoUpdated, t)
69
70 return videoUpdated
71 })
72
73 await autoBlacklistVideoIfNeeded({
74 video: videoUpdated,
75 user: undefined,
76 isRemote: true,
77 isNew: false,
78 transaction: undefined
79 })
80
81 // Notify our users?
82 if (this.wasPrivateVideo || this.wasUnlistedVideo) {
83 Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated)
84 }
85
86 if (videoUpdated.isLive) {
87 PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated)
88 PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
89 }
90
91 logger.info('Remote video with uuid %s updated', this.videoObject.uuid)
92
93 return videoUpdated
94 } catch (err) {
95 this.catchUpdateError(err)
96 }
97 }
98
99 // Check we can update the channel: we trust the remote server
100 private checkChannelUpdateOrThrow () {
101 if (!this.oldVideoChannel.Actor.serverId || !this.channel.Actor.serverId) {
102 throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
103 }
104
105 if (this.oldVideoChannel.Actor.serverId !== this.channel.Actor.serverId) {
106 throw new Error(`New channel ${this.channel.Actor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`)
107 }
108 }
109
110 private updateVideo (transaction: Transaction) {
111 const to = this.overrideTo || this.videoObject.to
112 const videoData = getVideoAttributesFromObject(this.channel, this.videoObject, to)
113 this.video.name = videoData.name
114 this.video.uuid = videoData.uuid
115 this.video.url = videoData.url
116 this.video.category = videoData.category
117 this.video.licence = videoData.licence
118 this.video.language = videoData.language
119 this.video.description = videoData.description
120 this.video.support = videoData.support
121 this.video.nsfw = videoData.nsfw
122 this.video.commentsEnabled = videoData.commentsEnabled
123 this.video.downloadEnabled = videoData.downloadEnabled
124 this.video.waitTranscoding = videoData.waitTranscoding
125 this.video.state = videoData.state
126 this.video.duration = videoData.duration
127 this.video.createdAt = videoData.createdAt
128 this.video.publishedAt = videoData.publishedAt
129 this.video.originallyPublishedAt = videoData.originallyPublishedAt
130 this.video.privacy = videoData.privacy
131 this.video.channelId = videoData.channelId
132 this.video.views = videoData.views
133 this.video.isLive = videoData.isLive
134
135 // Ensures we update the updated video attribute
136 this.video.changed('updatedAt', true)
137
138 return this.video.save({ transaction }) as Promise<MVideoFullLight>
139 }
140
141 private async setCaptions (videoUpdated: MVideoFullLight, t: Transaction) {
142 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
143
144 await this.insertOrReplaceCaptions(videoUpdated, t)
145 }
146
147 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction: Transaction) {
148 if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction)
149
150 // Delete existing live if it exists
151 await VideoLiveModel.destroy({
152 where: {
153 videoId: this.video.id
154 },
155 transaction
156 })
157
158 videoUpdated.VideoLive = null
159 }
160
161 private catchUpdateError (err: Error) {
162 if (this.video !== undefined && this.videoFieldsSave !== undefined) {
163 resetSequelizeInstance(this.video, this.videoFieldsSave)
164 }
165
166 // This is just a debug because we will retry the insert
167 logger.debug('Cannot update the remote video.', { err })
168 throw err
169 }
170}