aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-09-19 11:16:23 +0200
committerChocobozzz <me@florianbigard.com>2018-09-19 11:16:23 +0200
commit4157cdb13748cb6e8ce7081d062a8778554cc5a7 (patch)
tree149ee35079c1f81f1294f88aa0122dd5c4d55b22 /server/lib
parent96f29c0f6d2e623fb088e88200934c5df8da9924 (diff)
downloadPeerTube-4157cdb13748cb6e8ce7081d062a8778554cc5a7.tar.gz
PeerTube-4157cdb13748cb6e8ce7081d062a8778554cc5a7.tar.zst
PeerTube-4157cdb13748cb6e8ce7081d062a8778554cc5a7.zip
Refractor videos AP functions
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/process/process-announce.ts2
-rw-r--r--server/lib/activitypub/process/process-create.ts10
-rw-r--r--server/lib/activitypub/process/process-like.ts2
-rw-r--r--server/lib/activitypub/process/process-undo.ts6
-rw-r--r--server/lib/activitypub/process/process-update.ts4
-rw-r--r--server/lib/activitypub/video-comments.ts2
-rw-r--r--server/lib/activitypub/videos.ts420
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts4
8 files changed, 229 insertions, 221 deletions
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index 814556817..b968389b3 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -25,7 +25,7 @@ export {
25async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { 25async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
26 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id 26 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
27 27
28 const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) 28 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri })
29 29
30 return sequelizeTypescript.transaction(async t => { 30 return sequelizeTypescript.transaction(async t => {
31 // Add share entry 31 // Add share entry
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 32e555acf..99841da14 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -48,7 +48,7 @@ export {
48async function processCreateVideo (activity: ActivityCreate) { 48async function processCreateVideo (activity: ActivityCreate) {
49 const videoToCreateData = activity.object as VideoTorrentObject 49 const videoToCreateData = activity.object as VideoTorrentObject
50 50
51 const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) 51 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData })
52 52
53 return video 53 return video
54} 54}
@@ -59,7 +59,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
59 59
60 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) 60 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
61 61
62 const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) 62 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
63 63
64 return sequelizeTypescript.transaction(async t => { 64 return sequelizeTypescript.transaction(async t => {
65 const rate = { 65 const rate = {
@@ -86,7 +86,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
86async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { 86async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
87 const view = activity.object as ViewObject 87 const view = activity.object as ViewObject
88 88
89 const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) 89 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: view.object })
90 90
91 const actor = await ActorModel.loadByUrl(view.actor) 91 const actor = await ActorModel.loadByUrl(view.actor)
92 if (!actor) throw new Error('Unknown actor ' + view.actor) 92 if (!actor) throw new Error('Unknown actor ' + view.actor)
@@ -103,7 +103,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
103async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { 103async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
104 const cacheFile = activity.object as CacheFileObject 104 const cacheFile = activity.object as CacheFileObject
105 105
106 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) 106 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
107 107
108 await createCacheFile(cacheFile, video, byActor) 108 await createCacheFile(cacheFile, video, byActor)
109 109
@@ -120,7 +120,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat
120 const account = actor.Account 120 const account = actor.Account
121 if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) 121 if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
122 122
123 const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) 123 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object })
124 124
125 return sequelizeTypescript.transaction(async t => { 125 return sequelizeTypescript.transaction(async t => {
126 const videoAbuseData = { 126 const videoAbuseData = {
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index 9e1664fd8..631a9dde7 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
27 const byAccount = byActor.Account 27 const byAccount = byActor.Account
28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
29 29
30 const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) 30 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl })
31 31
32 return sequelizeTypescript.transaction(async t => { 32 return sequelizeTypescript.transaction(async t => {
33 const rate = { 33 const rate = {
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index 0eb5fa392..b78de6697 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -54,7 +54,7 @@ export {
54async function processUndoLike (actorUrl: string, activity: ActivityUndo) { 54async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
55 const likeActivity = activity.object as ActivityLike 55 const likeActivity = activity.object as ActivityLike
56 56
57 const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) 57 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object })
58 58
59 return sequelizeTypescript.transaction(async t => { 59 return sequelizeTypescript.transaction(async t => {
60 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 60 const byAccount = await AccountModel.loadByUrl(actorUrl, t)
@@ -78,7 +78,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
78async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { 78async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
79 const dislike = activity.object.object as DislikeObject 79 const dislike = activity.object.object as DislikeObject
80 80
81 const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) 81 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
82 82
83 return sequelizeTypescript.transaction(async t => { 83 return sequelizeTypescript.transaction(async t => {
84 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 84 const byAccount = await AccountModel.loadByUrl(actorUrl, t)
@@ -102,7 +102,7 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
102async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { 102async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
103 const cacheFileObject = activity.object.object as CacheFileObject 103 const cacheFileObject = activity.object.object as CacheFileObject
104 104
105 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) 105 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object })
106 106
107 return sequelizeTypescript.transaction(async t => { 107 return sequelizeTypescript.transaction(async t => {
108 const byActor = await ActorModel.loadByUrl(actorUrl) 108 const byActor = await ActorModel.loadByUrl(actorUrl)
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index d3af1a181..935da5a54 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -48,7 +48,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
48 return undefined 48 return undefined
49 } 49 }
50 50
51 const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) 51 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id })
52 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 52 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
53 53
54 return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) 54 return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to)
@@ -64,7 +64,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp
64 64
65 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 65 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
66 if (!redundancyModel) { 66 if (!redundancyModel) {
67 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) 67 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id })
68 return createCacheFile(cacheFileObject, video, byActor) 68 return createCacheFile(cacheFileObject, video, byActor)
69 } 69 }
70 70
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index ffbd3a64e..4ca8bf659 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
94 try { 94 try {
95 // Maybe it's a reply to a video? 95 // Maybe it's a reply to a video?
96 // If yes, it's done: we resolved all the thread 96 // If yes, it's done: we resolved all the thread
97 const { video } = await getOrCreateVideoAndAccountAndChannel(url) 97 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url })
98 98
99 if (comments.length !== 0) { 99 if (comments.length !== 0) {
100 const firstReply = comments[ comments.length - 1 ] 100 const firstReply = comments[ comments.length - 1 ]
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 5150c9975..5aabd3e0d 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -3,7 +3,7 @@ import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import { join } from 'path' 4import { join } from 'path'
5import * as request from 'request' 5import * as request from 'request'
6import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' 6import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index'
7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8import { VideoPrivacy } from '../../../shared/models/videos' 8import { VideoPrivacy } from '../../../shared/models/videos'
9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub
28import { createRates } from './video-rates' 28import { createRates } from './video-rates'
29import { addVideoShares, shareVideoByServerAndChannel } from './share' 29import { addVideoShares, shareVideoByServerAndChannel } from './share'
30import { AccountModel } from '../../models/account/account' 30import { AccountModel } from '../../models/account/account'
31import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
31 32
32async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 33async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
33 // If the video is not private and published, we federate it 34 // If the video is not private and published, we federate it
@@ -50,13 +51,24 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr
50 } 51 }
51} 52}
52 53
53function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { 54async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
54 const host = video.VideoChannel.Account.Actor.Server.host 55 const options = {
56 uri: videoUrl,
57 method: 'GET',
58 json: true,
59 activityPub: true
60 }
55 61
56 // We need to provide a callback, if no we could have an uncaught exception 62 logger.info('Fetching remote video %s.', videoUrl)
57 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { 63
58 if (err) reject(err) 64 const { response, body } = await doRequest(options)
59 }) 65
66 if (sanitizeAndCheckVideoTorrentObject(body) === false) {
67 logger.debug('Remote video JSON is not valid.', { body })
68 return { response, videoObject: undefined }
69 }
70
71 return { response, videoObject: body }
60} 72}
61 73
62async function fetchRemoteVideoDescription (video: VideoModel) { 74async function fetchRemoteVideoDescription (video: VideoModel) {
@@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) {
71 return body.description ? body.description : '' 83 return body.description ? body.description : ''
72} 84}
73 85
86function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
87 const host = video.VideoChannel.Account.Actor.Server.host
88
89 // We need to provide a callback, if no we could have an uncaught exception
90 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
91 if (err) reject(err)
92 })
93}
94
74function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { 95function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
75 const thumbnailName = video.getThumbnailName() 96 const thumbnailName = video.getThumbnailName()
76 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) 97 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
@@ -82,94 +103,6 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject)
82 return doRequestAndSaveToFile(options, thumbnailPath) 103 return doRequestAndSaveToFile(options, thumbnailPath)
83} 104}
84 105
85async function videoActivityObjectToDBAttributes (
86 videoChannel: VideoChannelModel,
87 videoObject: VideoTorrentObject,
88 to: string[] = []
89) {
90 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
91 const duration = videoObject.duration.replace(/[^\d]+/, '')
92
93 let language: string | undefined
94 if (videoObject.language) {
95 language = videoObject.language.identifier
96 }
97
98 let category: number | undefined
99 if (videoObject.category) {
100 category = parseInt(videoObject.category.identifier, 10)
101 }
102
103 let licence: number | undefined
104 if (videoObject.licence) {
105 licence = parseInt(videoObject.licence.identifier, 10)
106 }
107
108 const description = videoObject.content || null
109 const support = videoObject.support || null
110
111 return {
112 name: videoObject.name,
113 uuid: videoObject.uuid,
114 url: videoObject.id,
115 category,
116 licence,
117 language,
118 description,
119 support,
120 nsfw: videoObject.sensitive,
121 commentsEnabled: videoObject.commentsEnabled,
122 waitTranscoding: videoObject.waitTranscoding,
123 state: videoObject.state,
124 channelId: videoChannel.id,
125 duration: parseInt(duration, 10),
126 createdAt: new Date(videoObject.published),
127 publishedAt: new Date(videoObject.published),
128 // FIXME: updatedAt does not seems to be considered by Sequelize
129 updatedAt: new Date(videoObject.updated),
130 views: videoObject.views,
131 likes: 0,
132 dislikes: 0,
133 remote: true,
134 privacy
135 }
136}
137
138function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
139 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
140
141 if (fileUrls.length === 0) {
142 throw new Error('Cannot find video files for ' + videoCreated.url)
143 }
144
145 const attributes: VideoFileModel[] = []
146 for (const fileUrl of fileUrls) {
147 // Fetch associated magnet uri
148 const magnet = videoObject.url.find(u => {
149 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
150 })
151
152 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
153
154 const parsed = magnetUtil.decode(magnet.href)
155 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
156 throw new Error('Cannot parse magnet URI ' + magnet.href)
157 }
158
159 const attribute = {
160 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
161 infoHash: parsed.infoHash,
162 resolution: fileUrl.height,
163 size: fileUrl.size,
164 videoId: videoCreated.id,
165 fps: fileUrl.fps
166 } as VideoFileModel
167 attributes.push(attribute)
168 }
169
170 return attributes
171}
172
173function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { 106function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
174 const channel = videoObject.attributedTo.find(a => a.type === 'Group') 107 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
175 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) 108 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
@@ -177,51 +110,6 @@ function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject
177 return getOrCreateActorAndServerAndModel(channel.id) 110 return getOrCreateActorAndServerAndModel(channel.id)
178} 111}
179 112
180async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
181 logger.debug('Adding remote video %s.', videoObject.id)
182
183 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
184 const sequelizeOptions = { transaction: t }
185
186 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
187 const video = VideoModel.build(videoData)
188
189 const videoCreated = await video.save(sequelizeOptions)
190
191 // Process files
192 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
193 if (videoFileAttributes.length === 0) {
194 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
195 }
196
197 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
198 await Promise.all(videoFilePromises)
199
200 // Process tags
201 const tags = videoObject.tag.map(t => t.name)
202 const tagInstances = await TagModel.findOrCreateTags(tags, t)
203 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
204
205 // Process captions
206 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
207 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
208 })
209 await Promise.all(videoCaptionsPromises)
210
211 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
212
213 videoCreated.VideoChannel = channelActor.VideoChannel
214 return videoCreated
215 })
216
217 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
218 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
219
220 if (waitThumbnail === true) await p
221
222 return videoCreated
223}
224
225type SyncParam = { 113type SyncParam = {
226 likes: boolean 114 likes: boolean
227 dislikes: boolean 115 dislikes: boolean
@@ -230,28 +118,7 @@ type SyncParam = {
230 thumbnail: boolean 118 thumbnail: boolean
231 refreshVideo: boolean 119 refreshVideo: boolean
232} 120}
233async function getOrCreateVideoAndAccountAndChannel ( 121async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
234 videoObject: VideoTorrentObject | string,
235 syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
236) {
237 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
238
239 let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
240 if (videoFromDatabase) {
241 const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
242 if (syncParam.refreshVideo === true) videoFromDatabase = await p
243
244 return { video: videoFromDatabase }
245 }
246
247 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
248 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
249
250 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
251 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
252
253 // Process outside the transaction because we could fetch remote data
254
255 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) 122 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
256 123
257 const jobPayloads: ActivitypubHttpFetcherPayload[] = [] 124 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
@@ -285,54 +152,37 @@ async function getOrCreateVideoAndAccountAndChannel (
285 } 152 }
286 153
287 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) 154 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
288
289 return { video }
290} 155}
291 156
292async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { 157async function getOrCreateVideoAndAccountAndChannel (options: {
293 const options = { 158 videoObject: VideoTorrentObject | string,
294 uri: videoUrl, 159 syncParam?: SyncParam,
295 method: 'GET', 160 fetchType?: VideoFetchByUrlType
296 json: true, 161}) {
297 activityPub: true 162 // Default params
298 } 163 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
164 const fetchType = options.fetchType || 'all'
299 165
300 logger.info('Fetching remote video %s.', videoUrl) 166 // Get video url
167 const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id
301 168
302 const { response, body } = await doRequest(options) 169 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
170 if (videoFromDatabase) {
171 const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase, fetchType, syncParam)
172 if (syncParam.refreshVideo === true) videoFromDatabase = await p
303 173
304 if (sanitizeAndCheckVideoTorrentObject(body) === false) { 174 return { video: videoFromDatabase }
305 logger.debug('Remote video JSON is not valid.', { body })
306 return { response, videoObject: undefined }
307 } 175 }
308 176
309 return { response, videoObject: body } 177 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
310} 178 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
311
312async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
313 if (!video.isOutdated()) return video
314
315 try {
316 const { response, videoObject } = await fetchRemoteVideo(video.url)
317 if (response.statusCode === 404) {
318 // Video does not exist anymore
319 await video.destroy()
320 return undefined
321 }
322 179
323 if (videoObject === undefined) { 180 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
324 logger.warn('Cannot refresh remote video: invalid body.') 181 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
325 return video
326 }
327 182
328 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 183 await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
329 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
330 184
331 return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) 185 return { video }
332 } catch (err) {
333 logger.warn('Cannot refresh video.', { err })
334 return video
335 }
336} 186}
337 187
338async function updateVideoFromAP ( 188async function updateVideoFromAP (
@@ -433,12 +283,7 @@ export {
433 fetchRemoteVideoStaticFile, 283 fetchRemoteVideoStaticFile,
434 fetchRemoteVideoDescription, 284 fetchRemoteVideoDescription,
435 generateThumbnailFromUrl, 285 generateThumbnailFromUrl,
436 videoActivityObjectToDBAttributes, 286 getOrCreateVideoChannelFromVideoObject
437 videoFileActivityUrlToDBAttributes,
438 createVideo,
439 getOrCreateVideoChannelFromVideoObject,
440 addVideoShares,
441 createRates
442} 287}
443 288
444// --------------------------------------------------------------------------- 289// ---------------------------------------------------------------------------
@@ -448,3 +293,166 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo
448 293
449 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') 294 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
450} 295}
296
297async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
298 logger.debug('Adding remote video %s.', videoObject.id)
299
300 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
301 const sequelizeOptions = { transaction: t }
302
303 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
304 const video = VideoModel.build(videoData)
305
306 const videoCreated = await video.save(sequelizeOptions)
307
308 // Process files
309 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
310 if (videoFileAttributes.length === 0) {
311 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
312 }
313
314 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
315 await Promise.all(videoFilePromises)
316
317 // Process tags
318 const tags = videoObject.tag.map(t => t.name)
319 const tagInstances = await TagModel.findOrCreateTags(tags, t)
320 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
321
322 // Process captions
323 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
324 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
325 })
326 await Promise.all(videoCaptionsPromises)
327
328 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
329
330 videoCreated.VideoChannel = channelActor.VideoChannel
331 return videoCreated
332 })
333
334 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
335 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
336
337 if (waitThumbnail === true) await p
338
339 return videoCreated
340}
341
342async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFetchByUrlType, syncParam: SyncParam): Promise<VideoModel> {
343 // We need more attributes if the argument video was fetched with not enough joints
344 const video = fetchedType === 'all' ? videoArg : await VideoModel.loadByUrlAndPopulateAccount(videoArg.url)
345
346 if (!video.isOutdated()) return video
347
348 try {
349 const { response, videoObject } = await fetchRemoteVideo(video.url)
350 if (response.statusCode === 404) {
351 // Video does not exist anymore
352 await video.destroy()
353 return undefined
354 }
355
356 if (videoObject === undefined) {
357 logger.warn('Cannot refresh remote video: invalid body.')
358 return video
359 }
360
361 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
362 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
363
364 await updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
365 await syncVideoExternalAttributes(video, videoObject, syncParam)
366 } catch (err) {
367 logger.warn('Cannot refresh video.', { err })
368 return video
369 }
370}
371
372async function videoActivityObjectToDBAttributes (
373 videoChannel: VideoChannelModel,
374 videoObject: VideoTorrentObject,
375 to: string[] = []
376) {
377 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
378 const duration = videoObject.duration.replace(/[^\d]+/, '')
379
380 let language: string | undefined
381 if (videoObject.language) {
382 language = videoObject.language.identifier
383 }
384
385 let category: number | undefined
386 if (videoObject.category) {
387 category = parseInt(videoObject.category.identifier, 10)
388 }
389
390 let licence: number | undefined
391 if (videoObject.licence) {
392 licence = parseInt(videoObject.licence.identifier, 10)
393 }
394
395 const description = videoObject.content || null
396 const support = videoObject.support || null
397
398 return {
399 name: videoObject.name,
400 uuid: videoObject.uuid,
401 url: videoObject.id,
402 category,
403 licence,
404 language,
405 description,
406 support,
407 nsfw: videoObject.sensitive,
408 commentsEnabled: videoObject.commentsEnabled,
409 waitTranscoding: videoObject.waitTranscoding,
410 state: videoObject.state,
411 channelId: videoChannel.id,
412 duration: parseInt(duration, 10),
413 createdAt: new Date(videoObject.published),
414 publishedAt: new Date(videoObject.published),
415 // FIXME: updatedAt does not seems to be considered by Sequelize
416 updatedAt: new Date(videoObject.updated),
417 views: videoObject.views,
418 likes: 0,
419 dislikes: 0,
420 remote: true,
421 privacy
422 }
423}
424
425function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
426 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
427
428 if (fileUrls.length === 0) {
429 throw new Error('Cannot find video files for ' + videoCreated.url)
430 }
431
432 const attributes: VideoFileModel[] = []
433 for (const fileUrl of fileUrls) {
434 // Fetch associated magnet uri
435 const magnet = videoObject.url.find(u => {
436 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
437 })
438
439 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
440
441 const parsed = magnetUtil.decode(magnet.href)
442 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
443 throw new Error('Cannot parse magnet URI ' + magnet.href)
444 }
445
446 const attribute = {
447 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
448 infoHash: parsed.infoHash,
449 resolution: fileUrl.height,
450 size: fileUrl.size,
451 videoId: videoCreated.id,
452 fps: fileUrl.fps
453 } as VideoFileModel
454 attributes.push(attribute)
455 }
456
457 return attributes
458}
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 72d670277..42217c27c 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,10 +1,10 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { processActivities } from '../../activitypub/process' 3import { processActivities } from '../../activitypub/process'
4import { VideoModel } from '../../../models/video/video'
5import { addVideoShares, createRates } from '../../activitypub/videos'
6import { addVideoComments } from '../../activitypub/video-comments' 4import { addVideoComments } from '../../activitypub/video-comments'
7import { crawlCollectionPage } from '../../activitypub/crawl' 5import { crawlCollectionPage } from '../../activitypub/crawl'
6import { VideoModel } from '../../../models/video/video'
7import { addVideoShares, createRates } from '../../activitypub'
8 8
9type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' 9type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments'
10 10