diff options
Diffstat (limited to 'server/lib/activitypub/videos.ts')
-rw-r--r-- | server/lib/activitypub/videos.ts | 241 |
1 files changed, 148 insertions, 93 deletions
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index fac1d3fc7..388c31fe5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -5,29 +5,30 @@ import { join } from 'path' | |||
5 | import * as request from 'request' | 5 | import * as request from 'request' |
6 | import { ActivityIconObject, VideoState } from '../../../shared/index' | 6 | import { ActivityIconObject, VideoState } from '../../../shared/index' |
7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
8 | import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' | 8 | import { VideoPrivacy } from '../../../shared/models/videos' |
9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
10 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | 10 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' |
11 | import { retryTransactionWrapper } from '../../helpers/database-utils' | 11 | import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' |
12 | import { logger } from '../../helpers/logger' | 12 | import { logger } from '../../helpers/logger' |
13 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' | 13 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' |
14 | import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' | 14 | import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' |
15 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | ||
16 | import { ActorModel } from '../../models/activitypub/actor' | 15 | import { ActorModel } from '../../models/activitypub/actor' |
17 | import { TagModel } from '../../models/video/tag' | 16 | import { TagModel } from '../../models/video/tag' |
18 | import { VideoModel } from '../../models/video/video' | 17 | import { VideoModel } from '../../models/video/video' |
19 | import { VideoChannelModel } from '../../models/video/video-channel' | 18 | import { VideoChannelModel } from '../../models/video/video-channel' |
20 | import { VideoFileModel } from '../../models/video/video-file' | 19 | import { VideoFileModel } from '../../models/video/video-file' |
21 | import { VideoShareModel } from '../../models/video/video-share' | 20 | import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor' |
22 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
23 | import { addVideoComments } from './video-comments' | 21 | import { addVideoComments } from './video-comments' |
24 | import { crawlCollectionPage } from './crawl' | 22 | import { crawlCollectionPage } from './crawl' |
25 | import { sendCreateVideo, sendUpdateVideo } from './send' | 23 | import { sendCreateVideo, sendUpdateVideo } from './send' |
26 | import { shareVideoByServerAndChannel } from './index' | ||
27 | import { isArray } from '../../helpers/custom-validators/misc' | 24 | import { isArray } from '../../helpers/custom-validators/misc' |
28 | import { VideoCaptionModel } from '../../models/video/video-caption' | 25 | import { VideoCaptionModel } from '../../models/video/video-caption' |
29 | import { JobQueue } from '../job-queue' | 26 | import { JobQueue } from '../job-queue' |
30 | import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' | 27 | import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' |
28 | import { getUrlFromWebfinger } from '../../helpers/webfinger' | ||
29 | import { createRates } from './video-rates' | ||
30 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | ||
31 | import { AccountModel } from '../../models/account/account' | ||
31 | 32 | ||
32 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 33 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
33 | // If the video is not private and published, we federate it | 34 | // If the video is not private and published, we federate it |
@@ -180,15 +181,11 @@ function getOrCreateVideoChannel (videoObject: VideoTorrentObject) { | |||
180 | return getOrCreateActorAndServerAndModel(channel.id) | 181 | return getOrCreateActorAndServerAndModel(channel.id) |
181 | } | 182 | } |
182 | 183 | ||
183 | async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | 184 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { |
184 | logger.debug('Adding remote video %s.', videoObject.id) | 185 | logger.debug('Adding remote video %s.', videoObject.id) |
185 | 186 | ||
186 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | 187 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { |
187 | const sequelizeOptions = { | 188 | const sequelizeOptions = { transaction: t } |
188 | transaction: t | ||
189 | } | ||
190 | const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t) | ||
191 | if (videoFromDatabase) return videoFromDatabase | ||
192 | 189 | ||
193 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | 190 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) |
194 | const video = VideoModel.build(videoData) | 191 | const video = VideoModel.build(videoData) |
@@ -230,26 +227,32 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: | |||
230 | } | 227 | } |
231 | 228 | ||
232 | type SyncParam = { | 229 | type SyncParam = { |
233 | likes: boolean, | 230 | likes: boolean |
234 | dislikes: boolean, | 231 | dislikes: boolean |
235 | shares: boolean, | 232 | shares: boolean |
236 | comments: boolean, | 233 | comments: boolean |
237 | thumbnail: boolean | 234 | thumbnail: boolean |
235 | refreshVideo: boolean | ||
238 | } | 236 | } |
239 | async function getOrCreateAccountAndVideoAndChannel ( | 237 | async function getOrCreateVideoAndAccountAndChannel ( |
240 | videoObject: VideoTorrentObject | string, | 238 | videoObject: VideoTorrentObject | string, |
241 | syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } | 239 | syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } |
242 | ) { | 240 | ) { |
243 | const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id | 241 | const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id |
244 | 242 | ||
245 | const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) | 243 | let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) |
246 | if (videoFromDatabase) return { video: videoFromDatabase } | 244 | if (videoFromDatabase) { |
245 | const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) | ||
246 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
247 | |||
248 | return { video: videoFromDatabase } | ||
249 | } | ||
247 | 250 | ||
248 | const fetchedVideo = await fetchRemoteVideo(videoUrl) | 251 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) |
249 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | 252 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) |
250 | 253 | ||
251 | const channelActor = await getOrCreateVideoChannel(fetchedVideo) | 254 | const channelActor = await getOrCreateVideoChannel(fetchedVideo) |
252 | const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail) | 255 | const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) |
253 | 256 | ||
254 | // Process outside the transaction because we could fetch remote data | 257 | // Process outside the transaction because we could fetch remote data |
255 | 258 | ||
@@ -290,101 +293,153 @@ async function getOrCreateAccountAndVideoAndChannel ( | |||
290 | return { video } | 293 | return { video } |
291 | } | 294 | } |
292 | 295 | ||
293 | async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { | 296 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { |
294 | let rateCounts = 0 | 297 | const options = { |
295 | 298 | uri: videoUrl, | |
296 | await Bluebird.map(actorUrls, async actorUrl => { | 299 | method: 'GET', |
297 | try { | 300 | json: true, |
298 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | 301 | activityPub: true |
299 | const [ , created ] = await AccountVideoRateModel | 302 | } |
300 | .findOrCreate({ | 303 | |
301 | where: { | 304 | logger.info('Fetching remote video %s.', videoUrl) |
302 | videoId: video.id, | 305 | |
303 | accountId: actor.Account.id | 306 | const { response, body } = await doRequest(options) |
304 | }, | 307 | |
305 | defaults: { | 308 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { |
306 | videoId: video.id, | 309 | logger.debug('Remote video JSON is not valid.', { body }) |
307 | accountId: actor.Account.id, | 310 | return { response, videoObject: undefined } |
308 | type: rate | 311 | } |
309 | } | 312 | |
310 | }) | 313 | return { response, videoObject: body } |
311 | 314 | } | |
312 | if (created) rateCounts += 1 | 315 | |
313 | } catch (err) { | 316 | async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> { |
314 | logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) | 317 | if (!video.isOutdated()) return video |
318 | |||
319 | try { | ||
320 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
321 | if (response.statusCode === 404) { | ||
322 | // Video does not exist anymore | ||
323 | await video.destroy() | ||
324 | return undefined | ||
315 | } | 325 | } |
316 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
317 | 326 | ||
318 | logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) | 327 | if (videoObject === undefined) { |
328 | logger.warn('Cannot refresh remote video: invalid body.') | ||
329 | return video | ||
330 | } | ||
319 | 331 | ||
320 | // This is "likes" and "dislikes" | 332 | const channelActor = await getOrCreateVideoChannel(videoObject) |
321 | if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) | 333 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) |
334 | return updateVideoFromAP(video, videoObject, account.Actor, channelActor) | ||
322 | 335 | ||
323 | return | 336 | } catch (err) { |
337 | logger.warn('Cannot refresh video.', { err }) | ||
338 | return video | ||
339 | } | ||
324 | } | 340 | } |
325 | 341 | ||
326 | async function addVideoShares (shareUrls: string[], instance: VideoModel) { | 342 | async function updateVideoFromAP ( |
327 | await Bluebird.map(shareUrls, async shareUrl => { | 343 | video: VideoModel, |
328 | try { | 344 | videoObject: VideoTorrentObject, |
329 | // Fetch url | 345 | accountActor: ActorModel, |
330 | const { body } = await doRequest({ | 346 | channelActor: ActorModel, |
331 | uri: shareUrl, | 347 | overrideTo?: string[] |
332 | json: true, | 348 | ) { |
333 | activityPub: true | 349 | logger.debug('Updating remote video "%s".', videoObject.uuid) |
334 | }) | 350 | let videoFieldsSave: any |
335 | if (!body || !body.actor) throw new Error('Body of body actor is invalid') | 351 | |
352 | try { | ||
353 | const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => { | ||
354 | const sequelizeOptions = { | ||
355 | transaction: t | ||
356 | } | ||
336 | 357 | ||
337 | const actorUrl = body.actor | 358 | videoFieldsSave = video.toJSON() |
338 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | ||
339 | 359 | ||
340 | const entry = { | 360 | // Check actor has the right to update the video |
341 | actorId: actor.id, | 361 | const videoChannel = video.VideoChannel |
342 | videoId: instance.id, | 362 | if (videoChannel.Account.Actor.id !== accountActor.id) { |
343 | url: shareUrl | 363 | throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url) |
344 | } | 364 | } |
345 | 365 | ||
346 | await VideoShareModel.findOrCreate({ | 366 | const to = overrideTo ? overrideTo : videoObject.to |
347 | where: { | 367 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to) |
348 | url: shareUrl | 368 | video.set('name', videoData.name) |
349 | }, | 369 | video.set('uuid', videoData.uuid) |
350 | defaults: entry | 370 | video.set('url', videoData.url) |
351 | }) | 371 | video.set('category', videoData.category) |
352 | } catch (err) { | 372 | video.set('licence', videoData.licence) |
353 | logger.warn('Cannot add share %s.', shareUrl, { err }) | 373 | video.set('language', videoData.language) |
354 | } | 374 | video.set('description', videoData.description) |
355 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | 375 | video.set('support', videoData.support) |
356 | } | 376 | video.set('nsfw', videoData.nsfw) |
377 | video.set('commentsEnabled', videoData.commentsEnabled) | ||
378 | video.set('waitTranscoding', videoData.waitTranscoding) | ||
379 | video.set('state', videoData.state) | ||
380 | video.set('duration', videoData.duration) | ||
381 | video.set('createdAt', videoData.createdAt) | ||
382 | video.set('publishedAt', videoData.publishedAt) | ||
383 | video.set('views', videoData.views) | ||
384 | video.set('privacy', videoData.privacy) | ||
385 | video.set('channelId', videoData.channelId) | ||
386 | |||
387 | await video.save(sequelizeOptions) | ||
388 | |||
389 | // Don't block on request | ||
390 | generateThumbnailFromUrl(video, videoObject.icon) | ||
391 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
392 | |||
393 | // Remove old video files | ||
394 | const videoFileDestroyTasks: Bluebird<void>[] = [] | ||
395 | for (const videoFile of video.VideoFiles) { | ||
396 | videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) | ||
397 | } | ||
398 | await Promise.all(videoFileDestroyTasks) | ||
357 | 399 | ||
358 | async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> { | 400 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) |
359 | const options = { | 401 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) |
360 | uri: videoUrl, | 402 | await Promise.all(tasks) |
361 | method: 'GET', | ||
362 | json: true, | ||
363 | activityPub: true | ||
364 | } | ||
365 | 403 | ||
366 | logger.info('Fetching remote video %s.', videoUrl) | 404 | // Update Tags |
405 | const tags = videoObject.tag.map(tag => tag.name) | ||
406 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
407 | await video.$set('Tags', tagInstances, sequelizeOptions) | ||
367 | 408 | ||
368 | const { body } = await doRequest(options) | 409 | // Update captions |
410 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) | ||
369 | 411 | ||
370 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | 412 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { |
371 | logger.debug('Remote video JSON is not valid.', { body }) | 413 | return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) |
372 | return undefined | 414 | }) |
373 | } | 415 | await Promise.all(videoCaptionsPromises) |
416 | }) | ||
417 | |||
418 | logger.info('Remote video with uuid %s updated', videoObject.uuid) | ||
374 | 419 | ||
375 | return body | 420 | return updatedVideo |
421 | } catch (err) { | ||
422 | if (video !== undefined && videoFieldsSave !== undefined) { | ||
423 | resetSequelizeInstance(video, videoFieldsSave) | ||
424 | } | ||
425 | |||
426 | // This is just a debug because we will retry the insert | ||
427 | logger.debug('Cannot update the remote video.', { err }) | ||
428 | throw err | ||
429 | } | ||
376 | } | 430 | } |
377 | 431 | ||
378 | export { | 432 | export { |
433 | updateVideoFromAP, | ||
379 | federateVideoIfNeeded, | 434 | federateVideoIfNeeded, |
380 | fetchRemoteVideo, | 435 | fetchRemoteVideo, |
381 | getOrCreateAccountAndVideoAndChannel, | 436 | getOrCreateVideoAndAccountAndChannel, |
382 | fetchRemoteVideoStaticFile, | 437 | fetchRemoteVideoStaticFile, |
383 | fetchRemoteVideoDescription, | 438 | fetchRemoteVideoDescription, |
384 | generateThumbnailFromUrl, | 439 | generateThumbnailFromUrl, |
385 | videoActivityObjectToDBAttributes, | 440 | videoActivityObjectToDBAttributes, |
386 | videoFileActivityUrlToDBAttributes, | 441 | videoFileActivityUrlToDBAttributes, |
387 | getOrCreateVideo, | 442 | createVideo, |
388 | getOrCreateVideoChannel, | 443 | getOrCreateVideoChannel, |
389 | addVideoShares, | 444 | addVideoShares, |
390 | createRates | 445 | createRates |