1 import * as Bluebird from 'bluebird'
2 import * as sequelize from 'sequelize'
3 import * as magnetUtil from 'magnet-uri'
4 import { join } from 'path'
5 import * as request from 'request'
6 import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index'
7 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8 import { VideoPrivacy } from '../../../shared/models/videos'
9 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
10 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
11 import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
12 import { logger } from '../../helpers/logger'
13 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
14 import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
15 import { ActorModel } from '../../models/activitypub/actor'
16 import { TagModel } from '../../models/video/tag'
17 import { VideoModel } from '../../models/video/video'
18 import { VideoChannelModel } from '../../models/video/video-channel'
19 import { VideoFileModel } from '../../models/video/video-file'
20 import { getOrCreateActorAndServerAndModel } from './actor'
21 import { addVideoComments } from './video-comments'
22 import { crawlCollectionPage } from './crawl'
23 import { sendCreateVideo, sendUpdateVideo } from './send'
24 import { isArray } from '../../helpers/custom-validators/misc'
25 import { VideoCaptionModel } from '../../models/video/video-caption'
26 import { JobQueue } from '../job-queue'
27 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
28 import { createRates } from './video-rates'
29 import { addVideoShares, shareVideoByServerAndChannel } from './share'
30 import { AccountModel } from '../../models/account/account'
32 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
33 // If the video is not private and published, we federate it
34 if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) {
35 // Fetch more attributes that we will need to serialize in AP object
36 if (isArray(video.VideoCaptions) === false) {
37 video.VideoCaptions = await video.$get('VideoCaptions', {
38 attributes: [ 'language' ],
40 }) as VideoCaptionModel[]
44 // Now we'll add the video's meta data to our followers
45 await sendCreateVideo(video, transaction)
46 await shareVideoByServerAndChannel(video, transaction)
48 await sendUpdateVideo(video, transaction)
53 function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
54 const host = video.VideoChannel.Account.Actor.Server.host
56 // We need to provide a callback, if no we could have an uncaught exception
57 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
62 async function fetchRemoteVideoDescription (video: VideoModel) {
63 const host = video.VideoChannel.Account.Actor.Server.host
64 const path = video.getDescriptionPath()
66 uri: REMOTE_SCHEME.HTTP + '://' + host + path,
70 const { body } = await doRequest(options)
71 return body.description ? body.description : ''
74 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
75 const thumbnailName = video.getThumbnailName()
76 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
82 return doRequestAndSaveToFile(options, thumbnailPath)
85 async function videoActivityObjectToDBAttributes (
86 videoChannel: VideoChannelModel,
87 videoObject: VideoTorrentObject,
90 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
91 const duration = videoObject.duration.replace(/[^\d]+/, '')
93 let language: string | undefined
94 if (videoObject.language) {
95 language = videoObject.language.identifier
98 let category: number | undefined
99 if (videoObject.category) {
100 category = parseInt(videoObject.category.identifier, 10)
103 let licence: number | undefined
104 if (videoObject.licence) {
105 licence = parseInt(videoObject.licence.identifier, 10)
108 const description = videoObject.content || null
109 const support = videoObject.support || null
112 name: videoObject.name,
113 uuid: videoObject.uuid,
120 nsfw: videoObject.sensitive,
121 commentsEnabled: videoObject.commentsEnabled,
122 waitTranscoding: videoObject.waitTranscoding,
123 state: videoObject.state,
124 channelId: videoChannel.id,
125 duration: parseInt(duration, 10),
126 createdAt: new Date(videoObject.published),
127 publishedAt: new Date(videoObject.published),
128 // FIXME: updatedAt does not seems to be considered by Sequelize
129 updatedAt: new Date(videoObject.updated),
130 views: videoObject.views,
138 function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
139 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
141 if (fileUrls.length === 0) {
142 throw new Error('Cannot find video files for ' + videoCreated.url)
145 const attributes: VideoFileModel[] = []
146 for (const fileUrl of fileUrls) {
147 // Fetch associated magnet uri
148 const magnet = videoObject.url.find(u => {
149 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
152 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
154 const parsed = magnetUtil.decode(magnet.href)
155 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
156 throw new Error('Cannot parse magnet URI ' + magnet.href)
160 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
161 infoHash: parsed.infoHash,
162 resolution: fileUrl.height,
164 videoId: videoCreated.id,
167 attributes.push(attribute)
173 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
174 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
175 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
177 return getOrCreateActorAndServerAndModel(channel.id)
180 async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
181 logger.debug('Adding remote video %s.', videoObject.id)
183 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
184 const sequelizeOptions = { transaction: t }
186 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
187 const video = VideoModel.build(videoData)
189 const videoCreated = await video.save(sequelizeOptions)
192 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
193 if (videoFileAttributes.length === 0) {
194 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
197 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
198 await Promise.all(videoFilePromises)
201 const tags = videoObject.tag.map(t => t.name)
202 const tagInstances = await TagModel.findOrCreateTags(tags, t)
203 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
206 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
207 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
209 await Promise.all(videoCaptionsPromises)
211 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
213 videoCreated.VideoChannel = channelActor.VideoChannel
217 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
218 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
220 if (waitThumbnail === true) await p
231 refreshVideo: boolean
233 async function getOrCreateVideoAndAccountAndChannel (
234 videoObject: VideoTorrentObject | string,
235 syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
237 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
239 let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
240 if (videoFromDatabase) {
241 const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
242 if (syncParam.refreshVideo === true) videoFromDatabase = await p
244 return { video: videoFromDatabase }
247 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
248 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
250 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
251 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
253 // Process outside the transaction because we could fetch remote data
255 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
257 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
259 if (syncParam.likes === true) {
260 await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like'))
261 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
263 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
266 if (syncParam.dislikes === true) {
267 await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike'))
268 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
270 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
273 if (syncParam.shares === true) {
274 await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video))
275 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
277 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
280 if (syncParam.comments === true) {
281 await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video))
282 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
284 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
287 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
292 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
300 logger.info('Fetching remote video %s.', videoUrl)
302 const { response, body } = await doRequest(options)
304 if (sanitizeAndCheckVideoTorrentObject(body) === false) {
305 logger.debug('Remote video JSON is not valid.', { body })
306 return { response, videoObject: undefined }
309 return { response, videoObject: body }
312 async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
313 if (!video.isOutdated()) return video
316 const { response, videoObject } = await fetchRemoteVideo(video.url)
317 if (response.statusCode === 404) {
318 // Video does not exist anymore
319 await video.destroy()
323 if (videoObject === undefined) {
324 logger.warn('Cannot refresh remote video: invalid body.')
328 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
329 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
331 return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
333 logger.warn('Cannot refresh video.', { err })
338 async function updateVideoFromAP (
340 videoObject: VideoTorrentObject,
341 account: AccountModel,
342 channel: VideoChannelModel,
343 overrideTo?: string[]
345 logger.debug('Updating remote video "%s".', videoObject.uuid)
346 let videoFieldsSave: any
349 const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
350 const sequelizeOptions = {
354 videoFieldsSave = video.toJSON()
356 // Check actor has the right to update the video
357 const videoChannel = video.VideoChannel
358 if (videoChannel.Account.id !== account.id) {
359 throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
362 const to = overrideTo ? overrideTo : videoObject.to
363 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
364 video.set('name', videoData.name)
365 video.set('uuid', videoData.uuid)
366 video.set('url', videoData.url)
367 video.set('category', videoData.category)
368 video.set('licence', videoData.licence)
369 video.set('language', videoData.language)
370 video.set('description', videoData.description)
371 video.set('support', videoData.support)
372 video.set('nsfw', videoData.nsfw)
373 video.set('commentsEnabled', videoData.commentsEnabled)
374 video.set('waitTranscoding', videoData.waitTranscoding)
375 video.set('state', videoData.state)
376 video.set('duration', videoData.duration)
377 video.set('createdAt', videoData.createdAt)
378 video.set('publishedAt', videoData.publishedAt)
379 video.set('views', videoData.views)
380 video.set('privacy', videoData.privacy)
381 video.set('channelId', videoData.channelId)
383 await video.save(sequelizeOptions)
385 // Don't block on request
386 generateThumbnailFromUrl(video, videoObject.icon)
387 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
389 // Remove old video files
390 const videoFileDestroyTasks: Bluebird<void>[] = []
391 for (const videoFile of video.VideoFiles) {
392 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
394 await Promise.all(videoFileDestroyTasks)
396 const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject)
397 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
398 await Promise.all(tasks)
401 const tags = videoObject.tag.map(tag => tag.name)
402 const tagInstances = await TagModel.findOrCreateTags(tags, t)
403 await video.$set('Tags', tagInstances, sequelizeOptions)
406 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t)
408 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
409 return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t)
411 await Promise.all(videoCaptionsPromises)
414 logger.info('Remote video with uuid %s updated', videoObject.uuid)
418 if (video !== undefined && videoFieldsSave !== undefined) {
419 resetSequelizeInstance(video, videoFieldsSave)
422 // This is just a debug because we will retry the insert
423 logger.debug('Cannot update the remote video.', { err })
430 federateVideoIfNeeded,
432 getOrCreateVideoAndAccountAndChannel,
433 fetchRemoteVideoStaticFile,
434 fetchRemoteVideoDescription,
435 generateThumbnailFromUrl,
436 videoActivityObjectToDBAttributes,
437 videoFileActivityUrlToDBAttributes,
439 getOrCreateVideoChannelFromVideoObject,
444 // ---------------------------------------------------------------------------
446 function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
447 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
449 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')