1 import * as Bluebird from 'bluebird'
2 import * as sequelize from 'sequelize'
3 import * as magnetUtil from 'magnet-uri'
4 import { join } from 'path'
5 import * as request from 'request'
6 import { ActivityIconObject, VideoState } from '../../../shared/index'
7 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8 import { VideoPrivacy } from '../../../shared/models/videos'
9 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
10 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
11 import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
12 import { logger } from '../../helpers/logger'
13 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
14 import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
15 import { ActorModel } from '../../models/activitypub/actor'
16 import { TagModel } from '../../models/video/tag'
17 import { VideoModel } from '../../models/video/video'
18 import { VideoChannelModel } from '../../models/video/video-channel'
19 import { VideoFileModel } from '../../models/video/video-file'
20 import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor'
21 import { addVideoComments } from './video-comments'
22 import { crawlCollectionPage } from './crawl'
23 import { sendCreateVideo, sendUpdateVideo } from './send'
24 import { isArray } from '../../helpers/custom-validators/misc'
25 import { VideoCaptionModel } from '../../models/video/video-caption'
26 import { JobQueue } from '../job-queue'
27 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
28 import { getUrlFromWebfinger } from '../../helpers/webfinger'
29 import { createRates } from './video-rates'
30 import { addVideoShares, shareVideoByServerAndChannel } from './share'
31 import { AccountModel } from '../../models/account/account'
33 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
34 // If the video is not private and published, we federate it
35 if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) {
36 // Fetch more attributes that we will need to serialize in AP object
37 if (isArray(video.VideoCaptions) === false) {
38 video.VideoCaptions = await video.$get('VideoCaptions', {
39 attributes: [ 'language' ],
41 }) as VideoCaptionModel[]
45 // Now we'll add the video's meta data to our followers
46 await sendCreateVideo(video, transaction)
47 await shareVideoByServerAndChannel(video, transaction)
49 await sendUpdateVideo(video, transaction)
54 function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
55 const host = video.VideoChannel.Account.Actor.Server.host
57 // We need to provide a callback, if no we could have an uncaught exception
58 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
63 async function fetchRemoteVideoDescription (video: VideoModel) {
64 const host = video.VideoChannel.Account.Actor.Server.host
65 const path = video.getDescriptionPath()
67 uri: REMOTE_SCHEME.HTTP + '://' + host + path,
71 const { body } = await doRequest(options)
72 return body.description ? body.description : ''
75 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
76 const thumbnailName = video.getThumbnailName()
77 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
83 return doRequestAndSaveToFile(options, thumbnailPath)
86 async function videoActivityObjectToDBAttributes (
87 videoChannel: VideoChannelModel,
88 videoObject: VideoTorrentObject,
91 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
92 const duration = videoObject.duration.replace(/[^\d]+/, '')
94 let language: string | undefined
95 if (videoObject.language) {
96 language = videoObject.language.identifier
99 let category: number | undefined
100 if (videoObject.category) {
101 category = parseInt(videoObject.category.identifier, 10)
104 let licence: number | undefined
105 if (videoObject.licence) {
106 licence = parseInt(videoObject.licence.identifier, 10)
109 const description = videoObject.content || null
110 const support = videoObject.support || null
113 name: videoObject.name,
114 uuid: videoObject.uuid,
121 nsfw: videoObject.sensitive,
122 commentsEnabled: videoObject.commentsEnabled,
123 waitTranscoding: videoObject.waitTranscoding,
124 state: videoObject.state,
125 channelId: videoChannel.id,
126 duration: parseInt(duration, 10),
127 createdAt: new Date(videoObject.published),
128 publishedAt: new Date(videoObject.published),
129 // FIXME: updatedAt does not seems to be considered by Sequelize
130 updatedAt: new Date(videoObject.updated),
131 views: videoObject.views,
139 function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
140 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
141 const fileUrls = videoObject.url.filter(u => {
142 return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
145 if (fileUrls.length === 0) {
146 throw new Error('Cannot find video files for ' + videoCreated.url)
149 const attributes: VideoFileModel[] = []
150 for (const fileUrl of fileUrls) {
151 // Fetch associated magnet uri
152 const magnet = videoObject.url.find(u => {
153 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
156 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
158 const parsed = magnetUtil.decode(magnet.href)
159 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
160 throw new Error('Cannot parse magnet URI ' + magnet.href)
164 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
165 infoHash: parsed.infoHash,
166 resolution: fileUrl.height,
168 videoId: videoCreated.id,
171 attributes.push(attribute)
177 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
178 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
179 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
181 return getOrCreateActorAndServerAndModel(channel.id)
184 async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
185 logger.debug('Adding remote video %s.', videoObject.id)
187 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
188 const sequelizeOptions = { transaction: t }
190 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
191 const video = VideoModel.build(videoData)
193 const videoCreated = await video.save(sequelizeOptions)
196 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
197 if (videoFileAttributes.length === 0) {
198 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
201 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
202 await Promise.all(videoFilePromises)
205 const tags = videoObject.tag.map(t => t.name)
206 const tagInstances = await TagModel.findOrCreateTags(tags, t)
207 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
210 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
211 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
213 await Promise.all(videoCaptionsPromises)
215 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
217 videoCreated.VideoChannel = channelActor.VideoChannel
221 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
222 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
224 if (waitThumbnail === true) await p
235 refreshVideo: boolean
237 async function getOrCreateVideoAndAccountAndChannel (
238 videoObject: VideoTorrentObject | string,
239 syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
241 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
243 let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
244 if (videoFromDatabase) {
245 const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
246 if (syncParam.refreshVideo === true) videoFromDatabase = await p
248 return { video: videoFromDatabase }
251 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
252 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
254 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
255 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
257 // Process outside the transaction because we could fetch remote data
259 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
261 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
263 if (syncParam.likes === true) {
264 await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like'))
265 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
267 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
270 if (syncParam.dislikes === true) {
271 await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike'))
272 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
274 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
277 if (syncParam.shares === true) {
278 await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video))
279 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
281 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
284 if (syncParam.comments === true) {
285 await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video))
286 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
288 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
291 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
296 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
304 logger.info('Fetching remote video %s.', videoUrl)
306 const { response, body } = await doRequest(options)
308 if (sanitizeAndCheckVideoTorrentObject(body) === false) {
309 logger.debug('Remote video JSON is not valid.', { body })
310 return { response, videoObject: undefined }
313 return { response, videoObject: body }
316 async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
317 if (!video.isOutdated()) return video
320 const { response, videoObject } = await fetchRemoteVideo(video.url)
321 if (response.statusCode === 404) {
322 // Video does not exist anymore
323 await video.destroy()
327 if (videoObject === undefined) {
328 logger.warn('Cannot refresh remote video: invalid body.')
332 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
333 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
334 return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
337 logger.warn('Cannot refresh video.', { err })
342 async function updateVideoFromAP (
344 videoObject: VideoTorrentObject,
345 accountActor: ActorModel,
346 channelActor: ActorModel,
347 overrideTo?: string[]
349 logger.debug('Updating remote video "%s".', videoObject.uuid)
350 let videoFieldsSave: any
353 const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
354 const sequelizeOptions = {
358 videoFieldsSave = video.toJSON()
360 // Check actor has the right to update the video
361 const videoChannel = video.VideoChannel
362 if (videoChannel.Account.Actor.id !== accountActor.id) {
363 throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url)
366 const to = overrideTo ? overrideTo : videoObject.to
367 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to)
368 video.set('name', videoData.name)
369 video.set('uuid', videoData.uuid)
370 video.set('url', videoData.url)
371 video.set('category', videoData.category)
372 video.set('licence', videoData.licence)
373 video.set('language', videoData.language)
374 video.set('description', videoData.description)
375 video.set('support', videoData.support)
376 video.set('nsfw', videoData.nsfw)
377 video.set('commentsEnabled', videoData.commentsEnabled)
378 video.set('waitTranscoding', videoData.waitTranscoding)
379 video.set('state', videoData.state)
380 video.set('duration', videoData.duration)
381 video.set('createdAt', videoData.createdAt)
382 video.set('publishedAt', videoData.publishedAt)
383 video.set('views', videoData.views)
384 video.set('privacy', videoData.privacy)
385 video.set('channelId', videoData.channelId)
387 await video.save(sequelizeOptions)
389 // Don't block on request
390 generateThumbnailFromUrl(video, videoObject.icon)
391 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
393 // Remove old video files
394 const videoFileDestroyTasks: Bluebird<void>[] = []
395 for (const videoFile of video.VideoFiles) {
396 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
398 await Promise.all(videoFileDestroyTasks)
400 const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject)
401 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
402 await Promise.all(tasks)
405 const tags = videoObject.tag.map(tag => tag.name)
406 const tagInstances = await TagModel.findOrCreateTags(tags, t)
407 await video.$set('Tags', tagInstances, sequelizeOptions)
410 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t)
412 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
413 return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t)
415 await Promise.all(videoCaptionsPromises)
418 logger.info('Remote video with uuid %s updated', videoObject.uuid)
422 if (video !== undefined && videoFieldsSave !== undefined) {
423 resetSequelizeInstance(video, videoFieldsSave)
426 // This is just a debug because we will retry the insert
427 logger.debug('Cannot update the remote video.', { err })
434 federateVideoIfNeeded,
436 getOrCreateVideoAndAccountAndChannel,
437 fetchRemoteVideoStaticFile,
438 fetchRemoteVideoDescription,
439 generateThumbnailFromUrl,
440 videoActivityObjectToDBAttributes,
441 videoFileActivityUrlToDBAttributes,
443 getOrCreateVideoChannelFromVideoObject,