1 import * as Bluebird from 'bluebird'
2 import * as sequelize from 'sequelize'
3 import * as magnetUtil from 'magnet-uri'
4 import { join } from 'path'
5 import * as request from 'request'
6 import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index'
7 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8 import { VideoPrivacy } from '../../../shared/models/videos'
9 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
10 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
11 import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
12 import { logger } from '../../helpers/logger'
13 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
14 import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
15 import { ActorModel } from '../../models/activitypub/actor'
16 import { TagModel } from '../../models/video/tag'
17 import { VideoModel } from '../../models/video/video'
18 import { VideoChannelModel } from '../../models/video/video-channel'
19 import { VideoFileModel } from '../../models/video/video-file'
20 import { getOrCreateActorAndServerAndModel } from './actor'
21 import { addVideoComments } from './video-comments'
22 import { crawlCollectionPage } from './crawl'
23 import { sendCreateVideo, sendUpdateVideo } from './send'
24 import { isArray } from '../../helpers/custom-validators/misc'
25 import { VideoCaptionModel } from '../../models/video/video-caption'
26 import { JobQueue } from '../job-queue'
27 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
28 import { createRates } from './video-rates'
29 import { addVideoShares, shareVideoByServerAndChannel } from './share'
30 import { AccountModel } from '../../models/account/account'
31 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
33 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
34 // If the video is not private and published, we federate it
35 if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) {
36 // Fetch more attributes that we will need to serialize in AP object
37 if (isArray(video.VideoCaptions) === false) {
38 video.VideoCaptions = await video.$get('VideoCaptions', {
39 attributes: [ 'language' ],
41 }) as VideoCaptionModel[]
45 // Now we'll add the video's meta data to our followers
46 await sendCreateVideo(video, transaction)
47 await shareVideoByServerAndChannel(video, transaction)
49 await sendUpdateVideo(video, transaction)
54 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
62 logger.info('Fetching remote video %s.', videoUrl)
64 const { response, body } = await doRequest(options)
66 if (sanitizeAndCheckVideoTorrentObject(body) === false) {
67 logger.debug('Remote video JSON is not valid.', { body })
68 return { response, videoObject: undefined }
71 return { response, videoObject: body }
74 async function fetchRemoteVideoDescription (video: VideoModel) {
75 const host = video.VideoChannel.Account.Actor.Server.host
76 const path = video.getDescriptionAPIPath()
78 uri: REMOTE_SCHEME.HTTP + '://' + host + path,
82 const { body } = await doRequest(options)
83 return body.description ? body.description : ''
86 function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
87 const host = video.VideoChannel.Account.Actor.Server.host
89 // We need to provide a callback, if no we could have an uncaught exception
90 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
95 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
96 const thumbnailName = video.getThumbnailName()
97 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
103 return doRequestAndSaveToFile(options, thumbnailPath)
106 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
107 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
108 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
110 return getOrCreateActorAndServerAndModel(channel.id, 'all')
119 refreshVideo: boolean
121 async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
122 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
124 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
126 if (syncParam.likes === true) {
127 await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like'))
128 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
130 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
133 if (syncParam.dislikes === true) {
134 await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike'))
135 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
137 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
140 if (syncParam.shares === true) {
141 await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video))
142 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
144 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
147 if (syncParam.comments === true) {
148 await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video))
149 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
151 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
154 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
157 async function getOrCreateVideoAndAccountAndChannel (options: {
158 videoObject: VideoTorrentObject | string,
159 syncParam?: SyncParam,
160 fetchType?: VideoFetchByUrlType,
161 refreshViews?: boolean
164 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
165 const fetchType = options.fetchType || 'all'
166 const refreshViews = options.refreshViews || false
169 const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id
171 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
172 if (videoFromDatabase) {
173 const refreshOptions = {
174 video: videoFromDatabase,
175 fetchedType: fetchType,
179 const p = refreshVideoIfNeeded(refreshOptions)
180 if (syncParam.refreshVideo === true) videoFromDatabase = await p
182 return { video: videoFromDatabase }
185 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
186 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
188 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
189 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
191 await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
196 async function updateVideoFromAP (options: {
198 videoObject: VideoTorrentObject,
199 account: AccountModel,
200 channel: VideoChannelModel,
201 updateViews: boolean,
202 overrideTo?: string[]
204 logger.debug('Updating remote video "%s".', options.videoObject.uuid)
205 let videoFieldsSave: any
208 const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
209 const sequelizeOptions = {
213 videoFieldsSave = options.video.toJSON()
215 // Check actor has the right to update the video
216 const videoChannel = options.video.VideoChannel
217 if (videoChannel.Account.id !== options.account.id) {
218 throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
221 const to = options.overrideTo ? options.overrideTo : options.videoObject.to
222 const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to)
223 options.video.set('name', videoData.name)
224 options.video.set('uuid', videoData.uuid)
225 options.video.set('url', videoData.url)
226 options.video.set('category', videoData.category)
227 options.video.set('licence', videoData.licence)
228 options.video.set('language', videoData.language)
229 options.video.set('description', videoData.description)
230 options.video.set('support', videoData.support)
231 options.video.set('nsfw', videoData.nsfw)
232 options.video.set('commentsEnabled', videoData.commentsEnabled)
233 options.video.set('waitTranscoding', videoData.waitTranscoding)
234 options.video.set('state', videoData.state)
235 options.video.set('duration', videoData.duration)
236 options.video.set('createdAt', videoData.createdAt)
237 options.video.set('publishedAt', videoData.publishedAt)
238 options.video.set('privacy', videoData.privacy)
239 options.video.set('channelId', videoData.channelId)
241 if (options.updateViews === true) options.video.set('views', videoData.views)
242 await options.video.save(sequelizeOptions)
244 // Don't block on request
245 generateThumbnailFromUrl(options.video, options.videoObject.icon)
246 .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }))
249 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
250 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
252 // Remove video files that do not exist anymore
253 const destroyTasks = options.video.VideoFiles
254 .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
255 .map(f => f.destroy(sequelizeOptions))
256 await Promise.all(destroyTasks)
258 // Update or add other one
259 const upsertTasks = videoFileAttributes.map(a => VideoFileModel.upsert(a, sequelizeOptions))
260 await Promise.all(upsertTasks)
265 const tags = options.videoObject.tag.map(tag => tag.name)
266 const tagInstances = await TagModel.findOrCreateTags(tags, t)
267 await options.video.$set('Tags', tagInstances, sequelizeOptions)
272 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
274 const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
275 return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
277 await Promise.all(videoCaptionsPromises)
281 logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
285 if (options.video !== undefined && videoFieldsSave !== undefined) {
286 resetSequelizeInstance(options.video, videoFieldsSave)
289 // This is just a debug because we will retry the insert
290 logger.debug('Cannot update the remote video.', { err })
297 federateVideoIfNeeded,
299 getOrCreateVideoAndAccountAndChannel,
300 fetchRemoteVideoStaticFile,
301 fetchRemoteVideoDescription,
302 generateThumbnailFromUrl,
303 getOrCreateVideoChannelFromVideoObject
306 // ---------------------------------------------------------------------------
308 function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
309 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
311 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
314 async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
315 logger.debug('Adding remote video %s.', videoObject.id)
317 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
318 const sequelizeOptions = { transaction: t }
320 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
321 const video = VideoModel.build(videoData)
323 const videoCreated = await video.save(sequelizeOptions)
326 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
327 if (videoFileAttributes.length === 0) {
328 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
331 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
332 await Promise.all(videoFilePromises)
335 const tags = videoObject.tag.map(t => t.name)
336 const tagInstances = await TagModel.findOrCreateTags(tags, t)
337 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
340 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
341 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
343 await Promise.all(videoCaptionsPromises)
345 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
347 videoCreated.VideoChannel = channelActor.VideoChannel
351 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
352 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
354 if (waitThumbnail === true) await p
359 async function refreshVideoIfNeeded (options: {
361 fetchedType: VideoFetchByUrlType,
362 syncParam: SyncParam,
363 refreshViews: boolean
364 }): Promise<VideoModel> {
365 if (!options.video.isOutdated()) return options.video
367 // We need more attributes if the argument video was fetched with not enough joints
368 const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
371 const { response, videoObject } = await fetchRemoteVideo(video.url)
372 if (response.statusCode === 404) {
373 // Video does not exist anymore
374 await video.destroy()
378 if (videoObject === undefined) {
379 logger.warn('Cannot refresh remote video: invalid body.')
383 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
384 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
386 const updateOptions = {
390 channel: channelActor.VideoChannel,
391 updateViews: options.refreshViews
393 await retryTransactionWrapper(updateVideoFromAP, updateOptions)
394 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
396 logger.warn('Cannot refresh video.', { err })
401 async function videoActivityObjectToDBAttributes (
402 videoChannel: VideoChannelModel,
403 videoObject: VideoTorrentObject,
406 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
407 const duration = videoObject.duration.replace(/[^\d]+/, '')
409 let language: string | undefined
410 if (videoObject.language) {
411 language = videoObject.language.identifier
414 let category: number | undefined
415 if (videoObject.category) {
416 category = parseInt(videoObject.category.identifier, 10)
419 let licence: number | undefined
420 if (videoObject.licence) {
421 licence = parseInt(videoObject.licence.identifier, 10)
424 const description = videoObject.content || null
425 const support = videoObject.support || null
428 name: videoObject.name,
429 uuid: videoObject.uuid,
436 nsfw: videoObject.sensitive,
437 commentsEnabled: videoObject.commentsEnabled,
438 waitTranscoding: videoObject.waitTranscoding,
439 state: videoObject.state,
440 channelId: videoChannel.id,
441 duration: parseInt(duration, 10),
442 createdAt: new Date(videoObject.published),
443 publishedAt: new Date(videoObject.published),
444 // FIXME: updatedAt does not seems to be considered by Sequelize
445 updatedAt: new Date(videoObject.updated),
446 views: videoObject.views,
454 function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
455 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
457 if (fileUrls.length === 0) {
458 throw new Error('Cannot find video files for ' + videoCreated.url)
461 const attributes: VideoFileModel[] = []
462 for (const fileUrl of fileUrls) {
463 // Fetch associated magnet uri
464 const magnet = videoObject.url.find(u => {
465 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
468 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
470 const parsed = magnetUtil.decode(magnet.href)
471 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
472 throw new Error('Cannot parse magnet URI ' + magnet.href)
476 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
477 infoHash: parsed.infoHash,
478 resolution: fileUrl.height,
480 videoId: videoCreated.id,
483 attributes.push(attribute)