1 import * as Bluebird from 'bluebird'
2 import * as sequelize from 'sequelize'
3 import * as magnetUtil from 'magnet-uri'
4 import { join } from 'path'
5 import * as request from 'request'
6 import { ActivityIconObject, VideoState } from '../../../shared/index'
7 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8 import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
9 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
10 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
11 import { retryTransactionWrapper } from '../../helpers/database-utils'
12 import { logger } from '../../helpers/logger'
13 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
14 import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, STATIC_PATHS, VIDEO_MIMETYPE_EXT } from '../../initializers'
15 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
16 import { ActorModel } from '../../models/activitypub/actor'
17 import { TagModel } from '../../models/video/tag'
18 import { VideoModel } from '../../models/video/video'
19 import { VideoChannelModel } from '../../models/video/video-channel'
20 import { VideoFileModel } from '../../models/video/video-file'
21 import { VideoShareModel } from '../../models/video/video-share'
22 import { getOrCreateActorAndServerAndModel } from './actor'
23 import { addVideoComments } from './video-comments'
24 import { crawlCollectionPage } from './crawl'
25 import { sendCreateVideo, sendUpdateVideo } from './send'
26 import { shareVideoByServerAndChannel } from './index'
28 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
29 // If the video is not private and published, we federate it
30 if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) {
31 if (isNewVideo === true) {
32 // Now we'll add the video's meta data to our followers
33 await sendCreateVideo(video, transaction)
34 await shareVideoByServerAndChannel(video, transaction)
36 await sendUpdateVideo(video, transaction)
41 function fetchRemoteVideoPreview (video: VideoModel, reject: Function) {
42 const host = video.VideoChannel.Account.Actor.Server.host
43 const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
45 // We need to provide a callback, if no we could have an uncaught exception
46 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
51 async function fetchRemoteVideoDescription (video: VideoModel) {
52 const host = video.VideoChannel.Account.Actor.Server.host
53 const path = video.getDescriptionPath()
55 uri: REMOTE_SCHEME.HTTP + '://' + host + path,
59 const { body } = await doRequest(options)
60 return body.description ? body.description : ''
63 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
64 const thumbnailName = video.getThumbnailName()
65 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
71 return doRequestAndSaveToFile(options, thumbnailPath)
74 async function videoActivityObjectToDBAttributes (
75 videoChannel: VideoChannelModel,
76 videoObject: VideoTorrentObject,
79 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
80 const duration = videoObject.duration.replace(/[^\d]+/, '')
82 let language: string = null
83 if (videoObject.language) {
84 language = videoObject.language.identifier
87 let category: number = null
88 if (videoObject.category) {
89 category = parseInt(videoObject.category.identifier, 10)
92 let licence: number = null
93 if (videoObject.licence) {
94 licence = parseInt(videoObject.licence.identifier, 10)
97 const description = videoObject.content || null
98 const support = videoObject.support || null
101 name: videoObject.name,
102 uuid: videoObject.uuid,
109 nsfw: videoObject.sensitive,
110 commentsEnabled: videoObject.commentsEnabled,
111 waitTranscoding: videoObject.waitTranscoding,
112 state: videoObject.state,
113 channelId: videoChannel.id,
114 duration: parseInt(duration, 10),
115 createdAt: new Date(videoObject.published),
116 publishedAt: new Date(videoObject.published),
117 // FIXME: updatedAt does not seems to be considered by Sequelize
118 updatedAt: new Date(videoObject.updated),
119 views: videoObject.views,
127 function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
128 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
129 const fileUrls = videoObject.url.filter(u => {
130 return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
133 if (fileUrls.length === 0) {
134 throw new Error('Cannot find video files for ' + videoCreated.url)
137 const attributes = []
138 for (const fileUrl of fileUrls) {
139 // Fetch associated magnet uri
140 const magnet = videoObject.url.find(u => {
141 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width
144 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
146 const parsed = magnetUtil.decode(magnet.href)
147 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.href)
150 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
151 infoHash: parsed.infoHash,
152 resolution: fileUrl.width,
154 videoId: videoCreated.id
156 attributes.push(attribute)
162 function getOrCreateVideoChannel (videoObject: VideoTorrentObject) {
163 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
164 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
166 return getOrCreateActorAndServerAndModel(channel.id)
169 async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel) {
170 logger.debug('Adding remote video %s.', videoObject.id)
172 return sequelizeTypescript.transaction(async t => {
173 const sequelizeOptions = {
176 const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t)
177 if (videoFromDatabase) return videoFromDatabase
179 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
180 const video = VideoModel.build(videoData)
182 // Don't block on request
183 generateThumbnailFromUrl(video, videoObject.icon)
184 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
186 const videoCreated = await video.save(sequelizeOptions)
188 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
189 if (videoFileAttributes.length === 0) {
190 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
193 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
194 await Promise.all(tasks)
196 const tags = videoObject.tag.map(t => t.name)
197 const tagInstances = await TagModel.findOrCreateTags(tags, t)
198 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
200 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
202 videoCreated.VideoChannel = channelActor.VideoChannel
207 async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentObject | string, actor?: ActorModel) {
208 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
210 const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
211 if (videoFromDatabase) {
213 video: videoFromDatabase,
214 actor: videoFromDatabase.VideoChannel.Account.Actor,
215 channelActor: videoFromDatabase.VideoChannel.Actor
219 videoObject = await fetchRemoteVideo(videoUrl)
220 if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
223 const actorObj = videoObject.attributedTo.find(a => a.type === 'Person')
224 if (!actorObj) throw new Error('Cannot find associated actor to video ' + videoObject.url)
226 actor = await getOrCreateActorAndServerAndModel(actorObj.id)
229 const channelActor = await getOrCreateVideoChannel(videoObject)
231 const video = await retryTransactionWrapper(getOrCreateVideo, videoObject, channelActor)
233 // Process outside the transaction because we could fetch remote data
234 logger.info('Adding likes of video %s.', video.uuid)
235 await crawlCollectionPage<string>(videoObject.likes, (items) => createRates(items, video, 'like'))
237 logger.info('Adding dislikes of video %s.', video.uuid)
238 await crawlCollectionPage<string>(videoObject.dislikes, (items) => createRates(items, video, 'dislike'))
240 logger.info('Adding shares of video %s.', video.uuid)
241 await crawlCollectionPage<string>(videoObject.shares, (items) => addVideoShares(items, video))
243 logger.info('Adding comments of video %s.', video.uuid)
244 await crawlCollectionPage<string>(videoObject.comments, (items) => addVideoComments(items, video))
246 return { actor, channelActor, video }
249 async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
251 const tasks: Bluebird<number>[] = []
253 for (const actorUrl of actorUrls) {
254 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
255 const p = AccountVideoRateModel
258 accountId: actor.Account.id,
261 .then(() => rateCounts += 1)
266 await Promise.all(tasks)
268 logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
270 // This is "likes" and "dislikes"
271 await video.increment(rate + 's', { by: rateCounts })
276 async function addVideoShares (shareUrls: string[], instance: VideoModel) {
277 for (const shareUrl of shareUrls) {
279 const { body } = await doRequest({
284 if (!body || !body.actor) {
285 logger.warn('Cannot add remote share with url: %s, skipping...', shareUrl)
289 const actorUrl = body.actor
290 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
294 videoId: instance.id,
298 await VideoShareModel.findOrCreate({
307 async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> {
315 logger.info('Fetching remote video %s.', videoUrl)
317 const { body } = await doRequest(options)
319 if (sanitizeAndCheckVideoTorrentObject(body) === false) {
320 logger.debug('Remote video JSON is not valid.', { body })
328 federateVideoIfNeeded,
330 getOrCreateAccountAndVideoAndChannel,
331 fetchRemoteVideoPreview,
332 fetchRemoteVideoDescription,
333 generateThumbnailFromUrl,
334 videoActivityObjectToDBAttributes,
335 videoFileActivityUrlToDBAttributes,
337 getOrCreateVideoChannel,