server/lib/activitypub/videos.ts

   1 import * as Bluebird from 'bluebird'
   2 import * as sequelize from 'sequelize'
   3 import * as magnetUtil from 'magnet-uri'
   4 import * as request from 'request'
   5 import {
   6   ActivityPlaylistSegmentHashesObject,
   7   ActivityPlaylistUrlObject,
   8   ActivityUrlObject,
   9   ActivityVideoUrlObject,
  10   VideoState
  11 } from '../../../shared/index'
  12 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
  13 import { VideoPrivacy } from '../../../shared/models/videos'
  14 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
  15 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
  16 import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
  17 import { logger } from '../../helpers/logger'
  18 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
  19 import {
  20   ACTIVITY_PUB,
  21   MIMETYPES,
  22   P2P_MEDIA_LOADER_PEER_VERSION,
  23   PREVIEWS_SIZE,
  24   REMOTE_SCHEME,
  25   STATIC_PATHS
  26 } from '../../initializers/constants'
  27 import { TagModel } from '../../models/video/tag'
  28 import { VideoModel } from '../../models/video/video'
  29 import { VideoFileModel } from '../../models/video/video-file'
  30 import { getOrCreateActorAndServerAndModel } from './actor'
  31 import { addVideoComments } from './video-comments'
  32 import { crawlCollectionPage } from './crawl'
  33 import { sendCreateVideo, sendUpdateVideo } from './send'
  34 import { isArray } from '../../helpers/custom-validators/misc'
  35 import { VideoCaptionModel } from '../../models/video/video-caption'
  36 import { JobQueue } from '../job-queue'
  37 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
  38 import { createRates } from './video-rates'
  39 import { addVideoShares, shareVideoByServerAndChannel } from './share'
  40 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
  41 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
  42 import { Notifier } from '../notifier'
  43 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
  44 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
  45 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
  46 import { VideoShareModel } from '../../models/video/video-share'
  47 import { VideoCommentModel } from '../../models/video/video-comment'
  48 import { sequelizeTypescript } from '../../initializers/database'
  49 import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
  50 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
  51 import { join } from 'path'
  52 import { FilteredModelAttributes } from '../../typings/sequelize'
  53 import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
  54 import { ActorFollowScoreCache } from '../files-cache'
  55 import {
  56   MAccountIdActor,
  57   MChannelAccountLight,
  58   MChannelDefault,
  59   MChannelId,
  60   MVideo,
  61   MVideoAccountLight,
  62   MVideoAccountLightBlacklistAllFiles,
  63   MVideoAP,
  64   MVideoAPWithoutCaption,
  65   MVideoFile,
  66   MVideoFullLight,
  67   MVideoId,
  68   MVideoThumbnail
  69 } from '../../typings/models'
  70 import { MThumbnail } from '../../typings/models/video/thumbnail'
  71
  72 async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
  73   const video = videoArg as MVideoAP
  74
  75   if (
  76     // Check this is not a blacklisted video, or unfederated blacklisted video
  77     (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
  78     // Check the video is public/unlisted and published
  79     video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED
  80   ) {
  81     // Fetch more attributes that we will need to serialize in AP object
  82     if (isArray(video.VideoCaptions) === false) {
  83       video.VideoCaptions = await video.$get('VideoCaptions', {
  84         attributes: [ 'language' ],
  85         transaction
  86       }) as VideoCaptionModel[]
  87     }
  88
  89     if (isNewVideo) {
  90       // Now we'll add the video's meta data to our followers
  91       await sendCreateVideo(video, transaction)
  92       await shareVideoByServerAndChannel(video, transaction)
  93     } else {
  94       await sendUpdateVideo(video, transaction)
  95     }
  96   }
  97 }
  98
  99 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
 100   const options = {
 101     uri: videoUrl,
 102     method: 'GET',
 103     json: true,
 104     activityPub: true
 105   }
 106
 107   logger.info('Fetching remote video %s.', videoUrl)
 108
 109   const { response, body } = await doRequest(options)
 110
 111   if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
 112     logger.debug('Remote video JSON is not valid.', { body })
 113     return { response, videoObject: undefined }
 114   }
 115
 116   return { response, videoObject: body }
 117 }
 118
 119 async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
 120   const host = video.VideoChannel.Account.Actor.Server.host
 121   const path = video.getDescriptionAPIPath()
 122   const options = {
 123     uri: REMOTE_SCHEME.HTTP + '://' + host + path,
 124     json: true
 125   }
 126
 127   const { body } = await doRequest(options)
 128   return body.description ? body.description : ''
 129 }
 130
 131 function fetchRemoteVideoStaticFile (video: MVideoAccountLight, path: string, destPath: string) {
 132   const url = buildRemoteBaseUrl(video, path)
 133
 134   // We need to provide a callback, if no we could have an uncaught exception
 135   return doRequestAndSaveToFile({ uri: url }, destPath)
 136 }
 137
 138 function buildRemoteBaseUrl (video: MVideoAccountLight, path: string) {
 139   const host = video.VideoChannel.Account.Actor.Server.host
 140
 141   return REMOTE_SCHEME.HTTP + '://' + host + path
 142 }
 143
 144 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
 145   const channel = videoObject.attributedTo.find(a => a.type === 'Group')
 146   if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
 147
 148   if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
 149     throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
 150   }
 151
 152   return getOrCreateActorAndServerAndModel(channel.id, 'all')
 153 }
 154
 155 type SyncParam = {
 156   likes: boolean
 157   dislikes: boolean
 158   shares: boolean
 159   comments: boolean
 160   thumbnail: boolean
 161   refreshVideo?: boolean
 162 }
 163 async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
 164   logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
 165
 166   const jobPayloads: ActivitypubHttpFetcherPayload[] = []
 167
 168   if (syncParam.likes === true) {
 169     const handler = items => createRates(items, video, 'like')
 170     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
 171
 172     await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
 173       .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
 174   } else {
 175     jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
 176   }
 177
 178   if (syncParam.dislikes === true) {
 179     const handler = items => createRates(items, video, 'dislike')
 180     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
 181
 182     await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
 183       .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
 184   } else {
 185     jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
 186   }
 187
 188   if (syncParam.shares === true) {
 189     const handler = items => addVideoShares(items, video)
 190     const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
 191
 192     await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
 193       .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
 194   } else {
 195     jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
 196   }
 197
 198   if (syncParam.comments === true) {
 199     const handler = items => addVideoComments(items)
 200     const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
 201
 202     await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
 203       .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
 204   } else {
 205     jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
 206   }
 207
 208   await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
 209 }
 210
 211 function getOrCreateVideoAndAccountAndChannel (options: {
 212   videoObject: { id: string } | string,
 213   syncParam?: SyncParam,
 214   fetchType?: 'all',
 215   allowRefresh?: boolean
 216 }): Promise<{ video: MVideoAccountLightBlacklistAllFiles, created: boolean, autoBlacklisted?: boolean }>
 217 function getOrCreateVideoAndAccountAndChannel (options: {
 218   videoObject: { id: string } | string,
 219   syncParam?: SyncParam,
 220   fetchType?: VideoFetchByUrlType,
 221   allowRefresh?: boolean
 222 }): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }>
 223 async function getOrCreateVideoAndAccountAndChannel (options: {
 224   videoObject: { id: string } | string,
 225   syncParam?: SyncParam,
 226   fetchType?: VideoFetchByUrlType,
 227   allowRefresh?: boolean // true by default
 228 }): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> {
 229   // Default params
 230   const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
 231   const fetchType = options.fetchType || 'all'
 232   const allowRefresh = options.allowRefresh !== false
 233
 234   // Get video url
 235   const videoUrl = getAPId(options.videoObject)
 236
 237   let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
 238   if (videoFromDatabase) {
 239     if (videoFromDatabase.isOutdated() && allowRefresh === true) {
 240       const refreshOptions = {
 241         video: videoFromDatabase,
 242         fetchedType: fetchType,
 243         syncParam
 244       }
 245
 246       if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
 247       else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } })
 248     }
 249
 250     return { video: videoFromDatabase, created: false }
 251   }
 252
 253   const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
 254   if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
 255
 256   const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
 257   const videoChannel = actor.VideoChannel
 258   const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
 259
 260   await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
 261
 262   return { video: videoCreated, created: true, autoBlacklisted }
 263 }
 264
 265 async function updateVideoFromAP (options: {
 266   video: MVideoAccountLightBlacklistAllFiles,
 267   videoObject: VideoTorrentObject,
 268   account: MAccountIdActor,
 269   channel: MChannelDefault,
 270   overrideTo?: string[]
 271 }) {
 272   const { video, videoObject, account, channel, overrideTo } = options
 273
 274   logger.debug('Updating remote video "%s".', options.videoObject.uuid, { account, channel })
 275
 276   let videoFieldsSave: any
 277   const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE
 278   const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED
 279
 280   try {
 281     let thumbnailModel: MThumbnail
 282
 283     try {
 284       thumbnailModel = await createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE)
 285     } catch (err) {
 286       logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
 287     }
 288
 289     const videoUpdated = await sequelizeTypescript.transaction(async t => {
 290       const sequelizeOptions = { transaction: t }
 291
 292       videoFieldsSave = video.toJSON()
 293
 294       // Check actor has the right to update the video
 295       const videoChannel = video.VideoChannel
 296       if (videoChannel.Account.id !== account.id) {
 297         throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
 298       }
 299
 300       const to = overrideTo ? overrideTo : videoObject.to
 301       const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
 302       video.name = videoData.name
 303       video.uuid = videoData.uuid
 304       video.url = videoData.url
 305       video.category = videoData.category
 306       video.licence = videoData.licence
 307       video.language = videoData.language
 308       video.description = videoData.description
 309       video.support = videoData.support
 310       video.nsfw = videoData.nsfw
 311       video.commentsEnabled = videoData.commentsEnabled
 312       video.downloadEnabled = videoData.downloadEnabled
 313       video.waitTranscoding = videoData.waitTranscoding
 314       video.state = videoData.state
 315       video.duration = videoData.duration
 316       video.createdAt = videoData.createdAt
 317       video.publishedAt = videoData.publishedAt
 318       video.originallyPublishedAt = videoData.originallyPublishedAt
 319       video.privacy = videoData.privacy
 320       video.channelId = videoData.channelId
 321       video.views = videoData.views
 322
 323       const videoUpdated = await video.save(sequelizeOptions) as MVideoFullLight
 324
 325       if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
 326
 327       // FIXME: use icon URL instead
 328       const previewUrl = buildRemoteBaseUrl(videoUpdated, join(STATIC_PATHS.PREVIEWS, videoUpdated.getPreview().filename))
 329       const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 330       await videoUpdated.addAndSaveThumbnail(previewModel, t)
 331
 332       {
 333         const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject)
 334         const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
 335
 336         // Remove video files that do not exist anymore
 337         const destroyTasks = videoUpdated.VideoFiles
 338                                   .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
 339                                   .map(f => f.destroy(sequelizeOptions))
 340         await Promise.all(destroyTasks)
 341
 342         // Update or add other one
 343         const upsertTasks = videoFileAttributes.map(a => {
 344           return VideoFileModel.upsert<VideoFileModel>(a, { returning: true, transaction: t })
 345             .then(([ file ]) => file)
 346         })
 347
 348         videoUpdated.VideoFiles = await Promise.all(upsertTasks)
 349       }
 350
 351       {
 352         const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles)
 353         const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
 354
 355         // Remove video files that do not exist anymore
 356         const destroyTasks = videoUpdated.VideoStreamingPlaylists
 357                                   .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
 358                                   .map(f => f.destroy(sequelizeOptions))
 359         await Promise.all(destroyTasks)
 360
 361         // Update or add other one
 362         const upsertTasks = streamingPlaylistAttributes.map(a => {
 363           return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
 364                                .then(([ streamingPlaylist ]) => streamingPlaylist)
 365         })
 366
 367         videoUpdated.VideoStreamingPlaylists = await Promise.all(upsertTasks)
 368       }
 369
 370       {
 371         // Update Tags
 372         const tags = videoObject.tag.map(tag => tag.name)
 373         const tagInstances = await TagModel.findOrCreateTags(tags, t)
 374         await videoUpdated.$set('Tags', tagInstances, sequelizeOptions)
 375       }
 376
 377       {
 378         // Update captions
 379         await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
 380
 381         const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 382           return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, t)
 383         })
 384         await Promise.all(videoCaptionsPromises)
 385       }
 386
 387       return videoUpdated
 388     })
 389
 390     await autoBlacklistVideoIfNeeded({
 391       video: videoUpdated,
 392       user: undefined,
 393       isRemote: true,
 394       isNew: false,
 395       transaction: undefined
 396     })
 397
 398     if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) // Notify our users?
 399
 400     logger.info('Remote video with uuid %s updated', videoObject.uuid)
 401
 402     return videoUpdated
 403   } catch (err) {
 404     if (video !== undefined && videoFieldsSave !== undefined) {
 405       resetSequelizeInstance(video, videoFieldsSave)
 406     }
 407
 408     // This is just a debug because we will retry the insert
 409     logger.debug('Cannot update the remote video.', { err })
 410     throw err
 411   }
 412 }
 413
 414 async function refreshVideoIfNeeded (options: {
 415   video: MVideoThumbnail,
 416   fetchedType: VideoFetchByUrlType,
 417   syncParam: SyncParam
 418 }): Promise<MVideoThumbnail> {
 419   if (!options.video.isOutdated()) return options.video
 420
 421   // We need more attributes if the argument video was fetched with not enough joints
 422   const video = options.fetchedType === 'all'
 423     ? options.video as MVideoAccountLightBlacklistAllFiles
 424     : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
 425
 426   try {
 427     const { response, videoObject } = await fetchRemoteVideo(video.url)
 428     if (response.statusCode === 404) {
 429       logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
 430
 431       // Video does not exist anymore
 432       await video.destroy()
 433       return undefined
 434     }
 435
 436     if (videoObject === undefined) {
 437       logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
 438
 439       await video.setAsRefreshed()
 440       return video
 441     }
 442
 443     const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 444
 445     const updateOptions = {
 446       video,
 447       videoObject,
 448       account: channelActor.VideoChannel.Account,
 449       channel: channelActor.VideoChannel
 450     }
 451     await retryTransactionWrapper(updateVideoFromAP, updateOptions)
 452     await syncVideoExternalAttributes(video, videoObject, options.syncParam)
 453
 454     ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
 455
 456     return video
 457   } catch (err) {
 458     logger.warn('Cannot refresh video %s.', options.video.url, { err })
 459
 460     ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
 461
 462     // Don't refresh in loop
 463     await video.setAsRefreshed()
 464     return video
 465   }
 466 }
 467
 468 export {
 469   updateVideoFromAP,
 470   refreshVideoIfNeeded,
 471   federateVideoIfNeeded,
 472   fetchRemoteVideo,
 473   getOrCreateVideoAndAccountAndChannel,
 474   fetchRemoteVideoStaticFile,
 475   fetchRemoteVideoDescription,
 476   getOrCreateVideoChannelFromVideoObject
 477 }
 478
 479 // ---------------------------------------------------------------------------
 480
 481 function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
 482   const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
 483
 484   const urlMediaType = url.mediaType || url.mimeType
 485   return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
 486 }
 487
 488 function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
 489   const urlMediaType = url.mediaType || url.mimeType
 490
 491   return urlMediaType === 'application/x-mpegURL'
 492 }
 493
 494 function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
 495   const urlMediaType = tag.mediaType || tag.mimeType
 496
 497   return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
 498 }
 499
 500 async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) {
 501   logger.debug('Adding remote video %s.', videoObject.id)
 502
 503   const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
 504   const video = VideoModel.build(videoData) as MVideoThumbnail
 505
 506   const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE)
 507
 508   let thumbnailModel: MThumbnail
 509   if (waitThumbnail === true) {
 510     thumbnailModel = await promiseThumbnail
 511   }
 512
 513   const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
 514     const sequelizeOptions = { transaction: t }
 515
 516     const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
 517     videoCreated.VideoChannel = channel
 518
 519     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 520
 521     // FIXME: use icon URL instead
 522     const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
 523     const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 524     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 525
 526     // Process files
 527     const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
 528     if (videoFileAttributes.length === 0) {
 529       throw new Error('Cannot find valid files for video %s ' + videoObject.url)
 530     }
 531
 532     const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
 533     const videoFiles = await Promise.all(videoFilePromises)
 534
 535     const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
 536     const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
 537     const streamingPlaylists = await Promise.all(playlistPromises)
 538
 539     // Process tags
 540     const tags = videoObject.tag
 541                             .filter(t => t.type === 'Hashtag')
 542                             .map(t => t.name)
 543     const tagInstances = await TagModel.findOrCreateTags(tags, t)
 544     await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
 545
 546     // Process captions
 547     const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 548       return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
 549     })
 550     await Promise.all(videoCaptionsPromises)
 551
 552     videoCreated.VideoFiles = videoFiles
 553     videoCreated.VideoStreamingPlaylists = streamingPlaylists
 554     videoCreated.Tags = tagInstances
 555
 556     const autoBlacklisted = await autoBlacklistVideoIfNeeded({
 557       video: videoCreated,
 558       user: undefined,
 559       isRemote: true,
 560       isNew: true,
 561       transaction: t
 562     })
 563
 564     logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
 565
 566     return { autoBlacklisted, videoCreated }
 567   })
 568
 569   if (waitThumbnail === false) {
 570     promiseThumbnail.then(thumbnailModel => {
 571       thumbnailModel = videoCreated.id
 572
 573       return thumbnailModel.save()
 574     })
 575   }
 576
 577   return { autoBlacklisted, videoCreated }
 578 }
 579
 580 async function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoTorrentObject, to: string[] = []) {
 581   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
 582   const duration = videoObject.duration.replace(/[^\d]+/, '')
 583
 584   let language: string | undefined
 585   if (videoObject.language) {
 586     language = videoObject.language.identifier
 587   }
 588
 589   let category: number | undefined
 590   if (videoObject.category) {
 591     category = parseInt(videoObject.category.identifier, 10)
 592   }
 593
 594   let licence: number | undefined
 595   if (videoObject.licence) {
 596     licence = parseInt(videoObject.licence.identifier, 10)
 597   }
 598
 599   const description = videoObject.content || null
 600   const support = videoObject.support || null
 601
 602   return {
 603     name: videoObject.name,
 604     uuid: videoObject.uuid,
 605     url: videoObject.id,
 606     category,
 607     licence,
 608     language,
 609     description,
 610     support,
 611     nsfw: videoObject.sensitive,
 612     commentsEnabled: videoObject.commentsEnabled,
 613     downloadEnabled: videoObject.downloadEnabled,
 614     waitTranscoding: videoObject.waitTranscoding,
 615     state: videoObject.state,
 616     channelId: videoChannel.id,
 617     duration: parseInt(duration, 10),
 618     createdAt: new Date(videoObject.published),
 619     publishedAt: new Date(videoObject.published),
 620     originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null,
 621     // FIXME: updatedAt does not seems to be considered by Sequelize
 622     updatedAt: new Date(videoObject.updated),
 623     views: videoObject.views,
 624     likes: 0,
 625     dislikes: 0,
 626     remote: true,
 627     privacy
 628   }
 629 }
 630
 631 function videoFileActivityUrlToDBAttributes (video: MVideo, videoObject: VideoTorrentObject) {
 632   const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
 633
 634   if (fileUrls.length === 0) {
 635     throw new Error('Cannot find video files for ' + video.url)
 636   }
 637
 638   const attributes: FilteredModelAttributes<VideoFileModel>[] = []
 639   for (const fileUrl of fileUrls) {
 640     // Fetch associated magnet uri
 641     const magnet = videoObject.url.find(u => {
 642       const mediaType = u.mediaType || u.mimeType
 643       return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height
 644     })
 645
 646     if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
 647
 648     const parsed = magnetUtil.decode(magnet.href)
 649     if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
 650       throw new Error('Cannot parse magnet URI ' + magnet.href)
 651     }
 652
 653     const mediaType = fileUrl.mediaType || fileUrl.mimeType
 654     const attribute = {
 655       extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ],
 656       infoHash: parsed.infoHash,
 657       resolution: fileUrl.height,
 658       size: fileUrl.size,
 659       videoId: video.id,
 660       fps: fileUrl.fps || -1
 661     }
 662
 663     attributes.push(attribute)
 664   }
 665
 666   return attributes
 667 }
 668
 669 function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoTorrentObject, videoFiles: MVideoFile[]) {
 670   const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
 671   if (playlistUrls.length === 0) return []
 672
 673   const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
 674   for (const playlistUrlObject of playlistUrls) {
 675     const segmentsSha256UrlObject = playlistUrlObject.tag
 676                                                      .find(t => {
 677                                                        return isAPPlaylistSegmentHashesUrlObject(t)
 678                                                      }) as ActivityPlaylistSegmentHashesObject
 679     if (!segmentsSha256UrlObject) {
 680       logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
 681       continue
 682     }
 683
 684     const attribute = {
 685       type: VideoStreamingPlaylistType.HLS,
 686       playlistUrl: playlistUrlObject.href,
 687       segmentsSha256Url: segmentsSha256UrlObject.href,
 688       p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
 689       p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
 690       videoId: video.id
 691     }
 692
 693     attributes.push(attribute)
 694   }
 695
 696   return attributes
 697 }