server/lib/activitypub/videos.ts

   1 import * as Bluebird from 'bluebird'
   2 import * as sequelize from 'sequelize'
   3 import * as magnetUtil from 'magnet-uri'
   4 import * as request from 'request'
   5 import {
   6   ActivityPlaylistSegmentHashesObject,
   7   ActivityPlaylistUrlObject,
   8   ActivityUrlObject,
   9   ActivityVideoUrlObject,
  10   VideoState
  11 } from '../../../shared/index'
  12 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
  13 import { VideoPrivacy } from '../../../shared/models/videos'
  14 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
  15 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
  16 import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
  17 import { logger } from '../../helpers/logger'
  18 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
  19 import {
  20   ACTIVITY_PUB,
  21   MIMETYPES,
  22   P2P_MEDIA_LOADER_PEER_VERSION,
  23   PREVIEWS_SIZE,
  24   REMOTE_SCHEME,
  25   STATIC_PATHS
  26 } from '../../initializers/constants'
  27 import { ActorModel } from '../../models/activitypub/actor'
  28 import { TagModel } from '../../models/video/tag'
  29 import { VideoModel } from '../../models/video/video'
  30 import { VideoChannelModel } from '../../models/video/video-channel'
  31 import { VideoFileModel } from '../../models/video/video-file'
  32 import { getOrCreateActorAndServerAndModel } from './actor'
  33 import { addVideoComments } from './video-comments'
  34 import { crawlCollectionPage } from './crawl'
  35 import { sendCreateVideo, sendUpdateVideo } from './send'
  36 import { isArray } from '../../helpers/custom-validators/misc'
  37 import { VideoCaptionModel } from '../../models/video/video-caption'
  38 import { JobQueue } from '../job-queue'
  39 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
  40 import { createRates } from './video-rates'
  41 import { addVideoShares, shareVideoByServerAndChannel } from './share'
  42 import { AccountModel } from '../../models/account/account'
  43 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
  44 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
  45 import { Notifier } from '../notifier'
  46 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
  47 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
  48 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
  49 import { VideoShareModel } from '../../models/video/video-share'
  50 import { VideoCommentModel } from '../../models/video/video-comment'
  51 import { sequelizeTypescript } from '../../initializers/database'
  52 import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
  53 import { ThumbnailModel } from '../../models/video/thumbnail'
  54 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
  55 import { join } from 'path'
  56 import { FilteredModelAttributes } from '../../typings/sequelize'
  57 import { Hooks } from '../plugins/hooks'
  58 import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
  59 import { ActorFollowScoreCache } from '../files-cache'
  60
  61 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
  62   if (
  63     // Check this is not a blacklisted video, or unfederated blacklisted video
  64     (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
  65     // Check the video is public/unlisted and published
  66     video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED
  67   ) {
  68     // Fetch more attributes that we will need to serialize in AP object
  69     if (isArray(video.VideoCaptions) === false) {
  70       video.VideoCaptions = await video.$get('VideoCaptions', {
  71         attributes: [ 'language' ],
  72         transaction
  73       }) as VideoCaptionModel[]
  74     }
  75
  76     if (isNewVideo) {
  77       // Now we'll add the video's meta data to our followers
  78       await sendCreateVideo(video, transaction)
  79       await shareVideoByServerAndChannel(video, transaction)
  80     } else {
  81       await sendUpdateVideo(video, transaction)
  82     }
  83   }
  84 }
  85
  86 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
  87   const options = {
  88     uri: videoUrl,
  89     method: 'GET',
  90     json: true,
  91     activityPub: true
  92   }
  93
  94   logger.info('Fetching remote video %s.', videoUrl)
  95
  96   const { response, body } = await doRequest(options)
  97
  98   if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
  99     logger.debug('Remote video JSON is not valid.', { body })
 100     return { response, videoObject: undefined }
 101   }
 102
 103   return { response, videoObject: body }
 104 }
 105
 106 async function fetchRemoteVideoDescription (video: VideoModel) {
 107   const host = video.VideoChannel.Account.Actor.Server.host
 108   const path = video.getDescriptionAPIPath()
 109   const options = {
 110     uri: REMOTE_SCHEME.HTTP + '://' + host + path,
 111     json: true
 112   }
 113
 114   const { body } = await doRequest(options)
 115   return body.description ? body.description : ''
 116 }
 117
 118 function fetchRemoteVideoStaticFile (video: VideoModel, path: string, destPath: string) {
 119   const url = buildRemoteBaseUrl(video, path)
 120
 121   // We need to provide a callback, if no we could have an uncaught exception
 122   return doRequestAndSaveToFile({ uri: url }, destPath)
 123 }
 124
 125 function buildRemoteBaseUrl (video: VideoModel, path: string) {
 126   const host = video.VideoChannel.Account.Actor.Server.host
 127
 128   return REMOTE_SCHEME.HTTP + '://' + host + path
 129 }
 130
 131 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
 132   const channel = videoObject.attributedTo.find(a => a.type === 'Group')
 133   if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
 134
 135   if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
 136     throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
 137   }
 138
 139   return getOrCreateActorAndServerAndModel(channel.id, 'all')
 140 }
 141
 142 type SyncParam = {
 143   likes: boolean
 144   dislikes: boolean
 145   shares: boolean
 146   comments: boolean
 147   thumbnail: boolean
 148   refreshVideo?: boolean
 149 }
 150 async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
 151   logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
 152
 153   const jobPayloads: ActivitypubHttpFetcherPayload[] = []
 154
 155   if (syncParam.likes === true) {
 156     const handler = items => createRates(items, video, 'like')
 157     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
 158
 159     await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
 160       .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
 161   } else {
 162     jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
 163   }
 164
 165   if (syncParam.dislikes === true) {
 166     const handler = items => createRates(items, video, 'dislike')
 167     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
 168
 169     await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
 170       .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
 171   } else {
 172     jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
 173   }
 174
 175   if (syncParam.shares === true) {
 176     const handler = items => addVideoShares(items, video)
 177     const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
 178
 179     await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
 180       .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
 181   } else {
 182     jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
 183   }
 184
 185   if (syncParam.comments === true) {
 186     const handler = items => addVideoComments(items)
 187     const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
 188
 189     await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
 190       .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
 191   } else {
 192     jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
 193   }
 194
 195   await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
 196 }
 197
 198 async function getOrCreateVideoAndAccountAndChannel (options: {
 199   videoObject: { id: string } | string,
 200   syncParam?: SyncParam,
 201   fetchType?: VideoFetchByUrlType,
 202   allowRefresh?: boolean // true by default
 203 }) {
 204   // Default params
 205   const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
 206   const fetchType = options.fetchType || 'all'
 207   const allowRefresh = options.allowRefresh !== false
 208
 209   // Get video url
 210   const videoUrl = getAPId(options.videoObject)
 211
 212   let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
 213   if (videoFromDatabase) {
 214     if (videoFromDatabase.isOutdated() && allowRefresh === true) {
 215       const refreshOptions = {
 216         video: videoFromDatabase,
 217         fetchedType: fetchType,
 218         syncParam
 219       }
 220
 221       if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
 222       else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } })
 223     }
 224
 225     return { video: videoFromDatabase, created: false }
 226   }
 227
 228   const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
 229   if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
 230
 231   const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
 232   const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
 233
 234   await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
 235
 236   return { video: videoCreated, created: true, autoBlacklisted }
 237 }
 238
 239 async function updateVideoFromAP (options: {
 240   video: VideoModel,
 241   videoObject: VideoTorrentObject,
 242   account: AccountModel,
 243   channel: VideoChannelModel,
 244   overrideTo?: string[]
 245 }) {
 246   const { video, videoObject, account, channel, overrideTo } = options
 247
 248   logger.debug('Updating remote video "%s".', options.videoObject.uuid)
 249
 250   let videoFieldsSave: any
 251   const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE
 252   const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED
 253
 254   try {
 255     let thumbnailModel: ThumbnailModel
 256
 257     try {
 258       thumbnailModel = await createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE)
 259     } catch (err) {
 260       logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
 261     }
 262
 263     await sequelizeTypescript.transaction(async t => {
 264       const sequelizeOptions = { transaction: t }
 265
 266       videoFieldsSave = video.toJSON()
 267
 268       // Check actor has the right to update the video
 269       const videoChannel = video.VideoChannel
 270       if (videoChannel.Account.id !== account.id) {
 271         throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
 272       }
 273
 274       const to = overrideTo ? overrideTo : videoObject.to
 275       const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
 276       video.name = videoData.name
 277       video.uuid = videoData.uuid
 278       video.url = videoData.url
 279       video.category = videoData.category
 280       video.licence = videoData.licence
 281       video.language = videoData.language
 282       video.description = videoData.description
 283       video.support = videoData.support
 284       video.nsfw = videoData.nsfw
 285       video.commentsEnabled = videoData.commentsEnabled
 286       video.downloadEnabled = videoData.downloadEnabled
 287       video.waitTranscoding = videoData.waitTranscoding
 288       video.state = videoData.state
 289       video.duration = videoData.duration
 290       video.createdAt = videoData.createdAt
 291       video.publishedAt = videoData.publishedAt
 292       video.originallyPublishedAt = videoData.originallyPublishedAt
 293       video.privacy = videoData.privacy
 294       video.channelId = videoData.channelId
 295       video.views = videoData.views
 296
 297       await video.save(sequelizeOptions)
 298
 299       if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
 300
 301       // FIXME: use icon URL instead
 302       const previewUrl = buildRemoteBaseUrl(video, join(STATIC_PATHS.PREVIEWS, video.getPreview().filename))
 303       const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 304       await video.addAndSaveThumbnail(previewModel, t)
 305
 306       {
 307         const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject)
 308         const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
 309
 310         // Remove video files that do not exist anymore
 311         const destroyTasks = video.VideoFiles
 312                                   .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
 313                                   .map(f => f.destroy(sequelizeOptions))
 314         await Promise.all(destroyTasks)
 315
 316         // Update or add other one
 317         const upsertTasks = videoFileAttributes.map(a => {
 318           return VideoFileModel.upsert<VideoFileModel>(a, { returning: true, transaction: t })
 319             .then(([ file ]) => file)
 320         })
 321
 322         video.VideoFiles = await Promise.all(upsertTasks)
 323       }
 324
 325       {
 326         const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(video, videoObject, video.VideoFiles)
 327         const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
 328
 329         // Remove video files that do not exist anymore
 330         const destroyTasks = video.VideoStreamingPlaylists
 331                                   .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
 332                                   .map(f => f.destroy(sequelizeOptions))
 333         await Promise.all(destroyTasks)
 334
 335         // Update or add other one
 336         const upsertTasks = streamingPlaylistAttributes.map(a => {
 337           return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
 338                                .then(([ streamingPlaylist ]) => streamingPlaylist)
 339         })
 340
 341         video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
 342       }
 343
 344       {
 345         // Update Tags
 346         const tags = videoObject.tag.map(tag => tag.name)
 347         const tagInstances = await TagModel.findOrCreateTags(tags, t)
 348         await video.$set('Tags', tagInstances, sequelizeOptions)
 349       }
 350
 351       {
 352         // Update captions
 353         await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t)
 354
 355         const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 356           return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t)
 357         })
 358         video.VideoCaptions = await Promise.all(videoCaptionsPromises)
 359       }
 360     })
 361
 362     await autoBlacklistVideoIfNeeded({
 363       video,
 364       user: undefined,
 365       isRemote: true,
 366       isNew: false,
 367       transaction: undefined
 368     })
 369
 370     if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(video) // Notify our users?
 371
 372     logger.info('Remote video with uuid %s updated', videoObject.uuid)
 373   } catch (err) {
 374     if (video !== undefined && videoFieldsSave !== undefined) {
 375       resetSequelizeInstance(video, videoFieldsSave)
 376     }
 377
 378     // This is just a debug because we will retry the insert
 379     logger.debug('Cannot update the remote video.', { err })
 380     throw err
 381   }
 382 }
 383
 384 async function refreshVideoIfNeeded (options: {
 385   video: VideoModel,
 386   fetchedType: VideoFetchByUrlType,
 387   syncParam: SyncParam
 388 }): Promise<VideoModel> {
 389   if (!options.video.isOutdated()) return options.video
 390
 391   // We need more attributes if the argument video was fetched with not enough joints
 392   const video = options.fetchedType === 'all'
 393     ? options.video
 394     : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
 395
 396   try {
 397     const { response, videoObject } = await fetchRemoteVideo(video.url)
 398     if (response.statusCode === 404) {
 399       logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
 400
 401       // Video does not exist anymore
 402       await video.destroy()
 403       return undefined
 404     }
 405
 406     if (videoObject === undefined) {
 407       logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
 408
 409       await video.setAsRefreshed()
 410       return video
 411     }
 412
 413     const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 414     const account = await AccountModel.load(channelActor.VideoChannel.accountId)
 415
 416     const updateOptions = {
 417       video,
 418       videoObject,
 419       account,
 420       channel: channelActor.VideoChannel
 421     }
 422     await retryTransactionWrapper(updateVideoFromAP, updateOptions)
 423     await syncVideoExternalAttributes(video, videoObject, options.syncParam)
 424
 425     ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
 426
 427     return video
 428   } catch (err) {
 429     logger.warn('Cannot refresh video %s.', options.video.url, { err })
 430
 431     ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
 432
 433     // Don't refresh in loop
 434     await video.setAsRefreshed()
 435     return video
 436   }
 437 }
 438
 439 export {
 440   updateVideoFromAP,
 441   refreshVideoIfNeeded,
 442   federateVideoIfNeeded,
 443   fetchRemoteVideo,
 444   getOrCreateVideoAndAccountAndChannel,
 445   fetchRemoteVideoStaticFile,
 446   fetchRemoteVideoDescription,
 447   getOrCreateVideoChannelFromVideoObject
 448 }
 449
 450 // ---------------------------------------------------------------------------
 451
 452 function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
 453   const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
 454
 455   const urlMediaType = url.mediaType || url.mimeType
 456   return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
 457 }
 458
 459 function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
 460   const urlMediaType = url.mediaType || url.mimeType
 461
 462   return urlMediaType === 'application/x-mpegURL'
 463 }
 464
 465 function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
 466   const urlMediaType = tag.mediaType || tag.mimeType
 467
 468   return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
 469 }
 470
 471 async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
 472   logger.debug('Adding remote video %s.', videoObject.id)
 473
 474   const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
 475   const video = VideoModel.build(videoData)
 476
 477   const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE)
 478
 479   let thumbnailModel: ThumbnailModel
 480   if (waitThumbnail === true) {
 481     thumbnailModel = await promiseThumbnail
 482   }
 483
 484   const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
 485     const sequelizeOptions = { transaction: t }
 486
 487     const videoCreated = await video.save(sequelizeOptions)
 488     videoCreated.VideoChannel = channelActor.VideoChannel
 489
 490     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 491
 492     // FIXME: use icon URL instead
 493     const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
 494     const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 495     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 496
 497     // Process files
 498     const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
 499     if (videoFileAttributes.length === 0) {
 500       throw new Error('Cannot find valid files for video %s ' + videoObject.url)
 501     }
 502
 503     const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
 504     const videoFiles = await Promise.all(videoFilePromises)
 505
 506     const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
 507     const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
 508     const streamingPlaylists = await Promise.all(playlistPromises)
 509
 510     // Process tags
 511     const tags = videoObject.tag
 512                             .filter(t => t.type === 'Hashtag')
 513                             .map(t => t.name)
 514     const tagInstances = await TagModel.findOrCreateTags(tags, t)
 515     await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
 516
 517     // Process captions
 518     const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 519       return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
 520     })
 521     const captions = await Promise.all(videoCaptionsPromises)
 522
 523     video.VideoFiles = videoFiles
 524     video.VideoStreamingPlaylists = streamingPlaylists
 525     video.Tags = tagInstances
 526     video.VideoCaptions = captions
 527
 528     const autoBlacklisted = await autoBlacklistVideoIfNeeded({
 529       video,
 530       user: undefined,
 531       isRemote: true,
 532       isNew: true,
 533       transaction: t
 534     })
 535
 536     logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
 537
 538     return { autoBlacklisted, videoCreated }
 539   })
 540
 541   if (waitThumbnail === false) {
 542     promiseThumbnail.then(thumbnailModel => {
 543       thumbnailModel = videoCreated.id
 544
 545       return thumbnailModel.save()
 546     })
 547   }
 548
 549   return { autoBlacklisted, videoCreated }
 550 }
 551
 552 async function videoActivityObjectToDBAttributes (
 553   videoChannel: VideoChannelModel,
 554   videoObject: VideoTorrentObject,
 555   to: string[] = []
 556 ) {
 557   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
 558   const duration = videoObject.duration.replace(/[^\d]+/, '')
 559
 560   let language: string | undefined
 561   if (videoObject.language) {
 562     language = videoObject.language.identifier
 563   }
 564
 565   let category: number | undefined
 566   if (videoObject.category) {
 567     category = parseInt(videoObject.category.identifier, 10)
 568   }
 569
 570   let licence: number | undefined
 571   if (videoObject.licence) {
 572     licence = parseInt(videoObject.licence.identifier, 10)
 573   }
 574
 575   const description = videoObject.content || null
 576   const support = videoObject.support || null
 577
 578   return {
 579     name: videoObject.name,
 580     uuid: videoObject.uuid,
 581     url: videoObject.id,
 582     category,
 583     licence,
 584     language,
 585     description,
 586     support,
 587     nsfw: videoObject.sensitive,
 588     commentsEnabled: videoObject.commentsEnabled,
 589     downloadEnabled: videoObject.downloadEnabled,
 590     waitTranscoding: videoObject.waitTranscoding,
 591     state: videoObject.state,
 592     channelId: videoChannel.id,
 593     duration: parseInt(duration, 10),
 594     createdAt: new Date(videoObject.published),
 595     publishedAt: new Date(videoObject.published),
 596     originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null,
 597     // FIXME: updatedAt does not seems to be considered by Sequelize
 598     updatedAt: new Date(videoObject.updated),
 599     views: videoObject.views,
 600     likes: 0,
 601     dislikes: 0,
 602     remote: true,
 603     privacy
 604   }
 605 }
 606
 607 function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
 608   const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
 609
 610   if (fileUrls.length === 0) {
 611     throw new Error('Cannot find video files for ' + video.url)
 612   }
 613
 614   const attributes: FilteredModelAttributes<VideoFileModel>[] = []
 615   for (const fileUrl of fileUrls) {
 616     // Fetch associated magnet uri
 617     const magnet = videoObject.url.find(u => {
 618       const mediaType = u.mediaType || u.mimeType
 619       return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height
 620     })
 621
 622     if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
 623
 624     const parsed = magnetUtil.decode(magnet.href)
 625     if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
 626       throw new Error('Cannot parse magnet URI ' + magnet.href)
 627     }
 628
 629     const mediaType = fileUrl.mediaType || fileUrl.mimeType
 630     const attribute = {
 631       extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ],
 632       infoHash: parsed.infoHash,
 633       resolution: fileUrl.height,
 634       size: fileUrl.size,
 635       videoId: video.id,
 636       fps: fileUrl.fps || -1
 637     }
 638
 639     attributes.push(attribute)
 640   }
 641
 642   return attributes
 643 }
 644
 645 function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) {
 646   const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
 647   if (playlistUrls.length === 0) return []
 648
 649   const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
 650   for (const playlistUrlObject of playlistUrls) {
 651     const segmentsSha256UrlObject = playlistUrlObject.tag
 652                                                      .find(t => {
 653                                                        return isAPPlaylistSegmentHashesUrlObject(t)
 654                                                      }) as ActivityPlaylistSegmentHashesObject
 655     if (!segmentsSha256UrlObject) {
 656       logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
 657       continue
 658     }
 659
 660     const attribute = {
 661       type: VideoStreamingPlaylistType.HLS,
 662       playlistUrl: playlistUrlObject.href,
 663       segmentsSha256Url: segmentsSha256UrlObject.href,
 664       p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
 665       p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
 666       videoId: video.id
 667     }
 668
 669     attributes.push(attribute)
 670   }
 671
 672   return attributes
 673 }