server/lib/activitypub/videos.ts

   1 import * as Bluebird from 'bluebird'
   2 import * as sequelize from 'sequelize'
   3 import * as magnetUtil from 'magnet-uri'
   4 import * as request from 'request'
   5 import {
   6   ActivityHashTagObject,
   7   ActivityMagnetUrlObject,
   8   ActivityPlaylistSegmentHashesObject,
   9   ActivityPlaylistUrlObject, ActivitypubHttpFetcherPayload,
  10   ActivityTagObject,
  11   ActivityUrlObject,
  12   ActivityVideoUrlObject,
  13   VideoState
  14 } from '../../../shared/index'
  15 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
  16 import { VideoPrivacy } from '../../../shared/models/videos'
  17 import { isAPVideoFileMetadataObject, sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
  18 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
  19 import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
  20 import { logger } from '../../helpers/logger'
  21 import { doRequest } from '../../helpers/requests'
  22 import {
  23   ACTIVITY_PUB,
  24   MIMETYPES,
  25   P2P_MEDIA_LOADER_PEER_VERSION,
  26   PREVIEWS_SIZE,
  27   REMOTE_SCHEME,
  28   STATIC_PATHS,
  29   THUMBNAILS_SIZE
  30 } from '../../initializers/constants'
  31 import { TagModel } from '../../models/video/tag'
  32 import { VideoModel } from '../../models/video/video'
  33 import { VideoFileModel } from '../../models/video/video-file'
  34 import { getOrCreateActorAndServerAndModel } from './actor'
  35 import { addVideoComments } from './video-comments'
  36 import { crawlCollectionPage } from './crawl'
  37 import { sendCreateVideo, sendUpdateVideo } from './send'
  38 import { isArray } from '../../helpers/custom-validators/misc'
  39 import { VideoCaptionModel } from '../../models/video/video-caption'
  40 import { JobQueue } from '../job-queue'
  41 import { createRates } from './video-rates'
  42 import { addVideoShares, shareVideoByServerAndChannel } from './share'
  43 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
  44 import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
  45 import { Notifier } from '../notifier'
  46 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
  47 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
  48 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
  49 import { VideoShareModel } from '../../models/video/video-share'
  50 import { VideoCommentModel } from '../../models/video/video-comment'
  51 import { sequelizeTypescript } from '../../initializers/database'
  52 import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
  53 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
  54 import { join } from 'path'
  55 import { FilteredModelAttributes } from '../../typings/sequelize'
  56 import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
  57 import { ActorFollowScoreCache } from '../files-cache'
  58 import {
  59   MAccountIdActor,
  60   MChannelAccountLight,
  61   MChannelDefault,
  62   MChannelId,
  63   MStreamingPlaylist,
  64   MVideo,
  65   MVideoAccountLight,
  66   MVideoAccountLightBlacklistAllFiles,
  67   MVideoAP,
  68   MVideoAPWithoutCaption,
  69   MVideoFile,
  70   MVideoFullLight,
  71   MVideoId,
  72   MVideoImmutable,
  73   MVideoThumbnail
  74 } from '../../typings/models'
  75 import { MThumbnail } from '../../typings/models/video/thumbnail'
  76 import { maxBy, minBy } from 'lodash'
  77
  78 async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
  79   const video = videoArg as MVideoAP
  80
  81   if (
  82     // Check this is not a blacklisted video, or unfederated blacklisted video
  83     (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
  84     // Check the video is public/unlisted and published
  85     video.hasPrivacyForFederation() && video.state === VideoState.PUBLISHED
  86   ) {
  87     // Fetch more attributes that we will need to serialize in AP object
  88     if (isArray(video.VideoCaptions) === false) {
  89       video.VideoCaptions = await video.$get('VideoCaptions', {
  90         attributes: [ 'language' ],
  91         transaction
  92       })
  93     }
  94
  95     if (isNewVideo) {
  96       // Now we'll add the video's meta data to our followers
  97       await sendCreateVideo(video, transaction)
  98       await shareVideoByServerAndChannel(video, transaction)
  99     } else {
 100       await sendUpdateVideo(video, transaction)
 101     }
 102   }
 103 }
 104
 105 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
 106   const options = {
 107     uri: videoUrl,
 108     method: 'GET',
 109     json: true,
 110     activityPub: true
 111   }
 112
 113   logger.info('Fetching remote video %s.', videoUrl)
 114
 115   const { response, body } = await doRequest<any>(options)
 116
 117   if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
 118     logger.debug('Remote video JSON is not valid.', { body })
 119     return { response, videoObject: undefined }
 120   }
 121
 122   return { response, videoObject: body }
 123 }
 124
 125 async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
 126   const host = video.VideoChannel.Account.Actor.Server.host
 127   const path = video.getDescriptionAPIPath()
 128   const options = {
 129     uri: REMOTE_SCHEME.HTTP + '://' + host + path,
 130     json: true
 131   }
 132
 133   const { body } = await doRequest<any>(options)
 134   return body.description ? body.description : ''
 135 }
 136
 137 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
 138   const channel = videoObject.attributedTo.find(a => a.type === 'Group')
 139   if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
 140
 141   if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
 142     throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
 143   }
 144
 145   return getOrCreateActorAndServerAndModel(channel.id, 'all')
 146 }
 147
 148 type SyncParam = {
 149   likes: boolean
 150   dislikes: boolean
 151   shares: boolean
 152   comments: boolean
 153   thumbnail: boolean
 154   refreshVideo?: boolean
 155 }
 156 async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
 157   logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
 158
 159   const jobPayloads: ActivitypubHttpFetcherPayload[] = []
 160
 161   if (syncParam.likes === true) {
 162     const handler = items => createRates(items, video, 'like')
 163     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
 164
 165     await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
 166       .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes }))
 167   } else {
 168     jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
 169   }
 170
 171   if (syncParam.dislikes === true) {
 172     const handler = items => createRates(items, video, 'dislike')
 173     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
 174
 175     await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
 176       .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes }))
 177   } else {
 178     jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
 179   }
 180
 181   if (syncParam.shares === true) {
 182     const handler = items => addVideoShares(items, video)
 183     const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
 184
 185     await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
 186       .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
 187   } else {
 188     jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
 189   }
 190
 191   if (syncParam.comments === true) {
 192     const handler = items => addVideoComments(items)
 193     const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
 194
 195     await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
 196       .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
 197   } else {
 198     jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
 199   }
 200
 201   await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
 202 }
 203
 204 type GetVideoResult <T> = Promise<{
 205   video: T
 206   created: boolean
 207   autoBlacklisted?: boolean
 208 }>
 209
 210 type GetVideoParamAll = {
 211   videoObject: { id: string } | string
 212   syncParam?: SyncParam
 213   fetchType?: 'all'
 214   allowRefresh?: boolean
 215 }
 216
 217 type GetVideoParamImmutable = {
 218   videoObject: { id: string } | string
 219   syncParam?: SyncParam
 220   fetchType: 'only-immutable-attributes'
 221   allowRefresh: false
 222 }
 223
 224 type GetVideoParamOther = {
 225   videoObject: { id: string } | string
 226   syncParam?: SyncParam
 227   fetchType?: 'all' | 'only-video'
 228   allowRefresh?: boolean
 229 }
 230
 231 function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
 232 function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
 233 function getOrCreateVideoAndAccountAndChannel (
 234   options: GetVideoParamOther
 235 ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
 236 async function getOrCreateVideoAndAccountAndChannel (
 237   options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
 238 ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
 239   // Default params
 240   const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
 241   const fetchType = options.fetchType || 'all'
 242   const allowRefresh = options.allowRefresh !== false
 243
 244   // Get video url
 245   const videoUrl = getAPId(options.videoObject)
 246   let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
 247
 248   if (videoFromDatabase) {
 249     // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
 250     if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
 251       const refreshOptions = {
 252         video: videoFromDatabase as MVideoThumbnail,
 253         fetchedType: fetchType,
 254         syncParam
 255       }
 256
 257       if (syncParam.refreshVideo === true) {
 258         videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
 259       } else {
 260         await JobQueue.Instance.createJobWithPromise({
 261           type: 'activitypub-refresher',
 262           payload: { type: 'video', url: videoFromDatabase.url }
 263         })
 264       }
 265     }
 266
 267     return { video: videoFromDatabase, created: false }
 268   }
 269
 270   const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
 271   if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
 272
 273   const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
 274   const videoChannel = actor.VideoChannel
 275   const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
 276
 277   await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
 278
 279   return { video: videoCreated, created: true, autoBlacklisted }
 280 }
 281
 282 async function updateVideoFromAP (options: {
 283   video: MVideoAccountLightBlacklistAllFiles
 284   videoObject: VideoTorrentObject
 285   account: MAccountIdActor
 286   channel: MChannelDefault
 287   overrideTo?: string[]
 288 }) {
 289   const { video, videoObject, account, channel, overrideTo } = options
 290
 291   logger.debug('Updating remote video "%s".', options.videoObject.uuid, { account, channel })
 292
 293   let videoFieldsSave: any
 294   const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE
 295   const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED
 296
 297   try {
 298     let thumbnailModel: MThumbnail
 299
 300     try {
 301       thumbnailModel = await createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
 302     } catch (err) {
 303       logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
 304     }
 305
 306     const videoUpdated = await sequelizeTypescript.transaction(async t => {
 307       const sequelizeOptions = { transaction: t }
 308
 309       videoFieldsSave = video.toJSON()
 310
 311       // Check actor has the right to update the video
 312       const videoChannel = video.VideoChannel
 313       if (videoChannel.Account.id !== account.id) {
 314         throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
 315       }
 316
 317       const to = overrideTo || videoObject.to
 318       const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
 319       video.name = videoData.name
 320       video.uuid = videoData.uuid
 321       video.url = videoData.url
 322       video.category = videoData.category
 323       video.licence = videoData.licence
 324       video.language = videoData.language
 325       video.description = videoData.description
 326       video.support = videoData.support
 327       video.nsfw = videoData.nsfw
 328       video.commentsEnabled = videoData.commentsEnabled
 329       video.downloadEnabled = videoData.downloadEnabled
 330       video.waitTranscoding = videoData.waitTranscoding
 331       video.state = videoData.state
 332       video.duration = videoData.duration
 333       video.createdAt = videoData.createdAt
 334       video.publishedAt = videoData.publishedAt
 335       video.originallyPublishedAt = videoData.originallyPublishedAt
 336       video.privacy = videoData.privacy
 337       video.channelId = videoData.channelId
 338       video.views = videoData.views
 339
 340       const videoUpdated = await video.save(sequelizeOptions) as MVideoFullLight
 341
 342       if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
 343
 344       if (videoUpdated.getPreview()) {
 345         const previewUrl = videoUpdated.getPreview().getFileUrl(videoUpdated)
 346         const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 347         await videoUpdated.addAndSaveThumbnail(previewModel, t)
 348       }
 349
 350       {
 351         const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url)
 352         const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
 353
 354         // Remove video files that do not exist anymore
 355         const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
 356         await Promise.all(destroyTasks)
 357
 358         // Update or add other one
 359         const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
 360         videoUpdated.VideoFiles = await Promise.all(upsertTasks)
 361       }
 362
 363       {
 364         const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles)
 365         const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
 366
 367         // Remove video playlists that do not exist anymore
 368         const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
 369         await Promise.all(destroyTasks)
 370
 371         let oldStreamingPlaylistFiles: MVideoFile[] = []
 372         for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
 373           oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
 374         }
 375
 376         videoUpdated.VideoStreamingPlaylists = []
 377
 378         for (const playlistAttributes of streamingPlaylistAttributes) {
 379           const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
 380                                      .then(([ streamingPlaylist ]) => streamingPlaylist)
 381
 382           const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
 383             .map(a => new VideoFileModel(a))
 384           const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
 385           await Promise.all(destroyTasks)
 386
 387           // Update or add other one
 388           const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
 389           streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
 390
 391           videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
 392         }
 393       }
 394
 395       {
 396         // Update Tags
 397         const tags = videoObject.tag
 398                                 .filter(isAPHashTagObject)
 399                                 .map(tag => tag.name)
 400         const tagInstances = await TagModel.findOrCreateTags(tags, t)
 401         await videoUpdated.$set('Tags', tagInstances, sequelizeOptions)
 402       }
 403
 404       {
 405         // Update captions
 406         await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
 407
 408         const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 409           return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, c.url, t)
 410         })
 411         await Promise.all(videoCaptionsPromises)
 412       }
 413
 414       return videoUpdated
 415     })
 416
 417     await autoBlacklistVideoIfNeeded({
 418       video: videoUpdated,
 419       user: undefined,
 420       isRemote: true,
 421       isNew: false,
 422       transaction: undefined
 423     })
 424
 425     if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) // Notify our users?
 426
 427     logger.info('Remote video with uuid %s updated', videoObject.uuid)
 428
 429     return videoUpdated
 430   } catch (err) {
 431     if (video !== undefined && videoFieldsSave !== undefined) {
 432       resetSequelizeInstance(video, videoFieldsSave)
 433     }
 434
 435     // This is just a debug because we will retry the insert
 436     logger.debug('Cannot update the remote video.', { err })
 437     throw err
 438   }
 439 }
 440
 441 async function refreshVideoIfNeeded (options: {
 442   video: MVideoThumbnail
 443   fetchedType: VideoFetchByUrlType
 444   syncParam: SyncParam
 445 }): Promise<MVideoThumbnail> {
 446   if (!options.video.isOutdated()) return options.video
 447
 448   // We need more attributes if the argument video was fetched with not enough joints
 449   const video = options.fetchedType === 'all'
 450     ? options.video as MVideoAccountLightBlacklistAllFiles
 451     : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
 452
 453   try {
 454     const { response, videoObject } = await fetchRemoteVideo(video.url)
 455     if (response.statusCode === 404) {
 456       logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
 457
 458       // Video does not exist anymore
 459       await video.destroy()
 460       return undefined
 461     }
 462
 463     if (videoObject === undefined) {
 464       logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
 465
 466       await video.setAsRefreshed()
 467       return video
 468     }
 469
 470     const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 471
 472     const updateOptions = {
 473       video,
 474       videoObject,
 475       account: channelActor.VideoChannel.Account,
 476       channel: channelActor.VideoChannel
 477     }
 478     await retryTransactionWrapper(updateVideoFromAP, updateOptions)
 479     await syncVideoExternalAttributes(video, videoObject, options.syncParam)
 480
 481     ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
 482
 483     return video
 484   } catch (err) {
 485     logger.warn('Cannot refresh video %s.', options.video.url, { err })
 486
 487     ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
 488
 489     // Don't refresh in loop
 490     await video.setAsRefreshed()
 491     return video
 492   }
 493 }
 494
 495 export {
 496   updateVideoFromAP,
 497   refreshVideoIfNeeded,
 498   federateVideoIfNeeded,
 499   fetchRemoteVideo,
 500   getOrCreateVideoAndAccountAndChannel,
 501   fetchRemoteVideoDescription,
 502   getOrCreateVideoChannelFromVideoObject
 503 }
 504
 505 // ---------------------------------------------------------------------------
 506
 507 function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
 508   const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
 509
 510   const urlMediaType = url.mediaType
 511   return mimeTypes.includes(urlMediaType) && urlMediaType.startsWith('video/')
 512 }
 513
 514 function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
 515   return url && url.mediaType === 'application/x-mpegURL'
 516 }
 517
 518 function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
 519   return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
 520 }
 521
 522 function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
 523   return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
 524 }
 525
 526 function isAPHashTagObject (url: any): url is ActivityHashTagObject {
 527   return url && url.type === 'Hashtag'
 528 }
 529
 530 async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) {
 531   logger.debug('Adding remote video %s.', videoObject.id)
 532
 533   const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
 534   const video = VideoModel.build(videoData) as MVideoThumbnail
 535
 536   const promiseThumbnail = createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
 537     .catch(err => {
 538       logger.error('Cannot create miniature from url.', { err })
 539       return undefined
 540     })
 541
 542   let thumbnailModel: MThumbnail
 543   if (waitThumbnail === true) {
 544     thumbnailModel = await promiseThumbnail
 545   }
 546
 547   const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
 548     const sequelizeOptions = { transaction: t }
 549
 550     const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
 551     videoCreated.VideoChannel = channel
 552
 553     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 554
 555     const previewIcon = getPreviewFromIcons(videoObject)
 556     const previewUrl = previewIcon
 557       ? previewIcon.url
 558       : buildRemoteVideoBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
 559     const previewModel = createPlaceholderThumbnail(previewUrl, videoCreated, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 560
 561     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 562
 563     // Process files
 564     const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
 565
 566     const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
 567     const videoFiles = await Promise.all(videoFilePromises)
 568
 569     const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
 570     videoCreated.VideoStreamingPlaylists = []
 571
 572     for (const playlistAttributes of streamingPlaylistsAttributes) {
 573       const playlistModel = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t })
 574
 575       const playlistFiles = videoFileActivityUrlToDBAttributes(playlistModel, playlistAttributes.tagAPObject)
 576       const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
 577       playlistModel.VideoFiles = await Promise.all(videoFilePromises)
 578
 579       videoCreated.VideoStreamingPlaylists.push(playlistModel)
 580     }
 581
 582     // Process tags
 583     const tags = videoObject.tag
 584                             .filter(isAPHashTagObject)
 585                             .map(t => t.name)
 586     const tagInstances = await TagModel.findOrCreateTags(tags, t)
 587     await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
 588
 589     // Process captions
 590     const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 591       return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, c.url, t)
 592     })
 593     await Promise.all(videoCaptionsPromises)
 594
 595     videoCreated.VideoFiles = videoFiles
 596     videoCreated.Tags = tagInstances
 597
 598     const autoBlacklisted = await autoBlacklistVideoIfNeeded({
 599       video: videoCreated,
 600       user: undefined,
 601       isRemote: true,
 602       isNew: true,
 603       transaction: t
 604     })
 605
 606     logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
 607
 608     return { autoBlacklisted, videoCreated }
 609   })
 610
 611   if (waitThumbnail === false) {
 612     // Error is already caught above
 613     // eslint-disable-next-line @typescript-eslint/no-floating-promises
 614     promiseThumbnail.then(thumbnailModel => {
 615       if (!thumbnailModel) return
 616
 617       thumbnailModel = videoCreated.id
 618
 619       return thumbnailModel.save()
 620     })
 621   }
 622
 623   return { autoBlacklisted, videoCreated }
 624 }
 625
 626 function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoTorrentObject, to: string[] = []) {
 627   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
 628     ? VideoPrivacy.PUBLIC
 629     : VideoPrivacy.UNLISTED
 630
 631   const duration = videoObject.duration.replace(/[^\d]+/, '')
 632   const language = videoObject.language?.identifier
 633
 634   const category = videoObject.category
 635     ? parseInt(videoObject.category.identifier, 10)
 636     : undefined
 637
 638   const licence = videoObject.licence
 639     ? parseInt(videoObject.licence.identifier, 10)
 640     : undefined
 641
 642   const description = videoObject.content || null
 643   const support = videoObject.support || null
 644
 645   return {
 646     name: videoObject.name,
 647     uuid: videoObject.uuid,
 648     url: videoObject.id,
 649     category,
 650     licence,
 651     language,
 652     description,
 653     support,
 654     nsfw: videoObject.sensitive,
 655     commentsEnabled: videoObject.commentsEnabled,
 656     downloadEnabled: videoObject.downloadEnabled,
 657     waitTranscoding: videoObject.waitTranscoding,
 658     state: videoObject.state,
 659     channelId: videoChannel.id,
 660     duration: parseInt(duration, 10),
 661     createdAt: new Date(videoObject.published),
 662     publishedAt: new Date(videoObject.published),
 663
 664     originallyPublishedAt: videoObject.originallyPublishedAt
 665       ? new Date(videoObject.originallyPublishedAt)
 666       : null,
 667
 668     updatedAt: new Date(videoObject.updated),
 669     views: videoObject.views,
 670     likes: 0,
 671     dislikes: 0,
 672     remote: true,
 673     privacy
 674   }
 675 }
 676
 677 function videoFileActivityUrlToDBAttributes (
 678   videoOrPlaylist: MVideo | MStreamingPlaylist,
 679   urls: (ActivityTagObject | ActivityUrlObject)[]
 680 ) {
 681   const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
 682
 683   if (fileUrls.length === 0) return []
 684
 685   const attributes: FilteredModelAttributes<VideoFileModel>[] = []
 686   for (const fileUrl of fileUrls) {
 687     // Fetch associated magnet uri
 688     const magnet = urls.filter(isAPMagnetUrlObject)
 689                        .find(u => u.height === fileUrl.height)
 690
 691     if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
 692
 693     const parsed = magnetUtil.decode(magnet.href)
 694     if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
 695       throw new Error('Cannot parse magnet URI ' + magnet.href)
 696     }
 697
 698     // Fetch associated metadata url, if any
 699     const metadata = urls.filter(isAPVideoFileMetadataObject)
 700                          .find(u => {
 701                            return u.height === fileUrl.height &&
 702                              u.fps === fileUrl.fps &&
 703                              u.rel.includes(fileUrl.mediaType)
 704                          })
 705
 706     const mediaType = fileUrl.mediaType
 707     const attribute = {
 708       extname: MIMETYPES.VIDEO.MIMETYPE_EXT[mediaType],
 709       infoHash: parsed.infoHash,
 710       resolution: fileUrl.height,
 711       size: fileUrl.size,
 712       fps: fileUrl.fps || -1,
 713       metadataUrl: metadata?.href,
 714
 715       // This is a video file owned by a video or by a streaming playlist
 716       videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id,
 717       videoStreamingPlaylistId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
 718     }
 719
 720     attributes.push(attribute)
 721   }
 722
 723   return attributes
 724 }
 725
 726 function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoTorrentObject, videoFiles: MVideoFile[]) {
 727   const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
 728   if (playlistUrls.length === 0) return []
 729
 730   const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
 731   for (const playlistUrlObject of playlistUrls) {
 732     const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
 733
 734     let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
 735
 736     // FIXME: backward compatibility introduced in v2.1.0
 737     if (files.length === 0) files = videoFiles
 738
 739     if (!segmentsSha256UrlObject) {
 740       logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
 741       continue
 742     }
 743
 744     const attribute = {
 745       type: VideoStreamingPlaylistType.HLS,
 746       playlistUrl: playlistUrlObject.href,
 747       segmentsSha256Url: segmentsSha256UrlObject.href,
 748       p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
 749       p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
 750       videoId: video.id,
 751       tagAPObject: playlistUrlObject.tag
 752     }
 753
 754     attributes.push(attribute)
 755   }
 756
 757   return attributes
 758 }
 759
 760 function getThumbnailFromIcons (videoObject: VideoTorrentObject) {
 761   let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
 762   // Fallback if there are not valid icons
 763   if (validIcons.length === 0) validIcons = videoObject.icon
 764
 765   return minBy(validIcons, 'width')
 766 }
 767
 768 function getPreviewFromIcons (videoObject: VideoTorrentObject) {
 769   const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
 770
 771   // FIXME: don't put a fallback here for compatibility with PeerTube <2.2
 772
 773   return maxBy(validIcons, 'width')
 774 }